org.graylog.plugins.map.geoip.GeoIpResolverEngine Maven / Gradle / Ivy
/*
* Copyright (C) 2020 Graylog, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
* as published by MongoDB, Inc.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Server Side Public License for more details.
*
* You should have received a copy of the Server Side Public License
* along with this program. If not, see
* .
*/
package org.graylog.plugins.map.geoip;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Timer;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.net.InetAddresses;
import org.apache.commons.lang3.StringUtils;
import org.graylog.plugins.map.config.GeoIpResolverConfig;
import org.graylog.plugins.map.config.S3GeoIpFileService;
import org.graylog2.plugin.Message;
import org.graylog2.utilities.ReservedIpChecker;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.Nullable;
import java.net.InetAddress;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static com.codahale.metrics.MetricRegistry.name;
public class GeoIpResolverEngine {
private static final Logger LOG = LoggerFactory.getLogger(GeoIpResolverEngine.class);
/**
* This is a list of schema fields defined in package org.graylog.schema as of 2022-07-15 which we want to scan for IP addresses. If the schema changes, or we need to add/remove fields,
* this list must be updated, until a better way of defining schema fields is developed, that allows iteration.
*/
private static final String[] KNOWN_SCHEMA_IP_FIELDS = {org.graylog.schema.DestinationFields.DESTINATION_IP,
org.graylog.schema.DestinationFields.DESTINATION_NAT_IP,
org.graylog.schema.EventFields.EVENT_OBSERVER_IP,
org.graylog.schema.SourceFields.SOURCE_NAT_IP,
org.graylog.schema.NetworkFields.NETWORK_FORWARDED_IP,
org.graylog.schema.SourceFields.SOURCE_IP,
org.graylog.schema.HostFields.HOST_IP};
/**
* A mapping of fields (per the Graylog Schema) that to search that contain IP addresses. When the user opts to
* enforce the Graylog Schema ONLY these fields will be checked; otherwise, all message fields will be checked.
* to see if they have valid Geo IP information.
*
*
* The mapping is field name -> new message field prefix, where field_name is the field name expected in the message
* which will be searched in the GeoIP database, and the new message field prefix is the prefix for the new field with the GeoIP data
* that will be inserted into the message.
*
*/
private final Map ipAddressFields = Stream.of(KNOWN_SCHEMA_IP_FIELDS)
.collect(Collectors.toMap(e -> e, mapFieldNameToPrefix()));
private final GeoIpResolver ipLocationResolver;
private final GeoIpResolver ipAsnResolver;
private final boolean enabled;
private final boolean enforceGraylogSchema;
public GeoIpResolverEngine(GeoIpVendorResolverService resolverService, GeoIpResolverConfig config, S3GeoIpFileService s3GeoIpFileService,
MetricRegistry metricRegistry) {
Timer resolveTime = metricRegistry.timer(name(GeoIpResolverEngine.class, "resolveTime"));
enforceGraylogSchema = config.enforceGraylogSchema();
if (config.useS3()) {
config = config.toBuilder()
.asnDbPath(s3GeoIpFileService.getActiveAsnFile())
.cityDbPath(s3GeoIpFileService.getActiveCityFile())
.build();
}
ipLocationResolver = resolverService.createCityResolver(config, resolveTime);
ipAsnResolver = resolverService.createAsnResolver(config, resolveTime);
LOG.debug("Created Geo IP Resolvers for '{}'", config.databaseVendorType());
LOG.debug("'{}' Status Enabled: {}", ipLocationResolver.getClass().getSimpleName(), ipLocationResolver.isEnabled());
LOG.debug("'{}' Status Enabled: {}", ipAsnResolver.getClass().getSimpleName(), ipAsnResolver.isEnabled());
this.enabled = ipLocationResolver.isEnabled() || ipAsnResolver.isEnabled();
}
public boolean filter(Message message) {
if (!enabled) {
return false;
}
List ipFields = getIpAddressFields(message);
for (String key : ipFields) {
Object fieldValue = message.getField(key);
final InetAddress address = getValidRoutableInetAddress(fieldValue);
if (address == null) {
continue;
}
// For reserved IPs just mark as reserved. Otherwise, enforce Graylog schema on only relevant IP fields
// or add legacy fields on all IP fields in the message if enforcement is disabled.
final String prefix = enforceGraylogSchema ? ipAddressFields.getOrDefault(key, key) : key;
if (ReservedIpChecker.getInstance().isReservedIpAddress(address.getHostAddress())) {
message.addField(prefix + "_reserved_ip", true);
} else if (enforceGraylogSchema) {
addGIMGeoIpDataIfPresent(message, address, prefix);
} else {
addLegacyGeoIpDataIfPresent(message, address, prefix);
}
}
return true;
}
// Pre-4.3 logic for adding geo fields to message.
private void addLegacyGeoIpDataIfPresent(Message message, InetAddress address, String key) {
ipLocationResolver.getGeoIpData(address).ifPresent(locationInformation -> {
// We will store the coordinates as a "lat,long" string
message.addField(key + "_geolocation", locationInformation.latitude() + "," + locationInformation.longitude());
message.addField(key + "_country_code", locationInformation.countryIsoCode());
message.addField(key + "_city_name", locationInformation.cityName());
});
}
private void addGIMGeoIpDataIfPresent(Message message, InetAddress address, String newFieldPrefix) {
ipLocationResolver.getGeoIpData(address).ifPresent(locationInformation -> {
message.addField(newFieldPrefix + "_geo_coordinates", locationInformation.latitude() + "," + locationInformation.longitude());
message.addField(newFieldPrefix + "_geo_country_iso", locationInformation.countryIsoCode());
message.addField(newFieldPrefix + "_geo_city", locationInformation.cityName());
message.addField(newFieldPrefix + "_geo_region", locationInformation.region());
message.addField(newFieldPrefix + "_geo_timezone", locationInformation.timeZone());
if (areValidGeoNames(locationInformation.countryName())) {
message.addField(newFieldPrefix + "_geo_country", locationInformation.countryName());
}
if (areValidGeoNames(locationInformation.cityName(), locationInformation.countryIsoCode())) {
String name = String.format(Locale.ENGLISH, "%s, %s", locationInformation.cityName(), locationInformation.countryIsoCode());
message.addField(newFieldPrefix + "_geo_name", name);
}
});
ipAsnResolver.getGeoIpData(address).ifPresent(info -> {
message.addField(newFieldPrefix + "_as_organization", info.organization());
message.addField(newFieldPrefix + "_as_number", info.asn());
});
}
/**
* Get the message fields that will be checked for IP addresses.
*
*
* If the user has chosen NOT to enforce the Graylog Schema, then all fields will be checked as any field could
* have an IP address.
*
*
* @param message message
* @return a list of field that may have an IP address
*/
@VisibleForTesting
List getIpAddressFields(Message message) {
return message.getFieldNames()
.stream()
.filter(e -> (!enforceGraylogSchema || ipAddressFields.containsKey(e))
&& !e.startsWith(Message.INTERNAL_FIELD_PREFIX))
.collect(Collectors.toList());
}
private InetAddress getValidRoutableInetAddress(Object fieldValue) {
final InetAddress ipAddress;
if (fieldValue instanceof InetAddress) {
ipAddress = (InetAddress) fieldValue;
} else if (fieldValue instanceof String) {
ipAddress = getIpFromFieldValue((String) fieldValue);
} else {
ipAddress = null;
}
return ipAddress;
}
private boolean areValidGeoNames(String... names) {
for (String name : names) {
if (StringUtils.isBlank(name) || "N/A".equalsIgnoreCase(name)) {
return false;
}
}
return true;
}
@Nullable
@VisibleForTesting
InetAddress getIpFromFieldValue(String fieldValue) {
try {
return InetAddresses.forString(fieldValue.trim());
} catch (IllegalArgumentException e) {
// Do nothing, field is not an IP
}
return null;
}
private static Function mapFieldNameToPrefix() {
return string -> string.replace("_ip", "");
}
}