Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
org.graylog2.bundles.BundleImporter Maven / Gradle / Ivy
/**
* This file is part of Graylog.
*
* Graylog is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Graylog is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Graylog. If not, see createdGrokPatterns = new HashMap<>();
private final Map createdInputs = new HashMap<>();
private final Map createdOutputs = new HashMap<>();
private final Map createdStreams = new HashMap<>();
private final Map createdDashboards = new HashMap<>();
private final Map createdLookupTables= new HashMap<>();
private final Map createdLookupCaches = new HashMap<>();
private final Map createdLookupDataAdapters = new HashMap<>();
private final Map outputsByReferenceId = new HashMap<>();
private final Map streamsByReferenceId = new HashMap<>();
// TODO: This class has become HUGE, it should be split into smaller classes.
@Inject
public BundleImporter(final InputService inputService,
final InputRegistry inputRegistry,
final ExtractorFactory extractorFactory,
final ConverterFactory converterFactory,
final StreamService streamService,
final StreamRuleService streamRuleService,
final IndexSetRegistry indexSetRegistry,
final OutputService outputService,
final DashboardService dashboardService,
final DashboardWidgetCreator dashboardWidgetCreator,
final ServerStatus serverStatus,
final MessageInputFactory messageInputFactory,
final InputLauncher inputLauncher,
final GrokPatternService grokPatternService,
final DBLookupTableService dbLookupTableService,
final DBCacheService dbCacheService,
final DBDataAdapterService dbDataAdapterService,
final LookupTableService lookupTableService,
final TimeRangeFactory timeRangeFactory,
final ClusterEventBus clusterBus,
final ObjectMapper objectMapper,
@Named("daemonScheduler") ScheduledExecutorService scheduler) {
this.inputService = inputService;
this.inputRegistry = inputRegistry;
this.extractorFactory = extractorFactory;
this.converterFactory = converterFactory;
this.streamService = streamService;
this.streamRuleService = streamRuleService;
this.indexSetRegistry = indexSetRegistry;
this.outputService = outputService;
this.dashboardService = dashboardService;
this.dashboardWidgetCreator = dashboardWidgetCreator;
this.serverStatus = serverStatus;
this.messageInputFactory = messageInputFactory;
this.inputLauncher = inputLauncher;
this.grokPatternService = grokPatternService;
this.dbLookupTableService = dbLookupTableService;
this.dbCacheService = dbCacheService;
this.dbDataAdapterService = dbDataAdapterService;
this.lookupTableService = lookupTableService;
this.timeRangeFactory = timeRangeFactory;
this.clusterBus = clusterBus;
this.objectMapper = objectMapper;
this.scheduler = scheduler;
}
public void runImport(final ConfigurationBundle bundle, final String userName) {
final String bundleId = bundle.getId();
try {
createGrokPatterns(bundleId, bundle.getGrokPatterns());
createInputs(bundleId, bundle.getInputs(), userName);
createOutputs(bundleId, bundle.getOutputs(), userName);
createStreams(bundleId, bundle.getStreams(), userName);
createDashboards(bundleId, bundle.getDashboards(), userName);
createLookupCaches(bundleId, bundle.getLookupCaches());
createLookupDataAdapters(bundleId, bundle.getLookupDataAdapters());
// Lookup tables should be created last because they need caches and adapters
createLookupTables(bundleId, bundle.getLookupTables());
} catch (Exception e) {
LOG.error("Error while creating entities in content pack. Starting rollback.", e);
if (!rollback()) {
LOG.error("Rollback unsuccessful.");
}
throw new RuntimeException(e);
}
}
private boolean rollback() {
boolean success = true;
try {
deleteCreatedDashboards();
} catch (Exception e) {
LOG.error("Error while removing dashboards during rollback.", e);
success = false;
}
try {
deleteCreatedStreams();
} catch (Exception e) {
LOG.error("Error while removing streams during rollback.", e);
success = false;
}
try {
deleteCreatedOutputs();
} catch (Exception e) {
LOG.error("Error while removing outputs during rollback.", e);
success = false;
}
try {
deleteCreatedGrokPatterns();
} catch (Exception e) {
LOG.error("Error while removing grok patterns during rollback.", e);
success = false;
}
try {
deleteCreatedInputs();
} catch (Exception e) {
LOG.error("Error while removing inputs during rollback.", e);
success = false;
}
try {
deleteCreatedLookupTables();
} catch (Exception e) {
LOG.error("Error while removing lookup tables during rollback.", e);
success = false;
}
try {
deleteCreatedLookupCaches();
} catch (Exception e) {
LOG.error("Error while removing lookup caches during rollback.", e);
success = false;
}
try {
deleteCreatedLookupDataAdapters();
} catch (Exception e) {
LOG.error("Error while removing lookup data adapters during rollback.", e);
success = false;
}
return success;
}
private void deleteCreatedGrokPatterns() throws NotFoundException {
for (String grokPatternName : createdGrokPatterns.keySet()) {
final org.graylog2.grok.GrokPattern grokPattern = grokPatternService.load(grokPatternName);
if (grokPattern.id() != null) {
LOG.debug("Deleting grok pattern \"{}\" from database", grokPatternName);
grokPatternService.delete(grokPattern.id());
} else {
LOG.debug("Couldn't find grok pattern \"{}\" in database", grokPatternName);
}
}
}
private void deleteCreatedInputs() throws NotFoundException {
for (Map.Entry entry : createdInputs.entrySet()) {
final String inputId = entry.getKey();
final MessageInput messageInput = entry.getValue();
LOG.debug("Terminating message input {}", inputId);
inputRegistry.remove(messageInput);
final org.graylog2.inputs.Input input = inputService.find(messageInput.getId());
inputService.destroy(input);
}
}
private void deleteCreatedOutputs() throws NotFoundException {
for (Map.Entry entry : createdOutputs.entrySet()) {
LOG.debug("Deleting output {} from database", entry.getKey());
outputService.destroy(entry.getValue());
}
}
private void deleteCreatedStreams() throws NotFoundException {
for (Map.Entry entry : createdStreams.entrySet()) {
final String streamId = entry.getKey();
LOG.debug("Deleting stream {} from database", streamId);
streamService.destroy(entry.getValue());
clusterBus.post(streamId);
}
}
private void deleteCreatedDashboards() {
for (Map.Entry entry : createdDashboards.entrySet()) {
final String dashboardId = entry.getKey();
LOG.debug("Deleting dashboard {} from database", dashboardId);
dashboardService.destroy(entry.getValue());
}
}
private void deleteCreatedLookupTables() {
for (String id : createdLookupTables.keySet()) {
LOG.debug("Deleting lookup table {} from database", id);
dbLookupTableService.delete(id);
}
clusterBus.post(LookupTablesDeleted.create(createdLookupTables.values()));
}
private void deleteCreatedLookupCaches() {
for (String id : createdLookupCaches.keySet()) {
LOG.debug("Deleting lookup cache {} from database", id);
dbCacheService.delete(id);
}
}
private void deleteCreatedLookupDataAdapters() {
for (String id : createdLookupDataAdapters.keySet()) {
LOG.debug("Deleting lookup data adapter {} from database", id);
dbDataAdapterService.delete(id);
}
}
private void createGrokPatterns(final String bundleId, final Set grokPatterns) throws ValidationException {
for (final GrokPattern grokPattern : grokPatterns) {
final org.graylog2.grok.GrokPattern createdGrokPattern = createGrokPattern(bundleId, grokPattern);
createdGrokPatterns.put(grokPattern.name(), createdGrokPattern);
}
clusterBus.post(GrokPatternsChangedEvent.create(Collections.emptySet(), createdGrokPatterns.keySet()));
}
private org.graylog2.grok.GrokPattern createGrokPattern(String bundleId, GrokPattern grokPattern) throws ValidationException {
final org.graylog2.grok.GrokPattern pattern = org.graylog2.grok.GrokPattern.create(null, grokPattern.name(), grokPattern.pattern(), bundleId);
return grokPatternService.save(pattern);
}
private void createInputs(final String bundleId, final Set extractors, final String userName)
throws org.graylog2.plugin.inputs.Extractor.ReservedFieldException, org.graylog2.ConfigurationException,
ExtractorFactory.NoSuchExtractorException, NotFoundException, ValidationException {
for (Extractor extractor : extractors) {
addExtractor(messageInput, extractor, userName);
}
}
private void addExtractor(
final MessageInput messageInput,
final Extractor extractorDescription,
final String userName)
throws NotFoundException, ValidationException, org.graylog2.ConfigurationException,
ExtractorFactory.NoSuchExtractorException, org.graylog2.plugin.inputs.Extractor.ReservedFieldException {
this.validateExtractor(extractorDescription);
final String extractorId = UUID.randomUUID().toString();
final org.graylog2.plugin.inputs.Extractor extractor = extractorFactory.factory(
extractorId,
extractorDescription.getTitle(),
extractorDescription.getOrder(),
extractorDescription.getCursorStrategy(),
extractorDescription.getType(),
extractorDescription.getSourceField(),
extractorDescription.getTargetField(),
extractorDescription.getConfiguration(),
userName,
createConverters(extractorDescription.getConverters()),
extractorDescription.getConditionType(),
extractorDescription.getConditionValue()
);
org.graylog2.inputs.Input mongoInput = inputService.find(messageInput.getPersistId());
inputService.addExtractor(mongoInput, extractor);
}
private List createConverters(final List requestedConverters) {
final ImmutableList.Builder converters = ImmutableList.builder();
for (final Converter converter : requestedConverters) {
try {
converters.add(converterFactory.create(converter.getType(), converter.getConfiguration()));
} catch (ConverterFactory.NoSuchConverterException e) {
LOG.warn("No such converter [" + converter.getType() + "]. Skipping.", e);
} catch (org.graylog2.ConfigurationException e) {
LOG.warn("Missing configuration for [" + converter.getType() + "]. Skipping.", e);
}
}
return converters.build();
}
private void addStaticFields(final MessageInput messageInput, final Map staticFields)
throws NotFoundException, ValidationException {
for (Map.Entry staticField : staticFields.entrySet()) {
addStaticField(messageInput, staticField.getKey(), staticField.getValue());
}
}
private void addStaticField(final MessageInput messageInput, final String key, final String value)
throws ValidationException, NotFoundException {
// Check if key is a valid message key.
if (!Message.validKey(key)) {
final String errorMessage = "Invalid key: [" + key + "]";
LOG.error(errorMessage);
throw new ValidationException(errorMessage);
}
if (isNullOrEmpty(key) || isNullOrEmpty(value)) {
final String errorMessage = "Missing attributes: key=[" + key + "], value=[" + value + "]";
LOG.error(errorMessage);
throw new ValidationException(errorMessage);
}
if (Message.RESERVED_FIELDS.contains(key) && !Message.RESERVED_SETTABLE_FIELDS.contains(key)) {
final String errorMessage = "Cannot add static field. Field [" + key + "] is reserved.";
LOG.error(errorMessage);
throw new ValidationException(errorMessage);
}
messageInput.addStaticField(key, value);
org.graylog2.inputs.Input mongoInput = inputService.find(messageInput.getPersistId());
inputService.addStaticField(mongoInput, key, value);
}
private Map buildMongoDbInput(
final Input input,
final String userName,
final DateTime createdAt,
final String bundleId) {
final ImmutableMap.Builder inputData = ImmutableMap.builder();
inputData.put(MessageInput.FIELD_TITLE, input.getTitle());
inputData.put(MessageInput.FIELD_TYPE, input.getType());
inputData.put(MessageInput.FIELD_CREATOR_USER_ID, userName);
inputData.put(MessageInput.FIELD_CONFIGURATION, input.getConfiguration());
inputData.put(MessageInput.FIELD_CREATED_AT, createdAt);
inputData.put(MessageInput.FIELD_CONTENT_PACK, bundleId);
if (input.isGlobal()) {
inputData.put(MessageInput.FIELD_GLOBAL, true);
} else {
inputData.put(MessageInput.FIELD_NODE_ID, serverStatus.getNodeId().toString());
}
return inputData.build();
}
private void createOutputs(final String bundleId, final Set outputs, final String userName)
throws ValidationException {
for (final Output outputDescription : outputs) {
final org.graylog2.plugin.streams.Output output = createOutput(bundleId, outputDescription, userName);
createdOutputs.put(output.getId(), output);
}
}
private org.graylog2.plugin.streams.Output createOutput(final String bundleId, final Output outputDescription, final String userName)
throws ValidationException {
final String referenceId = outputDescription.getId();
final org.graylog2.plugin.streams.Output output = outputService.create(OutputImpl.create(
outputDescription.getId(),
outputDescription.getTitle(),
outputDescription.getType(),
userName,
outputDescription.getConfiguration(),
Tools.nowUTC().toDate(),
bundleId));
if (!isNullOrEmpty(referenceId)) {
outputsByReferenceId.put(referenceId, output);
}
return output;
}
private void createStreams(final String bundleId, final Set streams, final String userName)
throws ValidationException {
for (final Stream streamDescription : streams) {
final String referenceId = streamDescription.getId();
final org.graylog2.plugin.streams.Stream stream = createStream(bundleId, streamDescription, userName);
createdStreams.put(stream.getId(), stream);
if (!isNullOrEmpty(referenceId)) {
streamsByReferenceId.put(referenceId, stream);
}
}
final ImmutableSet streamIds = ImmutableSet.copyOf(createdStreams.keySet());
clusterBus.post(StreamsChangedEvent.create(streamIds));
}
private org.graylog2.plugin.streams.Stream createStream(final String bundleId, final Stream streamDescription, final String userName)
throws ValidationException {
// We cannot create streams without having a default index set.
final IndexSet indexSet = indexSetRegistry.getDefault();
final Map streamData = ImmutableMap.builder()
.put(StreamImpl.FIELD_TITLE, streamDescription.getTitle())
.put(StreamImpl.FIELD_DESCRIPTION, streamDescription.getDescription())
.put(StreamImpl.FIELD_DISABLED, streamDescription.isDisabled())
.put(StreamImpl.FIELD_MATCHING_TYPE, streamDescription.getMatchingType().name())
.put(StreamImpl.FIELD_CREATOR_USER_ID, userName)
.put(StreamImpl.FIELD_CREATED_AT, Tools.nowUTC())
.put(StreamImpl.FIELD_CONTENT_PACK, bundleId)
.put(StreamImpl.FIELD_DEFAULT_STREAM, streamDescription.isDefaultStream())
.put(StreamImpl.FIELD_INDEX_SET_ID, indexSet.getConfig().id())
.build();
final String defaultStreamId = org.graylog2.plugin.streams.Stream.DEFAULT_STREAM_ID;
final ObjectId id = streamDescription.isDefaultStream() ? new ObjectId(defaultStreamId) : new ObjectId(streamDescription.getId());
final org.graylog2.plugin.streams.Stream stream = new StreamImpl(id, streamData, Collections.emptyList(), Collections.emptySet(), indexSet);
final String streamId = streamService.save(stream);
if (streamDescription.getStreamRules() != null) {
for (StreamRule streamRule : streamDescription.getStreamRules()) {
final Map streamRuleData = ImmutableMap.builder()
.put(StreamRuleImpl.FIELD_TYPE, streamRule.getType().toInteger())
.put(StreamRuleImpl.FIELD_VALUE, streamRule.getValue())
.put(StreamRuleImpl.FIELD_FIELD, streamRule.getField())
.put(StreamRuleImpl.FIELD_INVERTED, streamRule.isInverted())
.put(StreamRuleImpl.FIELD_STREAM_ID, new ObjectId(streamId))
.put(StreamRuleImpl.FIELD_CONTENT_PACK, bundleId)
.put(StreamRuleImpl.FIELD_DESCRIPTION, streamRule.getDescription())
.build();
streamRuleService.save(new StreamRuleImpl(streamRuleData));
}
}
for (final String outputId : streamDescription.getOutputs()) {
if (isNullOrEmpty(outputId)) {
LOG.warn("Couldn't find referenced output <{}> for stream <{}>", outputId, streamDescription.getTitle());
} else {
streamService.addOutput(stream, outputsByReferenceId.get(outputId));
}
}
return stream;
}
private void createDashboards(final String bundleId, final Set dashboards, final String userName)
throws org.graylog2.dashboards.widgets.DashboardWidget.NoSuchWidgetTypeException, InvalidWidgetConfigurationException, InvalidRangeParametersException, ValidationException {
for (final Dashboard dashboard : dashboards) {
org.graylog2.dashboards.Dashboard createdDashboard = createDashboard(bundleId, dashboard, userName);
createdDashboards.put(createdDashboard.getId(), createdDashboard);
}
}
private org.graylog2.dashboards.Dashboard createDashboard(final String bundleId, final Dashboard dashboardDescription, final String userName)
throws ValidationException, org.graylog2.dashboards.widgets.DashboardWidget.NoSuchWidgetTypeException, InvalidRangeParametersException, InvalidWidgetConfigurationException {
// Create dashboard.
final Map dashboardData = new HashMap<>();
dashboardData.put(DashboardImpl.FIELD_TITLE, dashboardDescription.getTitle());
dashboardData.put(DashboardImpl.FIELD_DESCRIPTION, dashboardDescription.getDescription());
dashboardData.put(DashboardImpl.FIELD_CONTENT_PACK, bundleId);
dashboardData.put(DashboardImpl.FIELD_CREATOR_USER_ID, userName);
dashboardData.put(DashboardImpl.FIELD_CREATED_AT, Tools.nowUTC());
final org.graylog2.dashboards.Dashboard dashboard = new DashboardImpl(dashboardData);
final String dashboardId = dashboardService.save(dashboard);
final ImmutableList.Builder widgetPositions = ImmutableList.builder();
for (DashboardWidget dashboardWidget : dashboardDescription.getDashboardWidgets()) {
final org.graylog2.dashboards.widgets.DashboardWidget widget = createDashboardWidget(dashboardWidget, userName);
dashboardService.addWidget(dashboard, widget);
final WidgetPositionsRequest.WidgetPosition widgetPosition = WidgetPositionsRequest.WidgetPosition.create(widget.getId(),
dashboardWidget.getCol(), dashboardWidget.getRow(), dashboardWidget.getHeight(), dashboardWidget.getWidth());
widgetPositions.add(widgetPosition);
}
// FML: We need to reload the dashboard because not all fields (I'm looking at you, "widgets") is set in the
// Dashboard instance used before.
final org.graylog2.dashboards.Dashboard persistedDashboard;
try {
persistedDashboard = dashboardService.load(dashboardId);
dashboardService.updateWidgetPositions(persistedDashboard, WidgetPositionsRequest.create(widgetPositions.build()));
} catch (NotFoundException e) {
LOG.error("Failed to load dashboard with id " + dashboardId, e);
}
return dashboard;
}
@SuppressWarnings("unchecked")
private org.graylog2.dashboards.widgets.DashboardWidget createDashboardWidget(
final DashboardWidget dashboardWidget, final String userName)
throws InvalidRangeParametersException, org.graylog2.dashboards.widgets.DashboardWidget.NoSuchWidgetTypeException, InvalidWidgetConfigurationException {
final String type = dashboardWidget.getType();
final Map config = dashboardWidget.getConfiguration();
// Replace "stream_id" in config if it's set
final String streamReference = (String) config.get("stream_id");
if (!isNullOrEmpty(streamReference)) {
final org.graylog2.plugin.streams.Stream stream = streamsByReferenceId.get(streamReference);
if (null != stream) {
config.put("stream_id", stream.getId());
} else {
LOG.warn("Couldn't find referenced stream {}", streamReference);
}
}
// Build timerange.
final Map timerangeConfig = (Map) config.get("timerange");
final TimeRange timeRange = timeRangeFactory.create(timerangeConfig);
final String widgetId = UUID.randomUUID().toString();
return dashboardWidgetCreator.buildDashboardWidget(type,
widgetId, dashboardWidget.getDescription(), dashboardWidget.getCacheTime(),
config, timeRange, userName);
}
private void createLookupTables(String bundleId, Set lookupTables) {
for (LookupTableBundle bundle : lookupTables) {
final Optional cacheDto = dbCacheService.get(bundle.getCacheName());
final Optional adapterDto = dbDataAdapterService.get(bundle.getDataAdapterName());
if (!cacheDto.isPresent() || !adapterDto.isPresent()) {
LOG.warn("Skipping content pack import of lookup table <{}> ({}) due to missing cache or data adapter", bundle.getName(), bundle.getTitle());
continue;
}
final LookupTableDto dto = dbLookupTableService.save(LookupTableDto.builder()
.title(bundle.getTitle())
.description(bundle.getDescription())
.name(bundle.getName())
.cacheId(cacheDto.get().id())
.dataAdapterId(adapterDto.get().id())
.defaultSingleValue(bundle.getDefaultSingleValue())
.defaultSingleValueType(bundle.getDefaultSingleValueType())
.defaultMultiValue(bundle.getDefaultMultiValue())
.defaultMultiValueType(bundle.getDefaultMultiValueType())
.contentPack(bundleId)
.build());
createdLookupTables.put(dto.id(), dto);
}
if (!createdLookupDataAdapters.isEmpty()) {
clusterBus.post(LookupTablesUpdated.create(createdLookupTables.values()));
}
}
private void createLookupCaches(String bundleId, Set lookupCaches) {
for (LookupCacheBundle bundle : lookupCaches) {
final CacheDto dto = dbCacheService.save(CacheDto.builder()
.title(bundle.getTitle())
.description(bundle.getDescription())
.name(bundle.getName())
.contentPack(bundleId)
.config(objectMapper.convertValue(bundle.getConfig(), LookupCacheConfiguration.class))
.build());
createdLookupCaches.put(dto.id(), dto);
}
if (!createdLookupCaches.isEmpty()) {
clusterBus.post(CachesUpdated.create(createdLookupCaches.keySet()));
}
final Collection caches = lookupTableService.getCaches(createdLookupCaches.keySet());
final CountDownLatch latch = new CountDownLatch(caches.size());
caches.forEach(c -> c.addListener(new LatchUpdaterListener(latch), scheduler));
try {
if (!latch.await(30, TimeUnit.SECONDS)) {
LOG.warn("Starting imported Lookup Table Caches did not finish within 30 seconds. A server restart might be required for imported Lookup Tables to function.");
}
} catch (InterruptedException e) {
LOG.warn("Starting imported Lookup Table Caches did not finish properly. A server restart might be required for imported Lookup Tables to function: ", e);
}
}
private void createLookupDataAdapters(String bundleId, Set lookupDataAdapters) {
for (LookupDataAdapterBundle bundle : lookupDataAdapters) {
final DataAdapterDto dto = dbDataAdapterService.save(DataAdapterDto.builder()
.title(bundle.getTitle())
.description(bundle.getDescription())
.name(bundle.getName())
.contentPack(bundleId)
.config(objectMapper.convertValue(bundle.getConfig(), LookupDataAdapterConfiguration.class))
.build());
createdLookupDataAdapters.put(dto.id(), dto);
}
if (!createdLookupDataAdapters.isEmpty()) {
clusterBus.post(DataAdaptersUpdated.create(createdLookupDataAdapters.keySet()));
}
final Collection dataAdapters = lookupTableService.getDataAdapters(createdLookupDataAdapters.keySet());
final CountDownLatch latch = new CountDownLatch(dataAdapters.size());
dataAdapters.forEach(da -> da.addListener(new LatchUpdaterListener(latch), scheduler));
try {
if (!latch.await(30, TimeUnit.SECONDS)) {
LOG.warn("Starting imported Lookup Table Data Adapters did not finish within 30 seconds. A server restart might be required for imported Lookup Tables to function.");
}
} catch (InterruptedException e) {
LOG.warn("Starting imported Lookup Table Data Adapters did not finish properly. A server restart might be required for imported Lookup Tables to function: ", e);
}
}
}
