Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
org.graylog2.bundles.BundleImporter Maven / Gradle / Ivy
/**
* This file is part of Graylog.
*
* Graylog is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Graylog is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Graylog. If not, see createdGrokPatterns = new HashMap<>();
private final Map createdInputs = new HashMap<>();
private final Map createdOutputs = new HashMap<>();
private final Map createdStreams = new HashMap<>();
private final Map createdDashboards = new HashMap<>();
private final Map outputsByReferenceId = new HashMap<>();
private final Map streamsByReferenceId = new HashMap<>();
@Inject
public BundleImporter(final InputService inputService,
final InputRegistry inputRegistry,
final ExtractorFactory extractorFactory,
final StreamService streamService,
final StreamRuleService streamRuleService,
final OutputService outputService,
final DashboardService dashboardService,
final DashboardWidgetCreator dashboardWidgetCreator,
final ServerStatus serverStatus,
final Searches searches,
final MessageInputFactory messageInputFactory,
final InputLauncher inputLauncher,
final GrokPatternService grokPatternService,
final TimeRangeFactory timeRangeFactory) {
this.inputService = inputService;
this.inputRegistry = inputRegistry;
this.extractorFactory = extractorFactory;
this.streamService = streamService;
this.streamRuleService = streamRuleService;
this.outputService = outputService;
this.dashboardService = dashboardService;
this.dashboardWidgetCreator = dashboardWidgetCreator;
this.serverStatus = serverStatus;
this.searches = searches;
this.messageInputFactory = messageInputFactory;
this.inputLauncher = inputLauncher;
this.grokPatternService = grokPatternService;
this.timeRangeFactory = timeRangeFactory;
}
public void runImport(final ConfigurationBundle bundle, final String userName) {
final String bundleId = bundle.getId();
try {
createGrokPatterns(bundleId, bundle.getGrokPatterns());
createInputs(bundleId, bundle.getInputs(), userName);
createOutputs(bundleId, bundle.getOutputs(), userName);
createStreams(bundleId, bundle.getStreams(), userName);
createDashboards(bundleId, bundle.getDashboards(), userName);
} catch (Exception e) {
LOG.error("Error while creating entities in content pack. Starting rollback.", e);
if (!rollback()) {
LOG.error("Rollback unsuccessful.");
}
Throwables.propagate(e);
}
}
private boolean rollback() {
boolean success = true;
try {
deleteCreatedDashboards();
} catch (Exception e) {
LOG.error("Error while removing dashboards during rollback.", e);
success = false;
}
try {
deleteCreatedStreams();
} catch (Exception e) {
LOG.error("Error while removing streams during rollback.", e);
success = false;
}
try {
deleteCreatedOutputs();
} catch (Exception e) {
LOG.error("Error while removing outputs during rollback.", e);
success = false;
}
try {
deleteCreatedGrokPatterns();
} catch (Exception e) {
LOG.error("Error while removing grok patterns during rollback.", e);
success = false;
}
try {
deleteCreatedInputs();
} catch (Exception e) {
LOG.error("Error while removing inputs during rollback.", e);
success = false;
}
return success;
}
private void deleteCreatedGrokPatterns() throws NotFoundException {
for (String grokPatternName : createdGrokPatterns.keySet()) {
final org.graylog2.grok.GrokPattern grokPattern = grokPatternService.load(grokPatternName);
if(grokPattern.id != null) {
LOG.debug("Deleting grok pattern \"{}\" from database", grokPatternName);
grokPatternService.delete(grokPattern.id.toHexString());
} else {
LOG.debug("Couldn't find grok pattern \"{}\" in database", grokPatternName);
}
}
}
private void deleteCreatedInputs() throws NotFoundException {
for (Map.Entry entry : createdInputs.entrySet()) {
final String inputId = entry.getKey();
final MessageInput messageInput = entry.getValue();
LOG.debug("Terminating message input {}", inputId);
inputRegistry.remove(messageInput);
final org.graylog2.inputs.Input input = inputService.find(messageInput.getId());
inputService.destroy(input);
}
}
private void deleteCreatedOutputs() throws NotFoundException {
for (Map.Entry entry : createdOutputs.entrySet()) {
LOG.debug("Deleting output {} from database", entry.getKey());
outputService.destroy(entry.getValue());
}
}
private void deleteCreatedStreams() throws NotFoundException {
for (Map.Entry entry : createdStreams.entrySet()) {
LOG.debug("Deleting stream {} from database", entry.getKey());
streamService.destroy(entry.getValue());
}
}
private void deleteCreatedDashboards() {
for (Map.Entry entry : createdDashboards.entrySet()) {
final String dashboardId = entry.getKey();
LOG.debug("Deleting dashboard {} from database", dashboardId);
dashboardService.destroy(entry.getValue());
}
}
private void createGrokPatterns(final String bundleId, final Set grokPatterns) throws ValidationException {
for (final GrokPattern grokPattern : grokPatterns) {
final org.graylog2.grok.GrokPattern createdGrokPattern = createGrokPattern(bundleId, grokPattern);
createdGrokPatterns.put(grokPattern.name(), createdGrokPattern);
}
}
private org.graylog2.grok.GrokPattern createGrokPattern(String bundleId, GrokPattern grokPattern) throws ValidationException {
final org.graylog2.grok.GrokPattern pattern = new org.graylog2.grok.GrokPattern();
pattern.name = grokPattern.name();
pattern.pattern = grokPattern.pattern();
pattern.contentPack = bundleId;
return grokPatternService.save(pattern);
}
private void createInputs(final String bundleId, final Set extractors, final String userName)
throws org.graylog2.plugin.inputs.Extractor.ReservedFieldException, org.graylog2.ConfigurationException,
ExtractorFactory.NoSuchExtractorException, NotFoundException, ValidationException {
for (Extractor extractor : extractors) {
addExtractor(messageInput, extractor, userName);
}
}
private void addExtractor(
final MessageInput messageInput,
final Extractor extractorDescription,
final String userName)
throws NotFoundException, ValidationException, org.graylog2.ConfigurationException,
ExtractorFactory.NoSuchExtractorException, org.graylog2.plugin.inputs.Extractor.ReservedFieldException {
this.validateExtractor(extractorDescription);
final String extractorId = UUID.randomUUID().toString();
final org.graylog2.plugin.inputs.Extractor extractor = extractorFactory.factory(
extractorId,
extractorDescription.getTitle(),
extractorDescription.getOrder(),
extractorDescription.getCursorStrategy(),
extractorDescription.getType(),
extractorDescription.getSourceField(),
extractorDescription.getTargetField(),
extractorDescription.getConfiguration(),
userName,
createConverters(extractorDescription.getConverters()),
extractorDescription.getConditionType(),
extractorDescription.getConditionValue()
);
org.graylog2.inputs.Input mongoInput = inputService.find(messageInput.getPersistId());
inputService.addExtractor(mongoInput, extractor);
}
private List createConverters(final List requestedConverters) {
final ImmutableList.Builder converters = ImmutableList.builder();
for (final Converter converter : requestedConverters) {
try {
converters.add(ConverterFactory.factory(converter.getType(), converter.getConfiguration()));
} catch (ConverterFactory.NoSuchConverterException e) {
LOG.warn("No such converter [" + converter.getType() + "]. Skipping.", e);
} catch (org.graylog2.ConfigurationException e) {
LOG.warn("Missing configuration for [" + converter.getType() + "]. Skipping.", e);
}
}
return converters.build();
}
private void addStaticFields(final MessageInput messageInput, final Map staticFields)
throws NotFoundException, ValidationException {
for (Map.Entry staticField : staticFields.entrySet()) {
addStaticField(messageInput, staticField.getKey(), staticField.getValue());
}
}
private void addStaticField(final MessageInput messageInput, final String key, final String value)
throws ValidationException, NotFoundException {
// Check if key is a valid message key.
if (!Message.validKey(key)) {
final String errorMessage = "Invalid key: [" + key + "]";
LOG.error(errorMessage);
throw new ValidationException(errorMessage);
}
if (isNullOrEmpty(key) || isNullOrEmpty(value)) {
final String errorMessage = "Missing attributes: key=[" + key + "], value=[" + value + "]";
LOG.error(errorMessage);
throw new ValidationException(errorMessage);
}
if (Message.RESERVED_FIELDS.contains(key) && !Message.RESERVED_SETTABLE_FIELDS.contains(key)) {
final String errorMessage = "Cannot add static field. Field [" + key + "] is reserved.";
LOG.error(errorMessage);
throw new ValidationException(errorMessage);
}
messageInput.addStaticField(key, value);
org.graylog2.inputs.Input mongoInput = inputService.find(messageInput.getPersistId());
inputService.addStaticField(mongoInput, key, value);
}
private Map buildMongoDbInput(
final Input input,
final String userName,
final DateTime createdAt,
final String bundleId) {
final ImmutableMap.Builder inputData = ImmutableMap.builder();
inputData.put(MessageInput.FIELD_TITLE, input.getTitle());
inputData.put(MessageInput.FIELD_TYPE, input.getType());
inputData.put(MessageInput.FIELD_CREATOR_USER_ID, userName);
inputData.put(MessageInput.FIELD_CONFIGURATION, input.getConfiguration());
inputData.put(MessageInput.FIELD_CREATED_AT, createdAt);
inputData.put(MessageInput.FIELD_CONTENT_PACK, bundleId);
if (input.isGlobal()) {
inputData.put(MessageInput.FIELD_GLOBAL, true);
} else {
inputData.put(MessageInput.FIELD_NODE_ID, serverStatus.getNodeId().toString());
}
return inputData.build();
}
private void createOutputs(final String bundleId, final Set outputs, final String userName)
throws ValidationException {
for (final Output outputDescription : outputs) {
final org.graylog2.plugin.streams.Output output = createOutput(bundleId, outputDescription, userName);
createdOutputs.put(output.getId(), output);
}
}
private org.graylog2.plugin.streams.Output createOutput(final String bundleId, final Output outputDescription, final String userName)
throws ValidationException {
final String referenceId = outputDescription.getId();
final org.graylog2.plugin.streams.Output output = outputService.create(OutputImpl.create(
outputDescription.getId(),
outputDescription.getTitle(),
outputDescription.getType(),
userName,
outputDescription.getConfiguration(),
Tools.nowUTC().toDate(),
bundleId));
if (!isNullOrEmpty(referenceId)) {
outputsByReferenceId.put(referenceId, output);
}
return output;
}
private void createStreams(final String bundleId, final Set streams, final String userName)
throws ValidationException {
for (final Stream streamDescription : streams) {
final String referenceId = streamDescription.getId();
final org.graylog2.plugin.streams.Stream stream = createStream(bundleId, streamDescription, userName);
createdStreams.put(stream.getId(), stream);
if (!isNullOrEmpty(referenceId)) {
streamsByReferenceId.put(referenceId, stream);
}
}
}
private org.graylog2.plugin.streams.Stream createStream(final String bundleId, final Stream streamDescription, final String userName)
throws ValidationException {
final ImmutableMap.Builder streamData = ImmutableMap.builder();
streamData.put(StreamImpl.FIELD_TITLE, streamDescription.getTitle());
streamData.put(StreamImpl.FIELD_DESCRIPTION, streamDescription.getDescription());
streamData.put(StreamImpl.FIELD_DISABLED, streamDescription.isDisabled());
streamData.put(StreamImpl.FIELD_MATCHING_TYPE, streamDescription.getMatchingType().name());
streamData.put(StreamImpl.FIELD_CREATOR_USER_ID, userName);
streamData.put(StreamImpl.FIELD_CREATED_AT, Tools.nowUTC());
streamData.put(StreamImpl.FIELD_CONTENT_PACK, bundleId);
final org.graylog2.plugin.streams.Stream stream = streamService.create(streamData.build());
final String streamId = streamService.save(stream);
if (streamDescription.getStreamRules() != null) {
for (StreamRule streamRule : streamDescription.getStreamRules()) {
final ImmutableMap.Builder streamRuleData = ImmutableMap.builder();
streamRuleData.put(StreamRuleImpl.FIELD_TYPE, streamRule.getType().toInteger());
streamRuleData.put(StreamRuleImpl.FIELD_VALUE, streamRule.getValue());
streamRuleData.put(StreamRuleImpl.FIELD_FIELD, streamRule.getField());
streamRuleData.put(StreamRuleImpl.FIELD_INVERTED, streamRule.isInverted());
streamRuleData.put(StreamRuleImpl.FIELD_STREAM_ID, new ObjectId(streamId));
streamRuleData.put(StreamRuleImpl.FIELD_CONTENT_PACK, bundleId);
streamRuleData.put(StreamRuleImpl.FIELD_DESCRIPTION, streamRule.getDescription());
streamRuleService.save(new StreamRuleImpl(streamRuleData.build()));
}
}
for (final String outputId : streamDescription.getOutputs()) {
if (isNullOrEmpty(outputId)) {
LOG.warn("Couldn't find referenced output <{}> for stream <{}>", outputId, streamDescription.getTitle());
} else {
streamService.addOutput(stream, outputsByReferenceId.get(outputId));
}
}
return stream;
}
private void createDashboards(final String bundleId, final Set dashboards, final String userName)
throws org.graylog2.dashboards.widgets.DashboardWidget.NoSuchWidgetTypeException, InvalidWidgetConfigurationException, InvalidRangeParametersException, ValidationException {
for (final Dashboard dashboard : dashboards) {
org.graylog2.dashboards.Dashboard createdDashboard = createDashboard(bundleId, dashboard, userName);
createdDashboards.put(createdDashboard.getId(), createdDashboard);
}
}
private org.graylog2.dashboards.Dashboard createDashboard(final String bundleId, final Dashboard dashboardDescription, final String userName)
throws ValidationException, org.graylog2.dashboards.widgets.DashboardWidget.NoSuchWidgetTypeException, InvalidRangeParametersException, InvalidWidgetConfigurationException {
// Create dashboard.
final Map dashboardData = new HashMap<>();
dashboardData.put(DashboardImpl.FIELD_TITLE, dashboardDescription.getTitle());
dashboardData.put(DashboardImpl.FIELD_DESCRIPTION, dashboardDescription.getDescription());
dashboardData.put(DashboardImpl.FIELD_CONTENT_PACK, bundleId);
dashboardData.put(DashboardImpl.FIELD_CREATOR_USER_ID, userName);
dashboardData.put(DashboardImpl.FIELD_CREATED_AT, Tools.nowUTC());
final org.graylog2.dashboards.Dashboard dashboard = new DashboardImpl(dashboardData);
final String dashboardId = dashboardService.save(dashboard);
final ImmutableList.Builder widgetPositions = ImmutableList.builder();
for (DashboardWidget dashboardWidget : dashboardDescription.getDashboardWidgets()) {
final org.graylog2.dashboards.widgets.DashboardWidget widget = createDashboardWidget(dashboardWidget, userName);
dashboardService.addWidget(dashboard, widget);
final WidgetPositionsRequest.WidgetPosition widgetPosition = WidgetPositionsRequest.WidgetPosition.create(widget.getId(),
dashboardWidget.getCol(), dashboardWidget.getRow(), dashboardWidget.getHeight(), dashboardWidget.getWidth());
widgetPositions.add(widgetPosition);
}
// FML: We need to reload the dashboard because not all fields (I'm looking at you, "widgets") is set in the
// Dashboard instance used before.
final org.graylog2.dashboards.Dashboard persistedDashboard;
try {
persistedDashboard = dashboardService.load(dashboardId);
dashboardService.updateWidgetPositions(persistedDashboard, WidgetPositionsRequest.create(widgetPositions.build()));
} catch (NotFoundException e) {
LOG.error("Failed to load dashboard with id " + dashboardId, e);
}
return dashboard;
}
@SuppressWarnings("unchecked")
private org.graylog2.dashboards.widgets.DashboardWidget createDashboardWidget(
final DashboardWidget dashboardWidget, final String userName)
throws InvalidRangeParametersException, org.graylog2.dashboards.widgets.DashboardWidget.NoSuchWidgetTypeException, InvalidWidgetConfigurationException {
final String type = dashboardWidget.getType();
final Map config = dashboardWidget.getConfiguration();
// Replace "stream_id" in config if it's set
final String streamReference = (String) config.get("stream_id");
if (!isNullOrEmpty(streamReference)) {
final org.graylog2.plugin.streams.Stream stream = streamsByReferenceId.get(streamReference);
if (null != stream) {
config.put("stream_id", stream.getId());
} else {
LOG.warn("Couldn't find referenced stream {}", streamReference);
}
}
// Build timerange.
final Map timerangeConfig = (Map) config.get("timerange");
final TimeRange timeRange = timeRangeFactory.create(timerangeConfig);
final String widgetId = UUID.randomUUID().toString();
return dashboardWidgetCreator.buildDashboardWidget(type,
widgetId, dashboardWidget.getDescription(), dashboardWidget.getCacheTime(),
config, timeRange, userName);
}
}
