org.graylog2.streams.StreamServiceImpl Maven / Gradle / Ivy

Show more of this group Show more artifacts with this name
Show all versions of graylog2-server Show documentation
Graylog
There is a newer version: 6.0.6
/**
 * This file is part of Graylog.
 *
 * Graylog is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * Graylog is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with Graylog.  If not, see .
 */
package org.graylog2.streams;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.mongodb.BasicDBObject;
import com.mongodb.DBObject;
import com.mongodb.QueryBuilder;
import org.bson.types.ObjectId;
import org.graylog2.alarmcallbacks.AlarmCallbackConfiguration;
import org.graylog2.alarmcallbacks.AlarmCallbackConfigurationImpl;
import org.graylog2.alarmcallbacks.AlarmCallbackConfigurationService;
import org.graylog2.alarmcallbacks.EmailAlarmCallback;
import org.graylog2.alerts.Alert;
import org.graylog2.alerts.AlertService;
import org.graylog2.database.MongoConnection;
import org.graylog2.database.NotFoundException;
import org.graylog2.database.PersistedServiceImpl;
import org.graylog2.indexer.IndexSet;
import org.graylog2.indexer.MongoIndexSet;
import org.graylog2.indexer.indexset.IndexSetConfig;
import org.graylog2.indexer.indexset.IndexSetService;
import org.graylog2.notifications.Notification;
import org.graylog2.notifications.NotificationService;
import org.graylog2.plugin.Tools;
import org.graylog2.plugin.alarms.AlertCondition;
import org.graylog2.plugin.database.EmbeddedPersistable;
import org.graylog2.plugin.database.ValidationException;
import org.graylog2.plugin.streams.Output;
import org.graylog2.plugin.streams.Stream;
import org.graylog2.plugin.streams.StreamRule;
import org.graylog2.rest.resources.streams.requests.CreateStreamRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.annotation.Nullable;
import javax.inject.Inject;
import javax.ws.rs.BadRequestException;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;

import static com.google.common.base.Strings.isNullOrEmpty;

public class StreamServiceImpl extends PersistedServiceImpl implements StreamService {
    private static final Logger LOG = LoggerFactory.getLogger(StreamServiceImpl.class);
    private final StreamRuleService streamRuleService;
    private final AlertService alertService;
    private final OutputService outputService;
    private final IndexSetService indexSetService;
    private final MongoIndexSet.Factory indexSetFactory;
    private final NotificationService notificationService;
    private final AlarmCallbackConfigurationService alarmCallbackConfigurationService;

    @Inject
    public StreamServiceImpl(MongoConnection mongoConnection,
                             StreamRuleService streamRuleService,
                             AlertService alertService,
                             OutputService outputService,
                             IndexSetService indexSetService,
                             MongoIndexSet.Factory indexSetFactory,
                             NotificationService notificationService,
                             AlarmCallbackConfigurationService alarmCallbackConfigurationService) {
        super(mongoConnection);
        this.streamRuleService = streamRuleService;
        this.alertService = alertService;
        this.outputService = outputService;
        this.indexSetService = indexSetService;
        this.indexSetFactory = indexSetFactory;
        this.notificationService = notificationService;
        this.alarmCallbackConfigurationService = alarmCallbackConfigurationService;
    }

    @Nullable
    private IndexSet getIndexSet(DBObject dbObject) {
        return getIndexSet((String) dbObject.get(StreamImpl.FIELD_INDEX_SET_ID));
    }

    @Nullable
    private IndexSet getIndexSet(String id) {
        if (isNullOrEmpty(id)) {
            return null;
        }
        final Optional indexSetConfig = indexSetService.get(id);
        return indexSetConfig.flatMap(c -> Optional.of(indexSetFactory.create(c))).orElse(null);
    }

    public Stream load(ObjectId id) throws NotFoundException {
        final DBObject o = get(StreamImpl.class, id);

        if (o == null) {
            throw new NotFoundException("Stream <" + id + "> not found!");
        }

        final List streamRules = streamRuleService.loadForStreamId(id.toHexString());

        final Set outputs = loadOutputsForRawStream(o);

        @SuppressWarnings("unchecked")
        final Map fields = o.toMap();
        return new StreamImpl((ObjectId) o.get("_id"), fields, streamRules, outputs, getIndexSet(o));
    }

    @Override
    public Stream create(Map fields) {
        return new StreamImpl(fields, getIndexSet((String) fields.get(StreamImpl.FIELD_INDEX_SET_ID)));
    }

    @Override
    public Stream create(CreateStreamRequest cr, String userId) {
        Map streamData = Maps.newHashMap();
        streamData.put(StreamImpl.FIELD_TITLE, cr.title());
        streamData.put(StreamImpl.FIELD_DESCRIPTION, cr.description());
        streamData.put(StreamImpl.FIELD_CREATOR_USER_ID, userId);
        streamData.put(StreamImpl.FIELD_CREATED_AT, Tools.nowUTC());
        streamData.put(StreamImpl.FIELD_CONTENT_PACK, cr.contentPack());
        streamData.put(StreamImpl.FIELD_MATCHING_TYPE, cr.matchingType().toString());
        streamData.put(StreamImpl.FIELD_REMOVE_MATCHES_FROM_DEFAULT_STREAM, cr.removeMatchesFromDefaultStream());
        streamData.put(StreamImpl.FIELD_INDEX_SET_ID, cr.indexSetId());

        return create(streamData);
    }

    @Override
    public Stream load(String id) throws NotFoundException {
        try {
            return load(new ObjectId(id));
        } catch (IllegalArgumentException e) {
            throw new NotFoundException("Stream <" + id + "> not found!");
        }
    }

    @Override
    public List loadAllEnabled() {
        return loadAllEnabled(new HashMap<>());
    }

    public List loadAllEnabled(Map additionalQueryOpts) {
        additionalQueryOpts.put(StreamImpl.FIELD_DISABLED, false);

        return loadAll(additionalQueryOpts);
    }

    @Override
    public List loadAll() {
        return loadAll(Collections.emptyMap());
    }

    public List loadAll(Map additionalQueryOpts) {
        final DBObject query = new BasicDBObject(additionalQueryOpts);
        return loadAll(query);
    }

    private List loadAll(DBObject query) {
        final List results = query(StreamImpl.class, query);
        final List streamIds = results.stream()
                .map(o -> o.get("_id").toString())
                .collect(Collectors.toList());
        final Map> allStreamRules = streamRuleService.loadForStreamIds(streamIds);

        final ImmutableList.Builder streams = ImmutableList.builder();
        for (DBObject o : results) {
            final ObjectId objectId = (ObjectId) o.get("_id");
            final String id = objectId.toHexString();
            final List streamRules = allStreamRules.getOrDefault(id, Collections.emptyList());
            LOG.debug("Found {} rules for stream <{}>", streamRules.size(), id);

            final Set outputs = loadOutputsForRawStream(o);

            @SuppressWarnings("unchecked")
            final Map fields = o.toMap();

            streams.add(new StreamImpl(objectId, fields, streamRules, outputs, getIndexSet(o)));
        }

        return streams.build();
    }

    @Override
    public List loadAllWithConfiguredAlertConditions() {
        final DBObject query = QueryBuilder.start().and(
            QueryBuilder.start(StreamImpl.EMBEDDED_ALERT_CONDITIONS).exists(true).get(),
            QueryBuilder.start(StreamImpl.EMBEDDED_ALERT_CONDITIONS).not().size(0).get()
        ).get();

        return loadAll(query);
    }

    protected Set loadOutputsForRawStream(DBObject stream) {
        @SuppressWarnings("unchecked")
        List outputIds = (List) stream.get(StreamImpl.FIELD_OUTPUTS);

        Set result = new HashSet<>();
        if (outputIds != null)
            for (ObjectId outputId : outputIds)
                try {
                    result.add(outputService.load(outputId.toHexString()));
                } catch (NotFoundException e) {
                    LOG.warn("Non-existing output <{}> referenced from stream <{}>!", outputId.toHexString(), stream.get("_id"));
                }

        return result;
    }

    @Override
    public long count() {
        return totalCount(StreamImpl.class);
    }

    @Override
    public void destroy(Stream stream) throws NotFoundException {
        for (StreamRule streamRule : streamRuleService.loadForStream(stream)) {
            super.destroy(streamRule);
        }
        for (Notification notification : notificationService.all()) {
            Object rawValue = notification.getDetail("stream_id");
            if (rawValue != null && rawValue.toString().equals(stream.getId())) {
                LOG.debug("Removing notification that references stream: {}", notification);
                notificationService.destroy(notification);
            }
        }
        super.destroy(stream);
    }

    public void update(Stream stream, String title, String description) throws ValidationException {
        if (title != null) {
            stream.getFields().put(StreamImpl.FIELD_TITLE, title);
        }

        if (description != null) {
            stream.getFields().put(StreamImpl.FIELD_DESCRIPTION, description);
        }

        save(stream);
    }

    @Override
    public void pause(Stream stream) throws ValidationException {
        stream.setDisabled(true);
        save(stream);
    }

    @Override
    public void resume(Stream stream) throws ValidationException {
        stream.setDisabled(false);
        save(stream);
    }

    @Override
    public List getStreamRules(Stream stream) throws NotFoundException {
        return streamRuleService.loadForStream(stream);
    }

    @Override
    public List getAlertConditions(Stream stream) {
        List conditions = Lists.newArrayList();

        if (stream.getFields().containsKey(StreamImpl.EMBEDDED_ALERT_CONDITIONS)) {
            @SuppressWarnings("unchecked")
            final List alertConditions = (List) stream.getFields().get(StreamImpl.EMBEDDED_ALERT_CONDITIONS);
            for (BasicDBObject conditionFields : alertConditions) {
                try {
                    conditions.add(alertService.fromPersisted(conditionFields, stream));
                } catch (Exception e) {
                    LOG.error("Skipping alert condition.", e);
                }
            }
        }

        return conditions;
    }

    @Override
    public AlertCondition getAlertCondition(Stream stream, String conditionId) throws NotFoundException {
        if (stream.getFields().containsKey(StreamImpl.EMBEDDED_ALERT_CONDITIONS)) {
            @SuppressWarnings("unchecked")
            final List alertConditions = (List) stream.getFields().get(StreamImpl.EMBEDDED_ALERT_CONDITIONS);
            for (BasicDBObject conditionFields : alertConditions) {
                try {
                    if (conditionFields.get("id").equals(conditionId)) {
                        return alertService.fromPersisted(conditionFields, stream);
                    }
                } catch (Exception e) {
                    LOG.error("Skipping alert condition.", e);
                }
            }
        }

        throw new NotFoundException("Alert condition <" + conditionId + "> for stream <" + stream.getId() + "> not found");
    }

    @Override
    public void addAlertCondition(Stream stream, AlertCondition condition) throws ValidationException {
        embed(stream, StreamImpl.EMBEDDED_ALERT_CONDITIONS, (EmbeddedPersistable) condition);
    }

    @Override
    public void updateAlertCondition(Stream stream, AlertCondition condition) throws ValidationException {
        removeAlertCondition(stream, condition.getId());
        addAlertCondition(stream, condition);
    }

    @Override
    public void removeAlertCondition(Stream stream, String conditionId) {
        // Before deleting alert condition, resolve all its alerts.
        final List alerts = alertService.listForStreamIds(Collections.singletonList(stream.getId()), Alert.AlertState.UNRESOLVED, 0, 0);
        alerts.stream()
                .filter(alert -> alert.getConditionId().equals(conditionId))
                .forEach(alertService::resolveAlert);

        removeEmbedded(stream, StreamImpl.EMBEDDED_ALERT_CONDITIONS, conditionId);
    }

    @Override
    public void addAlertReceiver(Stream stream, String type, String name) {
        final List streamCallbacks = alarmCallbackConfigurationService.getForStream(stream);
        updateCallbackConfiguration("add", type, name, streamCallbacks);
    }

    @Override
    public void removeAlertReceiver(Stream stream, String type, String name) {
        final List streamCallbacks = alarmCallbackConfigurationService.getForStream(stream);
        updateCallbackConfiguration("remove", type, name, streamCallbacks);
    }

    // I tried to be sorry, really. https://www.youtube.com/watch?v=3KVyRqloGmk
    private void updateCallbackConfiguration(String action, String type, String entity, List streamCallbacks) {
        final AtomicBoolean ran = new AtomicBoolean(false);

        streamCallbacks.stream()
                .filter(callback -> callback.getType().equals(EmailAlarmCallback.class.getCanonicalName()))
                .forEach(callback -> {
                    ran.set(true);
                    final Map configuration = callback.getConfiguration();
                    String key;

                    if ("users".equals(type)) {
                        key = EmailAlarmCallback.CK_USER_RECEIVERS;
                    } else {
                        key = EmailAlarmCallback.CK_EMAIL_RECEIVERS;
                    }

                    @SuppressWarnings("unchecked")
                    final List recipients = (List) configuration.get(key);
                    if ("add".equals(action)) {
                        if (!recipients.contains(entity)) {
                            recipients.add(entity);
                        }
                    } else {
                        if (recipients.contains(entity)) {
                            recipients.remove(entity);
                        }
                    }
                    configuration.put(key, recipients);

                    final AlarmCallbackConfiguration updatedConfig = ((AlarmCallbackConfigurationImpl) callback).toBuilder()
                            .setConfiguration(configuration)
                            .build();
                    try {
                        alarmCallbackConfigurationService.save(updatedConfig);
                    } catch (ValidationException e) {
                        throw new BadRequestException("Unable to save alarm callback configuration", e);
                    }
                });

        if (!ran.get()) {
            throw new BadRequestException("Unable to " + action + " receiver: Stream has no email alarm callback.");
        }
    }


    @Override
    public void addOutput(Stream stream, Output output) {
        collection(stream).update(
                new BasicDBObject("_id", new ObjectId(stream.getId())),
                new BasicDBObject("$addToSet", new BasicDBObject(StreamImpl.FIELD_OUTPUTS, new ObjectId(output.getId())))
        );
    }

    @Override
    public void removeOutput(Stream stream, Output output) {
        collection(stream).update(
                new BasicDBObject("_id", new ObjectId(stream.getId())),
                new BasicDBObject("$pull", new BasicDBObject(StreamImpl.FIELD_OUTPUTS, new ObjectId(output.getId())))
        );
    }

    @Override
    public void removeOutputFromAllStreams(Output output) {
        ObjectId outputId = new ObjectId(output.getId());
        DBObject match = new BasicDBObject(StreamImpl.FIELD_OUTPUTS, outputId);
        DBObject modify = new BasicDBObject("$pull", new BasicDBObject(StreamImpl.FIELD_OUTPUTS, outputId));

        collection(StreamImpl.class).update(
                match, modify, false, true
        );
    }

    @Override
    public List loadAllWithIndexSet(String indexSetId) {
        final Map query = new BasicDBObject(StreamImpl.FIELD_INDEX_SET_ID, indexSetId);
        return loadAll(query);
    }
}