Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/**
* This file is part of Graylog.
*
* Graylog is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Graylog is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Graylog. If not, see .
*/
package org.graylog2.indexer.ranges;
import com.github.rholder.retry.Retryer;
import com.github.rholder.retry.RetryerBuilder;
import com.github.rholder.retry.StopStrategies;
import com.github.rholder.retry.WaitStrategies;
import com.google.common.base.Stopwatch;
import com.google.common.collect.ImmutableSortedSet;
import com.google.common.eventbus.AllowConcurrentEvents;
import com.google.common.eventbus.EventBus;
import com.google.common.eventbus.Subscribe;
import com.google.common.primitives.Ints;
import com.mongodb.BasicDBObject;
import com.mongodb.BasicDBObjectBuilder;
import org.bson.types.ObjectId;
import org.elasticsearch.indices.IndexClosedException;
import org.graylog2.bindings.providers.MongoJackObjectMapperProvider;
import org.graylog2.database.MongoConnection;
import org.graylog2.database.NotFoundException;
import org.graylog2.indexer.esplugin.IndexChangeMonitor;
import org.graylog2.indexer.esplugin.IndicesClosedEvent;
import org.graylog2.indexer.esplugin.IndicesDeletedEvent;
import org.graylog2.indexer.esplugin.IndicesReopenedEvent;
import org.graylog2.indexer.indices.Indices;
import org.graylog2.indexer.searches.TimestampStats;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.mongojack.DBCursor;
import org.mongojack.DBQuery;
import org.mongojack.JacksonDBCollection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import java.util.Iterator;
import java.util.SortedSet;
import java.util.concurrent.TimeUnit;
public class MongoIndexRangeService implements IndexRangeService {
private static final Logger LOG = LoggerFactory.getLogger(MongoIndexRangeService.class);
private static final String COLLECTION_NAME = "index_ranges";
private final Indices indices;
private final JacksonDBCollection collection;
@Inject
public MongoIndexRangeService(MongoConnection mongoConnection,
MongoJackObjectMapperProvider objectMapperProvider,
Indices indices,
EventBus eventBus) {
this.indices = indices;
this.collection = JacksonDBCollection.wrap(
mongoConnection.getDatabase().getCollection(COLLECTION_NAME),
MongoIndexRange.class,
ObjectId.class,
objectMapperProvider.get());
// This sucks. We need to bridge Elasticsearch's and our own Guice injector.
IndexChangeMonitor.setEventBus(eventBus);
eventBus.register(this);
collection.createIndex(new BasicDBObject(MongoIndexRange.FIELD_INDEX_NAME, 1));
collection.createIndex(BasicDBObjectBuilder.start()
.add(MongoIndexRange.FIELD_BEGIN, 1)
.add(MongoIndexRange.FIELD_END, 1)
.get());
}
@Override
public IndexRange get(String index) throws NotFoundException {
final DBQuery.Query query = DBQuery.and(
DBQuery.notExists("start"),
DBQuery.is(IndexRange.FIELD_INDEX_NAME, index));
final MongoIndexRange indexRange = collection.findOne(query);
if (indexRange == null) {
throw new NotFoundException("Index range for index <" + index + "> not found.");
}
return indexRange;
}
@Override
public SortedSet find(DateTime begin, DateTime end) {
final DBCursor indexRanges = collection.find(
DBQuery.and(
DBQuery.notExists("start"), // "start" has been used by the old index ranges in MongoDB
DBQuery.lessThanEquals(IndexRange.FIELD_BEGIN, end.getMillis()),
DBQuery.greaterThanEquals(IndexRange.FIELD_END, begin.getMillis())
)
);
return ImmutableSortedSet.copyOf(IndexRange.COMPARATOR, (Iterator) indexRanges);
}
@Override
public SortedSet findAll() {
return ImmutableSortedSet.copyOf(IndexRange.COMPARATOR, (Iterator) collection.find(DBQuery.notExists("start")));
}
@Override
public IndexRange calculateRange(String index) {
indices.waitForRecovery(index);
final DateTime now = DateTime.now(DateTimeZone.UTC);
final Stopwatch sw = Stopwatch.createStarted();
final TimestampStats stats = indices.timestampStatsOfIndex(index);
final int duration = Ints.saturatedCast(sw.stop().elapsed(TimeUnit.MILLISECONDS));
LOG.info("Calculated range of [{}] in [{}ms].", index, duration);
return MongoIndexRange.create(index, stats.min(), stats.max(), now, duration);
}
@Override
public void save(IndexRange indexRange) {
collection.remove(DBQuery.in(IndexRange.FIELD_INDEX_NAME, indexRange.indexName()));
collection.save(MongoIndexRange.create(indexRange));
}
@Subscribe
@AllowConcurrentEvents
public void handleIndexDeletion(IndicesDeletedEvent event) {
for (String index : event.indices()) {
LOG.debug("Index \"{}\" has been deleted. Removing index range.");
collection.remove(DBQuery.in(IndexRange.FIELD_INDEX_NAME, index));
}
}
@Subscribe
@AllowConcurrentEvents
public void handleIndexClosing(IndicesClosedEvent event) {
for (String index : event.indices()) {
LOG.debug("Index \"{}\" has been closed. Removing index range.");
collection.remove(DBQuery.in(IndexRange.FIELD_INDEX_NAME, index));
}
}
@Subscribe
@AllowConcurrentEvents
public void handleIndexReopening(IndicesReopenedEvent event) {
for (final String index : event.indices()) {
LOG.debug("Index \"{}\" has been reopened. Calculating index range.", index);
indices.waitForRecovery(index);
final Retryer retryer = RetryerBuilder.newBuilder()
.retryIfException(input -> !(input instanceof IndexClosedException))
.withWaitStrategy(WaitStrategies.exponentialWait())
.withStopStrategy(StopStrategies.stopAfterDelay(5, TimeUnit.MINUTES))
.build();
final IndexRange indexRange;
try {
indexRange = retryer.call(() -> calculateRange(index));
} catch (Exception e) {
if (e.getCause() instanceof IndexClosedException) {
LOG.debug("Couldn't calculate index range for closed index \"" + index + "\"", e.getCause());
return;
}
LOG.error("Couldn't calculate index range for index \"" + index + "\"", e.getCause());
throw new RuntimeException("Couldn't calculate index range for index \"" + index + "\"", e);
}
save(indexRange);
}
}
}
