rapture.series.mongo.MongoSeriesStore Maven / Gradle / Ivy
/**
* The MIT License (MIT)
*
* Copyright (c) 2011-2016 Incapture Technologies LLC
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in all
* copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package rapture.series.mongo;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import org.apache.log4j.Logger;
import com.google.common.base.Preconditions;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.mongodb.BasicDBObject;
import com.mongodb.DBCollection;
import com.mongodb.DBCursor;
import com.mongodb.DBObject;
import com.mongodb.MongoException;
import com.mongodb.WriteResult;
import rapture.common.Messages;
import rapture.common.RaptureFolderInfo;
import rapture.common.SeriesValue;
import rapture.common.exception.ExceptionToString;
import rapture.common.exception.RaptureExceptionFactory;
import rapture.dsl.serfun.DecimalSeriesValue;
import rapture.dsl.serfun.LongSeriesValue;
import rapture.dsl.serfun.StringSeriesValue;
import rapture.dsl.serfun.StructureSeriesValueImpl;
import rapture.mongodb.MongoDBFactory;
import rapture.mongodb.MongoRetryWrapper;
import rapture.series.SeriesPaginator;
import rapture.series.SeriesStore;
import rapture.series.children.ChildKeyUtil;
import rapture.series.children.ChildrenRepo;
/**
* Mongo implementation of the storage for series api.
*
* @author mel
*/
public class MongoSeriesStore implements SeriesStore {
private String instanceName = "default";
private String tableName;
public static final String ROWKEY = "row";
public static final String COLKEY = "col";
public static final String VALKEY = "val";
private Cache keyCache = CacheBuilder.newBuilder().expireAfterAccess(3, TimeUnit.SECONDS).build();
private final ChildrenRepo childrenRepo;
private static Logger log = Logger.getLogger(MongoSeriesStore.class);
private Messages mongoMsgCatalog;
public MongoSeriesStore() {
mongoMsgCatalog = new Messages("Mongo");
this.childrenRepo = new ChildrenRepo() {
@Override
public List getPoints(String key) {
return MongoSeriesStore.this.getPoints(key);
}
@Override
public boolean addPoint(String key, SeriesValue value) {
addPointToSeries(key, value);
return true;
}
@Override
public boolean dropPoints(String key, List points) {
return deletePointsFromSeriesByPointKey(key, points);
}
@Override
public void dropRow(String key) {
deletePointsFromSeries(key);
}
};
}
@Override
public void drop() {
DBCollection collection = getCollection(null);
collection.drop();
keyCache.invalidateAll();
}
@Override
public void addDoubleToSeries(String key, String column, double value) {
saveDBObject(key, column, value);
}
@Override
public void addLongToSeries(String key, String column, long value) {
saveDBObject(key, column, value);
}
@Override
public void addStringToSeries(String key, String column, String value) {
saveDBObject(key, column, "'" + value);
}
@Override
public void addStructureToSeries(String key, String column, String json) {
saveDBObject(key, column, json);
}
@Override
public void addPointToSeries(String key, SeriesValue value) {
if (value.isDouble()) addDoubleToSeries(key, value.getColumn(), value.asDouble());
else if (value.isLong()) addLongToSeries(key, value.getColumn(), value.asLong());
else if (value.isString()) addStringToSeries(key, value.getColumn(), value.asString());
else if (value.isStructure()) addStructureToSeries(key, value.getColumn(), value.asString());
else throw RaptureExceptionFactory.create(HttpURLConnection.HTTP_INTERNAL_ERROR, mongoMsgCatalog.getMessage("NoEncoder", value.asString()));
}
private void saveDBObject(String key, String column, Object val) {
registerKey(key);
DBCollection collection = getCollection(key);
DBObject dbkey = new BasicDBObject(ROWKEY, key).append(COLKEY, column);
DBObject dbval = new BasicDBObject(ROWKEY, key).append(COLKEY, column).append(VALKEY, val);
try {
WriteResult result = collection.update(dbkey, dbval, true, false);
log.info("Update " + (result.wasAcknowledged() ? "was" : "was not") + " acknowledged");
} catch (MongoException me) {
throw RaptureExceptionFactory.create(HttpURLConnection.HTTP_INTERNAL_ERROR, new ExceptionToString(me));
}
}
private static final String DIRECTORY_KEY = "..directory";
private Callable FALSE_CALL = new Callable() {
public Boolean call() {
return false;
}
};
private void registerKey(String key) {
if (DIRECTORY_KEY.equals(key) || ChildKeyUtil.isRowKey(key)) {
return;
} else try {
if (keyCache.get(key, FALSE_CALL) == false) {
addPointToSeries(DIRECTORY_KEY, new StringSeriesValue(".", key));
childrenRepo.registerParentage(key);
keyCache.put(key, true);
}
} catch (ExecutionException e) {
// this should be impossible
throw RaptureExceptionFactory.create(HttpURLConnection.HTTP_INTERNAL_ERROR, new ExceptionToString(e));
}
}
public void unregisterKey(String key) {
unregisterKey(key, false);
}
public void unregisterKey(String key, boolean isFolder) {
if (DIRECTORY_KEY.equals(key)) {
return;
}
deletePointsFromSeriesByPointKey(DIRECTORY_KEY, ImmutableList.of(key));
if (isFolder) childrenRepo.dropFolderEntry(key);
else childrenRepo.dropFileEntry(key);
keyCache.invalidate(key);
}
private void multiAdd(String key, List columns, List values, Callback c) {
boolean nullKey = false;
Preconditions.checkArgument(columns.size() == values.size());
Iterator col = columns.iterator();
Iterator val = values.iterator();
while (col.hasNext()) {
String column = col.next();
T value = val.next();
if (column == null) nullKey = true;
else c.go(key, column, value);
}
if (nullKey) throw RaptureExceptionFactory.create(HttpURLConnection.HTTP_INTERNAL_ERROR, mongoMsgCatalog.getMessage("NullKey"));
}
@Override
public void addDoublesToSeries(String key, List columns, List values) {
multiAdd(key, columns, values, multiDouble);
}
@Override
public void addLongsToSeries(String key, List columns, List values) {
multiAdd(key, columns, values, multiLong);
}
@Override
public void addStringsToSeries(String key, List columns, List values) {
multiAdd(key, columns, values, multiString);
}
@Override
public void addStructuresToSeries(String key, List columns, List values) {
multiAdd(key, columns, values, multiStruct);
}
@Override
public void addPointsToSeries(String key, List values) {
Iterator val = values.iterator();
while (val.hasNext()) {
multiPoint.go(key, null, val.next());
}
}
private interface Callback {
public void go(String key, String column, T value);
}
private Callback multiDouble = new Callback() {
public void go(String key, String column, Double value) {
addDoubleToSeries(key, column, value);
}
};
private Callback multiLong = new Callback() {
public void go(String key, String column, Long value) {
addLongToSeries(key, column, value);
}
};
private Callback multiString = new Callback() {
public void go(String key, String column, String value) {
addStringToSeries(key, column, value);
}
};
private Callback multiStruct = new Callback() {
public void go(String key, String column, String value) {
addStructureToSeries(key, column, value);
}
};
private Callback multiPoint = new Callback() {
public void go(String key, String column, SeriesValue value) {
if (value.isDouble()) addDoubleToSeries(key, value.getColumn(), value.asDouble());
else if (value.isLong()) addLongToSeries(key, value.getColumn(), value.asLong());
else if (value.isString()) addStringToSeries(key, value.getColumn(), value.asString());
else if (value.isStructure()) addStructureToSeries(key, value.getColumn(), value.asString());
}
};
@Override
public Boolean deletePointsFromSeriesByPointKey(String key, List pointKeys) {
DBCollection collection = getCollection(key);
for (String pointKey : pointKeys) {
DBObject victim = new BasicDBObject(ROWKEY, key).append(COLKEY, pointKey);
try {
WriteResult result = collection.remove(victim);
log.info("Remove " + (result.wasAcknowledged() ? "was" : "was not") + " acknowledged");
} catch (MongoException me) {
throw RaptureExceptionFactory.create(HttpURLConnection.HTTP_INTERNAL_ERROR, new ExceptionToString(me));
}
}
return true;
}
@Override
public void deletePointsFromSeries(String key) {
unregisterKey(key);
DBCollection collection = getCollection(key);
DBObject victim = new BasicDBObject(ROWKEY, key);
try {
WriteResult result = collection.remove(victim);
log.info("Remove " + (result.wasAcknowledged() ? "was" : "was not") + " acknowledged");
} catch (MongoException me) {
throw RaptureExceptionFactory.create(HttpURLConnection.HTTP_INTERNAL_ERROR, new ExceptionToString(me));
}
}
@Override
public List getPoints(final String key) {
MongoRetryWrapper> wrapper = new MongoRetryWrapper>() {
public DBCursor makeCursor() {
DBObject query = new BasicDBObject(ROWKEY, key);
DBCollection collection = getCollection(key);
return collection.find(query);
}
public List action(DBCursor cursor) {
List result = Lists.newArrayList();
for (DBObject entry : cursor) {
SeriesValue bolt = makeSeriesValue(entry);
result.add(bolt);
}
return result;
}
};
return wrapper.doAction();
}
private SeriesValue makeSeriesValue(DBObject entry) {
Object val = entry.get(VALKEY);
String col = (String) entry.get(COLKEY);
if (val instanceof Double) return new DecimalSeriesValue(((Double) val).doubleValue(), col);
if (val instanceof String) return decodeString((String) val, col);
if (val instanceof Long) return new LongSeriesValue(((Long) val).longValue(), col);
throw RaptureExceptionFactory.create(HttpURLConnection.HTTP_INTERNAL_ERROR, mongoMsgCatalog.getMessage("UnknownType"));
}
private SeriesValue decodeString(String val, String col) {
if (val.startsWith("'")) return new StringSeriesValue(val.substring(1), col);
try {
return StructureSeriesValueImpl.unmarshal(val, col);
} catch (IOException e) {
throw RaptureExceptionFactory.create(HttpURLConnection.HTTP_BAD_REQUEST, mongoMsgCatalog.getMessage("JsonError", val));
}
}
@Override
public List getPointsAfter(final String key, final String startColumn, final int maxNumber) {
MongoRetryWrapper> wrapper = new MongoRetryWrapper>() {
public DBCursor makeCursor() {
DBCollection collection = getCollection(key);
DBObject query = new BasicDBObject(ROWKEY, key).append(COLKEY, new BasicDBObject("$gte", startColumn));
return collection.find(query);
}
public List action(DBCursor cursor) {
List result = Lists.newArrayList();
int count = 0;
for (DBObject entry : cursor) {
if (count >= maxNumber) break;
SeriesValue bolt = makeSeriesValue(entry);
result.add(bolt);
count++;
}
return result;
}
};
return wrapper.doAction();
}
@Override
public List getPointsAfterReverse(final String key, final String startColumn, final int maxNumber) {
MongoRetryWrapper> wrapper = new MongoRetryWrapper>() {
public DBCursor makeCursor() {
DBCollection collection = getCollection(key);
DBObject query = new BasicDBObject(ROWKEY, key).append(COLKEY, new BasicDBObject("$lte", startColumn));
BasicDBObject sort = new BasicDBObject(COLKEY, -1);
return collection.find(query).sort(sort);
}
public List action(DBCursor cursor) {
int count = 0;
List result = Lists.newArrayList();
for (DBObject entry : cursor) {
if (count >= maxNumber) break;
SeriesValue bolt = makeSeriesValue(entry);
result.add(bolt);
count++;
}
return result;
}
};
return wrapper.doAction();
}
@Override
public List getPointsAfter(final String key, final String startColumn, final String endColumn, final int maxNumber) {
MongoRetryWrapper> wrapper = new MongoRetryWrapper>() {
public DBCursor makeCursor() {
DBCollection collection = getCollection(key);
DBObject query = new BasicDBObject(ROWKEY, key).append(COLKEY, new BasicDBObject("$gte", startColumn).append("$lte", endColumn));
return collection.find(query);
}
public List action(DBCursor cursor) {
int count = 0;
List result = Lists.newArrayList();
for (DBObject entry : cursor) {
if (count >= maxNumber) break;
SeriesValue bolt = makeSeriesValue(entry);
result.add(bolt);
count++;
}
return result;
}
};
return wrapper.doAction();
}
@Override
public void setInstanceName(String instanceName) {
this.instanceName = instanceName;
}
@Override
public void setConfig(Map config) {
// TODO MEL No options for you! We will choose everything.
tableName = config.get("prefix");
log.debug("Table name is " + tableName + ", instance name is " + instanceName);
if (tableName == null) {
throw RaptureExceptionFactory.create(HttpURLConnection.HTTP_BAD_REQUEST, mongoMsgCatalog.getMessage("NoPrefix"));
}
// WARNING: Fragile code assumes setInstanceName is called BEFORE
// setConfig
MongoDBFactory.getCollection(instanceName, tableName).createIndex(INDEX_KEYS, INDEX_OPTS);
}
@Override
public List getSeriesLike(String keyPrefix) {
throw new UnsupportedOperationException();
}
@Override
public Iterable getRangeAsIteration(String key, String startCol, String endCol, int pageSize) {
return new SeriesPaginator(key, startCol, endCol, pageSize, this);
}
@Override
public List getRangeAsList(String key, String startCol, String endCol) {
return getPointsAfter(key, startCol, endCol, Integer.MAX_VALUE);
}
@Override
public List listSeriesByUriPrefix(String folderName) {
return childrenRepo.getChildren(folderName);
}
private static final DBObject INDEX_KEYS = new BasicDBObject(ROWKEY, 1).append(COLKEY, 1);
private static final DBObject INDEX_OPTS = new BasicDBObject("unique", true);
private DBCollection getCollection(String checkAllTheCallersIfYouStopIgnoringThisParameter) {
DBCollection result = MongoDBFactory.getCollection(instanceName, tableName);
return result;
}
@Override
public SeriesValue getLastPoint(final String key) {
MongoRetryWrapper wrapper = new MongoRetryWrapper() {
public DBCursor makeCursor() {
DBCollection collection = getCollection(key);
DBObject query = new BasicDBObject(ROWKEY, key);
BasicDBObject sort = new BasicDBObject(COLKEY, -1);
return collection.find(query).sort(sort).limit(1);
}
public SeriesValue action(DBCursor cursor) {
return cursor.hasNext() ? makeSeriesValue(cursor.next()) : null;
}
};
return wrapper.doAction();
}
@Override
public void createSeries(String key) {
registerKey(key);
}
@Override
public void deleteSeries(String key) {
unregisterKey(key);
deletePointsFromSeries(key);
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy