me.lightspeed7.mongofs.MongoFileStore Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of mongoFS Show documentation
Show all versions of mongoFS Show documentation
An extension to the MongoDB Java Driver library that goes beyond what the GridFS feature supports.
Compressed file storage, zip files, temporary files
package me.lightspeed7.mongofs;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import me.lightspeed7.mongofs.url.MongoFileUrl;
import me.lightspeed7.mongofs.url.StorageFormat;
import me.lightspeed7.mongofs.util.ChunkSize;
import me.lightspeed7.mongofs.util.TimeMachine;
import org.bson.types.ObjectId;
import org.mongodb.CommandResult;
import org.mongodb.Document;
import org.mongodb.Index;
import org.mongodb.MongoCollection;
import org.mongodb.MongoCollectionOptions;
import org.mongodb.MongoCursor;
import org.mongodb.MongoDatabase;
import org.mongodb.MongoException;
import org.mongodb.MongoView;
import org.mongodb.OrderBy;
import org.mongodb.WriteResult;
import org.mongodb.diagnostics.Loggers;
import org.mongodb.diagnostics.logging.Logger;
import com.mongodb.DB;
import com.mongodb.ReadPreference;
import com.mongodb.WriteConcern;
public class MongoFileStore {
private static final Logger LOGGER = Loggers.getLogger("file");
private final MongoCollection filesCollection;
private final MongoCollection chunksCollection;
private MongoFileStoreConfig config;
/**
* Legacy CTOR, use this for the 2.12.x driver only
*
* @param database
* @param config
*/
public MongoFileStore(final DB database, final MongoFileStoreConfig config) {
this(new MongoDatabase(database), config);
}
/**
* CTOR
*
* @param database
* the MongoDB database to look for the collections in
* @param config
* the configuration for this file store
*/
public MongoFileStore(final MongoDatabase database, final MongoFileStoreConfig config) {
this.config = config;
// Determine which object to get readPreference and WriteConcern
WriteConcern writeConcern = config.getWriteConcern() == null // pull from database object
? database.getSurrogate().getWriteConcern()
: config.getWriteConcern();
ReadPreference readPreference = config.getReadPreference() == null // pull from database object
? database.getSurrogate().getReadPreference()
: config.getReadPreference();
// FILES
MongoCollectionOptions fileOptions = (MongoCollectionOptions) MongoCollectionOptions.builder().writeConcern(writeConcern)
.readPreference(readPreference).build();
filesCollection = database.getCollection(config.getBucket() + ".files", fileOptions);
// CHUNKS
MongoCollectionOptions chunksOptions = (MongoCollectionOptions) MongoCollectionOptions.builder()//
.writeConcern(writeConcern).readPreference(readPreference).build();
chunksCollection = database.getCollection(config.getBucket() + ".chunks", chunksOptions);
// INDEXES
// make sure the expiration index is present
// files go first, within a minute according to MongoDB
// then the chunks follow 1 minute after the file objects
checkForExpireAtIndex(filesCollection, 0);
checkForExpireAtIndex(chunksCollection, 60);
// make sure the zip manifest index exists
checkForManifestIndex(filesCollection);
// ensure standard indexes as long as collections are small
try {
if (getCollectionStats(filesCollection) < 1000) {
createIdIndexes(filesCollection, chunksCollection);
}
} catch (MongoException e) {
LOGGER.info(String.format("Unable to ensure indices on GridFS collections in database %s", //
filesCollection.getDatabase().getName()));
}
}
private int getCollectionStats(final MongoCollection coll) {
// { collStats: "collection" , scale : 1024 }
CommandResult result = coll.getDatabase().executeCommand(new Document("collStats", coll.getName()).append("scale", 1024));
return result.isOk() ? result.getResponse().getInteger("size").intValue() : 0;
}
private void checkForExpireAtIndex(final MongoCollection coll, final int secondsDelay) {
// check for existing
for (Document document : coll.tools().getIndexes()) {
if ("ttl".equals(document.get("name"))) {
return;
}
}
// build one
Index idx = Index.builder()//
.addKey("expireAt", OrderBy.ASC)//
.expireAfterSeconds(secondsDelay)//
.name("ttl")//
.sparse() //
.background(true)//
.build();
coll.tools().createIndexes(java.util.Collections.singletonList(idx));
}
private void checkForManifestIndex(final MongoCollection coll) {
String name = MongoFileConstants.manifestId.name();
// check for existing
for (Document document : coll.tools().getIndexes()) {
if (name.equals(document.get("name"))) {
return;
}
}
// build one
Index idx = Index.builder()//
.addKey(name, OrderBy.ASC)//
.name(name)//
.sparse() //
.background(true)//
.build();
coll.tools().createIndexes(java.util.Collections.singletonList(idx));
}
private void createIdIndexes(final MongoCollection fileColl, final MongoCollection chunksColl) {
Index filesIdx = Index.builder()//
.name("filename")//
.addKey("filename", OrderBy.ASC)//
.addKey("uploadDate", OrderBy.ASC).background(true)//
.build();
fileColl.tools().createIndexes(java.util.Collections.singletonList(filesIdx));
Index chunksIdx = Index.builder()//
.name("files_id")//
.addKey("files_id", OrderBy.ASC)//
.addKey("n", OrderBy.ASC).unique()//
.background(true)//
.build();
chunksColl.tools().createIndexes(java.util.Collections.singletonList(chunksIdx));
}
//
// public
// ///////////////
public ChunkSize getChunkSize() {
return config.getChunkSize();
}
MongoFileStoreConfig getConfig() {
return config;
}
/**
* Run a test command to the mongoDB to test connectivity and the server is running
*
* @return true if a connection could be made
*
* @throws MongoException
*/
public boolean validateConnection() {
try {
// String command = String.format(
// "{ touch: \"%s\", data: false, index: true }",
// config.getBucket() + ".files");
Document doc = new Document() //
.append("touch", config.getBucket() + ".files") //
.append("data", Boolean.FALSE) //
.append("index", Boolean.TRUE);
CommandResult commandResult = filesCollection.getDatabase().executeCommand(doc);
if (!commandResult.isOk()) {
throw new MongoException(commandResult.getErrorMessage());
}
return true;
} catch (Exception e) {
throw new MongoException("Unable to run command on server", e);
}
}
//
// writing
// //////////////////
/**
* Create a new file entry in the datastore, then a MongoFile object to start writing to it.
*
* NOTE : the system will determine if compression is needed
*
* @param filename
* the name of the new file
* @param mediaType
* the media type of the data
*
* @return a writer to write datq to for this file
*
* @throws IOException
* if an error occurs during reading and/or writing
* @throws IllegalArgumentException
* if required parameters are null
*/
public MongoFileWriter createNew(final String filename, final String mediaType) throws IOException {
return createNew(filename, mediaType, null, config.isCompressionEnabled());
}
/**
* Create a new file entry in the datastore, then a MongoFile object to start writing to it.
*
* NOTE : if compress = false and the media type is compressible, the file will not be stored compressed in the store
*
* @param filename
* the name of the new file
* @param mediaType
* the media type of the data
* @param expiresAt
* the date when the file should be expired
* @param compress
* should use compression if the mime type allows ( zip files will not be compressed even compress = true )
*
* @return a writer to write datq to for this file
*
* @throws IllegalStateException
* if compression is disabled in the configuration but ask for on the command line
* @throws IOException
* if an error occurs during reading and/or writing
* @throws IllegalArgumentException
* if required parameters are null
*
*/
public MongoFileWriter createNew(final String filename, final String mediaType, final Date expiresAt, final boolean compress)
throws IOException {
if (filename == null) {
throw new IllegalArgumentException("filename cannot be null");
}
if (mediaType == null) {
throw new IllegalArgumentException("mediaType cannot be null");
}
if (compress && !config.isCompressionEnabled()) {
throw new IllegalStateException("This data store has compression disabled");
}
// send wrapper object
StorageFormat format = StorageFormat.detect(compress, config.isEncryptionEnabled());
MongoFileUrl mongoFileUrl = MongoFileUrl.construct(new ObjectId(), filename, mediaType, format);
MongoFile mongoFile = new MongoFile(this, mongoFileUrl, config.getChunkSize().getChunkSize());
if (expiresAt != null) {
mongoFile.setExpiresAt(expiresAt);
}
return new MongoFileWriter(this, mongoFileUrl, mongoFile, chunksCollection);
}
/**
* Upload a file to the datastore from the filesystem
*
* @param file
* - the file object to get the data from
* @param mediaType
* the media type of the data
*
* @return the MongoFile object created for this file object
*
* @throws IllegalStateException
* if compression is disabled in the configuration but ask for on the command line
* @throws IOException
* if an error occurs during reading and/or writing
* @throws IllegalArgumentException
* if required parameters are null
* @throws FileNotFoundException
* if the file does not exist or cannot be read
*/
public MongoFile upload(final File file, final String mediaType) throws IOException {
return upload(file, mediaType, config.isCompressionEnabled(), null);
}
/**
* Upload a file to the datastore from the filesystem
*
* @param file
* - the file object to get the data from
* @param mediaType
* the media type of the data
* @param expiresAt
* the date in the future that the file should expire.
* @param compress
* allow compression to be used if applicable
*
* @return the MongoFile object created for this file object
*
* @throws IllegalStateException
* if compression is disabled in the configuration but ask for on the command line
* @throws IOException
* if an error occurs during reading and/or writing
* @throws IllegalArgumentException
* if required parameters are null
* @throws FileNotFoundException
* if the file does not exist or cannot be read
*/
public MongoFile upload(final File file, final String mediaType, final boolean compress, final Date expiresAt) throws IOException {
if (file == null) {
throw new IllegalArgumentException("passed in file cannot be null");
}
if (!file.exists()) {
throw new FileNotFoundException("File does not exist or cannot be read by this library");
}
FileInputStream inputStream = new FileInputStream(file);
try {
return upload(file.toPath().toString(), mediaType, expiresAt, compress, inputStream);
} finally {
inputStream.close();
}
}
/**
* Upload a file to the datastore from any InputStream
*
* @param filename
* the name of the file to use
* @param mediaType
* the media type of the data
* @param inputStream
* the stream object to read the data from
*
* @return the MongoFile object created for this file object
*
* @throws IllegalStateException
* if compression is disabled in the configuration but ask for on the command line
* @throws IOException
* if an error occurs during reading and/or writing
* @throws IllegalArgumentException
* if required parameters are null
*/
public MongoFile upload(final String filename, final String mediaType, final InputStream inputStream) throws IOException {
return upload(filename, mediaType, null, true, inputStream);
}
/**
* Upload a file to the datastore from any InputStream
*
* @param filename
* the name of the file to use
* @param mediaType
* the media type of the data
* @param expiresAt
* the date in the future that the file should expire.
* @param compress
* allow compression to be used if applicable
* @param inputStream
* the stream object to read the data from
*
* @return the MongoFile object created for this file object
*
* @throws IllegalStateException
* if compression is disabled in the configuration but ask for on the command line
* @throws IOException
* if an error occurs during reading and/or writing
* @throws IllegalArgumentException
* if required parameters are null
*
*/
public MongoFile upload(final String filename, final String mediaType, final Date expiresAt, final boolean compress,
final InputStream inputStream) throws IOException {
return createNew(filename, mediaType, expiresAt, compress).write(inputStream);
}
//
// manifest
// ////////////////////
public MongoManifest getManifest(final MongoFile file) {
if (file == null) {
throw new IllegalArgumentException("file cannot be null");
}
if (!file.isExpandedZipFile()) {
throw new IllegalStateException("");
}
//
Document query = new Document(MongoFileConstants.manifestId.name(), file.getId());
Document sort = new Document(MongoFileConstants.manifestNum.name(), 1);
MongoFileCursor mongoFileCursor = find(query, sort);
if (!mongoFileCursor.hasNext()) {
throw new IllegalStateException("Cannot generate manifest correctly");
}
// generate the manifest
MongoManifest manifest = new MongoManifest(mongoFileCursor.next());
while (mongoFileCursor.hasNext()) {
manifest.addMongoFile(mongoFileCursor.next());
}
return manifest;
}
public MongoManifest getManifest(final MongoFileUrl url) throws IOException {
if (url == null) {
throw new IllegalArgumentException("file cannot be null");
}
MongoFile mongoFile = findOne(url);
if (mongoFile == null) {
return null;
}
return getManifest(mongoFile);
}
//
// read
// ////////////////////
/**
* Returns a reader for the passed in URL
*
* @param url
*
* @return a reader object
*
* @throws MongoException
* @throws IllegalArgumentException
* if required parameters are null
*/
public MongoFile findOne(final URL url) {
if (url == null) {
throw new IllegalArgumentException("url cannot be null");
}
return findOne(MongoFileUrl.construct(url));
}
/**
* Returns a MongoFile for the passed in file url
*
* @param url
*
* @return a reader object
*
* @throws MongoException
* @throws IllegalArgumentException
* if required parameters are null
*/
public MongoFile findOne(final MongoFileUrl url) {
if (url == null) {
throw new IllegalArgumentException("url cannot be null");
}
MongoCursor cursor = filesCollection.find(//
new Document().append(MongoFileConstants._id.toString(), url.getMongoFileId())).get();
if (!cursor.hasNext()) {
return null;
}
Document file = deletedFileCheck(cursor.next());
return file == null ? null : new MongoFile(this, file);
}
/**
* Returns true is the file exists in the data store
*
* @param url
*
* @return true if the file exists in the datastore
*
* @throws MongoException
*/
public boolean exists(final MongoFileUrl url) {
if (url == null) {
throw new IllegalArgumentException("mongoFile cannot be null");
}
return findOne(url) != null;
}
/**
* REturns true if the file can be accessed from the database
*
* @param id
* @return true if file exists in the DataStore
*/
public boolean exists(final ObjectId id) {
return null != findOne(id);
}
/**
* finds one file matching the given id. Equivalent to findOne(id)
*
* @param id
* @return the MongoFile object
* @throws MongoException
*/
public MongoFile findOne(final ObjectId id) {
MongoCursor cursor = this.getFilesCollection().find(new Document("_id", id)).get();
if (!cursor.hasNext()) {
return null;
}
Document one = deletedFileCheck(cursor.next());
if (one == null) {
return null;
}
return new MongoFile(this, one);
}
/**
* finds a list of files matching the given filename
*
* @param filename
* @return the MongoFileCursor object
* @throws MongoException
*/
public MongoFileCursor find(final String filename) {
return find(filename, null);
}
/**
* finds a list of files matching the given filename
*
* @param filename
* @param sort
* @return the MongoFileCursor object
* @throws MongoException
*/
public MongoFileCursor find(final String filename, final Document sort) {
return find(new Document(MongoFileConstants.filename.toString(), filename), sort);
}
/**
* finds a list of files matching the given query
*
* @param query
* @return the MongoFileCursor object
* @throws MongoException
*/
public MongoFileCursor find(final Document query) {
return find(query, null);
}
/**
* finds a list of files matching the given query
*
* @param query
* @param sort
* @return the MongoFileCursor object
* @throws MongoException
*/
public MongoFileCursor find(final Document query, final Document sort) {
MongoView c = this.getFilesCollection().find(query);
if (sort != null) {
c.sort(sort);
}
MongoCursor cursor = c.get();
return new MongoFileCursor(this, cursor);
}
/**
* Find a file within a list of file uploaded from a given zip archive
*
* @param zipFile
* @return MongoZipArchiveQuery object
* @throws Exception
*/
public MongoZipArchiveQuery findInZipArchive(final MongoFile zipFile) throws Exception {
return findInZipArchive(zipFile.getURL());
}
/**
* Find a file within a list of file uploaded from a given zip archive url
*
* @param zipFileUrl
* @return MongoZipArchiveQuery object
* @throws Exception
*/
public MongoZipArchiveQuery findInZipArchive(final MongoFileUrl zipFileUrl) {
return new MongoZipArchiveQuery(this, zipFileUrl);
}
//
// remove methods
// ////////////////////
/**
* Give a file an expiration date so I can be removed and resources its recovered.
*
* Use the TimeMachine DSL to easily create expiration dates.
*
* This uses MongoDB's TTL indexes feature to allow a server background thread to remove the file. According to their documentation,
* this may not happen immediately at the time the file is set to expire.
*
*
* NOTE: The MongoFileStore has remove methods which perform immediate removal of the file in the MongoFileStore.
*
* @param file
* the MongoFile to fix
* @param when
* - the date to expire the file by
*
* @throws MalformedURLException
*/
public void expireFile(final MongoFile file, final Date when) throws MalformedURLException {
MongoFileUrl url = file.getURL();
Document filesQuery = new Document("_id", url.getMongoFileId());
Document chunksQuery = new Document("files_id", url.getMongoFileId());
setExpiresAt(filesQuery, chunksQuery, when, false);
}
/**
* Remove a file from the database identified by the given MongoFile
*
* NOTE: this is not asynchronous
*
* @param mongoFile
* @throws IllegalArgumentException
* @throws MongoException
* @throws IOException
*/
public void remove(final MongoFile mongoFile) throws IOException {
remove(mongoFile, false);
}
/**
* Remove a file from the database identified by the given MongoFile
*
* @param mongoFile
* @throws IllegalArgumentException
* @throws MongoException
* @throws IOException
*/
public void remove(final MongoFile mongoFile, final boolean async) throws IOException {
if (mongoFile == null) {
throw new IllegalArgumentException("mongoFile cannot be null");
}
remove(mongoFile.getURL(), async);
}
/**
* Remove a file from the datastore identified by the given MongoFileUrl
*
* NOTE: this is not asynchronous
*
* @param url
* @throws IllegalArgumentException
* @throws MongoException
*/
public void remove(final MongoFileUrl url) {
remove(url, false);
}
/**
* Remove a file from the datastore identified by the given MongoFileUrl
*
* @param url
* - the MongoFileUrl
* @param async
* - should the delete be asynchroized
*
* @throws IOException
* if an error occurs during reading and/or writing
* @throws IllegalArgumentException
* if required parameters are null
*/
public void remove(final MongoFileUrl url, final boolean async) {
if (url == null) {
throw new IllegalArgumentException("mongoFileUrl cannot be null");
}
remove(new Document("_id", url.getMongoFileId()), async);
}
/**
* Delete all files that match the given criteria
*
* NOTE: this is not asynchronous
*
* @param query
*/
public void remove(final Document query) {
remove(query, false);
}
/**
* Delete all files that match the given criteria
*
* This code was taken from -- https://github.com/mongodb/mongo-java-driver/pull/171
*
* @param query
* the selection criteria
* @param async
* - can the file be deleted asynchronously
* @throws IllegalArgumentException
* if required parameters are null
*/
public void remove(final Document query, final boolean async) {
if (query == null) {
throw new IllegalArgumentException("query can not be null");
}
// can't remove chunks without files_id thus keep them
List filesIds = new ArrayList();
for (MongoFile f : find(query)) {
// add all files in from the expanded zip file
if (f.isExpandedZipFile()) {
for (MongoFile f2 : this.getManifest(f).getFiles()) {
filesIds.add(f2.getId());
}
}
filesIds.add(f.getId());
}
Document filesQuery = new Document("_id", new Document("$in", filesIds));
Document chunksQuery = new Document("files_id", new Document("$in", filesIds));
// flag delete always for quick "logically" removal
setExpiresAt(filesQuery, chunksQuery, TimeMachine.now().backward(1).seconds().inTime(), true);
// do the real delete if requested
if (!async) {
// remove files from bucket
WriteResult writeResult = getFilesCollection().remove(filesQuery);
if (writeResult.getCount() > 0) {
// then remove chunks, for those file objects
getChunksCollection().remove(chunksQuery);
}
}
}
private void setExpiresAt(final Document filesQuery, final Document chunksQuery, final Date when, final boolean multi) {
// files collection
Document filesUpdate = new Document()//
.append(MongoFileConstants.expireAt.toString(), when)//
.append(MongoFileConstants.deleted.toString(), when.before(new Date())); //
filesUpdate = new Document().append("$set", filesUpdate);
getFilesCollection().find(filesQuery)//
.withWriteConcern(WriteConcern.JOURNALED)//
.update(filesUpdate);
// chunks collection - wait until the file objects are removed
Document chunksUpdate = new Document()//
.append(MongoFileConstants.expireAt.toString(), when);
chunksUpdate = new Document("$set", chunksUpdate);
getChunksCollection().find(chunksQuery)//
.withWriteConcern(WriteConcern.JOURNALED)//
.update(chunksUpdate);
}
private Document deletedFileCheck(final Document file) {
if (new MongoFile(this, file).isDeleted()) {
return null;
}
return file;
}
//
// collection getters
// /////////////////////////
/**
* The underlying MongoDB collection object for files
*
* @return the DBCollection object
*/
/* package */MongoCollection getFilesCollection() {
return filesCollection;
}
/**
* The underlying MongoDB collection object
*
* @return the DBCollection object
*/
/* package */MongoCollection getChunksCollection() {
return chunksCollection;
}
//
// toString
// //////////////////
@Override
public String toString() {
return String.format("MongoFileStore [filesCollection=%s, chunksCollection=%s,%n config=%s%n]", filesCollection, chunksCollection,
config.toString());
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy