All Downloads are FREE. Search and download functionalities are using the official Maven repository.
  • Log In
  • Register

    • org.infinispan.persistence.leveldb.LevelDBStore Maven / Gradle / Ivy

    There is a newer version: 9.1.7.Final
    Show newest version
    package org.infinispan.persistence.leveldb;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletionService;
import java.util.concurrent.Executor;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.Semaphore;

import org.infinispan.commons.CacheConfigurationException;
import org.infinispan.commons.configuration.ConfiguredBy;
import org.infinispan.commons.util.Util;
import org.infinispan.executors.ExecutorAllCompletionService;
import org.infinispan.filter.KeyFilter;
import org.infinispan.marshall.core.MarshalledEntry;
import org.infinispan.metadata.InternalMetadata;
import org.infinispan.persistence.PersistenceUtil;
import org.infinispan.persistence.TaskContextImpl;
import org.infinispan.persistence.leveldb.configuration.LevelDBStoreConfiguration;
import org.infinispan.persistence.leveldb.logging.Log;
import org.infinispan.persistence.spi.AdvancedLoadWriteStore;
import org.infinispan.persistence.spi.InitializationContext;
import org.infinispan.persistence.spi.PersistenceException;
import org.infinispan.util.logging.LogFactory;
import org.iq80.leveldb.CompressionType;
import org.iq80.leveldb.DB;
import org.iq80.leveldb.DBFactory;
import org.iq80.leveldb.DBIterator;
import org.iq80.leveldb.Options;
import org.iq80.leveldb.ReadOptions;

@ConfiguredBy(LevelDBStoreConfiguration.class)
public class LevelDBStore implements AdvancedLoadWriteStore {
   private static final Log log = LogFactory.getLog(LevelDBStore.class, Log.class);

   private static final String JNI_DB_FACTORY_CLASS_NAME = "org.fusesource.leveldbjni.JniDBFactory";
   private static final String JAVA_DB_FACTORY_CLASS_NAME = "org.iq80.leveldb.impl.Iq80DBFactory";
   private static final String[] DB_FACTORY_CLASS_NAMES = new String[] { JNI_DB_FACTORY_CLASS_NAME, JAVA_DB_FACTORY_CLASS_NAME };

   private LevelDBStoreConfiguration configuration;
   private BlockingQueue expiryEntryQueue;
   private DBFactory dbFactory;
   private DB db;
   private DB expiredDb;
   private InitializationContext ctx;
   private Semaphore semaphore;
   private volatile boolean stopped = true;

   @Override
   public void init(InitializationContext ctx) {
      this.configuration = ctx.getConfiguration();
      this.dbFactory = newDbFactory();
      this.ctx = ctx;
      this.semaphore = new Semaphore(Integer.MAX_VALUE, true);

      if (this.dbFactory == null) {
         throw log.cannotLoadlevelDBFactories(Arrays.toString(DB_FACTORY_CLASS_NAMES));
      }
      String dbFactoryClassName = this.dbFactory.getClass().getName();
      if (dbFactoryClassName.equals("org.iq80.leveldb.impl.Iq80DBFactory")) {
         log.infoUsingJavaDbFactory(dbFactoryClassName);
      } else {
         log.infoUsingJNIDbFactory(dbFactoryClassName);
      }
   }

   protected DBFactory newDbFactory() {
      switch (configuration.implementationType()) {
         case JNI: {
            return Util.getInstance(JNI_DB_FACTORY_CLASS_NAME, LevelDBStore.class.getClassLoader());
         }
         case JAVA: {
            return Util.getInstance(JAVA_DB_FACTORY_CLASS_NAME, LevelDBStore.class.getClassLoader());
         }
         default: {
            for (String className : DB_FACTORY_CLASS_NAMES) {
               try {
                  return Util.getInstance(className, LevelDBStore.class.getClassLoader());
               } catch (Throwable e) {
                  if (log.isDebugEnabled())
                     log.debugUnableToInstantiateDbFactory(className, e);
               }
            }
         }
      }
      return null;
   }

   @Override
   public void start()  {
      expiryEntryQueue = new LinkedBlockingQueue(configuration.expiryQueueSize());

      try {
         db = openDatabase(getQualifiedLocation(), dataDbOptions());
         expiredDb = openDatabase(getQualifiedExpiredLocation(), expiredDbOptions());
         stopped = false;
      } catch (IOException e) {
         throw new CacheConfigurationException("Unable to open database", e);
      }
   }

   private String sanitizedCacheName() {
      String cacheFileName = ctx.getCache().getName().replaceAll("[^a-zA-Z0-9-_\\.]", "_");
      return cacheFileName;
   }

   private String getQualifiedLocation() {
      return configuration.location() + sanitizedCacheName();
   }

   private String getQualifiedExpiredLocation() {
      return configuration.expiredLocation() + sanitizedCacheName();
   }

   private Options dataDbOptions() {
      Options options = new Options().createIfMissing(true);

      options.compressionType(CompressionType.valueOf(configuration.compressionType().name()));

      if (configuration.blockSize() != null) {
         options.blockSize(configuration.blockSize());
      }

      if (configuration.cacheSize() != null) {
         options.cacheSize(configuration.cacheSize());
      }

      return options;
   }

   private Options expiredDbOptions() {
      return new Options().createIfMissing(true);
   }

   /**
    * Creates database if it doesn't exist.
    */
   protected DB openDatabase(String location, Options options) throws IOException {
      File dir = new File(location);

      // LevelDB JNI Option createIfMissing doesn't seem to work properly
      dir.mkdirs();
      return dbFactory.open(dir, options);
   }

   protected void destroyDatabase(String location) throws IOException {
      File dir = new File(location);
      dbFactory.destroy(dir, new Options());
   }

   protected DB reinitDatabase(String location, Options options) throws IOException {
      destroyDatabase(location);
      return openDatabase(location, options);
   }

   protected void reinitAllDatabases() throws IOException {
      try {
         semaphore.acquire(Integer.MAX_VALUE);
      } catch (InterruptedException e) {
         throw new PersistenceException("Cannot acquire semaphore", e);
      }
      try {
         if (stopped) {
            throw new PersistenceException("LevelDB is stopped");
         }
         try {
            db.close();
         } catch (IOException e) {
            log.warnUnableToCloseDb(e);
         }
         try {
            expiredDb.close();
         } catch (IOException e) {
            log.warnUnableToCloseExpiredDb(e);
         }
         db = reinitDatabase(getQualifiedLocation(), dataDbOptions());
         expiredDb = reinitDatabase(getQualifiedExpiredLocation(), expiredDbOptions());
      } finally {
         semaphore.release(Integer.MAX_VALUE);
      }
   }

   @Override
   public void stop()  {
      try {
         semaphore.acquire(Integer.MAX_VALUE);
      } catch (InterruptedException e) {
         throw new PersistenceException("Cannot acquire semaphore", e);
      }
      try {
         try {
            db.close();
         } catch (IOException e) {
            log.warnUnableToCloseDb(e);
         }

         try {
            expiredDb.close();
         } catch (IOException e) {
            log.warnUnableToCloseExpiredDb(e);
         }
      } finally {
         stopped = true;
         semaphore.release(Integer.MAX_VALUE);
      }
   }

   @Override
   public void clear() {
      long count = 0;
      boolean destroyDatabase = false;
      try {
         semaphore.acquire();
      } catch (InterruptedException e) {
         throw new PersistenceException("Cannot acquire semaphore", e);
      }
      try {
         if (stopped) {
            throw new PersistenceException("LevelDB is stopped");
         }
         DBIterator it = db.iterator(new ReadOptions().fillCache(false));
         if (configuration.clearThreshold() <= 0) {
            try {
               for (it.seekToFirst(); it.hasNext(); ) {
                  Map.Entry entry = it.next();
                  db.delete(entry.getKey());
                  count++;

                  if (count > configuration.clearThreshold()) {
                     destroyDatabase = true;
                     break;
                  }
               }
            } finally {
               try {
                  it.close();
               } catch (IOException e) {
                  log.warnUnableToCloseDbIterator(e);
               }
            }
         } else {
            destroyDatabase = true;
         }
      } finally {
         semaphore.release();
      }

      if (destroyDatabase) {
         try {
            reinitAllDatabases();
         } catch (IOException e) {
            throw new PersistenceException(e);
         }
      }
   }

   @Override
   public int size() {
      return PersistenceUtil.count(this, null);
   }

   @Override
   public boolean contains(Object key) {
      try {
         return load(key) != null;
      } catch (Exception e) {
         throw new PersistenceException(e);
      }
   }

   @SuppressWarnings("unchecked")
   @Override
   public void process(KeyFilter keyFilter, CacheLoaderTask cacheLoaderTask, Executor executor, boolean loadValues, boolean loadMetadata) {

      int batchSize = 100;
      ExecutorAllCompletionService eacs = new ExecutorAllCompletionService(executor);
      final TaskContext taskContext = new TaskContextImpl();

      List> entries = new ArrayList>(batchSize);
      try {
         semaphore.acquire();
      } catch (InterruptedException e) {
         throw new PersistenceException("Cannot acquire semaphore: CacheStore is likely stopped.", e);
      }
      try {
         if (stopped) {
            throw new PersistenceException("LevelDB is stopped");
         }
         DBIterator it = db.iterator(new ReadOptions().fillCache(false));
         try {
            for (it.seekToFirst(); it.hasNext(); ) {
               Map.Entry entry = it.next();
               entries.add(entry);
               if (entries.size() == batchSize) {
                  final List> batch = entries;
                  entries = new ArrayList>(batchSize);
                  submitProcessTask(cacheLoaderTask, keyFilter, eacs, taskContext, batch, loadValues, loadMetadata);
               }
            }
            if (!entries.isEmpty()) {
               submitProcessTask(cacheLoaderTask, keyFilter, eacs, taskContext, entries, loadValues, loadMetadata);
            }

            eacs.waitUntilAllCompleted();
            if (eacs.isExceptionThrown()) {
               throw new PersistenceException("Execution exception!", eacs.getFirstException());
            }
         } catch (Exception e) {
            throw new PersistenceException(e);
         } finally {
            try {
               it.close();
            } catch (IOException e) {
               log.warnUnableToCloseDbIterator(e);
            }
         }
      } finally {
         semaphore.release();
      }
   }

   @SuppressWarnings("unchecked")
   private void submitProcessTask(final CacheLoaderTask cacheLoaderTask, final KeyFilter filter, CompletionService ecs,
                                  final TaskContext taskContext, final List> batch,
                                  final boolean loadValues, final boolean loadMetadata) {
      ecs.submit(new Callable() {
         @Override
         public Void call() throws Exception {
            try {
               long now = ctx.getTimeService().wallClockTime();
               for (Map.Entry pair : batch) {
                  if (taskContext.isStopped()) {break;}
                  Object key = unmarshall(pair.getKey());
                  if (filter == null || filter.accept(key)) {
                     MarshalledEntry entry = loadValues || loadMetadata ? (MarshalledEntry) unmarshall(pair.getValue()) : null;
                     boolean isExpired = entry != null && entry.getMetadata() != null && entry.getMetadata().isExpired(now);
                     if (!isExpired) {
                        if (!loadValues || !loadMetadata) {
                           entry = ctx.getMarshalledEntryFactory().newMarshalledEntry(
                                 key, loadValues ? entry.getValue() : null, loadMetadata ? entry.getMetadata() : null);
                        }
                        cacheLoaderTask.processEntry(entry, taskContext);
                     }
                  }
               }
            } catch (Exception e) {
               log.errorExecutingParallelStoreTask(e);
               throw e;
            }
            return null;
         }
      });
   }

   @Override
   public boolean delete(Object key)  {
      try {
         byte[] keyBytes = marshall(key);
         semaphore.acquire();
         try {
            if (stopped) {
               throw new PersistenceException("LevelDB is stopped");
            }
            if (db.get(keyBytes) == null) {
               return false;
            }
            db.delete(keyBytes);
         } finally {
            semaphore.release();
         }
         return true;
      } catch (Exception e) {
         throw new PersistenceException(e);
      }
   }

   @Override
   public void write(MarshalledEntry me)  {
      try {
         byte[] marshelledKey = marshall(me.getKey());
         byte[] marshalledEntry = marshall(me);
         semaphore.acquire();
         try {
            if (stopped) {
               throw new PersistenceException("LevelDB is stopped");
            }
            db.put(marshelledKey, marshalledEntry);
         } finally {
            semaphore.release();
         }
         InternalMetadata meta = me.getMetadata();
         if (meta != null && meta.expiryTime() > -1) {
            addNewExpiry(me);
         }
      } catch (Exception e) {
         throw new PersistenceException(e);
      }
   }

   @Override
   public MarshalledEntry load(Object key)  {
      try {
         byte[] marshalledEntry;
         semaphore.acquire();
         try {
            if (stopped) {
               throw new PersistenceException("LevelDB is stopped");
            }
            marshalledEntry = db.get(marshall(key));
         } finally {
            semaphore.release();
         }
         MarshalledEntry me = (MarshalledEntry) unmarshall(marshalledEntry);
         if (me == null) return null;

         InternalMetadata meta = me.getMetadata();
         if (meta != null && meta.isExpired(ctx.getTimeService().wallClockTime())) {
            return null;
         }
         return me;
      } catch (Exception e) {
         throw new PersistenceException(e);
      }
   }

   @SuppressWarnings("unchecked")
   @Override
   public void purge(Executor executor, PurgeListener purgeListener) {
      try {
         semaphore.acquire();
      } catch (InterruptedException e) {
         throw new PersistenceException("Cannot acquire semaphore: CacheStore is likely stopped.", e);
      }
      try {
         if (stopped) {
            throw new PersistenceException("LevelDB is stopped");
         }
         // Drain queue and update expiry tree
         List entries = new ArrayList();
         expiryEntryQueue.drainTo(entries);
         for (ExpiryEntry entry : entries) {
            final byte[] expiryBytes = marshall(entry.expiry);
            final byte[] keyBytes = marshall(entry.key);
            final byte[] existingBytes = expiredDb.get(expiryBytes);

            if (existingBytes != null) {
               // in the case of collision make the key a List ...
               final Object existing = unmarshall(existingBytes);
               if (existing instanceof List) {
                  ((List) existing).add(entry.key);
                  expiredDb.put(expiryBytes, marshall(existing));
               } else {
                  List al = new ArrayList(2);
                  al.add(existing);
                  al.add(entry.key);
                  expiredDb.put(expiryBytes, marshall(al));
               }
            } else {
               expiredDb.put(expiryBytes, keyBytes);
            }
         }

         List times = new ArrayList();
         List keys = new ArrayList();
         DBIterator it = expiredDb.iterator(new ReadOptions().fillCache(false));
         long now = ctx.getTimeService().wallClockTime();
         try {
            for (it.seekToFirst(); it.hasNext();) {
               Map.Entry entry = it.next();

               Long time = (Long) unmarshall(entry.getKey());
               if (time > now)
                  break;
               times.add(time);
               Object key = unmarshall(entry.getValue());
               if (key instanceof List)
                  keys.addAll((List) key);
               else
                  keys.add(key);
            }

            for (Long time : times) {
               expiredDb.delete(marshall(time));
            }

            if (!keys.isEmpty())
               log.debugf("purge (up to) %d entries", keys.size());
            int count = 0;
            for (Object key : keys) {
               byte[] keyBytes = marshall(key);

               byte[] b = db.get(keyBytes);
               if (b == null)
                  continue;
               MarshalledEntry me = (MarshalledEntry) ctx.getMarshaller().objectFromByteBuffer(b);
               // TODO race condition: the entry could be updated between the get and delete!
               if (me.getMetadata() != null && me.getMetadata().isExpired(now)) {
                  // somewhat inefficient to FIND then REMOVE...
                  db.delete(keyBytes);
                  purgeListener.entryPurged(key);
                  count++;
               }

            }
            if (count != 0)
               log.debugf("purged %d entries", count);
         } catch (Exception e) {
            throw new PersistenceException(e);
         } finally {
            try {
               it.close();
            } catch (IOException e) {
               log.warnUnableToCloseDbIterator(e);
            }
         }
      } catch (PersistenceException e) {
         throw e;
      } catch (Exception e) {
         throw new PersistenceException(e);
      } finally {
         semaphore.release();
      }
   }

   private byte[] marshall(Object entry) throws IOException, InterruptedException {
      return ctx.getMarshaller().objectToByteBuffer(entry);
   }

   private Object unmarshall(byte[] bytes) throws IOException, ClassNotFoundException {
      if (bytes == null)
         return null;

      return ctx.getMarshaller().objectFromByteBuffer(bytes);
   }

   private void addNewExpiry(MarshalledEntry entry) throws IOException {
      long expiry = entry.getMetadata().expiryTime();
      long maxIdle = entry.getMetadata().maxIdle();
      if (maxIdle > 0) {
         // Coding getExpiryTime() for transient entries has the risk of
         // being a moving target
         // which could lead to unexpected results, hence, InternalCacheEntry
         // calls are required
         expiry = maxIdle + ctx.getTimeService().wallClockTime();
      }
      Long at = expiry;
      Object key = entry.getKey();

      try {
         expiryEntryQueue.put(new ExpiryEntry(at, key));
      } catch (InterruptedException e) {
         Thread.currentThread().interrupt(); // Restore interruption status
      }
   }

   private static final class ExpiryEntry {
      private final Long expiry;
      private final Object key;

      private ExpiryEntry(long expiry, Object key) {
         this.expiry = expiry;
         this.key = key;
      }

      @Override
      public int hashCode() {
         final int prime = 31;
         int result = 1;
         result = prime * result + ((key == null) ? 0 : key.hashCode());
         return result;
      }

      @Override
      public boolean equals(Object obj) {
         if (this == obj)
            return true;
         if (obj == null)
            return false;
         if (getClass() != obj.getClass())
            return false;
         ExpiryEntry other = (ExpiryEntry) obj;
         if (key == null) {
            if (other.key != null)
               return false;
         } else if (!key.equals(other.key))
            return false;
         return true;
      }

   }

}
    
    
    
    

    




    
    
    © 2015 - 2025 Weber Informatics LLC | Privacy Policy