• Home
• com.arcadedb
• arcadedb-engine
• 24.1.1
• source code
• LocalDatabase.java

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

com.arcadedb.database.LocalDatabase Maven / Gradle / Ivy

/*
 * Copyright © 2021-present Arcade Data Ltd ([email protected])
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * SPDX-FileCopyrightText: 2021-present Arcade Data Ltd ([email protected])
 * SPDX-License-Identifier: Apache-2.0
 */
package com.arcadedb.database;

import com.arcadedb.ContextConfiguration;
import com.arcadedb.GlobalConfiguration;
import com.arcadedb.Profiler;
import com.arcadedb.database.async.DatabaseAsyncExecutor;
import com.arcadedb.database.async.DatabaseAsyncExecutorImpl;
import com.arcadedb.database.async.ErrorCallback;
import com.arcadedb.database.async.OkCallback;
import com.arcadedb.engine.Bucket;
import com.arcadedb.engine.ComponentFile;
import com.arcadedb.engine.Dictionary;
import com.arcadedb.engine.ErrorRecordCallback;
import com.arcadedb.engine.FileManager;
import com.arcadedb.engine.LocalBucket;
import com.arcadedb.engine.PageManager;
import com.arcadedb.engine.TransactionManager;
import com.arcadedb.engine.WALFile;
import com.arcadedb.engine.WALFileFactory;
import com.arcadedb.engine.WALFileFactoryEmbedded;
import com.arcadedb.exception.ArcadeDBException;
import com.arcadedb.exception.CommandExecutionException;
import com.arcadedb.exception.DatabaseIsClosedException;
import com.arcadedb.exception.DatabaseIsReadOnlyException;
import com.arcadedb.exception.DatabaseMetadataException;
import com.arcadedb.exception.DatabaseOperationException;
import com.arcadedb.exception.DuplicatedKeyException;
import com.arcadedb.exception.InvalidDatabaseInstanceException;
import com.arcadedb.exception.NeedRetryException;
import com.arcadedb.exception.TransactionException;
import com.arcadedb.graph.Edge;
import com.arcadedb.graph.GraphEngine;
import com.arcadedb.graph.MutableVertex;
import com.arcadedb.graph.Vertex;
import com.arcadedb.graph.VertexInternal;
import com.arcadedb.index.IndexCursor;
import com.arcadedb.index.IndexInternal;
import com.arcadedb.index.TypeIndex;
import com.arcadedb.index.lsm.LSMTreeIndexCompacted;
import com.arcadedb.index.lsm.LSMTreeIndexMutable;
import com.arcadedb.index.vector.HnswVectorIndex;
import com.arcadedb.log.LogManager;
import com.arcadedb.query.QueryEngine;
import com.arcadedb.query.QueryEngineManager;
import com.arcadedb.query.select.Select;
import com.arcadedb.query.sql.executor.ResultSet;
import com.arcadedb.query.sql.parser.ExecutionPlanCache;
import com.arcadedb.query.sql.parser.StatementCache;
import com.arcadedb.schema.DocumentType;
import com.arcadedb.schema.LocalDocumentType;
import com.arcadedb.schema.LocalSchema;
import com.arcadedb.schema.LocalVertexType;
import com.arcadedb.schema.Property;
import com.arcadedb.schema.Schema;
import com.arcadedb.schema.VertexType;
import com.arcadedb.security.SecurityDatabaseUser;
import com.arcadedb.security.SecurityManager;
import com.arcadedb.serializer.BinarySerializer;
import com.arcadedb.utility.FileUtils;
import com.arcadedb.utility.LockException;
import com.arcadedb.utility.MultiIterator;
import com.arcadedb.utility.RWLockContext;

import java.io.*;
import java.nio.channels.*;
import java.nio.file.*;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.*;
import java.util.concurrent.locks.*;
import java.util.logging.*;

/**
 * Local implementation of {@link Database}. It is based on files opened on the local file system.
 * 

 * Thread safe and therefore the same instance can be shared among threads.
 *
 * @author Luca Garulli ([email protected])
 */
public class LocalDatabase extends RWLockContext implements DatabaseInternal {
  public static final  int                                       EDGE_LIST_INITIAL_CHUNK_SIZE         = 64;
  public static final  int                                       MAX_RECOMMENDED_EDGE_LIST_CHUNK_SIZE = 8192;
  private static final Set                               SUPPORTED_FILE_EXT                   = Set.of(Dictionary.DICT_EXT,
      LocalBucket.BUCKET_EXT, LSMTreeIndexMutable.NOTUNIQUE_INDEX_EXT, LSMTreeIndexMutable.UNIQUE_INDEX_EXT,
      LSMTreeIndexCompacted.NOTUNIQUE_INDEX_EXT, LSMTreeIndexCompacted.UNIQUE_INDEX_EXT, HnswVectorIndex.FILE_EXT);
  public final         AtomicLong                                indexCompactions                     = new AtomicLong();
  protected final      String                                    name;
  protected final      ComponentFile.MODE                        mode;
  protected final      ContextConfiguration                      configuration;
  protected final      String                                    databasePath;
  protected final      BinarySerializer                          serializer;
  protected final      RecordFactory                             recordFactory                        = new RecordFactory();
  protected final      GraphEngine                               graphEngine;
  protected final      WALFileFactory                            walFactory;
  protected final      DocumentIndexer                           indexer;
  protected final      QueryEngineManager                        queryEngineManager;
  protected final      DatabaseStats                             stats                                = new DatabaseStats();
  protected            FileManager                               fileManager;
  protected            PageManager                               pageManager;
  protected            LocalSchema                               schema;
  protected            TransactionManager                        transactionManager;
  protected volatile   DatabaseAsyncExecutorImpl                 async                                = null;
  protected final      Lock                                      asyncLock                            = new ReentrantLock();
  protected            boolean                                   autoTransaction                      = false;
  protected volatile   boolean                                   open                                 = false;
  private              boolean                                   readYourWrites                       = true;
  private final        Map>> callbacks;
  private final        StatementCache                            statementCache;
  private final        ExecutionPlanCache                        executionPlanCache;
  private final        File                                      configurationFile;
  private              DatabaseInternal                          wrappedDatabaseInstance              = this;
  private              int                                       edgeListSize                         = EDGE_LIST_INITIAL_CHUNK_SIZE;
  private final        SecurityManager                           security;
  private final        Map                       wrappers                             = new HashMap<>();
  private              File                                      lockFile;
  private              RandomAccessFile                          lockFileIO;
  private              FileChannel                               lockFileIOChannel;
  private              FileLock                                  lockFileLock;
  private final        RecordEventsRegistry                      events                               = new RecordEventsRegistry();
  private final        ConcurrentHashMap    reusableQueryEngines                 = new ConcurrentHashMap<>();
  private              TRANSACTION_ISOLATION_LEVEL               transactionIsolationLevel            = TRANSACTION_ISOLATION_LEVEL.READ_COMMITTED;

  protected LocalDatabase(final String path, final ComponentFile.MODE mode, final ContextConfiguration configuration,
      final SecurityManager security, final Map>> callbacks) {
    try {
      this.mode = mode;
      this.configuration = configuration;
      this.security = security;
      this.callbacks = callbacks;
      this.serializer = new BinarySerializer(configuration);
      this.walFactory = mode == ComponentFile.MODE.READ_WRITE ? new WALFileFactoryEmbedded() : null;
      this.statementCache = new StatementCache(this, configuration.getValueAsInteger(GlobalConfiguration.SQL_STATEMENT_CACHE));
      this.executionPlanCache = new ExecutionPlanCache(this,
          configuration.getValueAsInteger(GlobalConfiguration.SQL_STATEMENT_CACHE));

      if (path.endsWith(File.separator))
        databasePath = path.substring(0, path.length() - 1);
      else
        databasePath = path;

      configurationFile = new File(databasePath + File.separator + "configuration.json");

      final int lastSeparatorPos = path.lastIndexOf(File.separator);
      if (lastSeparatorPos > -1)
        name = path.substring(lastSeparatorPos + 1);
      else
        name = path;

      checkDatabaseName();

      indexer = new DocumentIndexer(this);
      queryEngineManager = new QueryEngineManager();
      graphEngine = new GraphEngine(this);

    } catch (final Exception e) {
      if (e instanceof DatabaseOperationException)
        throw (DatabaseOperationException) e;

      throw new DatabaseOperationException("Error on creating new database instance", e);
    }
  }

  protected void open() {
    if (!new File(databasePath).exists())
      throw new DatabaseOperationException("Database '" + databasePath + "' does not exist");

    if (configurationFile.exists()) {
      try {
        final String content = FileUtils.readFileAsString(configurationFile);
        configuration.reset();
        configuration.fromJSON(content);
      } catch (final IOException e) {
        LogManager.instance().log(this, Level.SEVERE, "Error on loading configuration from file '%s'", e, configurationFile);
      }
    }

    openInternal();
  }

  protected void create() {
    final File databaseDirectory = new File(databasePath);
    if (new File(databaseDirectory, LocalSchema.SCHEMA_FILE_NAME).exists() || new File(databaseDirectory,
        LocalSchema.SCHEMA_PREV_FILE_NAME).exists())
      throw new DatabaseOperationException("Database '" + databasePath + "' already exists");

    if (!databaseDirectory.exists() && !databaseDirectory.mkdirs())
      throw new DatabaseOperationException("Cannot create directory '" + databasePath + "'");

    openInternal();

    schema.saveConfiguration();

    try {
      saveConfiguration();
    } catch (final IOException e) {
      LogManager.instance().log(this, Level.SEVERE, "Error on saving configuration to file '%s'", e, configurationFile);
    }
  }

  @Override
  public void drop() {
    checkDatabaseIsOpen();

    if (isTransactionActive())
      throw new DatabaseOperationException("Cannot drop the database in transaction");

    if (mode == ComponentFile.MODE.READ_ONLY)
      throw new DatabaseIsReadOnlyException("Cannot drop database");

    internalClose(true);

    executeInWriteLock(() -> {
      FileUtils.deleteRecursively(new File(databasePath));
      return null;
    });
  }

  @Override
  public void close() {
    internalClose(false);
  }

  /**
   * Test only API. Simulates a forced kill of the JVM leaving the database with the .lck file on the file system.
   */
  @Override
  public void kill() {
    if (async != null)
      async.kill();

    if (getTransaction().isActive())
      // ROLLBACK ANY PENDING OPERATION
      getTransaction().kill();

    try {
      schema.close();
      pageManager.kill();
      fileManager.close();
      transactionManager.kill();

      if (lockFile != null) {
        try {
          if (lockFileLock != null) {
            lockFileLock.release();
          }
          if (lockFileIOChannel != null)
            lockFileIOChannel.close();
          if (lockFileIO != null)
            lockFileIO.close();
        } catch (final IOException e) {
          // IGNORE IT
        }
      }

    } finally {
      open = false;
      Profiler.INSTANCE.unregisterDatabase(LocalDatabase.this);
    }
  }

  @Override
  public boolean isAsyncProcessing() {
    if (async != null) {
      asyncLock.lock();
      try {
        return async.isProcessing();
      } finally {
        asyncLock.unlock();
      }
    }
    return false;
  }

  public DatabaseAsyncExecutor async() {
    if (async == null) {
      asyncLock.lock();
      try {
        if (async == null)
          async = new DatabaseAsyncExecutorImpl(wrappedDatabaseInstance);
      } finally {
        asyncLock.unlock();
      }
    }
    return async;
  }

  @Override
  public Map getStats() {
    final Map map = stats.toMap();
    map.put("indexCompactions", indexCompactions.get());
    return map;
  }

  @Override
  public String getDatabasePath() {
    return databasePath;
  }

  @Override
  public String getCurrentUserName() {
    final DatabaseContext.DatabaseContextTL dbContext = DatabaseContext.INSTANCE.getContextIfExists(databasePath);
    if (dbContext == null)
      return null;
    final SecurityDatabaseUser user = dbContext.getCurrentUser();
    return user != null ? user.getName() : null;
  }

  @Override
  public int getNestedTransactions() {
    final DatabaseContext.DatabaseContextTL dbContext = DatabaseContext.INSTANCE.getContextIfExists(databasePath);
    if (dbContext != null)
      return dbContext.transactions.size();
    return 0;
  }

  public TransactionContext getTransactionIfExists() {
    final DatabaseContext.DatabaseContextTL dbContext = DatabaseContext.INSTANCE.getContextIfExists(databasePath);
    if (dbContext != null) {
      final TransactionContext tx = dbContext.getLastTransaction();
      if (tx != null) {
        final DatabaseInternal txDb = tx.getDatabase();
        if (txDb == null) {
          tx.rollback();
          throw new InvalidDatabaseInstanceException("Invalid transactional context (db is null)");
        }
        if (txDb.getEmbedded() != this) {
          try {
            DatabaseContext.INSTANCE.init(this);
          } catch (final Exception e) {
            // IGNORE IT
          }
          throw new InvalidDatabaseInstanceException("Invalid transactional context (different db)");
        }
        return tx;
      }
    }

    return null;
  }

  @Override
  public void begin() {
    begin(transactionIsolationLevel);
  }

  @Override
  public void begin(final TRANSACTION_ISOLATION_LEVEL isolationLevel) {
    executeInReadLock(() -> {
      checkDatabaseIsOpen();

      // FORCE THE RESET OF TL
      final DatabaseContext.DatabaseContextTL current = DatabaseContext.INSTANCE.getContext(databasePath);
      TransactionContext tx = current.getLastTransaction();
      if (tx.isActive()) {
        // CREATE A NESTED TX
        tx = new TransactionContext(getWrappedDatabaseInstance());
        current.pushTransaction(tx);
      }

      tx.begin(isolationLevel);

      return null;
    });
  }

  public void incrementStatsTxCommits() {
    stats.txCommits.incrementAndGet();
  }

  @Override
  public void commit() {
    stats.txCommits.incrementAndGet();

    executeInReadLock(() -> {
      checkTransactionIsActive(false);

      final DatabaseContext.DatabaseContextTL current = DatabaseContext.INSTANCE.getContext(LocalDatabase.this.getDatabasePath());
      try {
        current.getLastTransaction().commit();
      } finally {
        current.popIfNotLastTransaction();
      }

      return null;
    });
  }

  @Override
  public void rollback() {
    stats.txRollbacks.incrementAndGet();

    executeInReadLock(() -> {
      try {
        checkTransactionIsActive(false);

        final DatabaseContext.DatabaseContextTL current = DatabaseContext.INSTANCE.getContext(LocalDatabase.this.getDatabasePath());
        current.popIfNotLastTransaction().rollback();

      } catch (final TransactionException e) {
        // ALREADY ROLLED BACK
      }
      return null;
    });
  }

  @Override
  public void rollbackAllNested() {
    if (!isTransactionActive())
      return;

    stats.txRollbacks.incrementAndGet();

    executeInReadLock(() -> {
      final DatabaseContext.DatabaseContextTL current = DatabaseContext.INSTANCE.getContext(LocalDatabase.this.getDatabasePath());

      TransactionContext tx;
      while ((tx = current.popIfNotLastTransaction()) != null) {
        try {
          if (tx.isActive())
            tx.rollback();
          else
            break;

        } catch (final InvalidDatabaseInstanceException e) {
          current.popIfNotLastTransaction().rollback();
        } catch (final TransactionException e) {
          // ALREADY ROLLED BACK
        }
      }
      return null;
    });
  }

  @Override
  public long countBucket(final String bucketName) {
    stats.countBucket.incrementAndGet();
    return (Long) executeInReadLock((Callable