org.neo4j.kernel.recovery.TransactionLogsRecovery Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of neo4j-kernel Show documentation
Neo4j kernel is a lightweight, embedded Java database designed to store data structured as graphs rather than tables. For more information, see http://neo4j.org.
There is a newer version: 5.25.1
Show newest version
/*
 * Copyright (c) "Neo4j"
 * Neo4j Sweden AB [http://neo4j.com]
 *
 * This file is part of Neo4j.
 *
 * Neo4j is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see .
 */
package org.neo4j.kernel.recovery;

import java.nio.channels.ClosedByInterruptException;

import org.neo4j.common.ProgressReporter;
import org.neo4j.dbms.database.DatabaseStartAbortedException;
import org.neo4j.io.pagecache.context.CursorContext;
import org.neo4j.io.pagecache.tracing.PageCacheTracer;
import org.neo4j.kernel.database.Database;
import org.neo4j.kernel.impl.transaction.CommittedTransactionRepresentation;
import org.neo4j.kernel.impl.transaction.log.LogPosition;
import org.neo4j.kernel.impl.transaction.log.entry.LogEntryCommit;
import org.neo4j.kernel.lifecycle.Lifecycle;
import org.neo4j.kernel.lifecycle.LifecycleAdapter;
import org.neo4j.storageengine.api.TransactionIdStore;
import org.neo4j.time.Stopwatch;

import static java.lang.String.format;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static org.neo4j.kernel.recovery.Recovery.throwUnableToCleanRecover;
import static org.neo4j.storageengine.api.TransactionApplicationMode.RECOVERY;
import static org.neo4j.storageengine.api.TransactionApplicationMode.REVERSE_RECOVERY;

/**
 * This is the process of doing a recovery on the transaction log and store, and is executed
 * at startup of {@link Database}.
 */
public class TransactionLogsRecovery extends LifecycleAdapter
{
    private static final String REVERSE_RECOVERY_TAG = "restoreDatabase";
    private static final String RECOVERY_TAG = "recoverDatabase";
    private static final String RECOVERY_COMPLETED_TAG = "databaseRecoveryCompleted";

    private final RecoveryService recoveryService;
    private final RecoveryMonitor monitor;
    private final CorruptedLogsTruncator logsTruncator;
    private final Lifecycle schemaLife;
    private final ProgressReporter progressReporter;
    private final boolean failOnCorruptedLogFiles;
    private final RecoveryStartupChecker recoveryStartupChecker;
    private final PageCacheTracer pageCacheTracer;
    private final RecoveryPredicate recoveryPredicate;
    private int numberOfRecoveredTransactions;

    public TransactionLogsRecovery( RecoveryService recoveryService, CorruptedLogsTruncator logsTruncator, Lifecycle schemaLife, RecoveryMonitor monitor,
            ProgressReporter progressReporter, boolean failOnCorruptedLogFiles, RecoveryStartupChecker recoveryStartupChecker,
            RecoveryPredicate recoveryPredicate, PageCacheTracer pageCacheTracer )
    {
        this.recoveryService = recoveryService;
        this.monitor = monitor;
        this.logsTruncator = logsTruncator;
        this.schemaLife = schemaLife;
        this.progressReporter = progressReporter;
        this.failOnCorruptedLogFiles = failOnCorruptedLogFiles;
        this.recoveryStartupChecker = recoveryStartupChecker;
        this.pageCacheTracer = pageCacheTracer;
        this.recoveryPredicate = recoveryPredicate;
    }

    @Override
    public void init() throws Exception
    {
        RecoveryStartInformation recoveryStartInformation = recoveryService.getRecoveryStartInformation();
        if ( !recoveryStartInformation.isRecoveryRequired() )
        {
            schemaLife.init();
            return;
        }

        Stopwatch recoveryStartTime = Stopwatch.start();

        LogPosition recoveryStartPosition = recoveryStartInformation.getTransactionLogPosition();

        monitor.recoveryRequired( recoveryStartPosition );

        LogPosition recoveryToPosition = recoveryStartPosition;
        LogPosition lastTransactionPosition = recoveryStartPosition;
        LogEntryCommit lastTransaction = null;
        LogEntryCommit lastReversedTransaction = null;
        if ( !recoveryStartInformation.isMissingLogs() )
        {
            try
            {
                long lowestRecoveredTxId = TransactionIdStore.BASE_TX_ID;
                try ( var transactionsToRecover = recoveryService.getTransactionsInReverseOrder( recoveryStartPosition );
                      var recoveryVisitor = recoveryService.getRecoveryApplier( REVERSE_RECOVERY, pageCacheTracer, REVERSE_RECOVERY_TAG ) )
                {
                    while ( transactionsToRecover.next() )
                    {
                        recoveryStartupChecker.checkIfCanceled();
                        CommittedTransactionRepresentation transaction = transactionsToRecover.get();
                        if ( lastReversedTransaction == null )
                        {
                            lastReversedTransaction = transaction.getCommitEntry();
                            initProgressReporter( recoveryStartInformation, lastReversedTransaction );
                        }
                        recoveryVisitor.visit( transaction );
                        lowestRecoveredTxId = transaction.getCommitEntry().getTxId();
                        reportProgress();
                    }
                }

                monitor.reverseStoreRecoveryCompleted( lowestRecoveredTxId );

                // We cannot initialise the schema (tokens, schema cache, indexing service, etc.) until we have returned the store to a consistent state.
                // We need to be able to read the store before we can even figure out what indexes, tokens, etc. we have. Hence we defer the initialisation
                // of the schema life until after we've done the reverse recovery.
                schemaLife.init();

                boolean fullRecovery = true;
                try ( var transactionsToRecover = recoveryService.getTransactions( recoveryStartPosition );
                        var recoveryVisitor = recoveryService.getRecoveryApplier( RECOVERY, pageCacheTracer, RECOVERY_TAG ) )
                {
                    while ( fullRecovery && transactionsToRecover.next() )
                    {
                        var nextTransaction = transactionsToRecover.get();
                        if ( !recoveryPredicate.test( nextTransaction ) )
                        {
                            monitor.partialRecovery( recoveryPredicate, lastTransaction );
                            fullRecovery = false;
                            if ( lastTransaction == null )
                            {
                                // First transaction after checkpoint failed predicate test
                                // we can't always load transaction before checkpoint to check what values we had there since those logs may be pruned,
                                // but we will try to load first transaction before checkpoint to see if we just on the edge of provided criteria
                                // and will fail otherwise.
                                long beforeCheckpointTransaction = recoveryStartInformation.getFirstTxIdAfterLastCheckPoint() - 1;
                                if ( beforeCheckpointTransaction < TransactionIdStore.BASE_TX_ID )
                                {
                                    throw new RecoveryPredicateException( format(
                                            "Partial recovery criteria can't be satisfied. No transaction after checkpoint matching to provided " +
                                                    "criteria found and transaction before checkpoint is not valid. " +
                                                    "Transaction id before checkpoint: %d, criteria %s.",
                                            beforeCheckpointTransaction, recoveryPredicate.describe() ) );
                                }
                                try ( var beforeCheckpointCursor = recoveryService.getTransactions( beforeCheckpointTransaction ) )
                                {
                                    if ( beforeCheckpointCursor.next() )
                                    {
                                        CommittedTransactionRepresentation candidate = beforeCheckpointCursor.get();
                                        if ( !recoveryPredicate.test( candidate ) )
                                        {
                                            throw new RecoveryPredicateException( format(
                                                    "Partial recovery criteria can't be satisfied. " +
                                                    "Transaction after and before checkpoint does not satisfy provided recovery criteria. " +
                                                    "Observed transaction id: %d, recovery criteria: %s.", candidate.getCommitEntry().getTxId(),
                                                    recoveryPredicate.describe() ) );
                                        }
                                        lastTransaction = candidate.getCommitEntry();
                                        lastTransactionPosition = beforeCheckpointCursor.position();
                                    }
                                    else
                                    {
                                        throw new RecoveryPredicateException( format(
                                                "Partial recovery criteria can't be satisfied. No transaction after checkpoint matching " +
                                                "to provided criteria found and transaction before checkpoint not found. Recovery criteria: %s.",
                                                recoveryPredicate.describe() ) );
                                    }
                                }
                                catch ( RecoveryPredicateException re )
                                {
                                    throw re;
                                }
                                catch ( Exception e )
                                {
                                    throw new RecoveryPredicateException( format(
                                            "Partial recovery criteria can't be satisfied. No transaction after checkpoint matching " +
                                            "to provided criteria found and fail to read transaction before checkpoint. Recovery criteria: %s.",
                                            recoveryPredicate.describe() ), e );
                                }
                            }
                        }
                        else
                        {
                            recoveryStartupChecker.checkIfCanceled();
                            long txId = nextTransaction.getCommitEntry().getTxId();
                            recoveryVisitor.visit( nextTransaction );

                            lastTransaction = nextTransaction.getCommitEntry();
                            monitor.transactionRecovered( txId );
                            numberOfRecoveredTransactions++;
                            lastTransactionPosition = transactionsToRecover.position();
                            recoveryToPosition = lastTransactionPosition;
                            reportProgress();
                        }
                    }
                    recoveryToPosition = fullRecovery ? transactionsToRecover.position() : lastTransactionPosition;
                }
            }
            catch ( Error | ClosedByInterruptException | DatabaseStartAbortedException | RecoveryPredicateException e )
            {
                // We do not want to truncate logs based on these exceptions. Since users can influence them with config changes
                // the users are able to workaround this if truncations is really needed.
                throw e;
            }
            catch ( Throwable t )
            {
                if ( failOnCorruptedLogFiles )
                {
                    throwUnableToCleanRecover( t );
                }
                if ( lastTransaction != null )
                {
                    monitor.failToRecoverTransactionsAfterCommit( t, lastTransaction, recoveryToPosition );
                }
                else
                {
                    monitor.failToRecoverTransactionsAfterPosition( t, recoveryStartPosition );
                }
            }
            progressReporter.completed();
            logsTruncator.truncate( recoveryToPosition );
        }

        try ( var cursorContext = new CursorContext( pageCacheTracer.createPageCursorTracer( RECOVERY_COMPLETED_TAG ) ) )
        {
            final boolean missingLogs = recoveryStartInformation.isMissingLogs();
            recoveryService.transactionsRecovered( lastTransaction, lastTransactionPosition, recoveryToPosition,
                    recoveryStartInformation.getCheckpointPosition(), missingLogs, cursorContext );
        }
        monitor.recoveryCompleted( numberOfRecoveredTransactions, recoveryStartTime.elapsed( MILLISECONDS ) );
    }

    private void initProgressReporter( RecoveryStartInformation recoveryStartInformation, LogEntryCommit lastReversedTransaction )
    {
        long numberOfTransactionToRecover =
                getNumberOfTransactionToRecover( recoveryStartInformation, lastReversedTransaction );
        // since we will process each transaction twice (doing reverse and direct detour) we need to
        // multiply number of transactions that we want to recover by 2 to be able to report correct progress
        progressReporter.start( numberOfTransactionToRecover * 2 );
    }

    private void reportProgress()
    {
        progressReporter.progress( 1 );
    }

    private static long getNumberOfTransactionToRecover( RecoveryStartInformation recoveryStartInformation, LogEntryCommit lastReversedTransaction )
    {
        return lastReversedTransaction.getTxId() - recoveryStartInformation.getFirstTxIdAfterLastCheckPoint() + 1;
    }

    @Override
    public void start() throws Exception
    {
        schemaLife.start();
    }

    @Override
    public void stop() throws Exception
    {
        schemaLife.stop();
    }

    @Override
    public void shutdown() throws Exception
    {
        schemaLife.shutdown();
    }
}