org.neo4j.kernel.impl.api.index.IndexPopulationJob Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of neo4j-kernel Show documentation
Show all versions of neo4j-kernel Show documentation
Neo4j kernel is a lightweight, embedded Java database designed to
store data structured as graphs rather than tables. For more
information, see http://neo4j.org.
/*
* Copyright (c) "Neo4j"
* Neo4j Sweden AB [http://neo4j.com]
*
* This file is part of Neo4j.
*
* Neo4j is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see populatedIndexes = new ArrayList<>();
private volatile StoreScan storeScan;
private volatile boolean stopped;
/**
* The {@link JobHandle} that represents the scheduling of this index population job.
* This is used in the cancellation of the job.
*/
private volatile JobHandle jobHandle;
public IndexPopulationJob( MultipleIndexPopulator multiPopulator, IndexMonitor monitor, boolean verifyBeforeFlipping,
PageCacheTracer pageCacheTracer, MemoryTracker memoryTracker, String databaseName, Subject subject, EntityType populatedEntityType,
Config config )
{
this.multiPopulator = multiPopulator;
this.monitor = monitor;
this.verifyBeforeFlipping = verifyBeforeFlipping;
this.pageCacheTracer = pageCacheTracer;
this.memoryTracker = memoryTracker;
this.memoryAllocationTracker = new ThreadSafePeakMemoryTracker();
this.bufferFactory = new ByteBufferFactory( UnsafeDirectByteBufferAllocator::new, config.get( index_populator_block_size ).intValue() );
this.databaseName = databaseName;
this.subject = subject;
this.populatedEntityType = populatedEntityType;
}
/**
* Adds an {@link IndexPopulator} to be populated in this store scan. All participating populators must
* be added before calling {@link #run()}.
* @param populator {@link IndexPopulator} to participate.
* @param indexProxyStrategy {@link IndexProxyStrategy} meta information about index.
* @param flipper {@link FlippableIndexProxy} to call after a successful population.
* @param failedIndexProxyFactory {@link FailedIndexProxyFactory} to use after an unsuccessful population.
*/
MultipleIndexPopulator.IndexPopulation addPopulator( IndexPopulator populator, IndexProxyStrategy indexProxyStrategy,
FlippableIndexProxy flipper, FailedIndexProxyFactory failedIndexProxyFactory )
{
assert storeScan == null : "Population have already started, too late to add populators at this point";
populatedIndexes.add( indexProxyStrategy );
return this.multiPopulator.addPopulator( populator, indexProxyStrategy, flipper, failedIndexProxyFactory );
}
/**
* Scans the store using store view and populates all participating {@link IndexPopulator} with data relevant to
* each index.
* The scan continues as long as there's at least one non-failed populator.
*/
@Override
public void run()
{
try ( var cursorContext = new CursorContext( pageCacheTracer.createPageCursorTracer( INDEX_POPULATION_TAG ) ) )
{
if ( !multiPopulator.hasPopulators() )
{
return;
}
if ( storeScan != null )
{
throw new IllegalStateException( "Population already started." );
}
try
{
multiPopulator.create( cursorContext );
multiPopulator.resetIndexCounts( cursorContext );
monitor.indexPopulationScanStarting( multiPopulator.indexDescriptors() );
indexAllEntities( pageCacheTracer );
monitor.indexPopulationScanComplete();
if ( stopped )
{
multiPopulator.stop( cursorContext );
// We remain in POPULATING state
return;
}
multiPopulator.flipAfterStoreScan( verifyBeforeFlipping, cursorContext );
}
catch ( Throwable t )
{
multiPopulator.cancel( t, cursorContext );
}
}
finally
{
// will only close "additional" resources, not the actual populators, since that's managed by flip
Runnables.runAll( "Failed to close resources in IndexPopulationJob",
multiPopulator::close,
bufferFactory::close,
() -> monitor.populationJobCompleted( memoryAllocationTracker.peakMemoryUsage() ),
doneSignal::countDown );
}
}
private void indexAllEntities( PageCacheTracer cacheTracer ) throws IndexPopulationFailedKernelException
{
storeScan = multiPopulator.createStoreScan( cacheTracer );
storeScan.run( multiPopulator );
}
PopulationProgress getPopulationProgress( MultipleIndexPopulator.IndexPopulation indexPopulation )
{
if ( storeScan == null )
{
// indexing hasn't begun yet
return PopulationProgress.NONE;
}
PopulationProgress storeScanProgress = storeScan.getProgress();
return indexPopulation.progress( storeScanProgress );
}
/**
* Signal to stop index population.
* All populating indexes will remain in {@link InternalIndexState#POPULATING populating state} to be rebuilt on next db start up.
* Asynchronous call, need to {@link #awaitCompletion(long, TimeUnit) await completion}.
*/
public void stop()
{
// Stop the population
if ( storeScan != null )
{
stopped = true;
storeScan.stop();
jobHandle.cancel();
monitor.populationCancelled();
}
}
/**
* Stop population of specific index. Index will remain in {@link InternalIndexState#POPULATING populating state} to be rebuilt on next db start up.
* @param population {@link MultipleIndexPopulator.IndexPopulation} to be stopped.
*/
void stop( MultipleIndexPopulator.IndexPopulation population, CursorContext cursorContext )
{
multiPopulator.stop( population, cursorContext );
}
/**
* Stop population of specific index and drop it.
* @param population {@link MultipleIndexPopulator.IndexPopulation} to be dropped.
*/
void dropPopulation( MultipleIndexPopulator.IndexPopulation population )
{
multiPopulator.dropIndexPopulation( population );
}
/**
* A transaction happened that produced the given updates. Let this job incorporate its data,
* feeding it to the {@link IndexPopulator}.
*
* @param update {@link IndexEntryUpdate} to queue.
*/
public void update( IndexEntryUpdate update )
{
multiPopulator.queueConcurrentUpdate( update );
}
@Override
public String toString()
{
return getClass().getSimpleName() + "[populator:" + multiPopulator + "]";
}
/**
* Awaits completion of this population job, but waits maximum the given time.
*
* @param time time to wait at the most for completion. A value of 0 means indefinite wait.
* @param unit {@link TimeUnit unit} of the {@code time}.
* @return {@code true} if the job is still running when leaving this method, otherwise {@code false} meaning that the job is completed.
* @throws InterruptedException if the wait got interrupted.
*/
public boolean awaitCompletion( long time, TimeUnit unit ) throws InterruptedException
{
if ( time == 0 )
{
doneSignal.await();
return false;
}
boolean completed = doneSignal.await( time, unit );
return !completed;
}
/**
* Assign the job-handle that was created when this index population job was scheduled.
* This makes it possible to {@link JobHandle#cancel() cancel} the scheduled index population,
* making it never start, through {@link IndexPopulationJob#stop()}.
*/
public void setHandle( JobHandle handle )
{
this.jobHandle = handle;
}
public ByteBufferFactory bufferFactory()
{
return bufferFactory;
}
public MemoryTracker getMemoryTracker()
{
return memoryTracker;
}
public JobMonitoringParams getMonitoringParams()
{
return new JobMonitoringParams( subject, databaseName, getMonitoringDescription(), () ->
{
var stateDescriptionBuilder = new StringBuilder();
// Print index names only if there is more than 1,
// because if there is only one, its name will already be in the job description
if ( populatedIndexes.size() > 1 )
{
stateDescriptionBuilder.append( "Population of indexes " );
boolean first = true;
for ( var index : populatedIndexes )
{
if ( first )
{
first = false;
}
else
{
stateDescriptionBuilder.append( "," );
}
stateDescriptionBuilder.append( "'" )
.append( index.getIndexDescriptor().getName() )
.append( "'" );
}
stateDescriptionBuilder.append( "; " );
}
PopulationProgress populationProgress = PopulationProgress.NONE;
if ( storeScan != null )
{
populationProgress = storeScan.getProgress();
}
stateDescriptionBuilder.append( "Total progress: " )
.append( populationProgress.toIndexPopulationProgress().getCompletedPercentage() )
.append( "%" );
return stateDescriptionBuilder.toString();
} );
}
private String getMonitoringDescription()
{
if ( populatedIndexes.isEmpty() )
{
// this should not happen
// but it is better to show this over throwing an exception.
return "Empty index population";
}
if ( populatedIndexes.size() == 1 )
{
var index = populatedIndexes.get( 0 );
return "Population of index '" + index.getIndexDescriptor().getName() + "'";
}
return "Population of " + populatedIndexes.size() + " '" + populatedEntityType + "' indexes";
}
}