org.neo4j.consistency.checking.full.ConsistencyCheckTasks Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of neo4j-consistency-check Show documentation
Show all versions of neo4j-consistency-check Show documentation
Tool for checking consistency of a Neo4j data store.
/*
* Copyright (c) 2002-2020 "Neo4j,"
* Neo4j Sweden AB [http://neo4j.com]
*
* This file is part of Neo4j.
*
* Neo4j is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see createTasksForFullCheck( boolean checkLabelScanStore, boolean checkIndexes,
boolean checkGraph )
{
List tasks = new ArrayList<>();
if ( checkGraph )
{
MandatoryProperties mandatoryProperties = new MandatoryProperties( nativeStores );
StoreProcessor processor =
multiPass.processor( CheckStage.Stage1_NS_PropsLabels, PROPERTIES );
tasks.add( create( CheckStage.Stage1_NS_PropsLabels.name(), nativeStores.getNodeStore(),
processor, ROUND_ROBIN ) );
//RelationshipStore pass - check label counts using cached labels, check properties, skip nodes and relationships
processor = multiPass.processor( CheckStage.Stage2_RS_Labels, LABELS );
multiPass.reDecorateRelationship( processor, RelationshipRecordCheck.relationshipRecordCheckForwardPass() );
tasks.add( create( CheckStage.Stage2_RS_Labels.name(), nativeStores.getRelationshipStore(),
processor, ROUND_ROBIN ) );
//NodeStore pass - just cache nextRel and inUse
tasks.add( new CacheTask.CacheNextRel( CheckStage.Stage3_NS_NextRel, cacheAccess,
Scanner.scan( nativeStores.getNodeStore() ) ) );
//RelationshipStore pass - check nodes inUse, FirstInFirst, FirstInSecond using cached info
processor = multiPass.processor( CheckStage.Stage4_RS_NextRel, NODES );
multiPass.reDecorateRelationship( processor, RelationshipRecordCheck.relationshipRecordCheckBackwardPass(
new PropertyChain<>( mandatoryProperties.forRelationships( reporter ) ) ) );
tasks.add( create( CheckStage.Stage4_RS_NextRel.name(), nativeStores.getRelationshipStore(),
processor, ROUND_ROBIN ) );
//NodeStore pass - just cache nextRel and inUse
multiPass.reDecorateNode( processor, NodeRecordCheck.toCheckNextRel(), true );
multiPass.reDecorateNode( processor, NodeRecordCheck.toCheckNextRelationshipGroup(), false );
tasks.add( new CacheTask.CheckNextRel( CheckStage.Stage5_Check_NextRel, cacheAccess, nativeStores, processor ) );
// source chain
//RelationshipStore pass - forward scan of source chain using the cache.
processor = multiPass.processor( CheckStage.Stage6_RS_Forward, RELATIONSHIPS );
multiPass.reDecorateRelationship( processor,
RelationshipRecordCheck.relationshipRecordCheckSourceChain() );
tasks.add( create( CheckStage.Stage6_RS_Forward.name(), nativeStores.getRelationshipStore(),
processor, QueueDistribution.RELATIONSHIPS ) );
//RelationshipStore pass - reverse scan of source chain using the cache.
processor = multiPass.processor( CheckStage.Stage7_RS_Backward, RELATIONSHIPS );
multiPass.reDecorateRelationship( processor,
RelationshipRecordCheck.relationshipRecordCheckSourceChain() );
tasks.add( create( CheckStage.Stage7_RS_Backward.name(), nativeStores.getRelationshipStore(),
processor, QueueDistribution.RELATIONSHIPS ) );
//relationshipGroup
StoreProcessor relGrpProcessor = multiPass.processor( Stage.PARALLEL_FORWARD, RELATIONSHIP_GROUPS );
tasks.add( create( "RelationshipGroupStore-RelGrp", nativeStores.getRelationshipGroupStore(),
relGrpProcessor, ROUND_ROBIN ) );
PropertyReader propertyReader = new PropertyReader( nativeStores );
tasks.add( recordScanner( CheckStage.Stage8_PS_Props.name(),
new IterableStore<>( nativeStores.getNodeStore(), true ),
new PropertyAndNode2LabelIndexProcessor( reporter, checkIndexes ? indexes : null,
propertyReader, cacheAccess, mandatoryProperties.forNodes( reporter ) ),
CheckStage.Stage8_PS_Props, ROUND_ROBIN,
new IterableStore<>( nativeStores.getPropertyStore(), true ) ) );
// Checking that relationships are in their expected relationship indexes.
List relationshipIndexes = Iterables.stream( indexes.onlineRules() )
.filter( rule -> rule.schema().entityType() == EntityType.RELATIONSHIP )
.collect( Collectors.toList() );
if ( checkIndexes && !relationshipIndexes.isEmpty() )
{
tasks.add( recordScanner( CheckStage.Stage9_RS_Indexes.name(),
new IterableStore<>( nativeStores.getRelationshipStore(), true ),
new RelationshipIndexProcessor( reporter, indexes, propertyReader, relationshipIndexes ),
CheckStage.Stage9_RS_Indexes,
ROUND_ROBIN,
new IterableStore<>( nativeStores.getPropertyStore(), true ) ) );
}
tasks.add( create( "StringStore-Str", nativeStores.getStringStore(),
multiPass.processor( Stage.SEQUENTIAL_FORWARD, STRINGS ), ROUND_ROBIN ) );
tasks.add( create( "ArrayStore-Arrays", nativeStores.getArrayStore(),
multiPass.processor( Stage.SEQUENTIAL_FORWARD, ARRAYS ), ROUND_ROBIN ) );
}
// The schema store is verified in multiple passes that share state since it fits into memory
// and we care about the consistency of back references (cf. SemanticCheck)
// PASS 1: Dynamic record chains
tasks.add( create( "SchemaStore", nativeStores.getSchemaStore(), ROUND_ROBIN ) );
// PASS 2: Rule integrity and obligation build up
final SchemaRecordCheck schemaCheck =
new SchemaRecordCheck( new SchemaStorage( nativeStores.getSchemaStore() ), indexes );
tasks.add( new SchemaStoreProcessorTask<>( "SchemaStoreProcessor-check_rules", statistics, numberOfThreads,
nativeStores.getSchemaStore(), nativeStores, "check_rules",
schemaCheck, multiPartBuilder, cacheAccess, defaultProcessor, ROUND_ROBIN ) );
// PASS 3: Obligation verification and semantic rule uniqueness
tasks.add( new SchemaStoreProcessorTask<>( "SchemaStoreProcessor-check_obligations", statistics,
numberOfThreads, nativeStores.getSchemaStore(), nativeStores,
"check_obligations", schemaCheck.forObligationChecking(), multiPartBuilder, cacheAccess, defaultProcessor,
ROUND_ROBIN ) );
if ( checkGraph )
{
tasks.add( create( "RelationshipTypeTokenStore", nativeStores.getRelationshipTypeTokenStore(), ROUND_ROBIN ) );
tasks.add( create( "PropertyKeyTokenStore", nativeStores.getPropertyKeyTokenStore(), ROUND_ROBIN ) );
tasks.add( create( "LabelTokenStore", nativeStores.getLabelTokenStore(), ROUND_ROBIN ) );
tasks.add( create( "RelationshipTypeNameStore", nativeStores.getRelationshipTypeNameStore(), ROUND_ROBIN ) );
tasks.add( create( "PropertyKeyNameStore", nativeStores.getPropertyKeyNameStore(), ROUND_ROBIN ) );
tasks.add( create( "LabelNameStore", nativeStores.getLabelNameStore(), ROUND_ROBIN ) );
tasks.add( create( "NodeDynamicLabelStore", nativeStores.getNodeDynamicLabelStore(), ROUND_ROBIN ) );
}
ConsistencyReporter filteredReporter = multiPass.reporter( NODES );
if ( checkLabelScanStore )
{
long highId = nativeStores.getNodeStore().getHighId();
tasks.add( new LabelIndexDirtyCheckTask() );
tasks.add( recordScanner( "LabelScanStore",
new GapFreeAllEntriesLabelScanReader( labelScanStore.allNodeLabelRanges(), highId ),
new LabelScanDocumentProcessor( filteredReporter, new LabelScanCheck() ), Stage.SEQUENTIAL_FORWARD,
ROUND_ROBIN ) );
}
if ( checkIndexes )
{
tasks.add( new IndexDirtyCheckTask() );
TokenNameLookup tokenNameLookup = new NonTransactionalTokenNameLookup( tokenHolders, true /*include token ids too*/ );
for ( StoreIndexDescriptor indexRule : indexes.onlineRules() )
{
tasks.add( recordScanner( format( "Index_%d", indexRule.getId() ),
new IndexIterator( indexes.accessorFor( indexRule ) ),
new IndexEntryProcessor( filteredReporter, new IndexCheck( indexRule ), indexRule, tokenNameLookup ),
Stage.SEQUENTIAL_FORWARD, ROUND_ROBIN ) );
}
}
return tasks;
}
private RecordScanner recordScanner( String name,
BoundedIterable store, RecordProcessor processor, Stage stage,
QueueDistribution distribution,
@SuppressWarnings( "rawtypes" ) IterableStore... warmupStores )
{
return stage.isParallel()
? new ParallelRecordScanner<>( name, statistics, numberOfThreads, store, multiPartBuilder, processor,
cacheAccess, distribution, warmupStores )
: new SequentialRecordScanner<>( name, statistics, numberOfThreads, store, multiPartBuilder, processor,
warmupStores );
}
private StoreProcessorTask create( String name,
RecordStore input, QueueDistribution distribution )
{
return new StoreProcessorTask<>( name, statistics, numberOfThreads, input, nativeStores, name, multiPartBuilder,
cacheAccess, defaultProcessor, distribution );
}
private StoreProcessorTask create( String name,
RecordStore input, StoreProcessor processor, QueueDistribution distribution )
{
return new StoreProcessorTask<>( name, statistics, numberOfThreads, input, nativeStores, name, multiPartBuilder,
cacheAccess, processor, distribution );
}
private class LabelIndexDirtyCheckTask extends ConsistencyCheckerTask
{
LabelIndexDirtyCheckTask()
{
super( "Label index dirty check", Statistics.NONE, 1 );
}
@Override
public void run()
{
if ( labelScanStore instanceof NativeLabelScanStore )
{
if ( ((NativeLabelScanStore)labelScanStore).isDirty() )
{
reporter.report( new LabelScanIndex( labelScanStore.getLabelScanStoreFile() ), ConsistencyReport.LabelScanConsistencyReport.class,
RecordType.LABEL_SCAN_DOCUMENT ).dirtyIndex();
}
}
}
}
private class IndexDirtyCheckTask extends ConsistencyCheckerTask
{
IndexDirtyCheckTask()
{
super( "Indexes dirty check", Statistics.NONE, 1 );
}
@Override
public void run()
{
for ( StoreIndexDescriptor indexRule : indexes.onlineRules() )
{
if ( indexes.accessorFor( indexRule ).isDirty() )
{
reporter.report( new IndexRecord( indexRule ), ConsistencyReport.IndexConsistencyReport.class,
RecordType.INDEX ).dirtyIndex();
}
}
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy