org.neo4j.consistency.checker.RelationshipChainChecker Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of neo4j-consistency-check Show documentation
Show all versions of neo4j-consistency-check Show documentation
Tool for checking consistency of a Neo4j data store.
/*
* Copyright (c) "Neo4j"
* Neo4j Sweden AB [http://neo4j.com]
*
* This file is part of Neo4j.
*
* Neo4j is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see [] threadQueues = new ArrayBlockingQueue[numberOfChainCheckers];
BatchedRelationshipRecords[] threadBatches = new BatchedRelationshipRecords[numberOfChainCheckers];
for ( int i = 0; i < numberOfChainCheckers; i++ )
{
threadQueues[i] = new ArrayBlockingQueue<>( 20 );
threadBatches[i] = new BatchedRelationshipRecords();
workers[i] = relationshipVsRelationshipChecker( nodeIdRange, direction, relationshipStore, threadQueues[i], end, i );
}
// Record reader
workers[workers.length - 1] = () ->
{
RelationshipRecord relationship = relationshipStore.newRecord();
try ( var cursorContext = new CursorContext( context.pageCacheTracer.createPageCursorTracer( RELATIONSHIP_CONSISTENCY_CHECKER_TAG ) );
var cursor = relationshipStore.openPageCursorForReadingWithPrefetching( 0, cursorContext ) )
{
int recordsPerPage = relationshipStore.getRecordsPerPage();
long id = direction.startingId( highId );
while ( id >= 0 && id < highId && !context.isCancelled() )
{
for ( int i = 0; i < recordsPerPage && id >= 0 && id < highId; i++, id = direction.nextId( id ) )
{
relationshipStore.getRecordByCursor( id, relationship, FORCE, cursor );
localProgress.add( 1 );
if ( relationship.inUse() )
{
queueRelationshipCheck( threadQueues, threadBatches, relationship );
}
}
}
processLastRelationshipChecks( threadQueues, threadBatches, end );
localProgress.done();
}
};
Stopwatch stopwatch = Stopwatch.start();
cacheAccess.clearCache();
context.execution.runAll( getClass().getSimpleName() + "-" + direction.name(), workers );
detectSingleRelationshipChainInconsistencies( nodeIdRange );
context.paddedDebug( "%s %s took %s", this, direction, duration( stopwatch.elapsed( TimeUnit.MILLISECONDS ) ) );
}
@Override
public boolean shouldBeChecked( ConsistencyFlags flags )
{
return flags.isCheckGraph();
}
private void detectSingleRelationshipChainInconsistencies( LongRange nodeIdRange )
{
CacheAccess.Client client = cacheAccess.client();
try ( var cursorContext = new CursorContext( context.pageCacheTracer.createPageCursorTracer( SINGLE_RELATIONSHIP_CONSISTENCY_CHECKER_TAG ) ) )
{
for ( long nodeId = nodeIdRange.from(); nodeId < nodeIdRange.to(); nodeId++ )
{
boolean inUse = client.getBooleanFromCache( nodeId, SLOT_IN_USE );
boolean hasMultipleRelationships = client.getBooleanFromCache( nodeId, SLOT_HAS_MULTIPLE_RELATIONSHIPS );
if ( inUse && !hasMultipleRelationships )
{
long reference = client.getFromCache( nodeId, SLOT_REFERENCE );
long relationshipId = client.getFromCache( nodeId, SLOT_RELATIONSHIP_ID );
long sourceOrTarget = client.getFromCache( nodeId, SLOT_SOURCE_OR_TARGET );
long prevOrNext = client.getFromCache( nodeId, SLOT_PREV_OR_NEXT );
boolean isFirstInChain = client.getBooleanFromCache( nodeId, SLOT_FIRST_IN_CHAIN );
boolean consistent;
if ( prevOrNext == PREV )
{
// we don't know here if this chain belongs to a group and has external degrees, because if so it could have any value here
consistent = isFirstInChain;
}
else
{
consistent = NULL_REFERENCE.is( reference );
}
if ( !consistent )
{
RelationshipStore relationshipStore = context.neoStores.getRelationshipStore();
RelationshipRecord relationship = relationshipStore.getRecord( relationshipId, relationshipStore.newRecord(), FORCE, cursorContext );
RelationshipRecord referenceRelationship =
relationshipStore.getRecord( reference, relationshipStore.newRecord(), FORCE, cursorContext );
linkOf( sourceOrTarget == SOURCE, prevOrNext == PREV ).reportDoesNotReferenceBack( reporter, relationship, referenceRelationship );
}
}
}
}
}
private static RelationshipLink linkOf( boolean source, boolean prev )
{
if ( source )
{
return prev ? SOURCE_PREV : SOURCE_NEXT;
}
return prev ? TARGET_PREV : TARGET_NEXT;
}
private ThrowingRunnable relationshipVsRelationshipChecker( LongRange nodeIdRange, ScanDirection direction, RelationshipStore store,
ArrayBlockingQueue queue, AtomicBoolean end, int threadId )
{
final RelationshipRecord relationship = store.newRecord();
final RelationshipRecord otherRelationship = store.newRecord();
final CacheAccess.Client client = cacheAccess.client();
final RelationshipLink sourceCachePointer = direction.sourceLink;
final RelationshipLink targetCachePointer = direction.targetLink;
final long prevOrNext = direction.cacheSlot;
return () ->
{
try ( var cursorContext = new CursorContext( context.pageCacheTracer.createPageCursorTracer( RELATIONSHIP_CONSISTENCY_CHECKER_TAG ) );
var otherRelationshipCursor = store.openPageCursorForReading( 0, cursorContext ) )
{
while ( (!end.get() || !queue.isEmpty()) && !context.isCancelled() )
{
BatchedRelationshipRecords batch = queue.poll( 100, TimeUnit.MILLISECONDS );
if ( batch != null )
{
while ( batch.fillNext( relationship ) && !context.isCancelled() )
{
long firstNode = relationship.getFirstNode();
long secondNode = relationship.getSecondNode();
// Intentionally not checking nodes outside highId of node store because RelationshipChecker will spot this inconsistency
boolean processStartNode =
Math.abs( firstNode % numberOfChainCheckers ) == threadId && nodeIdRange.isWithinRangeExclusiveTo( firstNode );
boolean processEndNode =
Math.abs( secondNode % numberOfChainCheckers ) == threadId && nodeIdRange.isWithinRangeExclusiveTo( secondNode );
if ( processStartNode )
{
checkRelationshipLink( direction, SOURCE_PREV, relationship, client, otherRelationship, otherRelationshipCursor, store,
cursorContext );
checkRelationshipLink( direction, SOURCE_NEXT, relationship, client, otherRelationship, otherRelationshipCursor, store,
cursorContext );
}
if ( processEndNode )
{
checkRelationshipLink( direction, TARGET_PREV, relationship, client, otherRelationship, otherRelationshipCursor, store,
cursorContext );
checkRelationshipLink( direction, TARGET_NEXT, relationship, client, otherRelationship, otherRelationshipCursor, store,
cursorContext );
}
if ( processStartNode )
{
boolean wasInUse = client.getBooleanFromCache( firstNode, SLOT_IN_USE );
long link = sourceCachePointer.link( relationship );
if ( link < NULL_REFERENCE.longValue() )
{
sourceCachePointer.reportDoesNotReferenceBack( reporter, relationship, otherRelationship );
}
else
{
client.putToCache( firstNode, relationship.getId(), link, SOURCE, prevOrNext, 1,
longOf( wasInUse ), longOf( relationship.isFirstInFirstChain() ) );
}
}
if ( processEndNode )
{
boolean wasInUse = client.getBooleanFromCache( secondNode, SLOT_IN_USE );
long link = targetCachePointer.link( relationship );
if ( link < NULL_REFERENCE.longValue() )
{
targetCachePointer.reportDoesNotReferenceBack( reporter, relationship, otherRelationship );
}
else
{
client.putToCache( secondNode, relationship.getId(), link, TARGET, prevOrNext, 1,
longOf( wasInUse ), longOf( relationship.isFirstInSecondChain() ) );
}
}
}
}
}
}
};
}
private void checkRelationshipLink( ScanDirection direction, RelationshipLink link, RelationshipRecord relationshipCursor,
CacheAccess.Client client, RelationshipRecord otherRelationship, PageCursor otherRelationshipCursor, RelationshipStore store,
CursorContext cursorContext )
{
long relationshipId = relationshipCursor.getId();
long nodeId = link.node( relationshipCursor );
long linkId = link.link( relationshipCursor );
long fromCache = client.getFromCache( nodeId, SLOT_RELATIONSHIP_ID );
boolean cachedLinkInUse = client.getBooleanFromCache( nodeId, SLOT_IN_USE );
if ( !link.endOfChain( relationshipCursor ) && cachedLinkInUse )
{
if ( fromCache != linkId )
{
// We can't use the cache since it doesn't contain the relationship right before us in this chain
if ( direction.exclude( relationshipId, linkId ) )
{
return;
}
else if ( !NULL_REFERENCE.is( fromCache ) )
{
// Load it from store
store.getRecordByCursor( linkId, otherRelationship, FORCE, otherRelationshipCursor );
}
else
{
otherRelationship.clear();
link.reportDoesNotReferenceBack( reporter, recordLoader.relationship( relationshipCursor.getId(), cursorContext ), otherRelationship );
}
}
else
{
// OK good we can use the cached values representing a relationship right before us in this chain
otherRelationship.clear();
otherRelationship.setId( linkId );
long other = client.getFromCache( nodeId, SLOT_REFERENCE );
NodeLink nodeLink = client.getFromCache( nodeId, SLOT_SOURCE_OR_TARGET ) == SOURCE ? NodeLink.SOURCE : NodeLink.TARGET;
nodeLink.setNode( otherRelationship, nodeId );
link.setOther( otherRelationship, nodeLink, other );
otherRelationship.setInUse( client.getBooleanFromCache( nodeId, SLOT_IN_USE ) );
otherRelationship.setCreated();
}
checkRelationshipLink( direction, link, otherRelationship, relationshipId, nodeId, linkId, cursorContext );
}
}
private void checkRelationshipLink( ScanDirection direction, RelationshipLink thing, RelationshipRecord otherRelationship, long relationshipId, long nodeId,
long linkId, CursorContext cursorContext )
{
// Perform the checks
NodeLink nodeLink = NodeLink.select( otherRelationship, nodeId );
if ( nodeLink == null )
{
thing.reportOtherNode( reporter, recordLoader.relationship( relationshipId, cursorContext ), recordLoader.relationship( linkId, cursorContext ) );
}
else
{
if ( thing.other( otherRelationship, nodeLink ) != relationshipId )
{
// Read the relationship from store and do the check on that actual record instead, should happen rarely anyway
if ( otherRelationship.isCreated() )
{
recordLoader.relationship( otherRelationship, otherRelationship.getId(), cursorContext );
// Call this method one more time, now with !created
checkRelationshipLink( direction, thing, otherRelationship, relationshipId, nodeId, linkId, cursorContext );
return;
}
thing.reportDoesNotReferenceBack( reporter, recordLoader.relationship( relationshipId, cursorContext ),
recordLoader.relationship( linkId, cursorContext ) );
}
else
{
if ( !direction.exclude( relationshipId, linkId ) && !otherRelationship.inUse() )
{
thing.reportNotUsedRelationshipReferencedInChain( reporter, recordLoader.relationship( relationshipId, cursorContext ),
recordLoader.relationship( linkId, cursorContext ) );
}
}
}
}
private void queueRelationshipCheck( ArrayBlockingQueue[] threadQueues, BatchedRelationshipRecords[] threadBatches,
RelationshipRecord relationshipCursor ) throws InterruptedException
{
int sourceThread = (int) Math.abs( relationshipCursor.getFirstNode() % numberOfChainCheckers );
queueRelationshipCheck( threadQueues, threadBatches, relationshipCursor, sourceThread );
int targetThread = (int) Math.abs( relationshipCursor.getSecondNode() % numberOfChainCheckers );
if ( targetThread != sourceThread )
{
queueRelationshipCheck( threadQueues, threadBatches, relationshipCursor, targetThread );
}
}
private static void queueRelationshipCheck( ArrayBlockingQueue[] threadQueues, BatchedRelationshipRecords[] threadBatches,
RelationshipRecord relationshipCursor, int thread ) throws InterruptedException
{
if ( !threadBatches[thread].hasMoreSpace() )
{
threadQueues[thread].put( threadBatches[thread] );
threadBatches[thread] = new BatchedRelationshipRecords();
}
threadBatches[thread].add( relationshipCursor );
}
private static void processLastRelationshipChecks( ArrayBlockingQueue[] threadQueues,
BatchedRelationshipRecords[] threadBatches,
AtomicBoolean end ) throws Exception
{
for ( int i = 0; i < threadBatches.length; i++ )
{
if ( threadBatches[i].numberOfRelationships() > 0 )
{
threadQueues[i].put( threadBatches[i] );
}
}
end.set( true );
}
@Override
public String toString()
{
return String.format( "%s[highId:%d]",getClass().getSimpleName(), context.neoStores.getRelationshipStore().getHighId() );
}
private enum ScanDirection
{
FORWARD( SOURCE_PREV, TARGET_PREV, PREV )
{
@Override
boolean exclude( long id, long reference )
{
return !NULL_REFERENCE.is( reference ) && reference > id;
}
@Override
long nextId( long id )
{
return id + 1;
}
@Override
long startingId( long highId )
{
return 0;
}
},
BACKWARD( SOURCE_NEXT, TARGET_NEXT, NEXT )
{
@Override
boolean exclude( long id, long reference )
{
return !NULL_REFERENCE.is( reference ) && reference < id;
}
@Override
long nextId( long id )
{
return id - 1;
}
@Override
long startingId( long highId )
{
return highId - 1;
}
};
final RelationshipLink sourceLink;
final RelationshipLink targetLink;
final long cacheSlot;
ScanDirection( RelationshipLink sourceLink, RelationshipLink targetLink, long cacheSlot )
{
this.sourceLink = sourceLink;
this.targetLink = targetLink;
this.cacheSlot = cacheSlot;
}
abstract boolean exclude( long id, long reference );
abstract long nextId( long id );
abstract long startingId( long highId );
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy