org.neo4j.tooling.ImportTool Maven / Gradle / Ivy
/*
* Copyright (c) 2002-2016 "Neo Technology,"
* Network Engine for Objects in Lund AB [http://neotechnology.com]
*
* This file is part of Neo4j.
*
* Neo4j is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see ",
"Database directory to import into. " + "Must not contain existing database." ),
NODE_DATA( "nodes", null,
"[:Label1:Label2] \"" + MULTI_FILE_DELIMITER + "" + MULTI_FILE_DELIMITER + "...\"",
"Node CSV header and data. Multiple files will be logically seen as one big file "
+ "from the perspective of the importer. "
+ "The first line must contain the header. "
+ "Multiple data sources like these can be specified in one import, "
+ "where each data source has its own header. "
+ "Note that file groups must be enclosed in quotation marks.",
true ),
RELATIONSHIP_DATA( "relationships", null,
"[:RELATIONSHIP_TYPE] \"" + MULTI_FILE_DELIMITER + "" +
MULTI_FILE_DELIMITER + "...\"",
"Relationship CSV header and data. Multiple files will be logically seen as one big file "
+ "from the perspective of the importer. "
+ "The first line must contain the header. "
+ "Multiple data sources like these can be specified in one import, "
+ "where each data source has its own header. "
+ "Note that file groups must be enclosed in quotation marks.",
true ),
DELIMITER( "delimiter", null,
"",
"Delimiter character, or 'TAB', between values in CSV data. The default option is `" + COMMAS.delimiter() + "`." ),
ARRAY_DELIMITER( "array-delimiter", null,
"",
"Delimiter character, or 'TAB', between array elements within a value in CSV data. The default option is `" + COMMAS.arrayDelimiter() + "`." ),
QUOTE( "quote", null,
"",
"Character to treat as quotation character for values in CSV data. "
+ "The default option is `" + COMMAS.quotationCharacter() + "`. "
+ "Quotes inside quotes escaped like `\"\"\"Go away\"\", he said.\"` and "
+ "`\"\\\"Go away\\\", he said.\"` are supported. "
+ "If you have set \"`'`\" to be used as the quotation character, "
+ "you could write the previous example like this instead: " + "`'\"Go away\", he said.'`" ),
MULTILINE_FIELDS( "multiline-fields", org.neo4j.csv.reader.Configuration.DEFAULT.multilineFields(),
"",
"Whether or not fields from input source can span multiple lines, i.e. contain newline characters." ),
INPUT_ENCODING( "input-encoding", null,
"",
"Character set that input data is encoded in. Provided value must be one out of the available "
+ "character sets in the JVM, as provided by Charset#availableCharsets(). "
+ "If no input encoding is provided, the default character set of the JVM will be used." ),
IGNORE_EMPTY_STRINGS( "ignore-empty-strings", org.neo4j.csv.reader.Configuration.DEFAULT.emptyQuotedStringsAsNull(),
"",
"Whether or not empty string fields, i.e. \"\" from input source are ignored, i.e. treated as null." ),
ID_TYPE( "id-type", IdType.STRING,
"",
"One out of " + Arrays.toString( IdType.values() )
+ " and specifies how ids in node/relationship "
+ "input files are treated.\n"
+ IdType.STRING + ": arbitrary strings for identifying nodes.\n"
+ IdType.INTEGER + ": arbitrary integer values for identifying nodes.\n"
+ IdType.ACTUAL + ": (advanced) actual node ids. The default option is `" + IdType.STRING + "`." ),
PROCESSORS( "processors", null,
"",
"(advanced) Max number of processors used by the importer. Defaults to the number of "
+ "available processors reported by the JVM"
+ availableProcessorsHint()
+ ". There is a certain amount of minimum threads needed so for that reason there "
+ "is no lower bound for this value. For optimal performance this value shouldn't be "
+ "greater than the number of available processors." ),
STACKTRACE( "stacktrace", null,
"",
"Enable printing of error stack traces." ),
BAD_TOLERANCE( "bad-tolerance", 1000,
"",
"Number of bad entries before the import is considered failed. This tolerance threshold is "
+ "about relationships refering to missing nodes. Format errors in input data are "
+ "still treated as errors" ),
SKIP_BAD_RELATIONSHIPS( "skip-bad-relationships", Boolean.TRUE,
"",
"Whether or not to skip importing relationships that refers to missing node ids, i.e. either "
+ "start or end node id/group referring to node that wasn't specified by the "
+ "node input data. "
+ "Skipped nodes will be logged"
+ ", containing at most number of entites specified by " + BAD_TOLERANCE.key() + "." ),
SKIP_DUPLICATE_NODES( "skip-duplicate-nodes", Boolean.FALSE,
"",
"Whether or not to skip importing nodes that have the same id/group. In the event of multiple "
+ "nodes within the same group having the same id, the first encountered will be imported "
+ "whereas consecutive such nodes will be skipped. "
+ "Skipped nodes will be logged"
+ ", containing at most number of entities specified by " + BAD_TOLERANCE.key() + "." ),
IGNORE_EXTRA_COLUMNS( "ignore-extra-columns", Boolean.FALSE,
"",
"Whether or not to ignore extra columns in the data not specified by the header. "
+ "Skipped columns will be logged, containing at most number of entities specified by "
+ BAD_TOLERANCE.key() + "." ),
DATABASE_CONFIG( "db-config", null,
"",
"(advanced) File specifying database-specific configuration. For more information consult "
+ "manual about available configuration options for a neo4j configuration file. "
+ "Only configuration affecting store at time of creation will be read. "
+ "Examples of supported config are:\n"
+ GraphDatabaseSettings.dense_node_threshold.name() + "\n"
+ GraphDatabaseSettings.string_block_size.name() + "\n"
+ GraphDatabaseSettings.array_block_size.name() );
private final String key;
private final Object defaultValue;
private final String usage;
private final String description;
private final boolean keyAndUsageGoTogether;
Options( String key, Object defaultValue, String usage, String description )
{
this( key, defaultValue, usage, description, false );
}
Options( String key, Object defaultValue, String usage, String description, boolean keyAndUsageGoTogether )
{
this.key = key;
this.defaultValue = defaultValue;
this.usage = usage;
this.description = description;
this.keyAndUsageGoTogether = keyAndUsageGoTogether;
}
String key()
{
return key;
}
String argument()
{
return "--" + key();
}
void printUsage( PrintStream out )
{
out.println( argument() + spaceInBetweenArgumentAndUsage() + usage );
for ( String line : Args.splitLongLine( descriptionWithDefaultValue().replace( "`", "" ), 80 ) )
{
out.println( "\t" + line );
}
}
private String spaceInBetweenArgumentAndUsage()
{
return keyAndUsageGoTogether ? "" : " ";
}
String descriptionWithDefaultValue()
{
String result = description;
if ( defaultValue != null )
{
if ( !result.endsWith( "." ) )
{
result += ".";
}
result += " Default value: " + defaultValue;
}
return result;
}
String manPageEntry()
{
String filteredDescription = descriptionWithDefaultValue().replace( availableProcessorsHint(), "" );
String usageString = (usage.length() > 0) ? spaceInBetweenArgumentAndUsage() + usage : "";
return "*" + argument() + usageString + "*::\n" + filteredDescription + "\n\n";
}
String manualEntry()
{
return "[[import-tool-option-" + key() + "]]\n" + manPageEntry() + "//^\n\n";
}
Object defaultValue()
{
return defaultValue;
}
private static String availableProcessorsHint()
{
return " (in your case " + Runtime.getRuntime().availableProcessors() + ")";
}
}
/**
* Delimiter used between files in an input group.
*/
static final String MULTI_FILE_DELIMITER = ",";
/**
* Runs the import tool given the supplied arguments.
*
* @param incomingArguments arguments for specifying input and configuration for the import.
*/
public static void main( String[] incomingArguments ) throws IOException
{
main( incomingArguments, false );
}
/**
* Runs the import tool given the supplied arguments.
*
* @param incomingArguments arguments for specifying input and configuration for the import.
* @param defaultSettingsSuitableForTests default configuration geared towards unit/integration
* test environments, for example lower default buffer sizes.
*/
public static void main( String[] incomingArguments, boolean defaultSettingsSuitableForTests ) throws IOException
{
Args args = Args.parse( incomingArguments );
if ( ArrayUtil.isEmpty( incomingArguments ) || asksForUsage( args ) )
{
printUsage( System.out );
return;
}
FileSystemAbstraction fs = new DefaultFileSystemAbstraction();
File storeDir;
Collection> nodesFiles, relationshipsFiles;
boolean enableStacktrace;
Number processors = null;
Input input = null;
int badTolerance;
Charset inputEncoding;
boolean skipBadRelationships, skipDuplicateNodes, ignoreExtraColumns;
Config dbConfig;
OutputStream badOutput = null;
boolean success = false;
try
{
storeDir = args.interpretOption( Options.STORE_DIR.key(), Converters.mandatory(),
Converters.toFile(), Validators.DIRECTORY_IS_WRITABLE, Validators.CONTAINS_NO_EXISTING_DATABASE );
File badFile = new File( storeDir, BAD_FILE_NAME );
badOutput = new BufferedOutputStream( fs.openAsOutputStream( badFile, false ) );
nodesFiles = INPUT_FILES_EXTRACTOR.apply( args, Options.NODE_DATA.key() );
relationshipsFiles = INPUT_FILES_EXTRACTOR.apply( args, Options.RELATIONSHIP_DATA.key() );
validateInputFiles( nodesFiles, relationshipsFiles );
enableStacktrace = args.getBoolean( Options.STACKTRACE.key(), Boolean.FALSE, Boolean.TRUE );
processors = args.getNumber( Options.PROCESSORS.key(), null );
IdType idType = args.interpretOption( Options.ID_TYPE.key(),
withDefault( (IdType)Options.ID_TYPE.defaultValue() ), TO_ID_TYPE );
badTolerance = args.getNumber( Options.BAD_TOLERANCE.key(),
(Number) Options.BAD_TOLERANCE.defaultValue() ).intValue();
inputEncoding = Charset.forName( args.get( Options.INPUT_ENCODING.key(), defaultCharset().name() ) );
skipBadRelationships = args.getBoolean( Options.SKIP_BAD_RELATIONSHIPS.key(),
(Boolean)Options.SKIP_BAD_RELATIONSHIPS.defaultValue(), true );
skipDuplicateNodes = args.getBoolean( Options.SKIP_DUPLICATE_NODES.key(),
(Boolean)Options.SKIP_DUPLICATE_NODES.defaultValue(), true );
ignoreExtraColumns = args.getBoolean( Options.IGNORE_EXTRA_COLUMNS.key(),
(Boolean)Options.IGNORE_EXTRA_COLUMNS.defaultValue(), true );
Collector badCollector = badCollector( badOutput, badTolerance, collect( skipBadRelationships,
skipDuplicateNodes, ignoreExtraColumns ) );
input = new CsvInput( nodeData( inputEncoding, nodesFiles ), defaultFormatNodeFileHeader(),
relationshipData( inputEncoding, relationshipsFiles ), defaultFormatRelationshipFileHeader(),
idType, csvConfiguration( args, defaultSettingsSuitableForTests ), badCollector );
dbConfig = loadDbConfig( args.interpretOption( Options.DATABASE_CONFIG.key(), Converters.optional(),
Converters.toFile(), Validators.REGEX_FILE_EXISTS ) );
success = true;
}
catch ( IllegalArgumentException e )
{
throw andPrintError( "Input error", e, false );
}
catch ( IOException e )
{
throw andPrintError( "File error", e, false );
}
finally
{
if ( !success && badOutput != null )
{
badOutput.close();
}
}
LifeSupport life = new LifeSupport();
LogService logService = life.add( StoreLogService.inLogsDirectory( fs, storeDir ) );
life.start();
org.neo4j.unsafe.impl.batchimport.Configuration configuration =
importConfiguration( processors, defaultSettingsSuitableForTests, dbConfig );
BatchImporter importer = new ParallelBatchImporter( storeDir,
configuration,
logService,
ExecutionMonitors.defaultVisible(),
dbConfig );
printOverview( storeDir, nodesFiles, relationshipsFiles );
success = false;
try
{
importer.doImport( input );
success = true;
}
catch ( Exception e )
{
throw andPrintError( "Import error", e, enableStacktrace );
}
finally
{
input.badCollector().close();
badOutput.close();
if ( input.badCollector().badEntries() > 0 )
{
File badFile = new File( storeDir, BAD_FILE_NAME );
if ( badFile.exists() )
{
System.out.println(
"There were bad entries which were skipped and logged into " + badFile.getAbsolutePath() );
}
}
life.shutdown();
if ( !success )
{
try
{
StoreFile.fileOperation( FileOperation.DELETE, fs, storeDir, null,
Iterables.iterable( StoreFile.values() ),
false, ExistingTargetStrategy.FAIL, StoreFileType.values() );
}
catch ( IOException e )
{
System.err.println( "Unable to delete store files after an aborted import " + e );
if ( enableStacktrace )
{
e.printStackTrace();
}
}
}
}
}
private static Config loadDbConfig( File file ) throws IOException
{
return file != null && file.exists() ? new Config( MapUtil.load( file ) ) : Config.defaults();
}
private static void printOverview( File storeDir, Collection> nodesFiles,
Collection> relationshipsFiles )
{
System.out.println( "Neo4j version: " + Version.getKernel().getReleaseVersion() );
System.out.println( "Importing the contents of these files into " + storeDir + ":" );
printInputFiles( "Nodes", nodesFiles );
printInputFiles( "Relationships", relationshipsFiles );
System.out.println();
System.out.println( "Available memory:" );
printIndented( "Free machine memory: " + bytes( OsBeanUtil.getFreePhysicalMemory() ) );
printIndented( "Max heap memory : " + bytes( Runtime.getRuntime().maxMemory() ) );
System.out.println();
}
private static void printInputFiles( String name, Collection> files )
{
if ( files.isEmpty() )
{
return;
}
System.out.println( name + ":" );
int i = 0;
for ( Option group : files )
{
if ( i++ > 0 )
{
System.out.println();
}
if ( group.metadata() != null )
{
printIndented( ":" + group.metadata() );
}
for ( File file : group.value() )
{
printIndented( file );
}
}
}
private static void printIndented( Object value )
{
System.out.println( " " + value );
}
private static void validateInputFiles( Collection> nodesFiles,
Collection> relationshipsFiles )
{
if ( nodesFiles.isEmpty() )
{
if ( relationshipsFiles.isEmpty() )
{
throw new IllegalArgumentException( "No input specified, nothing to import" );
}
throw new IllegalArgumentException( "No node input specified, cannot import relationships without nodes" );
}
}
private static org.neo4j.unsafe.impl.batchimport.Configuration importConfiguration( final Number processors,
final boolean defaultSettingsSuitableForTests, final Config dbConfig )
{
return new org.neo4j.unsafe.impl.batchimport.Configuration.Default()
{
@Override
public long pageCacheMemory()
{
return defaultSettingsSuitableForTests ? mebiBytes( 8 ) : super.pageCacheMemory();
}
@Override
public int maxNumberOfProcessors()
{
return processors != null ? processors.intValue() : super.maxNumberOfProcessors();
}
@Override
public int denseNodeThreshold()
{
return dbConfig.get( GraphDatabaseSettings.dense_node_threshold );
}
};
}
private static String manualReference( ManualPage page, Anchor anchor )
{
// Docs are versioned major.minor-suffix, so drop the patch version.
String[] versionParts = Version.getKernel().getReleaseVersion().split("-");
versionParts[0] = versionParts[0].substring(0, 3);
String docsVersion = String.join("-", versionParts);
return " http://neo4j.com/docs/operations-manual/" + docsVersion + "/" +
page.getReference( anchor );
}
/**
* Method name looks strange, but look at how it's used and you'll see why it's named like that.
* @param stackTrace whether or not to also print the stack trace of the error.
*/
private static RuntimeException andPrintError( String typeOfError, Exception e, boolean stackTrace )
{
// List of common errors that can be explained to the user
if ( DuplicateInputIdException.class.equals( e.getClass() ) )
{
printErrorMessage( "Duplicate input ids that would otherwise clash can be put into separate id space, " +
"read more about how to use id spaces in the manual:" +
manualReference( ManualPage.IMPORT_TOOL_FORMAT, Anchor.ID_SPACES ), e, stackTrace );
}
else if ( MissingRelationshipDataException.class.equals( e.getClass() ) )
{
printErrorMessage( "Relationship missing mandatory field '" +
((MissingRelationshipDataException) e).getFieldType() + "', read more about " +
"relationship format in the manual: " +
manualReference( ManualPage.IMPORT_TOOL_FORMAT, Anchor.RELATIONSHIP ), e, stackTrace );
}
// This type of exception is wrapped since our input code throws InputException consistently,
// and so IllegalMultilineFieldException comes from the csv component, which has no access to InputException
// therefore it's wrapped.
else if ( Exceptions.contains( e, IllegalMultilineFieldException.class ) )
{
printErrorMessage( "Detected field which spanned multiple lines for an import where " +
Options.MULTILINE_FIELDS.argument() + "=false. If you know that your input data " +
"include fields containing new-line characters then import with this option set to " +
"true.", e, stackTrace );
}
// Fallback to printing generic error and stack trace
else
{
printErrorMessage( typeOfError + ": " + e.getMessage(), e, true );
}
System.err.println();
// Mute the stack trace that the default exception handler would have liked to print.
// Calling System.exit( 1 ) or similar would be convenient on one hand since we can set
// a specific exit code. On the other hand It's very inconvenient to have any System.exit
// call in code that is tested.
Thread.currentThread().setUncaughtExceptionHandler( ( t, e1 ) -> { /* Shhhh */ } );
return launderedException( e ); // throw in order to have process exit with !0
}
private static void printErrorMessage( String string, Exception e, boolean stackTrace )
{
System.err.println( string );
System.err.println( "Caused by:" + e.getMessage() );
if ( stackTrace )
{
e.printStackTrace( System.err );
}
}
private static Iterable>
relationshipData( final Charset encoding, Collection> relationshipsFiles )
{
return new IterableWrapper,Option>( relationshipsFiles )
{
@Override
protected DataFactory underlyingObjectToObject( Option group )
{
return data( defaultRelationshipType( group.metadata() ), encoding, group.value() );
}
};
}
private static Iterable> nodeData( final Charset encoding,
Collection> nodesFiles )
{
return new IterableWrapper,Option>( nodesFiles )
{
@Override
protected DataFactory underlyingObjectToObject( Option input )
{
Function decorator = input.metadata() != null
? additiveLabels( input.metadata().split( ":" ) )
: NO_NODE_DECORATOR;
return data( decorator, encoding, input.value() );
}
};
}
private static void printUsage( PrintStream out )
{
out.println( "Neo4j Import Tool" );
for ( String line : Args.splitLongLine( "neo4j-import is used to create a new Neo4j database "
+ "from data in CSV files. "
+
"See the chapter \"Import Tool\" in the Neo4j Manual for details on the CSV file format "
+ "- a special kind of header is required.", 80 ) )
{
out.println( "\t" + line );
}
out.println( "Usage:" );
for ( Options option : Options.values() )
{
option.printUsage( out );
}
out.println( "Example:");
out.print( Strings.joinAsLines(
TAB + "bin/neo4j-import --into retail.db --id-type string --nodes:Customer customers.csv ",
TAB + "--nodes products.csv --nodes orders_header.csv,orders1.csv,orders2.csv ",
TAB + "--relationships:CONTAINS order_details.csv ",
TAB + "--relationships:ORDERED customer_orders_header.csv,orders1.csv,orders2.csv" ) );
}
private static boolean asksForUsage( Args args )
{
for ( String orphan : args.orphans() )
{
if ( isHelpKey( orphan ) )
{
return true;
}
}
for ( Entry option : args.asMap().entrySet() )
{
if ( isHelpKey( option.getKey() ) )
{
return true;
}
}
return false;
}
private static boolean isHelpKey( String key )
{
return key.equals( "?" ) || key.equals( "help" );
}
private static Configuration csvConfiguration( Args args, final boolean defaultSettingsSuitableForTests )
{
final Configuration defaultConfiguration = COMMAS;
final Character specificDelimiter = args.interpretOption( Options.DELIMITER.key(),
Converters.optional(), CHARACTER_CONVERTER );
final Character specificArrayDelimiter = args.interpretOption( Options.ARRAY_DELIMITER.key(),
Converters.optional(), CHARACTER_CONVERTER );
final Character specificQuote = args.interpretOption( Options.QUOTE.key(), Converters.optional(),
CHARACTER_CONVERTER );
final Boolean multiLineFields = args.getBoolean( Options.MULTILINE_FIELDS.key(), null );
final Boolean emptyStringsAsNull = args.getBoolean( Options.IGNORE_EMPTY_STRINGS.key(), null );
return new Configuration.Default()
{
@Override
public char delimiter()
{
return specificDelimiter != null
? specificDelimiter.charValue()
: defaultConfiguration.delimiter();
}
@Override
public char arrayDelimiter()
{
return specificArrayDelimiter != null
? specificArrayDelimiter.charValue()
: defaultConfiguration.arrayDelimiter();
}
@Override
public char quotationCharacter()
{
return specificQuote != null
? specificQuote.charValue()
: defaultConfiguration.quotationCharacter();
}
@Override
public boolean multilineFields()
{
return multiLineFields != null
? multiLineFields.booleanValue()
: defaultConfiguration.multilineFields();
}
@Override
public boolean emptyQuotedStringsAsNull()
{
return emptyStringsAsNull != null
? emptyStringsAsNull.booleanValue()
: defaultConfiguration.emptyQuotedStringsAsNull();
}
@Override
public int bufferSize()
{
return defaultSettingsSuitableForTests ? 10_000 : super.bufferSize();
}
};
}
private static final Function TO_ID_TYPE = from -> IdType.valueOf( from.toUpperCase() );
private static final Function CHARACTER_CONVERTER = new CharacterConverter();
static final Validator FILES_EXISTS = files -> {
for ( File file : files )
{
if ( file.getName().startsWith( ":" ) )
{
warn( "It looks like you're trying to specify default label or relationship type (" +
file.getName() + "). Please put such directly on the key, f.ex. " +
Options.NODE_DATA.argument() + ":MyLabel" );
}
Validators.REGEX_FILE_EXISTS.validate( file );
}
};
private static final BiFunction>> INPUT_FILES_EXTRACTOR =
( args, key ) -> args.interpretOptionsWithMetadata( key, Converters.optional(),
Converters.toFiles( MULTI_FILE_DELIMITER, Converters.regexFiles( true ) ), FILES_EXISTS,
Validators.atLeast( "--" + key, 1 ) );
static void warn( String warning )
{
System.err.println( warning );
}
private enum ManualPage
{
IMPORT_TOOL_FORMAT( "import-tool-header-format.html" );
private final String page;
ManualPage( String page )
{
this.page = page;
}
public String getReference( Anchor anchor )
{
// As long as the the operations manual is single-page we only use the anchor.
return "#" + anchor.anchor;
}
}
private enum Anchor
{
ID_SPACES( "import-tool-id-spaces" ),
RELATIONSHIP( "import-tool-header-format-rels" );
private final String anchor;
Anchor( String anchor )
{
this.anchor = anchor;
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy