src.it.unimi.dsi.big.mg4j.document.FileSetDocumentCollection Maven / Gradle / Ivy
Show all versions of mg4j-big Show documentation
package it.unimi.dsi.big.mg4j.document;
/*
* MG4J: Managing Gigabytes for Java (big)
*
* Copyright (C) 2005-2011 Paolo Boldi and Sebastiano Vigna
*
* This library is free software; you can redistribute it and/or modify it
* under the terms of the GNU Lesser General Public License as published by the Free
* Software Foundation; either version 3 of the License, or (at your option)
* any later version.
*
* This library is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
* or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License
* for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, see This class provides a main method with a flexible syntax that serialises
* into a document collection a list of files given on the command line or
* piped into standard input. Optionally, you can provide a parallel list of URIs
* that will be associated with each file.
*
*
Warning: the number of file is limited by {@link Integer#MAX_VALUE}.
*/
public class FileSetDocumentCollection extends AbstractDocumentCollection implements Serializable {
private static final long serialVersionUID = 0L;
/** The files in this collection. */
private final String[] file;
/** URIs for each file in this collection, or null, in which case the filename will be used as URI. */
private final String[] uri;
/** The factory to be used by this collection. */
private final DocumentFactory factory;
/** The last returned file input stream. */
private InputStream last = NullInputStream.getInstance();
/** Builds a document collection corresponding to a given set of files specified as an array.
*
*
Beware. This class is not guaranteed to work if files are
* deleted or modified after creation!
*
* @param file an array containing the files that will be contained in the collection.
* @param factory the factory that will be used to create documents.
*/
public FileSetDocumentCollection( final String[] file, final DocumentFactory factory ) {
this( file, null, factory );
}
/** Builds a document collection corresponding to a given set of files specified as an array and
* a parallel array of URIs, one for each file.
*
*
Beware. This class is not guaranteed to work if files are
* deleted or modified after creation!
*
* @param file an array containing the files that will be contained in the collection.
* @param uri an array, parallel to file, containing URIs to be associated with each element of file.
* @param factory the factory that will be used to create documents.
*/
public FileSetDocumentCollection( final String[] file, final String uri[], final DocumentFactory factory ) {
this.file = file;
this.uri = uri;
this.factory = factory;
}
public DocumentFactory factory() {
return factory;
}
public long size() {
return file.length;
}
public Reference2ObjectMap,Object> metadata( final long index ) {
ensureDocumentIndex( index );
final Reference2ObjectArrayMap, Object> metadata = new Reference2ObjectArrayMap,Object>( 2 );
metadata.put( MetadataKeys.TITLE, file[ (int)index ] );
if ( uri != null ) metadata.put( MetadataKeys.URI, uri[ (int)index ] );
else metadata.put( MetadataKeys.URI, new File( file[ (int)index ] ).toURI().toString() );
metadata.put( MetadataKeys.MIMETYPE, MimeTypeResolver.getContentType( file[ (int)index ] ) );
return metadata;
}
public Document document( final long index ) throws IOException {
return factory.getDocument( stream( index ), metadata( index ) );
}
public InputStream stream( final long index ) throws IOException {
ensureDocumentIndex( index );
last.close();
return last = new FileInputStream( file[ (int)index ] );
}
public FileSetDocumentCollection copy() {
return new FileSetDocumentCollection( file, uri, factory.copy() );
}
public void close() throws IOException {
last.close();
super.close();
}
public static void main( final String[] arg ) throws IOException, JSAPException, InstantiationException, IllegalAccessException, InvocationTargetException, NoSuchMethodException {
SimpleJSAP jsap = new SimpleJSAP( FileSetDocumentCollection.class.getName(), "Saves a serialised document collection based on a set of files.",
new Parameter[] {
new FlaggedOption( "factory", MG4JClassParser.getParser(), IdentityDocumentFactory.class.getName(), JSAP.NOT_REQUIRED, 'f', "factory", "A document factory with a standard constructor." ),
new FlaggedOption( "property", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, 'p', "property", "A 'key=value' specification, or the name of a property file" ).setAllowMultipleDeclarations( true ),
new FlaggedOption( "uris", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, 'u', "uris", "A file containing a list of URIs in ASCII encoding, one per line, that will be associated with each file" ),
new UnflaggedOption( "collection", JSAP.STRING_PARSER, JSAP.REQUIRED, "The filename for the serialised collection." ),
new UnflaggedOption( "file", JSAP.STRING_PARSER, JSAP.NO_DEFAULT, JSAP.NOT_REQUIRED, JSAP.GREEDY, "A list of files that will be indexed. If missing, a list of files will be read from standard input." )
}
);
JSAPResult jsapResult = jsap.parse( arg );
if ( jsap.messagePrinted() ) return;
String uri[] = null;
if ( jsapResult.getString( "uris" ) != null ) {
Collection lines = new FileLinesCollection( jsapResult.getString( "uris" ), "ASCII" ).allLines();
uri = new String[ lines.size() ];
int i = 0;
for( Object l: lines ) uri[ i++ ] = l.toString();
}
final DocumentFactory factory = PropertyBasedDocumentFactory.getInstance( jsapResult.getClass( "factory" ), jsapResult.getStringArray( "property" ) );
String[] file = (String[])jsapResult.getObjectArray( "file", new String[ 0 ] );
if ( file.length == 0 ) {
final ObjectArrayList files = new ObjectArrayList();
BufferedReader bufferedReader = new BufferedReader( new InputStreamReader( System.in ) );
String s;
while( ( s = bufferedReader.readLine() ) != null ) files.add( s );
file = files.toArray( new String[ 0 ] );
}
if ( file.length == 0 ) System.err.println( "WARNING: empty file set." );
if ( uri != null && file.length != uri.length ) throw new IllegalArgumentException( "The number of files (" + file.length + ") and the number of URIs (" + uri.length + ") differ" );
BinIO.storeObject( new FileSetDocumentCollection( file, uri, factory ), jsapResult.getString( "collection" ) );
}
}