org.fbk.cit.hlt.thewikimachine.index.PageTrafficIndexer Maven / Gradle / Ivy

Go to download
/*
 * Copyright (2013) Fondazione Bruno Kessler (http://www.fbk.eu/)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.fbk.cit.hlt.thewikimachine.index;

import org.apache.commons.cli.*;
import org.apache.log4j.Logger;
import org.apache.log4j.PropertyConfigurator;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.fbk.cit.hlt.thewikimachine.index.util.SerialUtils;
import org.fbk.cit.hlt.thewikimachine.index.util.SingleValueIndexer;

import java.io.File;
import java.io.IOException;

/**
 * Created with IntelliJ IDEA.
 * User: giuliano
 * Date: 1/24/13
 * Time: 6:47 PM
 * To change this template use File | Settings | File Templates.
 *
 * @see org.fbk.cit.hlt.thewikimachine.wikipedia.WikipediaTrafficDownloader
 */
public class PageTrafficIndexer extends SingleValueIndexer {
	/**
	 * Define a static logger variable so that it references the
	 * Logger instance named PageTrafficIndexer.
	 */
	static Logger logger = Logger.getLogger(PageTrafficIndexer.class.getName());

	public static final String PAGE_FIELD_NAME = "page";

	public static final String PAGE_FREQ_FIELD_NAME = "freq";


	public static final int PAGE_COLUMN_INDEX = 1;

	public static final int FREQ_COLUMN_INDEX = 0;


	public PageTrafficIndexer(String indexName) throws IOException {
		super(indexName);
	}

	@Override
	public void index(String fileName) throws IOException {
		index(fileName, PAGE_COLUMN_INDEX);
	}

	@Override
	public void index(File file) throws IOException {
		index(file, PAGE_COLUMN_INDEX);
	}

	@Override
	public void add(String[] t) {

		Document doc = new Document();
		try {
			doc.add(new Field(PAGE_FIELD_NAME, t[PAGE_COLUMN_INDEX], Field.Store.YES, Field.Index.NOT_ANALYZED));
			doc.add(new Field(PAGE_FREQ_FIELD_NAME, SerialUtils.toByteArray(Integer.parseInt(t[FREQ_COLUMN_INDEX])), Field.Store.YES));

			indexWriter.addDocument(doc);
		} catch (IOException e) {
			logger.error(e);
		}
	}

	public static void main(String args[]) throws Exception {
		String logConfig = System.getProperty("log-config");
		if (logConfig == null) {
			logConfig = "configuration/log-config.txt";
		}


		PropertyConfigurator.configure(logConfig);
		Options options = new Options();
		try {
			Option indexNameOpt = OptionBuilder.withArgName("index").hasArg().withDescription("create an index with the specified name").isRequired().withLongOpt("index").create("i");
			Option inputFileOpt = OptionBuilder.withArgName("file").hasArg().withDescription("read the key/value pairs to index from the specified file").withLongOpt("file").create("f");
			//Option keyFieldNameOpt = OptionBuilder.withArgName("key-field-name").hasArg().withDescription("use the specified name for the field key").withLongOpt("key-field-name").create("k");
			//Option valueFieldNameOpt = OptionBuilder.withArgName("value-field-name").hasArg().withDescription("use the specified name for the field value").withLongOpt("value-field-name").create("v");
			//Option freqFileOpt = OptionBuilder.withArgName("key-freq").withDescription("key frequency file").withLongOpt("key-freq").create("f");

			options.addOption("h", "help", false, "print this message");
			options.addOption("v", "version", false, "output version information and exit");

			options.addOption(indexNameOpt);
			options.addOption(inputFileOpt);
			//options.addOption(keyFieldNameOpt);
			//options.addOption(valueFieldNameOpt);
			CommandLineParser parser = new PosixParser();
			CommandLine line = parser.parse(options, args);

			if (line.hasOption("help") || line.hasOption("version")) {
				throw new ParseException("");
			}

			PageTrafficIndexer ngramIndexer = new PageTrafficIndexer(line.getOptionValue("index"));
			ngramIndexer.index(line.getOptionValue("file"));
			ngramIndexer.close();
		} catch (ParseException e) {
			// oops, something went wrong
			if (e.getMessage().length() > 0) {
				System.out.println("Parsing failed: " + e.getMessage() + "\n");
			}
			HelpFormatter formatter = new HelpFormatter();
			formatter.printHelp(400, "java -cp dist/thewikimachine.jar org.fbk.cit.hlt.thewikimachine.index.PageTrafficIndexer", "\n", options, "\n", true);
		}
	}
}