org.fbk.cit.hlt.thewikimachine.index.PageTrafficSearcher Maven / Gradle / Ivy

Go to download
/*
 * Copyright (2013) Fondazione Bruno Kessler (http://www.fbk.eu/)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.fbk.cit.hlt.thewikimachine.index;

import org.apache.commons.cli.*;
import org.apache.log4j.Logger;
import org.apache.log4j.PropertyConfigurator;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermDocs;
import org.fbk.cit.hlt.thewikimachine.index.util.AbstractSearcher;
import org.fbk.cit.hlt.thewikimachine.index.util.SerialUtils;
import org.fbk.cit.hlt.thewikimachine.util.CharacterTable;
import org.fbk.cit.hlt.thewikimachine.util.StringTable;
import org.fbk.cit.hlt.thewikimachine.xmldump.AbstractWikipediaExtractor;

import java.io.*;
import java.text.DecimalFormat;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Pattern;

/**
 * Created with IntelliJ IDEA.
 * User: giuliano
 * Date: 1/22/13
 * Time: 6:11 PM
 * To change this template use File | Settings | File Templates.
 */
public class PageTrafficSearcher extends AbstractSearcher {
	/**
	 * Define a static logger variable so that it references the
	 * Logger instance named PageTrafficSearcher.
	 */
	static Logger logger = Logger.getLogger(PageTrafficSearcher.class.getName());

	public static final int DEFAULT_MIN_FREQ = 1000;

	public static final boolean DEFAULT_THREAD_SAFE = false;

	protected static DecimalFormat df = new DecimalFormat("###,###,###,###");

	private static DecimalFormat tf = new DecimalFormat("000,000,000.#");

	//
	private static DecimalFormat rf = new DecimalFormat("0.000000");


	private static Pattern tabPattern = Pattern.compile(StringTable.HORIZONTAL_TABULATION);

	protected boolean threadSafe;

	private Map cache;

	private Term keyTerm;

	public PageTrafficSearcher(String indexName) throws IOException {
		this(indexName, DEFAULT_THREAD_SAFE);
	}

	public PageTrafficSearcher(String indexName, boolean threadSafe) throws IOException {
		super(indexName);
		this.threadSafe = threadSafe;

		keyTerm = new Term(PageFreqIndexer.PAGE_FIELD_NAME, "");
		logger.debug(keyTerm);
	}

	public void loadCache() throws IOException {
		loadCache(DEFAULT_MIN_FREQ);
	}

	public void loadCache(int minFreq) throws IOException {
		logger.info("loading cache (freq >" + minFreq + ")...");

		long begin = System.nanoTime();

		if (threadSafe) {
			logger.info(this.getClass().getName() + "'s cache is thread safe");
			cache = Collections.synchronizedMap(new HashMap());
		}
		else {
			logger.warn(this.getClass().getName() + "'s cache isn't thread safe");
			cache = new HashMap();
		}

		String form;
		int freq = 0;
		Document doc;
		for (int i = 1; i < indexReader.numDocs(); i++) {
			doc = indexReader.document(i);
			form = doc.get(PageFreqIndexer.PAGE_FIELD_NAME);
			freq = SerialUtils.toInt(doc.getBinaryValue(PageFreqIndexer.PAGE_FREQ_FIELD_NAME));
			if (freq > minFreq) {
				cache.put(form, freq);
			}
			else {
				break;
			}

			if ((i % notificationPoint) == 0) {
				//System.out.print(CharacterTable.FULL_STOP);
				logger.debug(i + " keys read (" + cache.size() + ") " + new Date());
			}
		}

		System.out.print(CharacterTable.LINE_FEED);

		long end = System.nanoTime();
		logger.info(df.format(cache.size()) + " (" + df.format(indexReader.numDocs()) + ") keys cached in " + tf.format(end - begin) + " ns");
	}

	public int search(String key) {
		Integer result = null;
		if (cache != null) {
			result = cache.get(key);
		}

		//long end = System.nanoTime();
		if (result != null) {
			//logger.debug("found in cache in " + tf.format(end - begin) + " ns");
			return result;
		}

		try {
			//begin = System.nanoTime();
			TermDocs termDocs = indexReader.termDocs(keyTerm.createTerm(key));
			//end = System.nanoTime();
			//logger.debug("\"" + form + "\" sought in index in " + tf.format(end - begin) + " ns");

			int freq = 0;
			if (termDocs.next()) {
				//begin = System.nanoTime();
				Document doc = indexReader.document(termDocs.doc());
				freq = SerialUtils.toInt(doc.getBinaryValue(PageFreqIndexer.PAGE_FREQ_FIELD_NAME));

				result = freq;
				//end = System.nanoTime();
				//logger.debug("\"" + form + "\" deserialized in " + tf.format(end - begin) + " ns");

				return result;
			}
		} catch (IOException e) {
			logger.error(e);
		}

		return 0;
	}

	public void interactive() throws Exception {
		InputStreamReader indexReader = null;
		BufferedReader myInput = null;
		long begin = 0, end = 0;
		while (true) {
			System.out.println("\nPlease write a query and type  to continue (CTRL C to exit):");

			indexReader = new InputStreamReader(System.in);
			myInput = new BufferedReader(indexReader);
			String query = myInput.readLine().toString();
			begin = System.nanoTime();
			Integer e = search(query);
			end = System.nanoTime();
			logger.info("query\tfreq\tratio\ttime");
			logger.info(query + "\t" + df.format(e) + "\t" + tf.format(end - begin) + " ns");
		}
	}

	public static void main(String args[]) throws Exception {
		String logConfig = System.getProperty("log-config");
		if (logConfig == null) {
			logConfig = "configuration/log-config.txt";
		}

		PropertyConfigurator.configure(logConfig);
		Options options = new Options();
		try {
			Option indexNameOpt = OptionBuilder.withArgName("index").hasArg().withDescription("open an index with the specified name").isRequired().withLongOpt("index").create("i");
			Option interactiveModeOpt = OptionBuilder.withArgName("interactive-mode").withDescription("enter in the interactive mode").withLongOpt("interactive-mode").create("t");
			Option searchOpt = OptionBuilder.withArgName("search").hasArg().withDescription("search for the specified key").withLongOpt("search").create("s");
			//Option keyFieldNameOpt = OptionBuilder.withArgName("key-field-name").hasArg().withDescription("use the specified name for the field key").withLongOpt("key-field-name").create("k");
			//Option valueFieldNameOpt = OptionBuilder.withArgName("value-field-name").hasArg().withDescription("use the specified name for the field value").withLongOpt("value-field-name").create("v");
			Option minimumKeyFreqOpt = OptionBuilder.withArgName("minimum-freq").hasArg().withDescription("minimum key frequency of cached values (default is " + DEFAULT_MIN_FREQ + ")").withLongOpt("minimum-freq").create("m");
			Option notificationPointOpt = OptionBuilder.withArgName("int").hasArg().withDescription("receive notification every n pages (default is " + AbstractWikipediaExtractor.DEFAULT_NOTIFICATION_POINT + ")").withLongOpt("notification-point").create("b");
			options.addOption("h", "help", false, "print this message");
			options.addOption("v", "version", false, "output version information and exit");

			options.addOption(indexNameOpt);
			options.addOption(interactiveModeOpt);
			options.addOption(searchOpt);
			//options.addOption(keyFieldNameOpt);
			//options.addOption(valueFieldNameOpt);
			options.addOption(minimumKeyFreqOpt);
			options.addOption(notificationPointOpt);

			CommandLineParser parser = new PosixParser();
			CommandLine line = parser.parse(options, args);

			if (line.hasOption("help") || line.hasOption("version")) {
				throw new ParseException("");
			}

			int minFreq = DEFAULT_MIN_FREQ;
			if (line.hasOption("minimum-freq")) {
				minFreq = Integer.parseInt(line.getOptionValue("minimum-freq"));
			}

			int notificationPoint = AbstractWikipediaExtractor.DEFAULT_NOTIFICATION_POINT;
			if (line.hasOption("notification-point")) {
				notificationPoint = Integer.parseInt(line.getOptionValue("notification-point"));
			}

			PageTrafficSearcher ngramSearcher = new PageTrafficSearcher(line.getOptionValue("index"));
			ngramSearcher.setNotificationPoint(notificationPoint);
			/*logger.debug(line.getOptionValue("key-field-name") + "\t" + line.getOptionValue("value-field-name"));
			if (line.hasOption("key-field-name"))
			{
				ngramSearcher.setKeyFieldName(line.getOptionValue("key-field-name"));
			}
			if (line.hasOption("value-field-name"))
			{
				ngramSearcher.setValueFieldName(line.getOptionValue("value-field-name"));
			} */
			if (line.hasOption("minimum-freq")) {
				ngramSearcher.loadCache(minFreq);
			}
			if (line.hasOption("search")) {
				logger.debug("searching " + line.getOptionValue("search") + "...");
				int result = ngramSearcher.search(line.getOptionValue("search"));
				logger.info(result);
			}
			if (line.hasOption("interactive-mode")) {
				ngramSearcher.interactive();
			}
		} catch (ParseException e) {
			// oops, something went wrong
			if (e.getMessage().length() > 0) {
				System.out.println("Parsing failed: " + e.getMessage() + "\n");
			}
			HelpFormatter formatter = new HelpFormatter();
			formatter.printHelp(400, "java -cp dist/thewikimachine.jar org.fbk.cit.hlt.thewikimachine.index.PageTrafficSearcher", "\n", options, "\n", true);
		}
	}

}