eu.fbk.twm.wiki.wikipedia.CrossLanguageTextBuilder Maven / Gradle / Ivy

Go to download
/*
 * Copyright (2013) Fondazione Bruno Kessler (http://www.fbk.eu/)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package eu.fbk.twm.wiki.wikipedia;

import eu.fbk.twm.index.CrossLanguageSearcher;
import eu.fbk.twm.index.PageTextSearcher;
import eu.fbk.twm.utils.CharacterTable;
import eu.fbk.twm.utils.GenericFileUtils;
import eu.fbk.twm.utils.StringTable;
import org.apache.commons.cli.*;
import org.apache.log4j.Logger;
import org.apache.log4j.PropertyConfigurator;

import java.io.*;
import java.text.DecimalFormat;
import java.util.*;
import java.util.regex.Pattern;

/**
 * Created with IntelliJ IDEA.
 * User: giuliano
 * Date: 2/14/13
 * Time: 9:03 AM
 * To change this template use File | Settings | File Templates.
 *
 * Use for cross language corpus building and cross language lsa
 */
public class CrossLanguageTextBuilder {
	/**
	 * Define a static logger variable so that it references the
	 * Logger instance named CrossLanguageTextBuilder.
	 */
	static Logger logger = Logger.getLogger(CrossLanguageTextBuilder.class.getName());

	public static final int DEFAULT_TEXT_LENGTH = 5;

	public static final int DEFAULT_NUM_PAGES = 10000;

	public static final int DEFAULT_NOTIFICATION_POINT = 100;

	//private static Pattern spacePattern = Pattern.compile(StringTable.SPACE);

	private static Pattern tabPattern = Pattern.compile(StringTable.HORIZONTAL_TABULATION);

	private Map> languageResourceMap;

	private String[] languages;

	private CrossLanguageSearcher crossLanguageSearcher;

	private int notificationPoint;

	private static DecimalFormat df = new DecimalFormat("###,###,###,###");

	public static final char DEFAULT_COLUMN_SEPARATOR = CharacterTable.SPACE;

	public CrossLanguageTextBuilder(String wikipediaModelsDirName, String sourceLanguage, int numPages, int maxLength, String crossTextFileName) throws Exception {
		this(wikipediaModelsDirName, sourceLanguage, numPages, maxLength, crossTextFileName, DEFAULT_COLUMN_SEPARATOR);
	}

	public CrossLanguageTextBuilder(String wikipediaModelsDirName, String sourceLanguage, int numPages, int maxLength, String crossTextFileName, char columnSeparator) throws Exception {
		//this.sourceLanguage = sourceLanguage;
		notificationPoint = DEFAULT_NOTIFICATION_POINT;
		init(wikipediaModelsDirName);
		String pageFreqFileName = getResource(sourceLanguage, "page-freq");

		PrintWriter pw = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(crossTextFileName), "UTF-8")));
		Map pageTextSearcherMap = createPageTextSearcherMap(maxLength);

		String sourceLanguageCrossLanguageIndexName = getResource(sourceLanguage, "cross-lang-index");
		logger.info("reading cross language index from " + sourceLanguageCrossLanguageIndexName + "...");
		crossLanguageSearcher = new CrossLanguageSearcher(sourceLanguageCrossLanguageIndexName);

		logger.info("reading page/frequency pairs from " + pageFreqFileName + "...");
		LineNumberReader lnr = new LineNumberReader(new InputStreamReader(new FileInputStream(pageFreqFileName), "UTF-8"));
		String line;
		String s[];
		int count = 1;
		Map crossLanguageMap;
		String targetLanguage;
		String sourcePage = null, targetPage = null;
		String sourceText = null, targetText = null;
		StringBuilder title, text;
		Iterator crossLanguageIterator;
		PageTextSearcher pageTextSearcher;
		long begin = System.currentTimeMillis(), end = 0;
		while ((line = lnr.readLine()) != null) {

			if (count >= numPages) {
				logger.info("Exit after " + count + " lines (" + numPages + ")");
				break;
			}

			if ((count % notificationPoint) == 0) {
				end = System.currentTimeMillis();
				logger.info(count + "\t" + df.format(count) + "\t" + df.format(end - begin) + " ms " + new Date());
				begin = System.currentTimeMillis();
			}

			s = tabPattern.split(line);
			if (s.length == 2) {
				//write the first title
				sourcePage = s[1];
				//logger.debug(count + "\t" + sourcePage);
				crossLanguageMap = crossLanguageSearcher.search(sourcePage);
				if (crossLanguageMap != null) {
					title = new StringBuilder();
					text = new StringBuilder();
					title.append(sourcePage);
					sourceText = pageTextSearcherMap.get(sourceLanguage).search(sourcePage);
					text.append(sourceText);
					crossLanguageIterator = crossLanguageMap.keySet().iterator();
					for (int i = 0; crossLanguageIterator.hasNext(); i++) {
						targetLanguage = crossLanguageIterator.next();
						pageTextSearcher = pageTextSearcherMap.get(targetLanguage);
						if (pageTextSearcher != null) {
							targetPage = crossLanguageMap.get(targetLanguage);
							//logger.debug("\t" + i + "\t" + targetLanguage + "\t" + targetPage);
							title.append(CharacterTable.LOW_LINE);
							title.append(targetLanguage);

							//text.append(CharacterTable.SPACE);
							text.append(columnSeparator);

							//text.append(targetLanguage + "[");
							targetText = pageTextSearcherMap.get(targetLanguage).search(targetPage);
							text.append(targetText);
							//text.append("]");
						}
					}
					//pw.println(title.toString() + CharacterTable.SPACE + text.toString());
					pw.println(title.toString() + columnSeparator + text.toString());
					count++;
				}
			}

		}

		end = System.currentTimeMillis();
		logger.info(count + "\t" + df.format(count) + "\t" + df.format(end - begin) + " ms " + new Date());
		lnr.close();
		pw.close();
	}

	public int getNotificationPoint() {
		return notificationPoint;
	}

	public void setNotificationPoint(int notificationPoint) {
		this.notificationPoint = notificationPoint;
	}

	String getResource(String language, String key) {
		return languageResourceMap.get(language).get(key);
	}

	Map createPageTextSearcherMap(int maxLength) throws IOException {
		Map pageTextSearcherMap = new HashMap();
		String textIndex;
		PageTextSearcher pageTextSearcher;
		for (int i = 0; i < languages.length; i++) {
			textIndex = getResource(languages[i], "text-index");
			logger.info("reading text index from " + textIndex + "...");
			pageTextSearcher = new PageTextSearcher(textIndex);
			pageTextSearcher.setMaximunTextLength(maxLength);
			pageTextSearcherMap.put(languages[i], pageTextSearcher);
		}
		return pageTextSearcherMap;
	}

	void init(String wikipediaModelsDirName) {
		if (!wikipediaModelsDirName.endsWith(File.separator)) {
			wikipediaModelsDirName += File.separator;
		}

		logger.info("wikipedia models " + wikipediaModelsDirName);
		languages = new File(wikipediaModelsDirName).list();
		logger.info("supported languages");
		logger.info(Arrays.toString(languages));
		languageResourceMap = new TreeMap>();
		for (int i = 0; i < languages.length; i++) {


			Map resourceMap = null;
			try {
				resourceMap = GenericFileUtils.searchForFilesInTheSameFolder(wikipediaModelsDirName + languages[i], "text-index", "page-freq", "cross-lang-index");
				languageResourceMap.put(languages[i], resourceMap);
			} catch (IOException e) {
				logger.error(e);
			}
		}
	}

	// Returns true iff all languages are contained in the map; false otherwise.
	/*Set checkLanguages(String[] lang, Map map) {
		Set set = new HashSet();
		String title = null;
		for (int i = 1; i < lang.length; i++) {
			title = map.get(lang[i]);
			if (title == null) {
				//logger.warn(lang[i] + " is not present for " + map.values());
				//return false;
				set.add(lang[i]);
			}
		}
		return set;
	}

	String trim(String article, int maxLength) {
		//logger.debug("article.length " + article.length());
		char ch = 0;
		int length = 0;
		int start = article.indexOf(' ') + 1;

		for (int i = start; i < article.length(); i++) {
			ch = article.charAt(i);
			if (ch == ' ') {
				//logger.info(article.substring(start, i));
				length++;
			}

			if (length >= maxLength) {
				return article.substring(start, i);
			}
		}
		return article;
	} */

	public static void main(String[] args) throws Exception {
		String logConfig = System.getProperty("log-config");
		if (logConfig == null) {
			logConfig = "log-config.txt";
		}

		PropertyConfigurator.configure(logConfig);

		// time java org.fbk.cit.hlt.wm.util.CrossLanguageTextBuilder wikipedia/ en,it,fr,pt,de 20120601,20120510,20120627,20120627,20120116 10 10 prova.csv
		/*if (args.length != 6)
		{
			logger.error(args.length + " != 6");
			logger.info("java org.fbk.cit.hlt.wm.util.CrossLanguageTextBuilder in-wiki-wikipediaModelsDirName-dir lang-list version-list numPages max-length out-cross-lang-text-csv");
			System.exit(1);
		}
		String wikipediaModelsDirName = args[0];
		String[] lang = args[1].split(",");
		String[] version = args[2].split(",");
		int numPages = Integer.parseInt(args[3]);
		int maxLength = Integer.parseInt(args[4]);
		File crossTextFileName = new File(args[5]);

		new CrossLanguageTextBuilder(wikipediaModelsDirName, lang, version, numPages, maxLength, crossTextFileName);*/
		Options options = new Options();
		try {
			Option wikipediaModelDirOpt = OptionBuilder.withArgName("dir").hasArg().withDescription("wikiepedia model dir").isRequired().withLongOpt("model-dir").create("m");
			Option sourceLanguageOpt = OptionBuilder.withArgName("string").hasArg().withDescription("source language").isRequired().withLongOpt("source-language").create("s");
			Option textLengthOpt = OptionBuilder.withArgName("int").hasArg().withDescription("maximum length of the text (default is " + DEFAULT_TEXT_LENGTH + ")").withLongOpt("maximum-length").create("l");
			Option numPageOpt = OptionBuilder.withArgName("int").hasArg().withDescription("maximum number of pages to process (default is " + DEFAULT_NUM_PAGES + ")").withLongOpt("num-pages").create("p");
			Option crossTextOpt = OptionBuilder.withArgName("file").hasArg().withDescription("cross language text").isRequired().withLongOpt("cross-text").create("t");
			Option columnSeparatorOpt = OptionBuilder.withArgName("char").hasArg().withDescription("column separator").isRequired().withLongOpt("col-separator").create();
			Option notificationPointOpt = OptionBuilder.withArgName("int").hasArg().withDescription("receive notification every n pages (default is " + df.format(DEFAULT_NOTIFICATION_POINT) + ")").withLongOpt("notification-point").create("b");
			options.addOption("h", "help", false, "print this message");
			options.addOption("v", "version", false, "output version information and exit");

			options.addOption(wikipediaModelDirOpt);
			options.addOption(sourceLanguageOpt);
			options.addOption(textLengthOpt);
			options.addOption(numPageOpt);
			options.addOption(crossTextOpt);
			options.addOption(notificationPointOpt);
			options.addOption(columnSeparatorOpt);

			CommandLineParser parser = new PosixParser();
			CommandLine line = parser.parse(options, args);

			if (line.hasOption("help") || line.hasOption("version")) {
				throw new ParseException("");
			}

			int notificationPoint = DEFAULT_NOTIFICATION_POINT;
			if (line.hasOption("notification-point")) {
				notificationPoint = Integer.parseInt(line.getOptionValue("notification-point"));
			}

			int textLength = DEFAULT_TEXT_LENGTH;
			if (line.hasOption("maximum-length")) {
				textLength = Integer.parseInt(line.getOptionValue("maximum-length"));
			}

			char ColumnSeparator = DEFAULT_COLUMN_SEPARATOR;
			if (line.hasOption("col-separator")) {
				ColumnSeparator = line.getOptionValue("col-separator").charAt(0);
			}

			int numPages = DEFAULT_NUM_PAGES;
			if (line.hasOption("num-pages")) {
				numPages = Integer.parseInt(line.getOptionValue("num-pages"));
			}
			logger.debug(line.getOptionValue("model-dir") + "\t" + line.getOptionValue("source-language") + "\t" + numPages + "\t" + textLength + "\t" + line.getOptionValue("cross-text"));
			CrossLanguageTextBuilder crossLanguageTextBuilder = new CrossLanguageTextBuilder(line.getOptionValue("model-dir"), line.getOptionValue("source-language"), numPages, textLength, line.getOptionValue("cross-text"), ColumnSeparator);

		} catch (ParseException e) {
			// oops, something went wrong
			if (e.getMessage().length() > 0) {
				System.out.println("Parsing failed: " + e.getMessage() + "\n");
			}

			HelpFormatter formatter = new HelpFormatter();
			formatter.printHelp(400, "java -cp dist/thewikimachine.jar org.fbk.cit.hlt.thewikimachine.wikipedia.CrossLanguageTextBuilder", "\n", options, "\n", true);
		}

	}
}