eu.fbk.twm.wiki.xmldump.WikipediaIncomingOutgoingLinkExtractor Maven / Gradle / Ivy

Go to download
/*
 * Copyright (2013) Fondazione Bruno Kessler (http://www.fbk.eu/)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package eu.fbk.twm.wiki.xmldump;

import de.tudarmstadt.ukp.wikipedia.parser.Link;
import de.tudarmstadt.ukp.wikipedia.parser.ParsedPage;
import de.tudarmstadt.ukp.wikipedia.parser.Section;
import eu.fbk.twm.utils.*;
import eu.fbk.twm.wiki.xmldump.util.WikiMarkupParser;
import org.apache.commons.cli.*;
import org.apache.commons.cli.OptionBuilder;
import org.apache.log4j.Logger;
import org.apache.log4j.PropertyConfigurator;

import java.io.*;
import java.util.Date;
import java.util.List;
import java.util.Locale;
import java.util.regex.Matcher;

public class WikipediaIncomingOutgoingLinkExtractor extends AbstractWikipediaExtractor implements WikipediaExtractor {
	/**
	 * Define a static logger variable so that it references the
	 * Logger instance named WikipediaIncomingOutgoingLinkExtractor.
	 */
	static Logger logger = Logger.getLogger(WikipediaIncomingOutgoingLinkExtractor.class.getName());

	private PrintWriter incomingOutgoingWriter;

	private PageMap redirectPagesMap;

	private PageSet disambiguationPagesSet;

	private PageMap titleIdMap;

	public final static String NUM_SIGN = "#";


	public WikipediaIncomingOutgoingLinkExtractor(int numThreads, int numPages, Locale locale) throws IOException {
		super(numThreads, numPages, locale);
	}

	@Override
	public void start(ExtractorParameters extractorParameters) {
		try {
			redirectPagesMap = new PageMap(new File(extractorParameters.getWikipediaRedirFileName()));
			logger.info(redirectPagesMap.size() + " redirect pages");
			disambiguationPagesSet = new PageSet(new File(extractorParameters.getWikipediaDisambiguationFileName()));
			logger.info(disambiguationPagesSet.size() + " disambiguation pages");
			titleIdMap = new PageMap(new File(extractorParameters.getWikipediaTitleIdFileName()));
			logger.info(titleIdMap.size() + " title id pairs");

			incomingOutgoingWriter = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(extractorParameters.getWikipediaIncomingOutgoingFileName()), "UTF-8")));
		} catch (IOException e) {
			logger.error(e);
		}
		startProcess(extractorParameters.getWikipediaXmlFileName());
	}

	@Override
	public void filePage(String text, String title, int wikiID) {
		//To change body of implemented methods use File | Settings | File Templates.
	}

	@Override
	public void categoryPage(String text, String title, int wikiID) {
		//todo: verify here
		contentPage(text, title, wikiID);
	}

	@Override
	public void contentPage(String text, String title, int wikiID) {
		try {
			StringBuilder sb = new StringBuilder();
			sb.append(outgoingInternalLinks(text, title));
			//todo: weight also the categories?
			//sb.append(categoryLinks(text, title));

			if (sb.length() > 0) {
				synchronized (this) {
					incomingOutgoingWriter.print(sb.toString());
				}
			}
		} catch (Exception e) {
			logger.error("Error at page " + title + " (" + wikiID + ")");
			logger.error(e);
		}
	}

	private String categoryLinks(String text, String title) {
		//todo: replace the same code in pre-processing
		Matcher m = categoryPattern.matcher(text);
		StringBuilder buff = new StringBuilder();
		String category;
		int index = 1;
		String titleWikiID, categoryWikiID;
		titleWikiID = titleIdMap.get(title);
		if (titleWikiID == null) {
			return buff.toString();
		}
		while (m.find()) {
			int s = m.start(index);
			int e = m.end(index);
			category = text.substring(s, e);
			int j = category.indexOf(CharacterTable.VERTICAL_LINE);
			if (j != -1) {
				category = category.substring(0, j);
			}
			category = category.replace(CharacterTable.SPACE, CharacterTable.LOW_LINE);

			categoryWikiID = titleIdMap.get(category);
			if (categoryWikiID != null) {
				buff.append(title);
				buff.append(CharacterTable.HORIZONTAL_TABULATION);
				buff.append(category);
				buff.append(CharacterTable.HORIZONTAL_TABULATION);
				buff.append(categoryWikiID);
				buff.append(CharacterTable.LINE_FEED);
			}

		}
		return buff.toString();
	}

	/**
	 * Returns the outgoing links. From this...
	 */
	private String outgoingInternalLinks(String text, String title) throws IOException {
		StringBuilder sb = new StringBuilder();
		String titleWikiID = titleIdMap.get(title);
		if (titleWikiID == null) {
			return sb.toString();
		}
		// this trick allows pages interlinked to share interlinks
		sb.append(titleWikiID);
		sb.append(CharacterTable.HORIZONTAL_TABULATION);
		sb.append(title);
		sb.append(CharacterTable.HORIZONTAL_TABULATION);
		sb.append(title);
		sb.append(CharacterTable.HORIZONTAL_TABULATION);
		sb.append(titleWikiID);
		sb.append(CharacterTable.LINE_FEED);

		String targetWikiID;
		WikiMarkupParser wikiMarkupParser = WikiMarkupParser.getInstance();
		ParsedPage pp = wikiMarkupParser.parsePage(text);
		String target;
		String redirect;
		for (Section section : pp.getSections()) {
			List internalLinks = section.getLinks(Link.type.INTERNAL);
			for (Link link : internalLinks) {
				target = normalizePageName(link.getTarget());
				redirect = redirectPagesMap.get(target);
				if (redirect != null) {
					target = redirect;
				}
				targetWikiID = titleIdMap.get(target);
				if (targetWikiID == null) {
				}
				else if (target.startsWith(NUM_SIGN)) {
				}
				else if (disambiguationPagesSet.contains(target)) {
				}
				else {
					// outgoing
					sb.append(titleWikiID);
					sb.append(CharacterTable.HORIZONTAL_TABULATION);
					sb.append(title);
					sb.append(CharacterTable.HORIZONTAL_TABULATION);
					sb.append(target);
					sb.append(CharacterTable.HORIZONTAL_TABULATION);
					sb.append(targetWikiID);
					sb.append(CharacterTable.LINE_FEED);

					// incoming
					sb.append(targetWikiID);
					sb.append(CharacterTable.HORIZONTAL_TABULATION);
					sb.append(target);
					sb.append(CharacterTable.HORIZONTAL_TABULATION);
					sb.append(title);
					sb.append(CharacterTable.HORIZONTAL_TABULATION);
					sb.append(titleWikiID);
					sb.append(CharacterTable.LINE_FEED);
				}
			}
		}
		//logger.debug(title + "\t"+sb.toString());
		return sb.toString();
	}

	@Override
	public void disambiguationPage(String text, String title, int wikiID) {
		//To change body of implemented methods use File | Settings | File Templates.
	}

	@Override
	public void templatePage(String text, String title, int wikiID) {
		//To change body of implemented methods use File | Settings | File Templates.
	}

	@Override
	public void redirectPage(String text, String title, int wikiID) {
		//To change body of implemented methods use File | Settings | File Templates.
	}

	@Override
	public void portalPage(String text, String title, int wikiID) {
		//To change body of implemented methods use File | Settings | File Templates.
	}

	@Override
	public void projectPage(String text, String title, int wikiID) {
		//To change body of implemented methods use File | Settings | File Templates.
	}

	@Override
	public void endProcess() {
		super.endProcess();
		incomingOutgoingWriter.flush();
		incomingOutgoingWriter.close();
	}

	public static void main(String args[]) throws IOException {
		String logConfig = System.getProperty("log-config");
		if (logConfig == null) {
			logConfig = "configuration/log-config.txt";
		}

		PropertyConfigurator.configure(logConfig);

		Options options = new Options();
		try {
			Option wikipediaDumpOpt = OptionBuilder.withArgName("file").hasArg().withDescription("wikipedia xml dump file").isRequired().withLongOpt("wikipedia-dump").create("d");
			Option outputDirOpt = OptionBuilder.withArgName("dir").hasArg().withDescription("output directory in which to store output files").isRequired().withLongOpt("output-dir").create("o");
			Option numThreadOpt = OptionBuilder.withArgName("int").hasArg().withDescription("number of threads (default " + Defaults.DEFAULT_THREADS_NUMBER + ")").withLongOpt("num-threads").create("t");
			Option numPageOpt = OptionBuilder.withArgName("int").hasArg().withDescription("number of pages to process (default all)").withLongOpt("num-pages").create("p");
			Option notificationPointOpt = OptionBuilder.withArgName("int").hasArg().withDescription("receive notification every n pages (default " + Defaults.DEFAULT_NOTIFICATION_POINT + ")").withLongOpt("notification-point").create("n");

			options.addOption("h", "help", false, "print this message");
			options.addOption("v", "version", false, "output version information and exit");


			options.addOption(wikipediaDumpOpt);
			options.addOption(outputDirOpt);
			options.addOption(numThreadOpt);
			options.addOption(numPageOpt);
			options.addOption(notificationPointOpt);
			CommandLineParser parser = new PosixParser();
			CommandLine line = parser.parse(options, args);

			if (line.hasOption("help") || line.hasOption("version")) {
				throw new ParseException("");
			}

			int numThreads = Defaults.DEFAULT_THREADS_NUMBER;
			if (line.hasOption("num-threads")) {
				numThreads = Integer.parseInt(line.getOptionValue("num-threads"));
			}

			int numPages = Defaults.DEFAULT_NUM_PAGES;
			if (line.hasOption("num-pages")) {
				numPages = Integer.parseInt(line.getOptionValue("num-pages"));
			}

			int notificationPoint = Defaults.DEFAULT_NOTIFICATION_POINT;
			if (line.hasOption("notification-point")) {
				notificationPoint = Integer.parseInt(line.getOptionValue("notification-point"));
			}

			ExtractorParameters extractorParameters = new ExtractorParameters(line.getOptionValue("wikipedia-dump"), line.getOptionValue("output-dir"));
			logger.debug(extractorParameters);
			WikipediaExtractor wikipediaExtractor = new WikipediaIncomingOutgoingLinkExtractor(numThreads, numPages, extractorParameters.getLocale());
			wikipediaExtractor.setNotificationPoint(notificationPoint);
			wikipediaExtractor.start(extractorParameters);
		} catch (ParseException e) {
			// oops, something went wrong
			if (e.getMessage().length() > 0) {
				System.out.println("Parsing failed: " + e.getMessage() + "\n");
			}
			HelpFormatter formatter = new HelpFormatter();
			formatter.printHelp(200, "java -cp dist/thewikimachine.jar org.fbk.cit.hlt.thewikimachine.xmldump.WikipediaIncomingOutgoingLinkExtractor", "\n", options, "\n", true);
		} finally {
			logger.info("extraction ended " + new Date());
		}
	}
}