eu.fbk.twm.wiki.xmldump.WikipediaSectionTitleExtractor Maven / Gradle / Ivy

Go to download
/*
 * Copyright (2013) Fondazione Bruno Kessler (http://www.fbk.eu/)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package eu.fbk.twm.wiki.xmldump;

import de.tudarmstadt.ukp.wikipedia.parser.ParsedPage;
import de.tudarmstadt.ukp.wikipedia.parser.Section;
import eu.fbk.twm.utils.CharacterTable;
import eu.fbk.twm.utils.ExtractorParameters;
import eu.fbk.twm.utils.WikipediaExtractor;
import eu.fbk.twm.wiki.xmldump.util.WikiMarkupParser;
import org.apache.log4j.Logger;

import java.io.*;
import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class WikipediaSectionTitleExtractor extends AbstractWikipediaExtractor implements WikipediaExtractor {
	/**
	 * Define a static logger variable so that it references the
	 * Logger instance named WikipediaSectionTitleExtractor.
	 */
	static Logger logger = Logger.getLogger(WikipediaSectionTitleExtractor.class.getName());

	private PrintWriter sectionTitleWriter;
	private Pattern sectionTitleSkipPattern;

	public WikipediaSectionTitleExtractor(int numThreads, int numPages, Locale locale) {
		super(numThreads, numPages, locale);
	}


	@Override
	public void start(ExtractorParameters extractorParameters) {
		try {
			sectionTitleWriter = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(extractorParameters.getWikipediaSectionTitleFilePrefixName()), "UTF-8")));
		} catch (IOException e) {
			logger.error(e);
		}

		if (resources.getString("SECTION_TITLE_SKIP_PATTERN") != null) {
			sectionTitleSkipPattern = Pattern.compile(resources.getString("SECTION_TITLE_SKIP_PATTERN"), Pattern.CASE_INSENSITIVE);
		}

		startProcess(extractorParameters.getWikipediaXmlFileName());
	}

	@Override
	public void filePage(String text, String title, int wikiID) {
		//To change body of implemented methods use File | Settings | File Templates.
	}

	@Override
	public void categoryPage(String text, String title, int wikiID) {
	}

	@Override
	public void contentPage(String text, String title, int wikiID) {
		try {
			String s = sectionTitle(text, title);
			synchronized (this) {

				sectionTitleWriter.print(s);
			}

		} catch (IOException e) {
			logger.error(e);
		}
	}

	/**
	 * Returns the section titles. From this...
	 */
	public String sectionTitle(String text, String title) throws IOException {
		logger.debug(title);
		StringBuilder sb = new StringBuilder();
		WikiMarkupParser wikiMarkupParser = WikiMarkupParser.getInstance();
		ParsedPage parsedPage = wikiMarkupParser.parsePage(text);
		String sectionTitle;
		for (Section section : parsedPage.getSections()) {
			sectionTitle = section.getTitle();
			if (sectionTitle != null && sectionTitle.trim().length() > 0) {
				if (sectionTitleSkipPattern != null) {
					Matcher matcher = sectionTitleSkipPattern.matcher(sectionTitle);
					if (matcher.find()) {
						continue;
					}
				}

				sb.append(title);
				sb.append(CharacterTable.HORIZONTAL_TABULATION);
				sb.append(sectionTitle);
				sb.append(CharacterTable.LINE_FEED);
			}
		}
		return sb.toString();
	}

	@Override
	public void disambiguationPage(String text, String title, int wikiID) {
		//To change body of implemented methods use File | Settings | File Templates.
	}

	@Override
	public void templatePage(String text, String title, int wikiID) {
		//To change body of implemented methods use File | Settings | File Templates.
	}

	@Override
	public void redirectPage(String text, String title, int wikiID) {
		//To change body of implemented methods use File | Settings | File Templates.
	}

	@Override
	public void portalPage(String text, String title, int wikiID) {
		//To change body of implemented methods use File | Settings | File Templates.
	}

	@Override
	public void projectPage(String text, String title, int wikiID) {
		//To change body of implemented methods use File | Settings | File Templates.
	}

	@Override
	public void endProcess() {
		super.endProcess();
		sectionTitleWriter.flush();
		sectionTitleWriter.close();
	}

	/*public static void main(String argv[]) throws IOException
	{
		String logConfig = System.getProperty("log-config");
		if (logConfig == null) logConfig = "configuration/log-config.txt";

		PropertyConfigurator.configure(logConfig);

		if (argv.length < 7) {
			System.out.println("");
			System.out.println("USAGE:");
			System.out.println("");
			System.out.println("java -mx6G org.fbk.cit.hlt.thewikimachine.xmldump.WikipediaSectionTitleExtractor\n" +
							" in-wiki-xml -- Input XML file\n" +
							" locale -- Locale (used for resources)\n" +
							" [threads=1] -- Number of threads to use\n" +
							" [count=0] -- Lines to stop, 0 means never stop");
			System.out.println("");
			System.exit(1);
		}

		int parID = 0;

		String xin = argv[parID++];
		String xout = argv[parID++];
		Locale locale = new Locale(argv[parID++]);

		int threads = 1;
		if (argv.length > parID) {
			threads = Integer.parseInt(argv[parID++]);
		}

		int size = 0;
		if (argv.length > parID) {
			size = Integer.parseInt(argv[parID++]);
		}

		WikipediaSectionTitleExtractor sectionTitleWriter = new WikipediaSectionTitleExtractor(threads, size, locale);
		sectionTitleWriter.start(xin, xout);
	} */

}