eu.fbk.twm.wiki.xmldump.WikipediaTemplateExtractor Maven / Gradle / Ivy

Go to download
/*
 * Copyright (2013) Fondazione Bruno Kessler (http://www.fbk.eu/)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package eu.fbk.twm.wiki.xmldump;

import eu.fbk.twm.utils.ExtractorParameters;
import eu.fbk.twm.utils.FreqSet;
import eu.fbk.twm.utils.WikipediaExtractor;
import eu.fbk.twm.wiki.xmldump.util.WikiTemplate;
import eu.fbk.twm.wiki.xmldump.util.WikiTemplateParser;
import org.apache.log4j.Logger;

import java.io.*;
import java.util.*;

public class WikipediaTemplateExtractor extends AbstractWikipediaExtractor implements WikipediaExtractor {
	/**
	 * Define a static logger variable so that it references the
	 * Logger instance named WikipediaTemplateExtractor.
	 */
	static Logger logger = Logger.getLogger(WikipediaTemplateExtractor.class.getName());

	private PrintWriter templateNameWriter;

	private PrintWriter templateFreqWriter;

	private PrintWriter templateMapWriter;

	private PrintWriter templateMapWriterWithRepetitions;

	private PrintWriter templateMapWriterProp;

	private FreqSet templateFreqSet;

	public WikipediaTemplateExtractor(int numThreads, int numPages, Locale locale) {
		super(numThreads, numPages, locale);
		templateFreqSet = new FreqSet();

	}

	@Override
	public void start(ExtractorParameters extractorParameters) {
		// String prefix = extractorParameters.getWikipediaTemplateFilePrefixName();
		try {

			templateNameWriter = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(extractorParameters.getWikipediaTemplateFileNames().get("name")), "UTF-8")));
			templateFreqWriter = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(extractorParameters.getWikipediaTemplateFileNames().get("freq")), "UTF-8")));
			templateMapWriter = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(extractorParameters.getWikipediaTemplateFileNames().get("map")), "UTF-8")));
			templateMapWriterWithRepetitions = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(extractorParameters.getWikipediaTemplateFileNames().get("map-rep")), "UTF-8")));
			templateMapWriterProp = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(extractorParameters.getWikipediaTemplateFileNames().get("map-prop")), "UTF-8")));

		} catch (IOException e) {
			logger.error(e);
		}
		startProcess(extractorParameters.getWikipediaXmlFileName());
	}

	@Override
	public void filePage(String text, String title, int wikiID) {
		//To change body of implemented methods use File | Settings | File Templates.
	}

	@Override
	public void disambiguationPage(String text, String title, int wikiID) {
		//To change body of implemented methods use File | Settings | File Templates.
	}

	@Override
	public void categoryPage(String text, String title, int wikiID) {
		//To change body of implemented methods use File | Settings | File Templates.
	}

	@Override
	public void redirectPage(String text, String title, int wikiID) {
		//To change body of implemented methods use File | Settings | File Templates.
	}

	@Override
	public void portalPage(String text, String title, int wikiID) {
		//To change body of implemented methods use File | Settings | File Templates.
	}

	@Override
	public void projectPage(String text, String title, int wikiID) {
		//To change body of implemented methods use File | Settings | File Templates.
	}


	@Override
	public void templatePage(String text, String title, int wikiID) {
		synchronized (this) {
			templateNameWriter.println(title);
		}
	}

	@Override
	public void contentPage(String text, String title, int wikiID) {
		ArrayList listOfTemplates = WikiTemplateParser.parse(text, false);

		Set set = new HashSet();
		Set keySet = new HashSet();

		StringBuffer toBeWrittenMap = new StringBuffer();
		StringBuffer toBeWrittenMapRep = new StringBuffer();
		StringBuffer toBeWrittenMapProp = new StringBuffer();

		int i = 0;
		for (WikiTemplate t : listOfTemplates) {
			HashMap parts = t.getHashMapOfParts();
			Set keys = parts.keySet();
			String name = t.getFirstPart();
			if (name == null || name.length() == 0) {
				continue;
			}
			if (name.startsWith("#")) {
				continue;
			}
			name = normalizePageName(name.trim()).replace(' ', '_');
			String toBeWritten;

			if (!set.contains(name)) {
				toBeWritten = title + "\t" + name + "\t" + i + "\t" + wikiID;
				toBeWrittenMap.append(toBeWritten).append("\n");
				synchronized (this) {
					templateFreqSet.add(name);
				}
				set.add(name);
				i++;
			}
			toBeWritten = title + "\t" + name + "\t" + t.getPartsCount() + "\t" + t.getNlCount() + "\t" + t.getKeyValueParts();
			toBeWrittenMapRep.append(toBeWritten).append("\n");

			for (Object key : keys) {
				String keyName = (String) key;
				String keyNameToSave = name + ";" + keyName;
				if (!keySet.contains(keyNameToSave)) {
					toBeWritten = title + "\t" + name + "\t" + keyName;
					toBeWrittenMapProp.append(toBeWritten).append("\n");
					keySet.add(keyNameToSave);
				}
			}
		}

		synchronized (this) {
			templateMapWriter.print(toBeWrittenMap);
			templateMapWriterWithRepetitions.print(toBeWrittenMapRep);
			templateMapWriterProp.print(toBeWrittenMapProp);
		}
	}

	@Override
	public void endProcess() {
		super.endProcess();
		SortedMap> smap = templateFreqSet.toSortedMap();
		int count = 0;
		for (Integer freq : smap.keySet()) {
			List list = smap.get(freq);
			for (String aList : list) {
				templateFreqWriter.println(freq + "\t" + aList);
				count += freq;
			}
		}
		logger.info(count + " pages with at least one template");

		templateFreqWriter.println(count + " pages with at least one template");
		templateNameWriter.flush();
		templateNameWriter.close();
		templateFreqWriter.flush();
		templateFreqWriter.close();
		templateMapWriter.flush();
		templateMapWriter.close();
		templateMapWriterWithRepetitions.flush();
		templateMapWriterWithRepetitions.close();
		templateMapWriterProp.flush();
		templateMapWriterProp.close();

	}

	/*public static void main(String argv[]) throws IOException
	{
		String logConfig = System.getProperty("log-config");
		if (logConfig == null) logConfig = "configuration/log-config.txt";

		PropertyConfigurator.configure(logConfig);

		if (argv.length < 2) {
			System.out.println("");
			System.out.println("USAGE:");
			System.out.println("");
			System.out.println("java -mx6G org.fbk.cit.hlt.thewikimachine.xmldump.WikipediaTemplateExtractor\n" +
							" in-wiki-xml -- Input filePageCounter\n" +
							" root-templatePageCounter-files -- Output templatePageCounter filePageCounter\n" +
							" locale -- Locale (used for resources)\n" +
							" [threads=1] -- Number of threads to use\n" +
							" [countPageCounter=0] -- Lines to stop, 0 means never stop");
			System.out.println("");
			System.exit(1);
		}

		int parID = 0;

		String xin = argv[parID++];
		String xout = argv[parID++];
		Locale locale = new Locale(argv[parID++]);

		int threads = 1;
		if (argv.length > parID) {
			threads = Integer.parseInt(argv[parID++]);
		}

		int size = 0;
		if (argv.length > parID) {
			size = Integer.parseInt(argv[parID++]);
		}

		WikipediaTemplateExtractor writer = new WikipediaTemplateExtractor(threads, size, locale);
		writer.start(xin, xout);
	}*/

}