eu.fbk.twm.wiki.xmldump.DBpediaMappingsPropertiesExtractor Maven / Gradle / Ivy

Go to download
/*
 * Copyright (2013) Fondazione Bruno Kessler (http://www.fbk.eu/)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package eu.fbk.twm.wiki.xmldump;

import eu.fbk.twm.utils.ExtractorParameters;
import eu.fbk.twm.utils.WikipediaExtractor;
import eu.fbk.twm.wiki.xmldump.util.WikiTemplate;
import eu.fbk.twm.wiki.xmldump.util.WikiTemplateParser;
import org.apache.log4j.Logger;
import org.apache.log4j.PropertyConfigurator;

import java.io.*;
import java.net.URLDecoder;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class DBpediaMappingsPropertiesExtractor extends AbstractWikipediaExtractor implements WikipediaExtractor {
	/**
	 * Define a static logger variable so that it references the
	 * Logger instance named DBpediaMappingsExtractor.
	 */
	static Logger logger = Logger.getLogger(DBpediaMappingsPropertiesExtractor.class.getName());
	private boolean write;
	private HashMap ret = new HashMap();
	private HashSet mappedTemplates = new HashSet();
	private HashSet mappedTemplatesCaseSensitive = new HashSet();

	public HashSet getMappedTemplatesCaseSensitive() {
		return mappedTemplatesCaseSensitive;
	}

	public HashSet getMappedTemplates() {
		return mappedTemplates;
	}

	public HashMap getRet() {
		return ret;
	}

	private PrintWriter dbpediaWriter;

	public DBpediaMappingsPropertiesExtractor(Locale locale, int numThreads, int numPages) {
		super(numThreads, numPages, locale);
	}

	public void startWithoutWriting(String in) {
		write = false;
		ret = new HashMap();
		mappedTemplates = new HashSet();
		mappedTemplatesCaseSensitive = new HashSet();
		startProcess(in);
	}

	public void start(String in, String out) {
		//todo: rename

		write = true;
		try {
			dbpediaWriter = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(out), "UTF-8")));
		} catch (IOException e) {
			logger.error(e);
		}
		startProcess(in);
	}

	@Override
	public void contentPage(String text, String title, int wikiID) {

		Pattern pt = Pattern.compile(":(.*)");
		Matcher mt = pt.matcher(title);

		if (!mt.find()) {
			// System.out.println("Pattern not found");
			return;
		}
		title = mt.group(1);
		try {
			title = URLDecoder.decode(title, "UTF-8");
		} catch (Exception e) {
			return;
		}

		ArrayList listOfTemplates = WikiTemplateParser.parse(text, false);
		for (WikiTemplate t : listOfTemplates) {
			if (t.getFirstPart().equals("PropertyMapping")) {
				HashMap parts = t.getHashMapOfParts();
				if (parts.get("templateProperty") != null && parts.get("ontologyProperty") != null) {
					String part1 = title + "\t" + parts.get("templateProperty");
					String part2 = parts.get("ontologyProperty");

					ret.put(part1.toLowerCase(), part2);
					mappedTemplates.add(title.toLowerCase());
					mappedTemplatesCaseSensitive.add(title);

					if (write) {
						dbpediaWriter.append(part1).append("\t").append(part2).append("\n");
					}
				}
//				Pattern p = Pattern.compile("mapToClass\\s*=\\s*([0-9a-zA-Z]+)", Pattern.MULTILINE);
//				Matcher m = p.matcher(t.getContent());
//				if (m.find()) {
//					synchronized (this) {
//						logger.info(title + " ---> " + m.group(1));
//						dbpediaWriter.append(title).append("\t").append(m.group(1)).append("\n");
//					}
//				}
			}
		}
	}

	@Override
	public void filePage(String text, String title, int wikiID) {
		//To change body of implemented methods use File | Settings | File Templates.
	}

	@Override
	public void start(ExtractorParameters extractorParameters) {
		//To change body of implemented methods use File | Settings | File Templates.
	}

	@Override
	public void disambiguationPage(String text, String title, int wikiID) {
		//To change body of implemented methods use File | Settings | File Templates.
	}

	@Override
	public void categoryPage(String text, String title, int wikiID) {
		//To change body of implemented methods use File | Settings | File Templates.
	}

	@Override
	public void templatePage(String text, String title, int wikiID) {
		//To change body of implemented methods use File | Settings | File Templates.
	}

	@Override
	public void redirectPage(String text, String title, int wikiID) {
		//To change body of implemented methods use File | Settings | File Templates.
	}

	@Override
	public void portalPage(String text, String title, int wikiID) {
		//To change body of implemented methods use File | Settings | File Templates.
	}

	@Override
	public void projectPage(String text, String title, int wikiID) {
		//To change body of implemented methods use File | Settings | File Templates.
	}

	public void endProcess() {
		if (write) {
			dbpediaWriter.flush();
			dbpediaWriter.close();
		}
	}

	public static void main(String argv[]) throws IOException {
		String logConfig = System.getProperty("log-config");
		if (logConfig == null) {
			logConfig = "configuration/log-config.txt";
		}

		PropertyConfigurator.configure(logConfig);

		if (argv.length < 3) {
			System.out.println("");
			System.out.println("USAGE:");
			System.out.println("");
			System.out.println("java -mx6G org.fbk.cit.hlt.thewikimachine.xmldump.DBpediaMappingsExtractor\n" +
					" in-wiki-xml -- Input file\n" +
					" out-file -- Output file\n" +
					" locale -- Locale (used for resources)\n" +
					" [numThreads=1] -- Number of numThreads to use\n" +
					" [count=0] -- Lines to stop, 0 means never stop");
			System.out.println("");
			System.exit(1);
		}

		int parID = 0;

		String xin = argv[parID++];
		String xout = argv[parID++];
		Locale locale = new Locale(argv[parID++]);

		int numThreads = 1;
		if (argv.length > parID) {
			numThreads = Integer.parseInt(argv[parID++]);
		}

		int size = Integer.MAX_VALUE;
		if (argv.length > parID) {
			size = Integer.parseInt(argv[parID++]);
		}

		DBpediaMappingsPropertiesExtractor extractor = new DBpediaMappingsPropertiesExtractor(locale, numThreads, size);
		extractor.start(xin, xout);
	}

}