eu.fbk.twm.wiki.xmldump.WikipediaEncodingChecker Maven / Gradle / Ivy

Go to download
package eu.fbk.twm.wiki.xmldump;

import org.apache.commons.cli.*;
import org.apache.log4j.Logger;
import org.apache.log4j.PropertyConfigurator;

import java.io.*;
import java.util.Date;

/**
 * Created with IntelliJ IDEA.
 * User: giuliano
 * Date: 9/17/13
 * Time: 6:03 PM
 * To change this template use File | Settings | File Templates.
 */
public class WikipediaEncodingChecker {
	/**
	 * Define a static logger variable so that it references the
	 * Logger instance named WikipediaEncodingChecker.
	 */
	static Logger logger = Logger.getLogger(WikipediaEncodingChecker.class.getName());

	public static void main(String args[]) throws IOException {
		String logConfig = System.getProperty("log-config");
		if (logConfig == null) {
			logConfig = "configuration/log-config.txt";
		}

		PropertyConfigurator.configure(logConfig);

		Options options = new Options();
		try {
			Option wikipediaDumpOpt = OptionBuilder.withArgName("file").hasArg().withDescription("wikipedia xml dump file").isRequired().withLongOpt("wikipedia-dump").create("d");

			options.addOption("h", "help", false, "print this message");
			options.addOption("v", "version", false, "output version information and exit");


			options.addOption(wikipediaDumpOpt);

			CommandLineParser parser = new PosixParser();
			CommandLine line = parser.parse(options, args);

			String name = line.getOptionValue("wikipedia-dump");
			LineNumberReader reader = new LineNumberReader(new InputStreamReader(new FileInputStream(name), "UTF-8"));
			PrintWriter writer = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream("/data/corpora/wikipedia/enwiki-20161200-pages-articles.xml"), "UTF-8")));
			String l = null;
			int count = 0;
			boolean b = true;
			while (b) {
				try {
					l = reader.readLine();
				} catch (Exception e) {
					logger.error(count);
					logger.error(e);
				}

				if (l == null) {
					break;
				} else {
					writer.println(l);
				}
			  count++;

				if ((count % 100000) == 0) {
					logger.info(count + " " + new Date());
				}
			}

			logger.info("extraction ended " + new Date());

		} catch (ParseException e) {
			// oops, something went wrong
			System.out.println("Parsing failed: " + e.getMessage() + "\n");
			HelpFormatter formatter = new HelpFormatter();
			formatter.printHelp(400, "java -cp dist/thewikimachine.jar org.fbk.cit.hlt.thewikimachine.xmldump.WikipediaEncodingChecker", "\n", options, "\n", true);
		}
	}
}