eu.fbk.twm.wiki.xmldump.WikipediaOutgoingLinkExtractor Maven / Gradle / Ivy
/*
* Copyright (2013) Fondazione Bruno Kessler (http://www.fbk.eu/)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package eu.fbk.twm.wiki.xmldump;
import de.tudarmstadt.ukp.wikipedia.parser.Link;
import de.tudarmstadt.ukp.wikipedia.parser.ParsedPage;
import de.tudarmstadt.ukp.wikipedia.parser.Section;
import eu.fbk.twm.utils.*;
import eu.fbk.twm.wiki.xmldump.util.WikiMarkupParser;
import org.apache.commons.cli.*;
import org.apache.commons.cli.OptionBuilder;
import org.apache.log4j.Logger;
import org.apache.log4j.PropertyConfigurator;
import java.io.*;
import java.util.Date;
import java.util.List;
import java.util.Locale;
import java.util.regex.Matcher;
public class WikipediaOutgoingLinkExtractor extends AbstractWikipediaExtractor implements WikipediaExtractor {
/**
* Define a static logger variable so that it references the
* Logger instance named WikipediaOutgoingLinkExtractor.
*/
static Logger logger = Logger.getLogger(WikipediaOutgoingLinkExtractor.class.getName());
private PrintWriter outgoingWriter;
private PageMap redirectPagesMap;
private PageSet disambiguationPagesSet;
private PageMap titleIdMap;
public final static String NUM_SIGN = "#";
public WikipediaOutgoingLinkExtractor(int numThreads, int numPages, Locale locale) throws IOException {
super(numThreads, numPages, locale);
}
@Override
public void start(ExtractorParameters extractorParameters) {
try {
redirectPagesMap = new PageMap(new File(extractorParameters.getWikipediaRedirFileName()));
logger.info(redirectPagesMap.size() + " redirect pages");
disambiguationPagesSet = new PageSet(new File(extractorParameters.getWikipediaDisambiguationFileName()));
logger.info(disambiguationPagesSet.size() + " disambiguation pages");
titleIdMap = new PageMap(new File(extractorParameters.getWikipediaTitleIdFileName()));
logger.info(titleIdMap.size() + " title id pairs");
outgoingWriter = new PrintWriter(new BufferedWriter(new OutputStreamWriter(new FileOutputStream(extractorParameters.getWikipediaOutgoingFileName()), "UTF-8")));
} catch (IOException e) {
logger.error(e);
}
startProcess(extractorParameters.getWikipediaXmlFileName());
}
@Override
public void filePage(String text, String title, int wikiID) {
//To change body of implemented methods use File | Settings | File Templates.
}
@Override
public void categoryPage(String text, String title, int wikiID) {
//todo: verify here
contentPage(text, title, wikiID);
}
@Override
public void contentPage(String text, String title, int wikiID) {
try {
StringBuilder sb = new StringBuilder();
sb.append(outgoingInternalLinks(text, title));
sb.append(categoryLinks(text, title));
//logger.debug(title + "\t" + sb.toString());
if (sb.length() > 0) {
synchronized (this) {
outgoingWriter.print(sb.toString());
}
}
} catch (IOException e) {
logger.error(e);
}
}
private String categoryLinks(String text, String title) {
Matcher m = categoryPattern.matcher(text);
StringBuilder buff = new StringBuilder();
String category;
int index = 1;
String wikiID;
while (m.find()) {
int s = m.start(index);
int e = m.end(index);
category = text.substring(s, e);
int j = category.indexOf(CharacterTable.VERTICAL_LINE);
if (j != -1) {
category = category.substring(0, j);
}
category = category.replace(CharacterTable.SPACE, CharacterTable.LOW_LINE);
wikiID = titleIdMap.get(category);
if (wikiID != null) {
buff.append(title);
buff.append(CharacterTable.HORIZONTAL_TABULATION);
buff.append(category);
buff.append(CharacterTable.HORIZONTAL_TABULATION);
buff.append(wikiID);
buff.append(CharacterTable.LINE_FEED);
}
}
return buff.toString();
}
/**
* Returns the outgoing links. From this...
* links contained within templates are missing?
*/
private String outgoingInternalLinks(String text, String title) throws IOException {
StringBuilder sb = new StringBuilder();
String wikiID = titleIdMap.get(title);
if (wikiID == null) {
return sb.toString();
}
// this trick allows pages interlinked to share interlinks
sb.append(title);
sb.append(CharacterTable.HORIZONTAL_TABULATION);
sb.append(title);
sb.append(CharacterTable.HORIZONTAL_TABULATION);
sb.append(wikiID);
sb.append(CharacterTable.LINE_FEED);
WikiMarkupParser wikiMarkupParser = WikiMarkupParser.getInstance();
ParsedPage pp = wikiMarkupParser.parsePage(text);
String target;
String redirect;
for (Section section : pp.getSections()) {
List internalLinks = section.getLinks(Link.type.INTERNAL);
for (Link link : internalLinks) {
target = normalizePageName(link.getTarget());
redirect = redirectPagesMap.get(target);
if (redirect != null) {
target = redirect;
}
wikiID = titleIdMap.get(target);
if (wikiID == null) {
}
else if (target.startsWith(NUM_SIGN)) {
}
else if (disambiguationPagesSet.contains(target)) {
}
else {
// outgoing
sb.append(title);
sb.append(CharacterTable.HORIZONTAL_TABULATION);
sb.append(target);
sb.append(CharacterTable.HORIZONTAL_TABULATION);
sb.append(wikiID);
sb.append(CharacterTable.LINE_FEED);
}
}
}
return sb.toString();
}
@Override
public void disambiguationPage(String text, String title, int wikiID) {
//To change body of implemented methods use File | Settings | File Templates.
}
@Override
public void templatePage(String text, String title, int wikiID) {
//To change body of implemented methods use File | Settings | File Templates.
}
@Override
public void redirectPage(String text, String title, int wikiID) {
//To change body of implemented methods use File | Settings | File Templates.
}
@Override
public void portalPage(String text, String title, int wikiID) {
//To change body of implemented methods use File | Settings | File Templates.
}
@Override
public void projectPage(String text, String title, int wikiID) {
//To change body of implemented methods use File | Settings | File Templates.
}
@Override
public void endProcess() {
super.endProcess();
outgoingWriter.flush();
outgoingWriter.close();
}
public static void main(String args[]) throws IOException {
String logConfig = System.getProperty("log-config");
if (logConfig == null) {
logConfig = "configuration/log-config.txt";
}
PropertyConfigurator.configure(logConfig);
Options options = new Options();
try {
Option wikipediaDumpOpt = OptionBuilder.withArgName("file").hasArg().withDescription("wikipedia xml dump file").isRequired().withLongOpt("wikipedia-dump").create("d");
Option outputDirOpt = OptionBuilder.withArgName("dir").hasArg().withDescription("output directory in which to store output files").isRequired().withLongOpt("output-dir").create("o");
Option numThreadOpt = OptionBuilder.withArgName("int").hasArg().withDescription("number of threads (default " + Defaults.DEFAULT_THREADS_NUMBER
+ ")").withLongOpt("num-threads").create("t");
Option numPageOpt = OptionBuilder.withArgName("int").hasArg().withDescription("number of pages to process (default all)").withLongOpt("num-pages").create("p");
Option notificationPointOpt = OptionBuilder.withArgName("int").hasArg().withDescription("receive notification every n pages (default " + Defaults.DEFAULT_NOTIFICATION_POINT + ")").withLongOpt("notification-point").create("n");
options.addOption("h", "help", false, "print this message");
options.addOption("v", "version", false, "output version information and exit");
options.addOption(wikipediaDumpOpt);
options.addOption(outputDirOpt);
options.addOption(numThreadOpt);
options.addOption(numPageOpt);
options.addOption(notificationPointOpt);
CommandLineParser parser = new PosixParser();
CommandLine line = parser.parse(options, args);
if (line.hasOption("help") || line.hasOption("version")) {
throw new ParseException("");
}
int numThreads = Defaults.DEFAULT_THREADS_NUMBER;
if (line.hasOption("num-threads")) {
numThreads = Integer.parseInt(line.getOptionValue("num-threads"));
}
int numPages = Defaults.DEFAULT_NUM_PAGES;
if (line.hasOption("num-pages")) {
numPages = Integer.parseInt(line.getOptionValue("num-pages"));
}
int notificationPoint = Defaults.DEFAULT_NOTIFICATION_POINT;
if (line.hasOption("notification-point")) {
notificationPoint = Integer.parseInt(line.getOptionValue("notification-point"));
}
ExtractorParameters extractorParameters = new ExtractorParameters(line.getOptionValue("wikipedia-dump"), line.getOptionValue("output-dir"));
logger.debug(extractorParameters);
WikipediaExtractor wikipediaExtractor = new WikipediaOutgoingLinkExtractor(numThreads, numPages, extractorParameters.getLocale());
wikipediaExtractor.setNotificationPoint(notificationPoint);
wikipediaExtractor.start(extractorParameters);
} catch (ParseException e) {
// oops, something went wrong
if (e.getMessage().length() > 0) {
System.out.println("Parsing failed: " + e.getMessage() + "\n");
}
HelpFormatter formatter = new HelpFormatter();
formatter.printHelp(200, "java -cp dist/thewikimachine.jar org.fbk.cit.hlt.thewikimachine.xmldump.WikipediaOutgoingLinkExtractor", "\n", options, "\n", true);
} finally {
logger.info("extraction ended " + new Date());
}
}
}© 2015 - 2025 Weber Informatics LLC | Privacy Policy