
com.digitalpebble.behemoth.uima.UIMAProcessor Maven / Gradle / Ivy
The newest version!
package com.digitalpebble.behemoth.uima;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.Reporter;
import org.apache.uima.UIMAFramework;
import org.apache.uima.analysis_engine.AnalysisEngine;
import org.apache.uima.cas.CAS;
import org.apache.uima.cas.FSIterator;
import org.apache.uima.cas.Feature;
import org.apache.uima.cas.Type;
import org.apache.uima.cas.impl.AnnotationImpl;
import org.apache.uima.cas.text.AnnotationFS;
import org.apache.uima.jcas.tcas.Annotation;
import org.apache.uima.pear.tools.PackageBrowser;
import org.apache.uima.pear.tools.PackageInstaller;
import org.apache.uima.resource.ResourceSpecifier;
import org.apache.uima.util.XMLInputSource;
import com.digitalpebble.behemoth.BehemothDocument;
import com.digitalpebble.behemoth.DocumentProcessor;
public class UIMAProcessor implements DocumentProcessor {
private static final Logger LOG = LoggerFactory
.getLogger(UIMAProcessor.class);
private AnalysisEngine tae;
private CAS cas;
private URL urlPEAR = null;
private Configuration config;
private boolean storeshortnames = true;
private List uimatypes = new ArrayList();
private Map> featfilts = new HashMap>();
public UIMAProcessor(URL appliPath) {
urlPEAR = appliPath;
}
public void close() {
if (cas != null)
cas.release();
if (tae != null)
tae.destroy();
}
public BehemothDocument[] process(BehemothDocument behemoth,
Reporter reporter) {
if (reporter != null)
reporter.setStatus("UIMA : " + behemoth.getUrl().toString());
// generate a CAS from the input document
cas.reset();
try {
// does the input document have a some text?
// if not - skip it
if (behemoth.getText() == null) {
LOG.debug(behemoth.getUrl().toString() + " has null text");
} else {
// detect language if specified by user
String lang = this.config.get("uima.language", "en");
cas.setDocumentLanguage(lang);
cas.setDocumentText(behemoth.getText());
// process it
tae.process(cas);
convertCASToBehemoth(cas, behemoth, reporter);
}
} catch (Exception e) {
if (reporter != null)
reporter.incrCounter("UIMA", "Exception", 1);
LOG.error(behemoth.getUrl().toString(), e);
}
if (reporter != null)
reporter.incrCounter("UIMA", "Document", 1);
// return the modified document
return new BehemothDocument[] { behemoth };
}
public Configuration getConf() {
return config;
}
public void setConf(Configuration conf) {
long initStart = System.currentTimeMillis();
config = conf;
storeshortnames = config.getBoolean("uima.store.short.names", true);
File pearFile = new File(urlPEAR.getPath());
PackageBrowser instPear = PackageInstaller.installPackage(
pearFile.getParentFile(), pearFile, true);
// get the resources required for the AnalysisEngine
org.apache.uima.resource.ResourceManager rsrcMgr = UIMAFramework
.newDefaultResourceManager();
// Create analysis engine from the installed PEAR package using
// the created PEAR specifier
XMLInputSource in;
try {
in = new XMLInputSource(instPear.getComponentPearDescPath());
ResourceSpecifier specifier = UIMAFramework.getXMLParser()
.parseResourceSpecifier(in);
tae = UIMAFramework.produceAnalysisEngine(specifier, rsrcMgr, null);
cas = tae.newCAS();
} catch (Exception e) {
throw new RuntimeException(e);
}
String[] featuresFilters = this.config.get("uima.features.filter", "")
.split(",");
// the featurefilters have the following form : Type:featureName
// we group them by annotation type
for (String ff : featuresFilters) {
String[] fp = ff.split(":");
if (fp.length != 2)
continue;
Set features = featfilts.get(fp[0]);
if (features == null) {
features = new HashSet();
featfilts.put(fp[0], features);
}
Feature f = cas.getTypeSystem().getFeatureByFullName(ff);
if (f != null)
features.add(f);
}
String[] annotTypes = this.config.get("uima.annotations.filter", "")
.split(",");
uimatypes = new ArrayList(annotTypes.length);
for (String type : annotTypes) {
Type aType = cas.getTypeSystem().getType(type);
uimatypes.add(aType);
}
long initEnd = System.currentTimeMillis();
LOG.info("Initialisation of UIMAProcessor done in "
+ (initEnd - initStart) + " msec");
}
/** convert the annotations from the CAS into the Behemoth format **/
private void convertCASToBehemoth(CAS uimadoc,
com.digitalpebble.behemoth.BehemothDocument behemoth,
Reporter reporter) {
String[] annotTypes = config.get("uima.annotations.filter", "").split(
",");
List uimatypes = new ArrayList(annotTypes.length);
for (String type : annotTypes) {
Type aType = cas.getTypeSystem().getType(type);
uimatypes.add(aType);
}
FSIterator annotIterator = cas.getAnnotationIndex()
.iterator();
while (annotIterator.hasNext()) {
AnnotationFS annotation = annotIterator.next();
if (!uimatypes.contains(annotation.getType()))
continue;
String atype = annotation.getType().toString();
// wanted type -> generate an annotation for it
if (reporter != null)
reporter.incrCounter("UIMA", atype, 1);
com.digitalpebble.behemoth.Annotation target = new com.digitalpebble.behemoth.Annotation();
// short version?
if (storeshortnames)
target.setType(annotation.getType().getShortName());
else
target.setType(atype);
target.setStart(annotation.getBegin());
target.setEnd(annotation.getEnd());
// now get the features for this annotation
Set possiblefeatures = featfilts.get(atype);
if (possiblefeatures != null) {
// iterate on the expected features
Iterator possiblefeaturesIterator = possiblefeatures
.iterator();
while (possiblefeaturesIterator.hasNext()) {
Feature expectedFeature = possiblefeaturesIterator.next();
String fvalue = annotation
.getFeatureValueAsString(expectedFeature);
if (fvalue == null)
fvalue = "null";
// always use the short names for the features
target.getFeatures().put(expectedFeature.getShortName(),
fvalue);
}
}
// add current annotation to Behemoth document
behemoth.getAnnotations().add(target);
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy