edu.stanford.nlp.patterns.GetPatternsFromDataMultiClass Maven / Gradle / Ivy
Show all versions of stanford-corenlp Show documentation
package edu.stanford.nlp.patterns;
import java.io.*;
import java.lang.reflect.Constructor;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.sql.SQLException;
import java.text.DecimalFormat;
import java.util.*;
import java.util.Map.Entry;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Function;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import javax.json.*;
import edu.stanford.nlp.io.IOUtils;
import edu.stanford.nlp.io.RegExFileFilter;
import edu.stanford.nlp.ling.CoreAnnotations;
import edu.stanford.nlp.ling.CoreLabel;
import edu.stanford.nlp.ling.CoreAnnotations.GoldAnswerAnnotation;
import edu.stanford.nlp.ling.IndexedWord;
import edu.stanford.nlp.ling.tokensregex.Env;
import edu.stanford.nlp.ling.tokensregex.TokenSequencePattern;
import edu.stanford.nlp.patterns.dep.DataInstanceDep;
import edu.stanford.nlp.patterns.surface.*;
import edu.stanford.nlp.patterns.ConstantsAndVariables.ScorePhraseMeasures;
import edu.stanford.nlp.pipeline.Annotation;
import edu.stanford.nlp.pipeline.StanfordCoreNLP;
import edu.stanford.nlp.semgraph.SemanticGraph;
import edu.stanford.nlp.semgraph.SemanticGraphEdge;
import edu.stanford.nlp.sequences.IOBUtils;
import edu.stanford.nlp.stats.ClassicCounter;
import edu.stanford.nlp.stats.Counter;
import edu.stanford.nlp.stats.Counters;
import edu.stanford.nlp.stats.TwoDimensionalCounter;
import edu.stanford.nlp.trees.GrammaticalRelation;
import edu.stanford.nlp.trees.Tree;
import edu.stanford.nlp.trees.TreeCoreAnnotations.TreeAnnotation;
import edu.stanford.nlp.util.*;
import edu.stanford.nlp.util.PriorityQueue;
import edu.stanford.nlp.util.TypesafeMap.Key;
import edu.stanford.nlp.util.logging.Redwood;
import org.joda.time.Interval;
import org.joda.time.Period;
/**
* Given text and a seed list, this class gives more words like the seed words
* by learning surface word or dependency patterns.
*
*
* The multi-threaded class ({@code nthread} parameter for number of
* threads) takes as input.
*
* To use the default options, run
*
* {@code java -mx1000m edu.stanford.nlp.patterns.GetPatternsFromDataMultiClass -file text_file -seedWordsFiles label1,seedwordlist1;label2,seedwordlist2;... -outDir output_directory (optional)}
*
*
* {@code fileFormat}: (Optional) Default is text. Valid values are text
* (or txt) and ser, where the serialized file is of the type {@code Map>}.
*
* {@code file}: (Required) Input file(s) (default assumed text). Can be
* one or more of (concatenated by comma or semi-colon): file, directory, files
* with regex in the filename (for example: "mydir/health-.*-processed.txt")
*
* {@code seedWordsFiles}: (Required)
* label1,file_seed_words1;label2,file_seed_words2;... where file_seed_words are
* files with list of seed words, one in each line
*
* {@code outDir}: (Optional) output directory where visualization/output
* files are stored
*
* For other flags, see individual comments for each flag.
*
*
* To use a properties file, see
* projects/core/data/edu/stanford/nlp/patterns/surface/example.properties or patterns/example.properties (depends on which codebase you are using)
* as an example for the flags and their brief descriptions. Run the code as:
* {@code java -mx1000m -cp classpath edu.stanford.nlp.patterns.GetPatternsFromDataMultiClass -props dir-as-above/example.properties}
*
*
* IMPORTANT: Many flags are described in the classes
* {@link ConstantsAndVariables}, {@link edu.stanford.nlp.patterns.surface.CreatePatterns}, and
* {@link PhraseScorer}.
*
* @author Sonal Gupta ([email protected])
*/
public class GetPatternsFromDataMultiClass implements Serializable {
/** A logger for this class */
private static Redwood.RedwoodChannels log = Redwood.channels(GetPatternsFromDataMultiClass.class);
private static final long serialVersionUID = 1L;
//public Map>> patternsForEachToken = null;
private PatternsForEachToken patsForEachToken = null;
public Map> wordsForOtherClass = null;
// String channelNameLogger = "patterns";
/**
*
* RlogF is from Riloff 1996, when R's denominator is (pos+neg+unlabeled)
*
* RlogFPosNeg is when the R's denominator is just (pos+negative) examples
*
* PosNegOdds is just the ratio of number of positive words to number of
* negative
*
* PosNegUnlabOdds is just the ratio of number of positive words to number of
* negative (unlabeled words + negative)
*
* RatioAll is pos/(neg+pos+unlabeled)
*
* YanGarber02 is the modified version presented in
* "Unsupervised Learning of Generalized Names"
*
* LOGREG is learning a logistic regression classifier to combine weights to
* score a phrase (Same as PhEvalInPat, except score of an unlabeled phrase is
* computed using a logistic regression classifier)
*
* LOGREGlogP is learning a logistic regression classifier to combine weights
* to score a phrase (Same as PhEvalInPatLogP, except score of an unlabeled
* phrase is computed using a logistic regression classifier)
*
* SqrtAllRatio is the pattern scoring used in Gupta et al. JAMIA 2014 paper
*
* Below F1SeedPattern and BPB based on paper
* "Unsupervised Method for Automatics Construction of a disease dictionary..."
*
* Precision, Recall, and FMeasure (controlled by fbeta flag) is ranking the patterns using
* their precision, recall and F_beta measure
*/
public enum PatternScoring {
F1SeedPattern, RlogF, RlogFPosNeg, RlogFUnlabNeg, RlogFNeg, PhEvalInPat, PhEvalInPatLogP, PosNegOdds,
YanGarber02, PosNegUnlabOdds, RatioAll, LOGREG, LOGREGlogP, SqrtAllRatio, LinICML03, kNN
}
enum WordScoring {
BPB, WEIGHTEDNORM
}
private Map writtenPatInJustification = new HashMap<>();
private Map> learnedPatterns = new HashMap<>();
//Same as learnedPatterns but with iteration information
private Map>> learnedPatternsEachIter = new HashMap<>();
Map> matchedSeedWords = new HashMap<>();
public Map> wordsPatExtracted = new HashMap<>();
Properties props;
public ScorePhrases scorePhrases;
public ConstantsAndVariables constVars;
public CreatePatterns createPats;
private final DecimalFormat df = new DecimalFormat("#.##");
private boolean notComputedAllPatternsYet = true;
/*
* when there is only one label
*/
public GetPatternsFromDataMultiClass(Properties props, Map sents, Set seedSet, boolean labelUsingSeedSets,
String answerLabel) throws IOException, InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException,
NoSuchMethodException, SecurityException, InterruptedException, ExecutionException, ClassNotFoundException {
this(props, sents, seedSet, labelUsingSeedSets, PatternsAnnotations.PatternLabel1.class, answerLabel);
}
@SuppressWarnings("rawtypes")
public GetPatternsFromDataMultiClass(Properties props, Map sents, Set seedSet, boolean labelUsingSeedSets,
Class answerClass, String answerLabel) throws IOException, InstantiationException, IllegalAccessException, IllegalArgumentException,
InvocationTargetException, NoSuchMethodException, SecurityException, InterruptedException, ExecutionException, ClassNotFoundException {
this.props = props;
Map>> ansCl = new HashMap<>();
ansCl.put(answerLabel, answerClass);
Map generalizeClasses = new HashMap<>();
Map> ignoreClasses = new HashMap<>();
ignoreClasses.put(answerLabel, new HashMap<>());
Map> seedSets = new HashMap<>();
seedSets.put(answerLabel, seedSet);
setUpConstructor(sents, seedSets, labelUsingSeedSets, ansCl, generalizeClasses, ignoreClasses);
}
@SuppressWarnings("rawtypes")
public GetPatternsFromDataMultiClass(Properties props, Map sents, Set seedSet, boolean labelUsingSeedSets,
String answerLabel, Map generalizeClasses, Map ignoreClasses) throws IOException, InstantiationException,
IllegalAccessException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException, InterruptedException,
ExecutionException, ClassNotFoundException {
this(props, sents, seedSet, labelUsingSeedSets, PatternsAnnotations.PatternLabel1.class, answerLabel, generalizeClasses, ignoreClasses);
}
@SuppressWarnings("rawtypes")
public GetPatternsFromDataMultiClass(Properties props, Map sents, Set seedSet, boolean labelUsingSeedSets,
Class answerClass, String answerLabel, Map generalizeClasses, Map ignoreClasses) throws IOException,
InstantiationException, IllegalAccessException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException,
InterruptedException, ExecutionException, ClassNotFoundException {
this.props = props;
Map>> ansCl = new HashMap<>();
ansCl.put(answerLabel, answerClass);
Map> iC = new HashMap<>();
iC.put(answerLabel, ignoreClasses);
Map> seedSets = new HashMap<>();
seedSets.put(answerLabel, seedSet);
setUpConstructor(sents, seedSets, labelUsingSeedSets, ansCl, generalizeClasses, iC);
}
@SuppressWarnings("rawtypes")
public GetPatternsFromDataMultiClass(Properties props, Map sents, Map> seedSets,
boolean labelUsingSeedSets) throws IOException, InstantiationException, IllegalAccessException, IllegalArgumentException,
InvocationTargetException, NoSuchMethodException, SecurityException, ClassNotFoundException, InterruptedException, ExecutionException {
this.props = props;
Map>> ansCl = new HashMap<>();
Map gC = new HashMap<>();
Map> iC = new HashMap<>();
int i = 1;
for (String label : seedSets.keySet()) {
String ansclstr = "edu.stanford.nlp.patterns.PatternsAnnotations$PatternLabel" + i;
ansCl.put(label, (Class>) Class.forName(ansclstr));
iC.put(label, new HashMap<>());
i++;
}
setUpConstructor(sents, seedSets, labelUsingSeedSets, ansCl, gC, iC);
}
@SuppressWarnings("rawtypes")
public GetPatternsFromDataMultiClass(Properties props, Map sents, Map> seedSets,
boolean labelUsingSeedSets, Map>> answerClass) throws IOException, InstantiationException,
IllegalAccessException, IllegalArgumentException, InvocationTargetException, NoSuchMethodException, SecurityException, InterruptedException,
ExecutionException, ClassNotFoundException {
this(props, sents, seedSets, labelUsingSeedSets, answerClass, new HashMap<>(), new HashMap<>());
}
/**
* Generalize classes basically maps label strings to a map of generalized
* strings and the corresponding class ignoreClasses have to be boolean.
*
* @throws IOException
* @throws SecurityException
* @throws NoSuchMethodException
* @throws InvocationTargetException
* @throws IllegalArgumentException
* @throws IllegalAccessException
* @throws InstantiationException
* @throws ExecutionException
* @throws InterruptedException
* @throws ClassNotFoundException
*/
@SuppressWarnings("rawtypes")
public GetPatternsFromDataMultiClass(Properties props, Map sents, Map> seedSets,
boolean labelUsingSeedSets, Map>> answerClass, Map generalizeClasses,
Map> ignoreClasses) throws IOException, InstantiationException, IllegalAccessException, IllegalArgumentException,
InvocationTargetException, NoSuchMethodException, SecurityException, InterruptedException, ExecutionException, ClassNotFoundException {
this.props = props;
if (ignoreClasses.isEmpty()) {
for (String label : seedSets.keySet())
ignoreClasses.put(label, new HashMap<>());
}
setUpConstructor(sents, seedSets, labelUsingSeedSets, answerClass, generalizeClasses, ignoreClasses);
}
@SuppressWarnings("rawtypes")
private void setUpConstructor(Map sents, Map> seedSets, boolean labelUsingSeedSets,
Map>> answerClass, Map generalizeClasses,
Map> ignoreClasses) throws IOException, InstantiationException, IllegalAccessException, IllegalArgumentException,
InvocationTargetException, NoSuchMethodException, SecurityException, InterruptedException, ExecutionException, ClassNotFoundException {
Data.sents = sents;
ArgumentParser.fillOptions(Data.class, props);
ArgumentParser.fillOptions(ConstantsAndVariables.class, props);
PatternFactory.setUp(props, PatternFactory.PatternType.valueOf(props.getProperty(Flags.patternType)), seedSets.keySet());
constVars = new ConstantsAndVariables(props, seedSets, answerClass, generalizeClasses, ignoreClasses);
if (constVars.writeMatchedTokensFiles && constVars.batchProcessSents) {
throw new RuntimeException(
"writeMatchedTokensFiles and batchProcessSents cannot be true at the same time (not implemented; also doesn't make sense to save a large sentences json file)");
}
if (constVars.debug < 1) {
Redwood.hideChannelsEverywhere(ConstantsAndVariables.minimaldebug);
}
if (constVars.debug < 2) {
Redwood.hideChannelsEverywhere(Redwood.DBG);
}
constVars.justify = true;
if (constVars.debug < 3) {
constVars.justify = false;
}
if (constVars.debug < 4) {
Redwood.hideChannelsEverywhere(ConstantsAndVariables.extremedebug);
}
Redwood.log(Redwood.DBG, "Running with debug output");
Redwood.log(ConstantsAndVariables.extremedebug, "Running with extreme debug output");
wordsPatExtracted = new HashMap<>();
for (String label : answerClass.keySet()) {
wordsPatExtracted.put(label, new TwoDimensionalCounter<>());
}
scorePhrases = new ScorePhrases(props, constVars);
createPats = new CreatePatterns(props, constVars);
assert !(constVars.doNotApplyPatterns && (PatternFactory.useStopWordsBeforeTerm || PatternFactory.numWordsCompoundMax > 1)) : " Cannot have both doNotApplyPatterns and (useStopWordsBeforeTerm true or numWordsCompound > 1)!";
if(constVars.invertedIndexDirectory == null){
File f = File.createTempFile("inv","index");
f.deleteOnExit();
f.mkdir();
constVars.invertedIndexDirectory = f.getAbsolutePath();
}
Set extremelySmallStopWordsList = CollectionUtils.asSet(".", ",", "in", "on", "of", "a", "the", "an");
//Function to use to how to add CoreLabels to index
Function> transformCoreLabelToString = l -> {
Map add = new HashMap<>();
for (Class gn: constVars.getGeneralizeClasses().values()) {
Object b = l.get(gn);
if (b != null && !b.toString().equals(constVars.backgroundSymbol)) {
add.put(Token.getKeyForClass(gn),b.toString());
}
}
return add;
};
boolean createIndex = false;
if (constVars.loadInvertedIndex)
constVars.invertedIndex = SentenceIndex.loadIndex(constVars.invertedIndexClass, props, extremelySmallStopWordsList, constVars.invertedIndexDirectory, transformCoreLabelToString);
else {
constVars.invertedIndex = SentenceIndex.createIndex(constVars.invertedIndexClass, null, props, extremelySmallStopWordsList, constVars.invertedIndexDirectory, transformCoreLabelToString);
createIndex = true;
}
int totalNumSents = 0;
boolean computeDataFreq = false;
if (Data.rawFreq == null) {
Data.rawFreq = new ClassicCounter<>();
computeDataFreq = true;
}
ConstantsAndVariables.DataSentsIterator iter = new ConstantsAndVariables.DataSentsIterator(constVars.batchProcessSents);
while(iter.hasNext()){
Pair, File> sentsIter = iter.next();
Map sentsf = sentsIter.first();
if(constVars.batchProcessSents) {
for (Entry en : sentsf.entrySet()) {
Data.sentId2File.put(en.getKey(), sentsIter.second());
}
}
totalNumSents += sentsf.size();
if(computeDataFreq){
Data.computeRawFreqIfNull(sentsf, PatternFactory.numWordsCompoundMax);
}
Redwood.log(Redwood.DBG, "Initializing sents size " + sentsf.size()
+ " sentences, either by labeling with the seed set or just setting the right classes");
for (String l : constVars.getAnswerClass().keySet()) {
Redwood.log(Redwood.DBG, "labelUsingSeedSets is " + labelUsingSeedSets + " and seed set size for " + l + " is " + (seedSets == null?"null":seedSets.get(l).size()));
Set seed = seedSets == null || !labelUsingSeedSets ? new HashSet<>() : (seedSets.containsKey(l) ? seedSets.get(l)
: new HashSet<>());
if(!matchedSeedWords.containsKey(l)){
matchedSeedWords.put(l, new ClassicCounter<>());
}
Counter matched = runLabelSeedWords(sentsf, constVars.getAnswerClass().get(l), l, seed, constVars, labelUsingSeedSets);
System.out.println("matched phrases for " + l + " is " + matched);
matchedSeedWords.get(l).addAll(matched);
if (constVars.addIndvWordsFromPhrasesExceptLastAsNeg) {
Redwood.log(ConstantsAndVariables.minimaldebug, "adding indv words from phrases except last as neg");
Set otherseed = new HashSet<>();
if(labelUsingSeedSets){
for (CandidatePhrase s : seed) {
String[] t = s.getPhrase().split("\\s+");
for (int i = 0; i < t.length - 1; i++) {
if (!seed.contains(t[i])) {
otherseed.add(CandidatePhrase.createOrGet(t[i]));
}
}
}
}
runLabelSeedWords(sentsf, PatternsAnnotations.OtherSemanticLabel.class, "OTHERSEM", otherseed, constVars, labelUsingSeedSets);
}
}
if (labelUsingSeedSets && constVars.getOtherSemanticClassesWords() != null) {
String l = "OTHERSEM";
if(!matchedSeedWords.containsKey(l)){
matchedSeedWords.put(l, new ClassicCounter<>());
}
matchedSeedWords.get(l).addAll(runLabelSeedWords(sentsf, PatternsAnnotations.OtherSemanticLabel.class, l, constVars.getOtherSemanticClassesWords(), constVars, labelUsingSeedSets));
}
if(constVars.removeOverLappingLabelsFromSeed){
removeOverLappingLabels(sentsf);
}
if(createIndex)
constVars.invertedIndex.add(sentsf, true);
if(sentsIter.second().exists()){
Redwood.log(Redwood.DBG, "Saving the labeled seed sents (if given the option) to the same file " + sentsIter.second());
IOUtils.writeObjectToFile(sentsf, sentsIter.second());
}
}
Redwood.log(Redwood.DBG, "Done loading/creating inverted index of tokens and labeling data with total of "
+ constVars.invertedIndex.size() + " sentences");
//If the scorer class is LearnFeatWt then individual word class is added as a feature
if (scorePhrases.phraseScorerClass.equals(ScorePhrasesAverageFeatures.class) && (constVars.usePatternEvalWordClass || constVars.usePhraseEvalWordClass)) {
if (constVars.externalFeatureWeightsDir == null) {
File f = File.createTempFile("tempfeat", ".txt");
f.delete();
f.deleteOnExit();
constVars.externalFeatureWeightsDir = f.getAbsolutePath();
}
IOUtils.ensureDir(new File(constVars.externalFeatureWeightsDir));
for (String label : seedSets.keySet()) {
String externalFeatureWeightsFileLabel = constVars.externalFeatureWeightsDir + "/" + label;
File f = new File(externalFeatureWeightsFileLabel);
if (!f.exists()) {
Redwood.log(Redwood.DBG, "externalweightsfile for the label " + label + " does not exist: learning weights!");
LearnImportantFeatures lmf = new LearnImportantFeatures();
ArgumentParser.fillOptions(lmf, props);
lmf.answerClass = answerClass.get(label);
lmf.answerLabel = label;
lmf.setUp();
lmf.getTopFeatures(new ConstantsAndVariables.DataSentsIterator(constVars.batchProcessSents), constVars.perSelectRand, constVars.perSelectNeg,
externalFeatureWeightsFileLabel);
}
Counter distSimWeightsLabel = new ClassicCounter<>();
for (String line : IOUtils.readLines(externalFeatureWeightsFileLabel)) {
String[] t = line.split(":");
if (!t[0].startsWith("Cluster"))
continue;
String s = t[0].replace("Cluster-", "");
Integer clusterNum = Integer.parseInt(s);
distSimWeightsLabel.setCount(clusterNum, Double.parseDouble(t[1]));
}
constVars.distSimWeights.put(label, distSimWeightsLabel);
}
}
// computing semantic odds values
if (constVars.usePatternEvalSemanticOdds || constVars.usePhraseEvalSemanticOdds) {
Counter dictOddsWeightsLabel = new ClassicCounter<>();
Counter otherSemanticClassFreq = new ClassicCounter<>();
for (CandidatePhrase s : constVars.getOtherSemanticClassesWords()) {
for (String s1 : StringUtils.getNgrams(Arrays.asList(s.getPhrase().split("\\s+")), 1, PatternFactory.numWordsCompoundMax))
otherSemanticClassFreq.incrementCount(CandidatePhrase.createOrGet(s1));
}
otherSemanticClassFreq = Counters.add(otherSemanticClassFreq, 1.0);
// otherSemanticClassFreq.setDefaultReturnValue(1.0);
Map> labelDictNgram = new HashMap<>();
for (String label : seedSets.keySet()) {
Counter classFreq = new ClassicCounter<>();
for (CandidatePhrase s : seedSets.get(label)) {
for (String s1 : StringUtils.getNgrams(Arrays.asList(s.getPhrase().split("\\s+")), 1, PatternFactory.numWordsCompoundMax))
classFreq.incrementCount(CandidatePhrase.createOrGet(s1));
}
classFreq = Counters.add(classFreq, 1.0);
labelDictNgram.put(label, classFreq);
// classFreq.setDefaultReturnValue(1.0);
}
for (String label : seedSets.keySet()) {
Counter otherLabelFreq = new ClassicCounter<>();
for (String label2 : seedSets.keySet()) {
if (label.equals(label2))
continue;
otherLabelFreq.addAll(labelDictNgram.get(label2));
}
otherLabelFreq.addAll(otherSemanticClassFreq);
dictOddsWeightsLabel = Counters.divisionNonNaN(labelDictNgram.get(label), otherLabelFreq);
constVars.dictOddsWeights.put(label, dictOddsWeightsLabel);
}
}
//Redwood.log(Redwood.DBG, "All options are:" + "\n" + Maps.toString(getAllOptions(), "","","\t","\n"));
}
public PatternsForEachToken getPatsForEachToken() {
return patsForEachToken;
}
/**
* If a token is labeled for two or more labels, then keep the one that has the longest matching phrase. For example, "lung" as BODYPART label and "lung cancer" as DISEASE label,
* keep only the DISEASE label for "lung". For this to work, you need to have {@code PatternsAnnotations.Ln} set, which is already done in runLabelSeedWords function.
*/
private void removeOverLappingLabels(Map sents){
for(Map.Entry sentEn: sents.entrySet()){
for(CoreLabel l : sentEn.getValue().getTokens()){
Map longestMatchingMap = l.get(PatternsAnnotations.LongestMatchedPhraseForEachLabel.class);
String longestMatchingString = "";
String longestMatchingLabel = null;
for(Map.Entry en: longestMatchingMap.entrySet()){
if(en.getValue().getPhrase().length() > longestMatchingString.length()){
longestMatchingLabel = en.getKey();
longestMatchingString = en.getValue().getPhrase();
}
}
if(longestMatchingLabel != null){
if(!"OTHERSEM".equals(longestMatchingLabel))
l.set(PatternsAnnotations.OtherSemanticLabel.class, constVars.backgroundSymbol);
for(Entry>> en: constVars.getAnswerClass().entrySet()) {
if (!en.getKey().equals(longestMatchingLabel)){
l.set(en.getValue(), constVars.backgroundSymbol);
}
else
l.set(en.getValue(), en.getKey());
}
}
}
}
}
public static Map runPOSNERParseOnTokens(Map sents, Properties propsoriginal){
PatternFactory.PatternType type = PatternFactory.PatternType.valueOf(propsoriginal.getProperty(Flags.patternType));
Properties props = new Properties();
List anns = new ArrayList<>();
anns.add("pos");
anns.add("lemma");
boolean useTargetParserParentRestriction = Boolean.parseBoolean(propsoriginal.getProperty(Flags.useTargetParserParentRestriction));
boolean useTargetNERRestriction = Boolean.parseBoolean(propsoriginal.getProperty(Flags.useTargetNERRestriction));
String posModelPath = props.getProperty(Flags.posModelPath);
String numThreads = propsoriginal.getProperty(Flags.numThreads);
if (useTargetParserParentRestriction){
anns.add("parse");
} else if(type.equals(PatternFactory.PatternType.DEP))
anns.add("depparse");
if (useTargetNERRestriction) {
anns.add("ner");
}
props.setProperty("annotators", StringUtils.join(anns, ","));
props.setProperty("parse.maxlen", "80");
props.setProperty("nthreads", numThreads);
props.setProperty("threads", numThreads);
// props.put( "tokenize.options",
// "ptb3Escaping=false,normalizeParentheses=false,escapeForwardSlashAsterisk=false");
if (posModelPath != null) {
props.setProperty("pos.model", posModelPath);
}
StanfordCoreNLP pipeline = new StanfordCoreNLP(props, false);
Redwood.log(Redwood.DBG, "Annotating text");
for(Map.Entry en: sents.entrySet()) {
List temp = new ArrayList<>();
CoreMap s= new ArrayCoreMap();
s.set(CoreAnnotations.TokensAnnotation.class, en.getValue().getTokens());
temp.add(s);
Annotation doc = new Annotation(temp);
try {
pipeline.annotate(doc);
if (useTargetParserParentRestriction)
inferParentParseTag(s.get(TreeAnnotation.class));
} catch (Exception e) {
log.warn("Ignoring error: for sentence " + StringUtils.joinWords(en.getValue().getTokens(), " "));
log.warn(e);
}
}
Redwood.log(Redwood.DBG, "Done annotating text");
return sents;
}
public static Map runPOSNEROnTokens(List sentsCM, String posModelPath, boolean useTargetNERRestriction,
String prefix, boolean useTargetParserParentRestriction, String numThreads, PatternFactory.PatternType type) {
Annotation doc = new Annotation(sentsCM);
Properties props = new Properties();
List anns = new ArrayList<>();
anns.add("pos");
anns.add("lemma");
if (useTargetParserParentRestriction){
anns.add("parse");
} else if(type.equals(PatternFactory.PatternType.DEP))
anns.add("depparse");
if (useTargetNERRestriction) {
anns.add("ner");
}
props.setProperty("annotators", StringUtils.join(anns, ","));
props.setProperty("parse.maxlen", "80");
props.setProperty("nthreads", numThreads);
props.setProperty("threads", numThreads);
// props.put( "tokenize.options",
// "ptb3Escaping=false,normalizeParentheses=false,escapeForwardSlashAsterisk=false");
if (posModelPath != null) {
props.setProperty("pos.model", posModelPath);
}
StanfordCoreNLP pipeline = new StanfordCoreNLP(props, false);
Redwood.log(Redwood.DBG, "Annotating text");
pipeline.annotate(doc);
Redwood.log(Redwood.DBG, "Done annotating text");
Map sents = new HashMap<>();
for (CoreMap s : doc.get(CoreAnnotations.SentencesAnnotation.class)) {
if (useTargetParserParentRestriction)
inferParentParseTag(s.get(TreeAnnotation.class));
DataInstance d = DataInstance.getNewInstance(type, s);
sents.put(prefix + s.get(CoreAnnotations.DocIDAnnotation.class), d);
}
return sents;
}
static StanfordCoreNLP pipeline = null;
public static int tokenize(Iterator textReader, String posModelPath, boolean lowercase, boolean useTargetNERRestriction, String sentIDPrefix,
boolean useTargetParserParentRestriction, String numThreads, boolean batchProcessSents, int numMaxSentencesPerBatchFile,
File saveSentencesSerDirFile, Map sents, int numFilesTillNow, PatternFactory.PatternType type) throws InterruptedException, ExecutionException,
IOException {
if (pipeline == null) {
Properties props = new Properties();
List anns = new ArrayList<>();
anns.add("tokenize");
anns.add("ssplit");
anns.add("pos");
anns.add("lemma");
if (useTargetParserParentRestriction){
anns.add("parse");
}
if(type.equals(PatternFactory.PatternType.DEP))
anns.add("depparse");
if (useTargetNERRestriction) {
anns.add("ner");
}
props.setProperty("annotators", StringUtils.join(anns, ","));
props.setProperty("parse.maxlen", "80");
if(numThreads != null)
props.setProperty("threads", numThreads);
props.setProperty("tokenize.options", "ptb3Escaping=false,normalizeParentheses=false,escapeForwardSlashAsterisk=false");
if (posModelPath != null) {
props.setProperty("pos.model", posModelPath);
}
pipeline = new StanfordCoreNLP(props);
}
String text = "";
int numLines = 0;
while(textReader.hasNext()) {
String line = textReader.next();
numLines ++;
if (batchProcessSents && numLines > numMaxSentencesPerBatchFile) {
break;
}
if (lowercase)
line = line.toLowerCase();
text += line+"\n";
}
Annotation doc = new Annotation(text);
pipeline.annotate(doc);
int i = -1;
for (CoreMap s : doc.get(CoreAnnotations.SentencesAnnotation.class)) {
i++;
if (useTargetParserParentRestriction)
inferParentParseTag(s.get(TreeAnnotation.class));
DataInstance d = DataInstance.getNewInstance(type, s);
sents.put(sentIDPrefix + i, d);
// if (batchProcessSents && sents.size() >= numMaxSentencesPerBatchFile) {
// numFilesTillNow++;
// File file = new File(saveSentencesSerDirFile + "/sents_" + numFilesTillNow);
// IOUtils.writeObjectToFile(sents, file);
// sents = new HashMap();
// Data.sentsFiles.add(file);
// }
}
Redwood.log(Redwood.DBG, "Done annotating text with " + i + " sentences");
if (sents.size() > 0 && batchProcessSents) {
numFilesTillNow++;
File file = new File(saveSentencesSerDirFile + "/sents_" + numFilesTillNow);
IOUtils.writeObjectToFile(sents, file);
Data.sentsFiles.add(file);
for(String sentid: sents.keySet()) {
assert !Data.sentId2File.containsKey(sentid) : "Data.sentId2File already contains " + sentid + ". Make sure sentIds are unique!";
Data.sentId2File.put(sentid, file);
}
sents.clear();
}
// not lugging around sents if batch processing
if (batchProcessSents)
sents = null;
return numFilesTillNow;
}
/*
public static int tokenize(String text, String posModelPath, boolean lowercase, boolean useTargetNERRestriction, String sentIDPrefix,
boolean useTargetParserParentRestriction, String numThreads, boolean batchProcessSents, int numMaxSentencesPerBatchFile,
File saveSentencesSerDirFile, Map sents, int numFilesTillNow) throws InterruptedException, ExecutionException,
IOException {
if (pipeline == null) {
Properties props = new Properties();
List anns = new ArrayList();
anns.add("tokenize");
anns.add("ssplit");
anns.add("pos");
anns.add("lemma");
if (useTargetParserParentRestriction) {
anns.add("parse");
}
if (useTargetNERRestriction) {
anns.add("ner");
}
props.setProperty("annotators", StringUtils.join(anns, ","));
props.setProperty("parse.maxlen", "80");
props.setProperty("threads", numThreads);
props.put("tokenize.options", "ptb3Escaping=false,normalizeParentheses=false,escapeForwardSlashAsterisk=false");
if (posModelPath != null) {
props.setProperty("pos.model", posModelPath);
}
pipeline = new StanfordCoreNLP(props);
}
if (lowercase)
text = text.toLowerCase();
Annotation doc = new Annotation(text);
pipeline.annotate(doc);
Redwood.log(Redwood.DBG, "Done annotating text");
int i = -1;
for (CoreMap s : doc.get(CoreAnnotations.SentencesAnnotation.class)) {
i++;
if (useTargetParserParentRestriction)
inferParentParseTag(s.get(TreeAnnotation.class));
sents.put(sentIDPrefix + i, s.get(CoreAnnotations.TokensAnnotation.class));
if (batchProcessSents && sents.size() >= numMaxSentencesPerBatchFile) {
numFilesTillNow++;
File file = new File(saveSentencesSerDirFile + "/sents_" + numFilesTillNow);
IOUtils.writeObjectToFile(sents, file);
sents = new HashMap();
Data.sentsFiles.add(file);
}
}
if (sents.size() > 0 && batchProcessSents) {
numFilesTillNow++;
File file = new File(saveSentencesSerDirFile + "/sents_" + numFilesTillNow);
IOUtils.writeObjectToFile(sents, file);
Data.sentsFiles.add(file);
sents.clear();
}
// not lugging around sents if batch processing
if (batchProcessSents)
sents = null;
return numFilesTillNow;
}
*/
private static void inferParentParseTag(Tree tree) {
String grandstr = tree.value();
for (Tree child : tree.children()) {
for (Tree grand : child.children()) {
if (grand.isLeaf()) {
((CoreLabel) grand.label()).set(CoreAnnotations.GrandparentAnnotation.class, grandstr);
}
}
inferParentParseTag(child);
}
}
/**
* If l1 is a part of l2, it finds the starting index of l1 in l2 If l1 is not
* a sub-array of l2, then it returns -1 note that l2 should have the exact
* elements and order as in l1
*
* @param l1 array you want to find in l2
* @param l2
* @return starting index of the sublist
*/
public static List getSubListIndex(String[] l1, String[] l2, String[] subl2, Set doNotLabelTheseWords, HashSet seenFuzzyMatches,
int minLen4Fuzzy, boolean fuzzyMatch, boolean ignoreCaseSeedMatch) {
if (l1.length > l2.length)
return null;
EditDistance editDistance = new EditDistance(true);
List allIndices = new ArrayList<>();
boolean matched = false;
int index = -1;
int lastUnmatchedIndex = 0;
for (int i = 0; i < l2.length;) {
for (int j = 0; j < l1.length;) {
boolean d1 = false, d2 = false;
boolean compareFuzzy = true;
if (!fuzzyMatch || doNotLabelTheseWords.contains(l2[i]) || doNotLabelTheseWords.contains(subl2[i]) || l2[i].length() <= minLen4Fuzzy || subl2[i].length() <= minLen4Fuzzy)
compareFuzzy = false;
if (compareFuzzy == false || l1[j].length() <= minLen4Fuzzy) {
d1 = (ignoreCaseSeedMatch && l1[j].equalsIgnoreCase(l2[i])) || l1[j].equals(l2[i]);
if (!d1 && fuzzyMatch)
d2 = (ignoreCaseSeedMatch && subl2[i].equalsIgnoreCase(l1[j])) || subl2[i].equals(l1[j]);
} else {
String combo = l1[j] + "#" + l2[i];
if ((ignoreCaseSeedMatch && l1[j].equalsIgnoreCase(l2[i])) || l1[j].equals(l2[i]) || seenFuzzyMatches.contains(combo))
d1 = true;
else {
d1 = editDistance.score(l1[j], l2[i]) <= 1;
if (!d1) {
String combo2 = l1[j] + "#" + subl2[i];
if ((ignoreCaseSeedMatch && l1[j].equalsIgnoreCase(subl2[i]) )||l1[j].equals(subl2[i]) || seenFuzzyMatches.contains(combo2))
d2 = true;
else {
d2 = editDistance.score(l1[j], subl2[i]) <= 1;
if (d2) {
// System.out.println(l1[j] + " matched with " + subl2[i]);
seenFuzzyMatches.add(combo2);
}
}
} else if (d1) {
// System.out.println(l1[j] + " matched with " + l2[i]);
seenFuzzyMatches.add(combo);
}
}
}
// if (l1[j].equals(l2[i]) || subl2[i].equals(l1[j])) {
if (d1 || d2) {
index = i;
i++;
j++;
if (j == l1.length) {
matched = true;
break;
}
} else {
j = 0;
i = lastUnmatchedIndex + 1;
lastUnmatchedIndex = i;
index = -1;
if (lastUnmatchedIndex == l2.length)
break;
}
if (i >= l2.length) {
index = -1;
break;
}
}
if (i == l2.length || matched) {
if (index >= 0)
// index = index - l1.length + 1;
allIndices.add(index - l1.length + 1);
matched = false;
lastUnmatchedIndex = index;
// break;
}
}
// get starting point
return allIndices;
}
//if matchcontextlowercase is on, transform that. escape the word etc. Useful for pattern matching later on
private static Function stringTransformationFunction = new Function() {
@Override
public String apply(CoreLabel l) {
String s;
if(PatternFactory.useLemmaContextTokens){
s = l.lemma();
assert s!=null : "Lemma is null and useLemmaContextTokens is true";
}
else
s= l.word();
if(ConstantsAndVariables.matchLowerCaseContext)
s = s.toLowerCase();
assert s!= null;
return s;
}
};
public static List> getThreadBatches(List keyset, int numThreads){
int num;
if (numThreads == 1)
num = keyset.size();
else
num = keyset.size() / (numThreads - 1);
Redwood.log(ConstantsAndVariables.extremedebug, "keyset size is " + keyset.size());
List> threadedSentIds = new ArrayList<>();
for (int i = 0; i < numThreads; i++) {
List keys = keyset.subList(i * num, Math.min(keyset.size(), (i + 1) * num));
threadedSentIds.add(keys);
Redwood.log(ConstantsAndVariables.extremedebug, "assigning from " + i * num + " till " + Math.min(keyset.size(), (i + 1) * num));
}
return threadedSentIds;
}
/** Warning: sets labels of words that are not in the given seed set as O!!!
* */
public static Counter runLabelSeedWords(Map sents, Class answerclass, String label, Collection seedWords, ConstantsAndVariables constVars, boolean overwriteExistingLabels)
throws InterruptedException, ExecutionException, IOException {
Redwood.log(Redwood.DBG,"ignoreCaseSeedMatch is " + constVars.ignoreCaseSeedMatch);
List> threadedSentIds = getThreadBatches(new ArrayList<>(sents.keySet()), constVars.numThreads);
ExecutorService executor = Executors.newFixedThreadPool(constVars.numThreads);
List, Counter>>> list = new ArrayList<>();
Counter matchedPhrasesCounter = new ClassicCounter<>();
for (List keys: threadedSentIds) {
Callable, Counter>> task = new LabelWithSeedWords(seedWords, sents, keys, answerclass, label, constVars.fuzzyMatch, constVars.minLen4FuzzyForPattern, constVars.backgroundSymbol, constVars.getEnglishWords(),
stringTransformationFunction, constVars.writeMatchedTokensIdsForEachPhrase, overwriteExistingLabels, constVars.patternType, constVars.ignoreCaseSeedMatch);
Pair, Counter> sentsi = executor.submit(task).get();
sents.putAll(sentsi.first());
matchedPhrasesCounter.addAll(sentsi.second());
}
executor.shutdown();
Redwood.log("extremedebug","Matched phrases freq is " + matchedPhrasesCounter);
return matchedPhrasesCounter;
}
public static void getFeatures(SemanticGraph graph, IndexedWord vertex, boolean isHead, Collection features, GrammaticalRelation reln){
if(isHead){
List> pt = graph.parentPairs(vertex);
for(Pair en: pt) {
features.add("PARENTREL-" + en.first());
}
} else{
//find the relation to the parent
if(reln == null){
List parents = graph.getOutEdgesSorted(vertex);
if(parents.size() > 0)
reln = parents.get(0).getRelation();
}
if(reln != null)
features.add("REL-" + reln.getShortName());
}
//System.out.println("For graph " + graph.toFormattedString() + " and vertex " + vertex + " the features are " + features);
}
/**
* Warning: sets labels of words that are not in the given seed set as O!!!
*/
@SuppressWarnings("rawtypes")
public static class LabelWithSeedWords implements Callable, Counter>> {
Map seedwordsTokens = new HashMap<>();
Map sents;
List keyset;
Class labelClass;
HashSet seenFuzzyMatches = new HashSet<>();
String label;
int minLen4FuzzyForPattern;
String backgroundSymbol = "O";
Set doNotLabelDictWords = null;
Function stringTransformation;
boolean writeMatchedTokensIdsForEachPhrase = false;
boolean overwriteExistingLabels;
PatternFactory.PatternType patternType;
boolean fuzzyMatch = false;
Map ignoreCaseSeedMatch;
public LabelWithSeedWords(Collection seedwords, Map sents, List keyset, Class labelclass, String label, boolean fuzzyMatch,
int minLen4FuzzyForPattern, String backgroundSymbol, Set doNotLabelDictWords,
Function stringTransformation, boolean writeMatchedTokensIdsForEachPhrase, boolean overwriteExistingLabels, PatternFactory.PatternType type,
Map ignoreCaseSeedMatch) {
for (CandidatePhrase s : seedwords)
this.seedwordsTokens.put(s, s.getPhrase().split("\\s+"));
this.sents = sents;
this.keyset = keyset;
this.labelClass = labelclass;
this.label = label;
this.minLen4FuzzyForPattern= minLen4FuzzyForPattern;
this.backgroundSymbol = backgroundSymbol;
this.doNotLabelDictWords = doNotLabelDictWords;
this.stringTransformation = stringTransformation;
this.writeMatchedTokensIdsForEachPhrase = writeMatchedTokensIdsForEachPhrase;
this.overwriteExistingLabels = overwriteExistingLabels;
this.patternType = type;
this.fuzzyMatch = fuzzyMatch;
this.ignoreCaseSeedMatch = ignoreCaseSeedMatch;
}
@SuppressWarnings("unchecked")
@Override
public Pair,Counter> call() {
Map newsent = new HashMap<>();
Counter matchedPhrasesCounter = new ClassicCounter<>();
for (String k : keyset) {
DataInstance sent = sents.get(k);
List tokensCore = sent.getTokens();
SemanticGraph graph = null;
if(patternType.equals(PatternFactory.PatternType.DEP)){
graph = ((DataInstanceDep)sent).getGraph();
}
String[] tokens = new String[tokensCore.size()];
String[] tokenslemma = new String[tokensCore.size()];
int num = 0;
for (CoreLabel l : tokensCore) {
//Setting the processedTextAnnotation, used in indexing and pattern matching
l.set(PatternsAnnotations.ProcessedTextAnnotation.class, stringTransformation.apply(l));
tokens[num] = l.word();
if(fuzzyMatch && l.lemma() == null)
throw new RuntimeException("how come lemma is null");
tokenslemma[num] = l.lemma();
num++;
}
boolean[] labels = new boolean[tokens.length];
CollectionValuedMap matchedPhrases = new CollectionValuedMap<>();
Map longestMatchedPhrases = new HashMap<>();
for (Entry sEn : seedwordsTokens.entrySet()) {
String[] s = sEn.getValue();
CandidatePhrase sc = sEn.getKey();
List indices = getSubListIndex(s, tokens, tokenslemma, doNotLabelDictWords, seenFuzzyMatches,
minLen4FuzzyForPattern, fuzzyMatch, (ignoreCaseSeedMatch.containsKey(label) ? Boolean.valueOf(ignoreCaseSeedMatch.get(label)) : false));
if (indices != null && !indices.isEmpty()){
String ph = StringUtils.join(s, " ");
sc.addFeature("LENGTH-" + s.length, 1.0);
Collection features = new ArrayList<>();
for (int index : indices){
if(graph != null){
GetPatternsFromDataMultiClass.getFeatures(graph, graph.getNodeByIndex(index + 1), true, features, null);
}
if(writeMatchedTokensIdsForEachPhrase) {
addToMatchedTokensByPhrase(ph, k, index, s.length);
}
for (int i = 0; i < s.length; i++) {
matchedPhrases.add(index + i, sc);
if(graph != null){
try{
GetPatternsFromDataMultiClass.getFeatures(graph, graph.getNodeByIndex(index+ i + 1), false, features, null);
} catch(Exception e) { log.warn(e); }
}
CandidatePhrase longPh = longestMatchedPhrases.get(index+i);
longPh = longPh != null && longPh.getPhrase().length() > sc.getPhrase().length() ? longPh: sc;
longestMatchedPhrases.put(index+i, longPh);
labels[index + i] = true;
}
}
sc.addFeatures(features);
}
}
int i = -1;
for (CoreLabel l : sent.getTokens()) {
i++;
//The second clause is for old sents ser files compatibility reason
if (!l.containsKey(PatternsAnnotations.MatchedPhrases.class) || !(PatternsAnnotations.MatchedPhrases.class.isInstance(l.get(PatternsAnnotations.MatchedPhrases.class))))
l.set(PatternsAnnotations.MatchedPhrases.class, new CollectionValuedMap<>());
if(!l.containsKey(PatternsAnnotations.LongestMatchedPhraseForEachLabel.class))
l.set(PatternsAnnotations.LongestMatchedPhraseForEachLabel.class, new HashMap<>());
if (labels[i]) {
l.set(labelClass, label);
//set whether labeled by the seeds or not
if(!l.containsKey(PatternsAnnotations.SeedLabeledOrNot.class))
l.set(PatternsAnnotations.SeedLabeledOrNot.class, new HashMap<>());
l.get(PatternsAnnotations.SeedLabeledOrNot.class).put(labelClass, true);
CandidatePhrase longestMatchingPh = l.get(PatternsAnnotations.LongestMatchedPhraseForEachLabel.class).get(label);
assert longestMatchedPhrases.containsKey(i);
longestMatchingPh = (longestMatchingPh != null && (longestMatchingPh.getPhrase().length() > longestMatchedPhrases.get(i).getPhrase().length())) ? longestMatchingPh : longestMatchedPhrases.get(i);
l.get(PatternsAnnotations.LongestMatchedPhraseForEachLabel.class).put(label, longestMatchingPh);
matchedPhrasesCounter.incrementCount(longestMatchingPh, 1.0);
l.get(PatternsAnnotations.MatchedPhrases.class).addAll(label, matchedPhrases.get(i));
Redwood.log(ConstantsAndVariables.extremedebug, "labeling " + l.word() + " or its lemma " + l.lemma() + " as " + label
+ " because of the dict phrases " + matchedPhrases.get(i));
} else if(overwriteExistingLabels)
l.set(labelClass, backgroundSymbol);
}
newsent.put(k, sent);
}
return new Pair(newsent, matchedPhrasesCounter);
}
}
private static void addToMatchedTokensByPhrase(String ph, String sentid, int index, int length){
if(!Data.matchedTokensForEachPhrase.containsKey(ph))
Data.matchedTokensForEachPhrase.put(ph, new HashMap<>());
Map> matcheds = Data.matchedTokensForEachPhrase.get(ph);
if(!matcheds.containsKey(sentid))
matcheds.put(sentid, new ArrayList<>());
for (int i = 0; i < length; i++)
matcheds.get(sentid).add(index + i);
}
public Map> patternsandWords = null;
//public Map> allPatternsandWords = null;
public Map> currentPatternWeights = null;
//deleteExistingIndex is def false for the second call to this function
public void processSents(Map sents, Boolean deleteExistingIndex) throws IOException, ClassNotFoundException {
if (constVars.computeAllPatterns) {
props.setProperty("createTable", deleteExistingIndex.toString());
props.setProperty("deleteExisting", deleteExistingIndex.toString());
props.setProperty("createPatLuceneIndex", deleteExistingIndex.toString());
Redwood.log(Redwood.DBG, "Computing all patterns");
createPats.getAllPatterns(sents, props, constVars.storePatsForEachToken);
}
else
Redwood.log(Redwood.DBG, "Reading patterns from existing dir");
props.setProperty("createTable", "false");
props.setProperty("deleteExisting","false");
props.setProperty("createPatLuceneIndex","false");
}
private void readSavedPatternsAndIndex() throws IOException, ClassNotFoundException {
if(!constVars.computeAllPatterns) {
assert constVars.allPatternsDir != null : "allPatternsDir flag cannot be empty if computeAllPatterns is false!";
//constVars.setPatternIndex(PatternIndex.load(constVars.allPatternsDir, constVars.storePatsIndex));
if(constVars.storePatsForEachToken.equals(ConstantsAndVariables.PatternForEachTokenWay.MEMORY))
patsForEachToken.load(constVars.allPatternsDir);
}
}
@SuppressWarnings({ "unchecked" })
public Counter getPatterns(String label, Set alreadyIdentifiedPatterns, E p0, Counter p0Set,
Set ignorePatterns) throws IOException, ClassNotFoundException {
TwoDimensionalCounter patternsandWords4Label = new TwoDimensionalCounter<>();
TwoDimensionalCounter negPatternsandWords4Label = new TwoDimensionalCounter<>();
//TwoDimensionalCounter posnegPatternsandWords4Label = new TwoDimensionalCounter();
TwoDimensionalCounter unLabeledPatternsandWords4Label = new TwoDimensionalCounter<>();
//TwoDimensionalCounter negandUnLabeledPatternsandWords4Label = new TwoDimensionalCounter();
//TwoDimensionalCounter allPatternsandWords4Label = new TwoDimensionalCounter();
Set allCandidatePhrases = new HashSet<>();
ConstantsAndVariables.DataSentsIterator sentsIter = new ConstantsAndVariables.DataSentsIterator(constVars.batchProcessSents);
boolean firstCallToProcessSents = true;
while(sentsIter.hasNext()){
Pair, File> sentsPair = sentsIter.next();
if(notComputedAllPatternsYet){
//in the first iteration
processSents(sentsPair.first(), firstCallToProcessSents);
firstCallToProcessSents = false;
if(patsForEachToken == null){
//in the first iteration, for the first file
patsForEachToken = PatternsForEachToken.getPatternsInstance(props, constVars.storePatsForEachToken);
readSavedPatternsAndIndex();
}
}
this.calculateSufficientStats(sentsPair.first(), patsForEachToken, label, patternsandWords4Label, negPatternsandWords4Label, unLabeledPatternsandWords4Label, allCandidatePhrases);
}
notComputedAllPatternsYet = false;
if (constVars.computeAllPatterns){
if(constVars.storePatsForEachToken.equals(ConstantsAndVariables.PatternForEachTokenWay.DB))
patsForEachToken.createIndexIfUsingDBAndNotExists();
// String systemdir = System.getProperty("java.io.tmpdir");
// File tempFile= File.createTempFile("patterns", ".tmp", new File(systemdir));
// tempFile.deleteOnExit();
// tempFile.delete();
// constVars.allPatternsDir = tempFile.getAbsolutePath();
if(constVars.allPatternsDir != null){
IOUtils.ensureDir(new File(constVars.allPatternsDir));
patsForEachToken.save(constVars.allPatternsDir);
}
//savePatternIndex(constVars.allPatternsDir);
}
patsForEachToken.close();
//This is important. It makes sure that we don't recompute patterns in every iteration!
constVars.computeAllPatterns = false;
if (patternsandWords == null)
patternsandWords = new HashMap<>();
if (currentPatternWeights == null)
currentPatternWeights = new HashMap<>();
Counter currentPatternWeights4Label = new ClassicCounter<>();
Set removePats = enforceMinSupportRequirements(patternsandWords4Label, unLabeledPatternsandWords4Label);
Counters.removeKeys(patternsandWords4Label, removePats);
Counters.removeKeys(unLabeledPatternsandWords4Label, removePats);
Counters.removeKeys(negPatternsandWords4Label, removePats);
ScorePatterns scorePatterns;
Class patternscoringclass = getPatternScoringClass(constVars.patternScoring);
if (patternscoringclass != null && patternscoringclass.equals(ScorePatternsF1.class)) {
scorePatterns = new ScorePatternsF1(constVars, constVars.patternScoring, label, allCandidatePhrases, patternsandWords4Label, negPatternsandWords4Label,
unLabeledPatternsandWords4Label, props, p0Set, p0);
Counter finalPat = scorePatterns.score();
Counters.removeKeys(finalPat, alreadyIdentifiedPatterns);
Counters.retainNonZeros(finalPat);
Counters.retainTop(finalPat, constVars.numPatterns);
if (Double.isNaN(Counters.max(finalPat)))
throw new RuntimeException("how is the value NaN");
Redwood.log(ConstantsAndVariables.minimaldebug, "Selected Patterns: " + finalPat);
return finalPat;
} else if (patternscoringclass != null && patternscoringclass.equals(ScorePatternsRatioModifiedFreq.class)) {
scorePatterns = new ScorePatternsRatioModifiedFreq(constVars, constVars.patternScoring, label, allCandidatePhrases, patternsandWords4Label,
negPatternsandWords4Label, unLabeledPatternsandWords4Label, phInPatScoresCache, scorePhrases, props);
} else if (patternscoringclass != null && patternscoringclass.equals(ScorePatternsFreqBased.class)) {
scorePatterns = new ScorePatternsFreqBased(constVars, constVars.patternScoring, label, allCandidatePhrases, patternsandWords4Label, negPatternsandWords4Label,
unLabeledPatternsandWords4Label, props);
} else if (constVars.patternScoring.equals(PatternScoring.kNN)) {
try {
Class clazz = (Class) Class.forName("edu.stanford.nlp.patterns.ScorePatternsKNN");
Constructor ctor = clazz.getConstructor(ConstantsAndVariables.class, PatternScoring.class, String.class, Set.class,
TwoDimensionalCounter.class, TwoDimensionalCounter.class, TwoDimensionalCounter.class, ScorePhrases.class, Properties.class);
scorePatterns = ctor.newInstance(constVars, constVars.patternScoring, label, allCandidatePhrases, patternsandWords4Label, negPatternsandWords4Label,
unLabeledPatternsandWords4Label, scorePhrases, props);
} catch (ClassNotFoundException e) {
throw new RuntimeException("kNN pattern scoring is not released yet. Stay tuned.");
} catch (NoSuchMethodException | InvocationTargetException | InstantiationException | IllegalAccessException e) {
throw new RuntimeException("newinstance of kNN not created", e);
}
} else {
throw new RuntimeException(constVars.patternScoring + " is not implemented (check spelling?). ");
}
scorePatterns.setUp(props);
currentPatternWeights4Label = scorePatterns.score();
Redwood.log(ConstantsAndVariables.extremedebug, "patterns counter size is " + currentPatternWeights4Label.size());
if (ignorePatterns != null && !ignorePatterns.isEmpty()) {
Counters.removeKeys(currentPatternWeights4Label, ignorePatterns);
Redwood.log(ConstantsAndVariables.extremedebug, "Removing patterns from ignorePatterns of size " + ignorePatterns.size()
+ ". New patterns size " + currentPatternWeights4Label.size());
}
if (alreadyIdentifiedPatterns != null && !alreadyIdentifiedPatterns.isEmpty()) {
Redwood.log(ConstantsAndVariables.extremedebug, "Patterns size is " + currentPatternWeights4Label.size());
Counters.removeKeys(currentPatternWeights4Label, alreadyIdentifiedPatterns);
Redwood.log(ConstantsAndVariables.extremedebug, "Removing already identified patterns of size " + alreadyIdentifiedPatterns.size()
+ ". New patterns size " + currentPatternWeights4Label.size());
}
PriorityQueue q = Counters.toPriorityQueue(currentPatternWeights4Label);
int num = 0;
Counter chosenPat = new ClassicCounter<>();
Set removePatterns = new HashSet<>();
Set removeIdentifiedPatterns = null;
while (num < constVars.numPatterns && !q.isEmpty()) {
E pat = q.removeFirst();
//E pat = constVars.getPatternIndex().get(patindex);
if (currentPatternWeights4Label.getCount(pat) < constVars.thresholdSelectPattern) {
Redwood.log(Redwood.DBG, "The max weight of candidate patterns is " + df.format(currentPatternWeights4Label.getCount(pat))
+ " so not adding anymore patterns");
break;
}
boolean notchoose = false;
if (!unLabeledPatternsandWords4Label.containsFirstKey(pat) || unLabeledPatternsandWords4Label.getCounter(pat).isEmpty()) {
Redwood.log(ConstantsAndVariables.extremedebug, "Removing pattern " + pat + " because it has no unlab support; pos words: "
+ patternsandWords4Label.getCounter(pat));
notchoose = true;
continue;
}
Set removeChosenPats = null;
if (!notchoose) {
if (alreadyIdentifiedPatterns != null) {
for (E p : alreadyIdentifiedPatterns) {
if (Pattern.subsumes(constVars.patternType, pat, p)) {
// if (pat.getNextContextStr().contains(p.getNextContextStr()) &&
// pat.getPrevContextStr().contains(p.getPrevContextStr())) {
Redwood.log(ConstantsAndVariables.extremedebug, "Not choosing pattern " + pat
+ " because it is contained in or contains the already chosen pattern " + p);
notchoose = true;
break;
}
int rest = pat.equalContext(p);
// the contexts dont match
if (rest == Integer.MAX_VALUE)
continue;
// if pat is less restrictive, remove p and add pat!
if (rest < 0) {
if(removeIdentifiedPatterns == null)
removeIdentifiedPatterns = new HashSet<>();
removeIdentifiedPatterns.add(p);
} else {
notchoose = true;
break;
}
}
}
}
// In this iteration:
if (!notchoose) {
for (Pattern p : chosenPat.keySet()) {
//E p = constVars.getPatternIndex().get(pindex);
boolean removeChosenPatFlag = false;
if (Pattern.sameGenre(constVars.patternType, pat, p)) {
if(Pattern.subsumes(constVars.patternType, pat, p)){
Redwood.log(ConstantsAndVariables.extremedebug, "Not choosing pattern " + pat
+ " because it is contained in or contains the already chosen pattern " + p);
notchoose = true;
break;
}
else if (E.subsumes(constVars.patternType, p, pat)) {
//subsume is true even if equal context
//check if equal context
int rest = pat.equalContext(p);
// the contexts do not match
if (rest == Integer.MAX_VALUE)
{
Redwood.log(ConstantsAndVariables.extremedebug, "Not choosing pattern " + p
+ " because it is contained in or contains another chosen pattern in this iteration " + pat);
removeChosenPatFlag = true;
}
// if pat is less restrictive, remove p from chosen patterns and
// add pat!
else if (rest < 0) {
removeChosenPatFlag = true;
} else {
notchoose = true;
break;
}
}
if (removeChosenPatFlag) {
if(removeChosenPats == null)
removeChosenPats = new HashSet<>();
removeChosenPats.add(pat);
num--;
}
}
}
}
if (notchoose) {
Redwood.log(Redwood.DBG, "Not choosing " + pat + " for whatever reason!");
continue;
}
if (removeChosenPats != null) {
Redwood.log(ConstantsAndVariables.extremedebug, "Removing already chosen patterns in this iteration " + removeChosenPats + " in favor of "
+ pat);
Counters.removeKeys(chosenPat, removeChosenPats);
}
if (removeIdentifiedPatterns != null) {
Redwood.log(ConstantsAndVariables.extremedebug, "Removing already identified patterns " + removeIdentifiedPatterns + " in favor of " + pat);
removePatterns.addAll(removeIdentifiedPatterns);
}
chosenPat.setCount(pat, currentPatternWeights4Label.getCount(pat));
num++;
}
this.removeLearnedPatterns(label, removePatterns);
Redwood.log(Redwood.DBG, "final size of the patterns is " + chosenPat.size());
Redwood.log(ConstantsAndVariables.minimaldebug, "\n\n## Selected Patterns for " + label + "##\n");
List> chosenPatSorted = Counters.toSortedListWithCounts(chosenPat);
for (Pair en : chosenPatSorted)
Redwood.log(ConstantsAndVariables.minimaldebug, en.first() + ":" + df.format(en.second) + "\n");
if (constVars.outDir != null && !constVars.outDir.isEmpty()) {
CollectionValuedMap posWords = new CollectionValuedMap<>();
for (Entry> en : patternsandWords4Label.entrySet()) {
posWords.addAll(en.getKey(), en.getValue().keySet());
}
CollectionValuedMap negWords = new CollectionValuedMap<>();
for (Entry> en : negPatternsandWords4Label.entrySet()) {
negWords.addAll(en.getKey(), en.getValue().keySet());
}
CollectionValuedMap unlabWords = new CollectionValuedMap<>();
for (Entry> en : unLabeledPatternsandWords4Label.entrySet()) {
unlabWords.addAll(en.getKey(), en.getValue().keySet());
}
if (constVars.outDir != null) {
String outputdir = constVars.outDir + "/" + constVars.identifier + "/" + label;
Redwood.log(ConstantsAndVariables.minimaldebug, "Saving output in " + outputdir);
IOUtils.ensureDir(new File(outputdir));
String filename = outputdir + "/patterns" + ".json";
JsonArrayBuilder obj = Json.createArrayBuilder();
if (writtenPatInJustification.containsKey(label) && writtenPatInJustification.get(label)) {
JsonReader jsonReader = Json.createReader(new BufferedInputStream(new FileInputStream(filename)));
JsonArray objarr = jsonReader.readArray();
jsonReader.close();
for (JsonValue o : objarr)
obj.add(o);
} else
obj = Json.createArrayBuilder();
JsonObjectBuilder objThisIter = Json.createObjectBuilder();
for (Pair pat : chosenPatSorted) {
JsonObjectBuilder o = Json.createObjectBuilder();
JsonArrayBuilder pos = Json.createArrayBuilder();
JsonArrayBuilder neg = Json.createArrayBuilder();
JsonArrayBuilder unlab = Json.createArrayBuilder();
for (CandidatePhrase w : posWords.get(pat.first()))
pos.add(w.getPhrase());
for (CandidatePhrase w : negWords.get(pat.first()))
neg.add(w.getPhrase());
for (CandidatePhrase w : unlabWords.get(pat.first()))
unlab.add(w.getPhrase());
o.add("Positive", pos);
o.add("Negative", neg);
o.add("Unlabeled", unlab);
o.add("Score", pat.second());
objThisIter.add(pat.first().toStringSimple(), o);
}
obj.add(objThisIter.build());
IOUtils.ensureDir(new File(filename).getParentFile());
IOUtils.writeStringToFile(StringUtils.normalize(StringUtils.toAscii(obj.build().toString())), filename, "ASCII");
writtenPatInJustification.put(label, true);
}
}
if (constVars.justify) {
Redwood.log(Redwood.DBG, "Justification for Patterns:");
for (E key : chosenPat.keySet()) {
Redwood.log(Redwood.DBG, "\nPattern: " + key);
Redwood.log(
Redwood.DBG,
"Positive Words:"
+ Counters.toSortedString(patternsandWords4Label.getCounter(key), patternsandWords4Label.getCounter(key).size(), "%1$s:%2$f", ";"));
Redwood.log(
Redwood.DBG,
"Negative Words:"
+ Counters.toSortedString(negPatternsandWords4Label.getCounter(key), negPatternsandWords4Label.getCounter(key).size(), "%1$s:%2$f",
";"));
Redwood.log(
Redwood.DBG,
"Unlabeled Words:"
+ Counters.toSortedString(unLabeledPatternsandWords4Label.getCounter(key), unLabeledPatternsandWords4Label.getCounter(key).size(),
"%1$s:%2$f", ";"));
}
}
//allPatternsandWords.put(label, allPatternsandWords4Label);
patternsandWords.put(label, patternsandWords4Label);
currentPatternWeights.put(label, currentPatternWeights4Label);
return chosenPat;
}
// private void savePatternIndex(String dir ) throws IOException {
// if(dir != null) {
// IOUtils.ensureDir(new File(dir));
// constVars.getPatternIndex().save(dir);
// }
// //patsForEachToken.savePatternIndex(constVars.getPatternIndex(), dir);
//
// }
public static Class getPatternScoringClass(PatternScoring patternScoring) {
if (patternScoring.equals(PatternScoring.F1SeedPattern)) {
return ScorePatternsF1.class;
} else if (patternScoring.equals(PatternScoring.PosNegUnlabOdds) || patternScoring.equals(PatternScoring.PosNegOdds)
|| patternScoring.equals(PatternScoring.RatioAll) || patternScoring.equals(PatternScoring.PhEvalInPat)
|| patternScoring.equals(PatternScoring.PhEvalInPatLogP) || patternScoring.equals(PatternScoring.LOGREG)
|| patternScoring.equals(PatternScoring.LOGREGlogP) || patternScoring.equals(PatternScoring.SqrtAllRatio)) {
return ScorePatternsRatioModifiedFreq.class;
} else if (patternScoring.equals(PatternScoring.RlogF) || patternScoring.equals(PatternScoring.RlogFPosNeg)
|| patternScoring.equals(PatternScoring.RlogFUnlabNeg) || patternScoring.equals(PatternScoring.RlogFNeg)
|| patternScoring.equals(PatternScoring.YanGarber02) || patternScoring.equals(PatternScoring.LinICML03)) {
return ScorePatternsFreqBased.class;
} else {
return null;
}
}
private static AtomicInteger numCallsToCalStats = new AtomicInteger();
private static List> splitIntoNumThreadsWithSampling(List c, int n, int numThreads) {
if (n < 0)
throw new IllegalArgumentException("n < 0: " + n);
if (n > c.size())
throw new IllegalArgumentException("n > size of collection: " + n + ", " + c.size());
List> resultAll = new ArrayList<>(numThreads);
int num;
if (numThreads == 1)
num = n;
else
num = n / (numThreads - 1);
System.out.println("shuffled " + c.size() + " sentences and selecting " + num + " sentences per thread");
List result = new ArrayList<>(num);
int totalitems = 0;
int nitem = 0;
Random r = new Random(numCallsToCalStats.incrementAndGet());
boolean[] added = new boolean[c.size()];
// Arrays.fill(added, false); // not needed; get false by default
while(totalitems < n){
//find the new sample index
int index;
do{
index = r.nextInt(c.size());
}while(added[index]);
added[index] = true;
E c1 = c.get(index);
if(nitem == num){
resultAll.add(result);
result = new ArrayList<>(num);
nitem= 0;
}
result.add(c1);
totalitems++;
nitem ++;
}
if(!result.isEmpty())
resultAll.add(result);
return resultAll;
}
//for each pattern, it calculates positive, negative, and unlabeled words
private void calculateSufficientStats(Map sents,
PatternsForEachToken patternsForEachToken, String label,
TwoDimensionalCounter patternsandWords4Label,
TwoDimensionalCounter negPatternsandWords4Label,
TwoDimensionalCounter unLabeledPatternsandWords4Label, Set allCandidatePhrases) {
Redwood.log(Redwood.DBG,"calculating sufficient stats");
patternsForEachToken.setupSearch();
// calculating the sufficient statistics
Class answerClass4Label = constVars.getAnswerClass().get(label);
int sampleSize = constVars.sampleSentencesForSufficientStats == 1.0 ? sents.size(): (int) Math.round(constVars.sampleSentencesForSufficientStats*sents.size());
List> sampledSentIds = splitIntoNumThreadsWithSampling(CollectionUtils.toList(sents.keySet()), sampleSize, constVars.numThreads);
Redwood.log(Redwood.DBG,"sampled " + sampleSize + " sentences (" + constVars.sampleSentencesForSufficientStats*100 + "%)");
ExecutorService executor = Executors.newFixedThreadPool(constVars.numThreads);
List>, List>, List>>>> list = new ArrayList<>();
for (List sampledSents : sampledSentIds) {
Callable>, List>, List>>> task = new CalculateSufficientStatsThreads(patternsForEachToken, sampledSents, sents, label, answerClass4Label);
Future>, List>, List>>> submit = executor.submit(task);
list.add(submit);
}
// Now retrieve the result
for (Future>, List>, List>>> future : list) {
try {
Triple>, List>, List>> stats = future.get();
addStats(patternsandWords4Label, stats.first());
addStats(negPatternsandWords4Label, stats.second());
addStats(unLabeledPatternsandWords4Label, stats.third());
} catch (Exception e) {
executor.shutdownNow();
throw new RuntimeException(e);
}
}
executor.shutdown();
}
private void addStats(TwoDimensionalCounter pw, List> v) {
for(Pair w: v){
pw.incrementCount(w.first(), w.second());
}
}
private class CalculateSufficientStatsThreads implements Callable{
private final Map sents;
private final PatternsForEachToken patternsForEachToken;
private final Collection sentIds;
private final String label;
private final Class answerClass4Label;
public CalculateSufficientStatsThreads(PatternsForEachToken patternsForEachToken, Collection sentIds, Map sents,String label, Class answerClass4Label){
this.patternsForEachToken = patternsForEachToken;
this.sentIds = sentIds;
this.sents = sents;
this.label = label;
this.answerClass4Label = answerClass4Label;
}
@Override
public Triple>, List>, List>> call() throws Exception {
List> posWords = new ArrayList<>();
List> negWords = new ArrayList<>();
List> unlabWords = new ArrayList<>();
for(String sentId: sentIds){
Map> pat4Sent = patternsForEachToken.getPatternsForAllTokens(sentId);
if (pat4Sent == null) {
throw new RuntimeException("How come there are no patterns for " + sentId);
}
DataInstance sent = sents.get(sentId);
List tokens = sent.getTokens();
for (int i = 0; i < tokens.size(); i++) {
CoreLabel token = tokens.get(i);
//Map> matchedPhrases = token.get(PatternsAnnotations.MatchedPhrases.class);
CandidatePhrase tokenWordOrLemma = CandidatePhrase.createOrGet(token.word());
CandidatePhrase longestMatchingPhrase;
if (constVars.useMatchingPhrase) {
Map longestMatchingPhrases = token.get(PatternsAnnotations.LongestMatchedPhraseForEachLabel.class);
longestMatchingPhrase = longestMatchingPhrases.get(label);
longestMatchingPhrase = (longestMatchingPhrase !=null && (longestMatchingPhrase.getPhrase().length() > tokenWordOrLemma.getPhrase().length()))? longestMatchingPhrase : tokenWordOrLemma;
/*if (matchedPhrases != null && !matchedPhrases.isEmpty()) {
for (String s : matchedPhrases) {
if (s.equals(tokenWordOrLemma)) {
longestMatchingPhrase = tokenWordOrLemma;
break;
}
if (longestMatchingPhrase == null || longestMatchingPhrase.length() > s.length()) {
longestMatchingPhrase = s;
}
}
} else {
longestMatchingPhrase = tokenWordOrLemma;
}*/
} else
longestMatchingPhrase = tokenWordOrLemma;
Set pats = pat4Sent.get(i);
//make a copy of pats because we are changing numwordscompound etc.
Set newpats = new HashSet();
boolean changedpats = false;
for (E s : pats) {
if(s instanceof SurfacePattern){
changedpats = true;
SurfacePattern snew = ((SurfacePattern) s).copyNewToken();
snew.setNumWordsCompound(PatternFactory.numWordsCompoundMapped.get(label));
newpats.add(snew);
}
}
if(changedpats)
pats = newpats;
//This happens when dealing with the collapseddependencies
if (pats == null) {
if(!constVars.patternType.equals(PatternFactory.PatternType.DEP))
throw new RuntimeException("Why are patterns null for sentence " + sentId + " and token " + i + "(" + tokens.get(i) + "). pat4Sent has token ids " + pat4Sent.keySet() +
(constVars.batchProcessSents ? "" : ". The sentence is " + Data.sents.get(sentId)) + ". If you have changed parameters, recompute all patterns.");
continue;
}
// Set prevPat = pat.first();
// Set nextPat = pat.second();
// Set prevnextPat = pat.third();
if (PatternFactory.ignoreWordRegex.matcher(token.word()).matches())
continue;
// if the target word/phrase does not satisfy the POS requirement
String tag = token.tag();
if (constVars.allowedTagsInitials != null && constVars.allowedTagsInitials.containsKey(label)) {
boolean use = false;
for (String allowed : constVars.allowedTagsInitials.get(label)) {
if (tag.startsWith(allowed)) {
use = true;
break;
}
}
if (!use)
continue;
}
// if the target word/phrase does not satisfy the NER requirements
String nertag = token.ner();
if (constVars.allowedNERsforLabels != null && constVars.allowedNERsforLabels.containsKey(label)) {
if (!constVars.allowedNERsforLabels.get(label).contains(nertag)) {
continue;
}
}
if (token.get(answerClass4Label).equals(label)) {
// Positive
for (E s : pats) {
posWords.add(new Pair<>(s, longestMatchingPhrase));
}
} else {
// Negative or unlabeled
boolean negToken = false;
Map ignore = constVars.getIgnoreWordswithClassesDuringSelection().get(label);
for (Class igCl : ignore.keySet())
if ((Boolean) token.get(igCl)) {
negToken = true;
break;
}
if (!negToken)
if (constVars.getOtherSemanticClassesWords().contains(token.word()) || constVars.getOtherSemanticClassesWords().contains(token.lemma()))
negToken = true;
if(!negToken){
for(String labelA : constVars.getLabels()){
if(!labelA.equals(label)){
if(constVars.getSeedLabelDictionary().get(labelA).contains(longestMatchingPhrase) || constVars.getSeedLabelDictionary().get(labelA).contains(tokenWordOrLemma)
|| constVars.getLearnedWords(labelA).containsKey(longestMatchingPhrase) || constVars.getLearnedWords(labelA).containsKey(tokenWordOrLemma)){
negToken = true;
break;
}
}
}
}
for (E sindex : pats) {
if (negToken) {
negWords.add(new Pair<>(sindex, longestMatchingPhrase));
} else {
unlabWords.add(new Pair<>(sindex, longestMatchingPhrase));
}
}
}
}
}
return new Triple>, List>, List>>(posWords, negWords, unlabWords);
}
}
private Set enforceMinSupportRequirements(TwoDimensionalCounter patternsandWords4Label,
TwoDimensionalCounter unLabeledPatternsandWords4Label) {
Set remove = new HashSet<>();
for (Entry> en : patternsandWords4Label.entrySet()) {
if (en.getValue().size() < constVars.minPosPhraseSupportForPat) {
remove.add(en.getKey());
}
}
int numRemoved = remove.size();
Redwood.log(Redwood.DBG, "Removing " + numRemoved + " patterns that do not meet minPosPhraseSupportForPat requirement of >= "
+ constVars.minPosPhraseSupportForPat);
for (Entry> en : unLabeledPatternsandWords4Label.entrySet()) {
if (en.getValue().size() < constVars.minUnlabPhraseSupportForPat) {
remove.add(en.getKey());
}
}
Redwood.log(Redwood.DBG, "Removing " + (remove.size() - numRemoved) + " patterns that do not meet minUnlabPhraseSupportForPat requirement of >= "
+ constVars.minUnlabPhraseSupportForPat);
return remove;
}
// void removeLearnedPattern(String label, E p) {
// this.learnedPatterns.get(label).remove(p);
// if (wordsPatExtracted.containsKey(label))
// for (Entry> en : this.wordsPatExtracted.get(label).entrySet()) {
// en.getValue().remove(p);
// }
// }
private void removeLearnedPatterns(String label, Collection pats) {
Counters.removeKeys(this.learnedPatterns.get(label), pats);
for(Map.Entry> en: this.learnedPatternsEachIter.get(label).entrySet())
Counters.removeKeys(en.getValue(), pats);
if (wordsPatExtracted.containsKey(label))
for (Entry> en : this.wordsPatExtracted.get(label).entrySet()) {
Counters.removeKeys(en.getValue(), pats);
}
}
public static Counter normalizeSoftMaxMinMaxScores(Counter scores, boolean minMaxNorm, boolean softmax, boolean oneMinusSoftMax) {
double minScore = Double.MAX_VALUE, maxScore = Double.MIN_VALUE;
Counter newscores = new ClassicCounter<>();
if (softmax) {
for (Entry en : scores.entrySet()) {
Double score = null;
if (oneMinusSoftMax)
score = (1 / (1 + Math.exp(Math.min(7, en.getValue()))));
else
score = (1 / (1 + Math.exp(-1 * Math.min(7, en.getValue()))));
if (score < minScore)
minScore = score;
if (score > maxScore)
maxScore = score;
newscores.setCount(en.getKey(), score);
}
} else {
newscores.addAll(scores);
minScore = Counters.min(newscores);
maxScore = Counters.max(newscores);
}
if (minMaxNorm) {
for (Entry en : newscores.entrySet()) {
double score;
if (minScore == maxScore)
score = minScore;
else
score = (en.getValue() - minScore + 1e-10) / (maxScore - minScore);
newscores.setCount(en.getKey(), score);
}
}
return newscores;
}
public TwoDimensionalCounter phInPatScoresCache = new TwoDimensionalCounter<>();
public void labelWords(String label, Map sents, Collection identifiedWords) throws IOException {
CollectionValuedMap> matchedTokensByPat = new CollectionValuedMap<>();
labelWords(label, sents, identifiedWords, null, matchedTokensByPat);
}
public void labelWords(String label, Map sents, Collection