edu.stanford.nlp.ling.tokensregex.SequencePattern Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of stanford-parser Show documentation
Stanford Parser processes raw text in English, Chinese, German, Arabic, and French, and extracts constituency parse trees.
The newest version!
package edu.stanford.nlp.ling.tokensregex;

import edu.stanford.nlp.util.*;

import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.util.*;
import java.util.function.Function;

/**
 * Generic Sequence Pattern for regular expressions.
 *
 * 
 * Similar to Java's {@link java.util.regex.Pattern} except it is for sequences over arbitrary types T instead
 *  of just characters.
 *
 * 
 A regular expression must first be compiled into
 * an instance of this class.  The resulting pattern can then be used to create
 * a {@link SequenceMatcher} object that can match arbitrary sequences of type T
 * against the regular expression.  All of the state involved in performing a match
 * resides in the matcher, so many matchers can share the same pattern.
 *
 * 

 * To support sequence matching on a new type T, the following is needed:
 * 

 *   Implement a {@link NodePattern for matching type T}
 *   Optionally define a language for node matches and implement {@link SequencePattern.Parser} to compile a
 *       regular expression into a SequencePattern.
 *   
 *   Optionally implement a {@link MultiPatternMatcher.NodePatternTrigger}
 *        for optimizing matches across multiple patterns
 *   Optionally implement a {@link NodesMatchChecker} to support backreferences
 * 
 * See {@link TokenSequencePattern} for an example of how this class can be extended
 * to support a specific type {@code T}.
 * 
 * To use
 * 
{@code
 *   SequencePattern p = SequencePattern.compile("....");
 *   SequenceMatcher m = p.getMatcher(tokens);
 *   while (m.find()) ....
 * }
 *
 *
 * 
 * To support a new type {@code T}:
 * 

 *  For a type {@code T} to be matchable, it has to have a corresponding {@code NodePattern} that indicates
 *    whether a node is matched or not  (see {@code CoreMapNodePattern} for example)
 *  To compile a string into corresponding pattern, will need to create a parser
 *    (see inner class {@code Parser}, {@code TokenSequencePattern} and {@code TokenSequenceParser.jj})
 * 
 *
 * 
 * SequencePattern supports the following standard regex features:
 * 

 *  Concatenation 
 *  Or 
 *  Groups  (capturing  / noncapturing )  
 *  Quantifiers (greedy / nongreedy) 
 * 
 *
 * 
 * SequencePattern also supports the following less standard features:
 * 

 *  Environment (see {@link Env}) with respect to which the patterns are compiled
 *  Binding of variables
 * 
Use {@link Env} to bind variables for use when compiling patterns
 * 
Can also bind names to groups (see {@link SequenceMatchResult} for accessor methods to retrieve matched groups)
 * 
 *  Backreference matches - need to specify how back references are to be matched using {@link NodesMatchChecker} 
 *  Multinode matches - for matching of multiple nodes using non-regex (at least not regex over nodes) patterns
 *                        (need to have corresponding {@link MultiNodePattern},
 *                         see {@link MultiCoreMapNodePattern} for example) 
 *  Conjunctions - conjunctions of sequence patterns (works for some cases)
 * 
 *
 * Note that this and the inherited classes do not implement any custom equals and hashCode functions.
 *
 * @author Angel Chang
 * @see SequenceMatcher
 */
public class SequencePattern implements Serializable {

  // TODO:
  //  1. Validate backref capture groupid
  //  2. Actions
  //  3. Inconsistent templating with T
  //  4. Update TokensSequenceParser to handle backref of other attributes (\9{attr1,attr2,...})
  //  5. Improve nested capture groups (in matchresult) for other node types such as conjunctions/disjunctions

  private static final long serialVersionUID = 3484918485303693833L;

  private String patternStr;
  private PatternExpr patternExpr;
  private SequenceMatchAction action;

  State root;
  int totalGroups = 0;

  // binding of group number to variable name
  VarGroupBindings varGroupBindings;

  // Priority associated with the pattern (higher priority patterns should take precedence over lower priority ones)
  double priority = 0.0;
  // Weight associated with the pattern
  double weight = 0.0;

  protected SequencePattern(SequencePattern.PatternExpr nodeSequencePattern) {
    this(null, nodeSequencePattern);
  }

  protected SequencePattern(String patternStr, SequencePattern.PatternExpr nodeSequencePattern) {
    this(patternStr, nodeSequencePattern, null);
  }

  protected SequencePattern(String patternStr, SequencePattern.PatternExpr nodeSequencePattern,
                            SequenceMatchAction action) {
    this.patternStr = patternStr;
    this.patternExpr = nodeSequencePattern;
    this.action = action;

    nodeSequencePattern = new GroupPatternExpr(nodeSequencePattern, true);
    nodeSequencePattern = nodeSequencePattern.optimize();
    this.totalGroups = nodeSequencePattern.assignGroupIds(0);
    Frag f = nodeSequencePattern.build();
    f.connect(MATCH_STATE);
    this.root = f.start;
    varGroupBindings = new VarGroupBindings(totalGroups+1);
    nodeSequencePattern.updateBindings(varGroupBindings);
  }

  @Override
  public String toString() {
    return this.pattern();
  }

  public  SequencePattern transform(NodePatternTransformer transformer) {
    if (action != null) {
      throw new UnsupportedOperationException("transform on actions not yet implemented");
    }
    SequencePattern.PatternExpr transformedPattern = this.patternExpr.transform(transformer);
    // TODO: Make string unique by indicating this pattern was transformed
    return new SequencePattern<>(this.patternStr, transformedPattern, null);
  }

  public String pattern() {
    return patternStr;
  }

  protected PatternExpr getPatternExpr() {
    return patternExpr;
  }

  public double getPriority() {
    return priority;
  }

  public void setPriority(double priority) {
    this.priority = priority;
  }

  public double getWeight() {
    return weight;
  }

  public void setWeight(double weight) {
    this.weight = weight;
  }

  public SequenceMatchAction getAction() {
    return action;
  }

  public void setAction(SequenceMatchAction action) {
    this.action = action;
  }

  public int getTotalGroups() {
    return totalGroups;
  }

  // Compiles string (regex) to NFA for doing pattern simulation
  public static  SequencePattern compile(Env env, String string) {
    try {
      Pair> p = env.parser.parseSequenceWithAction(env, string);
      return new SequencePattern<>(string, p.first(), p.second());
    } catch (Exception ex) {
      throw new RuntimeException("Error compiling " + string + " using environment " + env);
    }
    //throw new UnsupportedOperationException("Compile from string not implemented");
  }

  protected static  SequencePattern compile(SequencePattern.PatternExpr nodeSequencePattern) {
    return new SequencePattern<>(nodeSequencePattern);
  }

  public SequenceMatcher getMatcher(List tokens) {
    return new SequenceMatcher<>(this, tokens);
  }

  public  OUT findNodePattern(Function, OUT> filter) {
    Queue todo = new LinkedList<>();
    Set seen = new HashSet<>();
    todo.add(root);
    seen.add(root);
    while (!todo.isEmpty()) {
      State state = todo.poll();
      if (state instanceof NodePatternState) {
        NodePattern pattern = ((NodePatternState) state).pattern;
        OUT res = filter.apply(pattern);
        if (res != null) return res;
      }
      if (state.next != null) {
        for (State s: state.next) {
          if (!seen.contains(s)) { seen.add(s); todo.add(s); }
        }
      }
    }
    return null;
  }

  public  Collection findNodePatterns(Function, OUT> filter, boolean allowOptional, boolean allowBranching) {
    List outList = new ArrayList<>();
    Queue todo = new LinkedList<>();
    Set seen = new HashSet<>();
    todo.add(root);
    seen.add(root);
    while (!todo.isEmpty()) {
      State state = todo.poll();
      if ((allowOptional || !state.isOptional) && (state instanceof NodePatternState)) {
        NodePattern pattern = ((NodePatternState) state).pattern;
        OUT res = filter.apply(pattern);
        if (res != null) {
          outList.add(res);
        }
      }
      if (state.next != null) {
        boolean addNext = allowBranching || state.next.size() == 1;
        if (addNext) {
          for (State s : state.next) {
            if (!seen.contains(s)) {
              seen.add(s);
              todo.add(s);
            }
          }
        }
      }
    }
    return outList;
  }
  // Parses string to PatternExpr
  public interface Parser {
    SequencePattern.PatternExpr parseSequence(Env env, String s) throws Exception;
    Pair> parseSequenceWithAction(Env env, String s) throws Exception;
    SequencePattern.PatternExpr parseNode(Env env, String s) throws Exception;
  }

  // Binding of variable names to groups
  // matches the group indices
  static class VarGroupBindings {

    final String[] varnames;  // Assumes number of groups low

    protected VarGroupBindings(int size) {
      varnames = new String[size];
    }

    protected void set(int index, String name) {
      varnames[index] = name;
    }
  }

  // Interface indicating when two nodes match
  protected static interface NodesMatchChecker {
    public boolean matches(T o1, T o2);
  }

  public static final NodesMatchChecker