org.snapscript.parse.TokenIndexer Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of snap Show documentation
Show all versions of snap Show documentation
Dynamic scripting for the JVM
package org.snapscript.parse;
import static org.snapscript.parse.TokenType.BINARY;
import static org.snapscript.parse.TokenType.DECIMAL;
import static org.snapscript.parse.TokenType.HEXIDECIMAL;
import static org.snapscript.parse.TokenType.IDENTIFIER;
import static org.snapscript.parse.TokenType.LITERAL;
import static org.snapscript.parse.TokenType.QUALIFIER;
import static org.snapscript.parse.TokenType.SPACE;
import static org.snapscript.parse.TokenType.TEMPLATE;
import static org.snapscript.parse.TokenType.TEXT;
import static org.snapscript.parse.TokenType.TYPE;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
public class TokenIndexer {
private final List values;
private final LineExtractor extractor;
private final GrammarIndexer indexer;
private final TextReader reader;
private final short[] lines;
public TokenIndexer(GrammarIndexer indexer, String resource, char[] original, char[] source, short[] lines, short[] types, int count) {
this.extractor = new LineExtractor(resource, original, count);
this.reader = new TextReader(source, types);
this.values = new ArrayList();
this.indexer = indexer;
this.lines = lines;
}
public short[] index(List tokens) {
if(values.isEmpty()) {
List literals = indexer.list();
for(String literal : literals) {
LiteralValue value = new LiteralValue(literal);
if(!value.isEmpty()) {
values.add(value);
}
}
Collections.sort(values);
}
return scan(tokens);
}
private short[] scan(List tokens) {
int count = reader.count();
while (true) {
int mark = reader.mark();
if(mark >= count) {
return create(tokens);
}
int line = lines[mark];
Token token = literal(line);
if (token == null) {
token = space(line);
}
if (token == null) {
token = template(line);
}
if (token == null) {
token = text(line);
}
if(token == null) {
token = type(line);
}
if(token == null) {
token = identifier(line);
}
if(token == null) {
token = binary(line);
}
if(token == null) {
token = hexidecimal(line);
}
if(token == null) {
token = decimal(line);
}
if(token == null) {
throw new ParseException("Could not parse token at line " + lines[mark]);
}
tokens.add(token);
}
}
private short[] create(List tokens) {
int length = tokens.size();
if(length > 0) {
short[] masks = new short[length];
for(int i = 0; i < length; i++) {
Token token = tokens.get(i);
if(token != null) {
masks[i] = token.getType();
}
}
return masks;
}
return new short[]{};
}
private Token type(int number) {
Line line = extractor.extract(number);
String token = reader.type();
if (token != null) {
return new StringToken(token, line, TYPE.mask | QUALIFIER.mask | IDENTIFIER.mask);
}
return null;
}
private Token identifier(int number) {
Line line = extractor.extract(number);
String token = reader.identifier();
if (token != null) {
return new StringToken(token, line, IDENTIFIER.mask | QUALIFIER.mask);
}
return null;
}
private Token decimal(int number) {
Line line = extractor.extract(number);
Number token = reader.decimal();
if (token != null) {
return new NumberToken(token, line, DECIMAL.mask);
}
return null;
}
private Token binary(int number) {
Line line = extractor.extract(number);
Number token = reader.binary();
if (token != null) {
return new NumberToken(token, line, BINARY.mask | DECIMAL.mask);
}
return null;
}
private Token hexidecimal(int number) {
Line line = extractor.extract(number);
Number token = reader.hexidecimal();
if (token != null) {
return new NumberToken(token, line, HEXIDECIMAL.mask | DECIMAL.mask);
}
return null;
}
private Token template(int number) {
Line line = extractor.extract(number);
String token = reader.template();
if (token != null) {
return new StringToken(token, line, TEMPLATE.mask);
}
return null;
}
private Token text(int number) {
Line line = extractor.extract(number);
String token = reader.text();
if (token != null) {
return new StringToken(token, line, TEXT.mask);
}
return null;
}
private Token space(int number) {
Line line = extractor.extract(number);
Character token = reader.space();
if (token != null) {
return new CharacterToken(token, line, SPACE.mask);
}
return null;
}
private Token literal(int number) {
Line line = extractor.extract(number);
int count = values.size();
for (int i = 0; i < count; i++) {
int mark = reader.mark();
LiteralValue literal = values.get(i);
if (reader.literal(literal.text)) {
char last = literal.text[literal.text.length - 1];
char peek = reader.peek();
if (identifier(last) && identifier(peek)) {
reader.reset(mark);
} else {
if(identifier(last) && special(peek)) {
return new StringToken(literal.value, line, LITERAL.mask | IDENTIFIER.mask | QUALIFIER.mask);
}
return new StringToken(literal.value, line, LITERAL.mask);
}
}
}
return null;
}
private boolean identifier(char value) {
if(value >= 'a' && value <='z') {
return true;
}
if(value >= 'A' && value <= 'Z') {
return true;
}
return value >= '0' && value <= '9';
}
private boolean special(char value) {
switch(value) {
case ')': case '(':
case '.': case '?':
case ',': case '=':
case ';': case ':':
case '+':
return true;
}
return false;
}
private static class LiteralValue implements Comparable {
private final String value;
private final char[] text;
private LiteralValue(String value) {
this.text = value.toCharArray();
this.value = value;
}
@Override
public int compareTo(LiteralValue other) {
if(text.length < other.text.length) {
return 1;
}
if(text.length == other.text.length) {
return 0;
}
return -1;
}
public boolean isEmpty() {
return text.length == 0;
}
@Override
public String toString() {
return value;
}
}
}