org.unlaxer.Token Maven / Gradle / Ivy

Go to download
package org.unlaxer;

import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;

import org.unlaxer.listener.OutputLevel;
import org.unlaxer.parser.Parser;
import org.unlaxer.reducer.TagBasedReducer.NodeKind;
import org.unlaxer.util.FactoryBoundCache;

public class Token implements Serializable{
	
	private static final long serialVersionUID = -2232289508694932061L;

	static final FactoryBoundCache empties = 
			new FactoryBoundCache<>(RangedString::new);
	
	static final FactoryBoundCache displayString = 
			new FactoryBoundCache<>(
					token->TokenPrinter.get(token,0,OutputLevel.detail,false));

	
	public final Optional tokenString;
	public final Parser parser;
	public final Range tokenRange;
	
	public Optional parent;
	private final List originalChildren;
	public  final List filteredChildren;
	public final TokenKind tokenKind;
	
	
	public Token(TokenKind tokenKind , RangedString token, Parser parser) {
		this(tokenKind , token , parser , new ArrayList<>());
	}
	
	public Token(TokenKind tokenKind , List tokens , Parser parser , int position) {
		this(tokenKind , 
			createRangedString(tokens , position),
			parser,
			tokens);
	}
	
	public Token(TokenKind tokenKind , RangedString token, Parser parser , List children) {
		super();
		this.tokenKind = tokenKind;
		this.tokenString = token.token;
		this.tokenRange = token.range;
		this.parser = parser;
		this.originalChildren = children;
		parent= Optional.empty();
		children.stream().forEach(child->child.parent = Optional.of(this));
		this.filteredChildren = children.stream()
			.filter(childToken->false == childToken.parser.hasTag(NodeKind.notNode.getTag()))
			.collect(Collectors.toList());
	}
	
	public static Token empty(TokenKind tokenKind , int position , Parser parser){
		return new Token(tokenKind , empties.get(position),parser);
	}
	
	public Optional getToken() {
		return tokenString;
	}
	
	public Range getTokenRange() {
		return tokenRange;
	}
	
	public RangedString getRangedString(){
		return new RangedString(tokenRange , tokenString);
	}
	
	public Parser getParser(){
		return parser;
	}

	static RangedString createRangedString(List tokens, int position){
		
		if(tokens.isEmpty()){
			return new RangedString(position);
		}
		
		Optional token = Optional.of(
			tokens.stream()
				.map(Token::getToken)
				.filter(Optional::isPresent)
				.map(Optional::get)
				.collect(Collectors.joining()));
		
		int startIndex = tokens.get(0).tokenRange.startIndexInclusive;
		int endIndex = tokens.get(tokens.size()-1).tokenRange.endIndexExclusive;
		return new RangedString(new Range(startIndex, endIndex), token);
	}
	
	public List flatten(){
		List list = new ArrayList();
		list.add(this);
		for(Token child :originalChildren){
			list.addAll(child.flatten());
		}
		return list;
	}
	
	@Override
	public String toString() {
		return displayString.get(this);
	}
	
	public boolean isTerminalSymbol(){
		return parser.forTerminalSymbol();
	}

	public TokenKind getTokenKind() {
		return tokenKind;
	}
	
	public Token newWithReplacedParser(Parser replace){
		if(false == originalChildren.isEmpty()){
			throw new IllegalArgumentException("not support collected token");
		}
		return new Token(tokenKind,new RangedString(tokenRange, tokenString),replace);
	}
}