org.freshmarker.TokenLineNormalizer Maven / Gradle / Ivy

Go to download

Show more of this group Show more artifacts with this name
Show all versions of freshmarker Show documentation

A simple, small but powerful template engine based loosely on the FreeMarker syntax. FreshMarker is implemented in Java 21 and supports the `java.time` API and Records.

There is a newer version: 1.6.9

Show newest version

package org.freshmarker;

import ftl.Node;
import ftl.Token;
import ftl.Token.TokenType;
import ftl.ast.Root;

import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.Set;

public final class TokenLineNormalizer {

    private static final Set NON_TAG_TOKEN_TYPES = Set.of(TokenType.INTERPOLATE, TokenType.PRINTABLE_CHARS);

    private boolean containNonTag;
    private Token first;

    public void normalize(Root root) {
        root.getAllTokens(false).stream().map(x -> (Token) x).forEach(this::normalizeWhitespaces);
    }

    private void cleanUpLine(Token whitespaceToken) {
        if (!whitespaceToken.toString().endsWith("\n")) {
            addWhitespaceTokenToLine(whitespaceToken);
            return;
        }
        if (first == null || containNonTag) {
            clearLine();
            return;
        }
        getFirstAsWhitespace().ifPresent(f -> f.getParent().remove(f));
        whitespaceToken.getParent().remove(whitespaceToken);
        clearLine();
    }

    private void normalizeWhitespaces(Token token) {
        if (token.getType() == TokenType.EOF) {
            Token previous = token.getPrevious();
            if (previous.getType() == TokenType.WHITESPACE && !previous.toString().contains("\n")) {
                Token previous1 = previous.getPrevious();
                if (previous1.getType() != TokenType.WHITESPACE) {
                    previous.getParent().remove(previous);
                }
            }
        }
        if (token.getType() != TokenType.WHITESPACE) {
            addNonWhitespaceTokenToLine(token);
            return;
        }
        String image = token.toString();
        int index = image.indexOf("\n");
        if (index == -1 || index == image.length() - 1) {
            cleanUpLine(token);
            return;
        }
        int beginOffset = token.getBeginOffset();
        int endOffset = token.getEndOffset();
        Node parent = token.getParent();
        int tokenIndex = parent.indexOf(token);
        parent.remove(token);
        int start = beginOffset;
        List list = new ArrayList<>();
        for (int i = 0; i < image.length(); i++) {
            char c = image.charAt(i);
            if (c == '\n') {
                Token newToken = newToken(token, start, beginOffset + i + 1);
                list.add(newToken);
                parent.add(tokenIndex, newToken);
                tokenIndex++;
                start = beginOffset + i + 1;
            }
        }
        if (start <= endOffset) {
            Token newToken = newToken(token, start, endOffset);
            list.add(newToken);
            parent.add(tokenIndex, newToken);
        }
        list.forEach(this::cleanUpLine);
    }

    private void clearLine() {
        first = null;
        containNonTag = false;
    }

    private void addWhitespaceTokenToLine(Token token) {
        containNonTag = first != null;
        first = first == null ? token : first;
    }

    private void addNonWhitespaceTokenToLine(Token token) {
        containNonTag = containNonTag || NON_TAG_TOKEN_TYPES.contains(token.getType());
        first = first == null ? token : first;
    }

    private Optional getFirstAsWhitespace() {
        return Optional.ofNullable(first).filter(f -> f.getType() == TokenType.WHITESPACE);
    }

    private Token newToken(Token token, int beginOffset, int endOffset) {
        return Token.newToken(TokenType.WHITESPACE, token.getTokenSource(), beginOffset, endOffset);
    }
}