All Downloads are FREE. Search and download functionalities are using the official Maven repository.

r.freemarker.2.3.28.source-code.FTL.jj Maven / Gradle / Ivy

Go to download

FreeMarker is a "template engine"; a generic tool to generate text output based on templates.

There is a newer version: 2.3.34
Show newest version
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 * 
 *   http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

options
{
    STATIC = false;
    UNICODE_INPUT = true;
    // DEBUG_TOKEN_MANAGER = true;
    // DEBUG_PARSER = true;
}

PARSER_BEGIN(FMParser)

package freemarker.core;

import freemarker.template.*;
import freemarker.template.utility.*;
import java.io.*;
import java.util.*;
import static freemarker.template.Configuration.*;

/**
 * This class is generated by JavaCC from a grammar file.
 */
public class FMParser {

    private static final int ITERATOR_BLOCK_KIND_LIST = 0; 
    private static final int ITERATOR_BLOCK_KIND_FOREACH = 1; 
    private static final int ITERATOR_BLOCK_KIND_ITEMS = 2; 
    private static final int ITERATOR_BLOCK_KIND_USER_DIRECTIVE = 3; 

    private static class ParserIteratorBlockContext {
        /**
         * loopVarName in <#list ... as loopVarName> or <#items as loopVarName>; null after we left the nested
         * block of #list or #items, respectively.
         */
        private String loopVarName;
        
        /**
         * loopVar1Name in <#list ... as k, loopVar2Name> or <#items as k, loopVar2Name>; null after we left the nested
         * block of #list or #items, respectively.
         */
        private String loopVar2Name;
        
        /**
         * See the ITERATOR_BLOCK_KIND_... costants.
         */
        private int kind;
        
        /**
         * Is this a key-value pair listing? When there's a nested #items, it's only set there. 
         */
        private boolean hashListing;
    }

    private Template template;

    private boolean stripWhitespace, stripText, preventStrippings;
    private int incompatibleImprovements;
    private OutputFormat outputFormat;
    private int autoEscapingPolicy;
    private boolean autoEscaping;
    private ParserConfiguration pCfg;

    /** Keeps track of #list and #foreach nesting. */
    private List iteratorBlockContexts;
    
    /**
     * Keeps track of the nesting depth of directives that support #break.
     */
    private int breakableDirectiveNesting;
    
    /**
     * Keeps track of the nesting depth of directives that support #continue.
     */
    private int continuableDirectiveNesting;
    
    private boolean inMacro, inFunction;
    private LinkedList escapes = new LinkedList();
    private int mixedContentNesting; // for stripText

    /**
     * Create an FM expression parser using a string.
     *
     * @Deprecated This is an internal API of FreeMarker; can be removed any time.
     */
    static public FMParser createExpressionParser(String s) {
        SimpleCharStream scs = new SimpleCharStream(new StringReader(s), 1, 1, s.length());
        FMParserTokenManager token_source = new FMParserTokenManager(scs);
        token_source.SwitchTo(FMParserConstants.FM_EXPRESSION);
        FMParser parser = new FMParser(token_source);
        token_source.setParser(parser);
        return parser;
    }

    /**
     * Constructs a new parser object.
     * 
     * @param template
     *            The template associated with this parser.
     * @param reader
     *            The character stream to use as input
     * @param strictSyntaxMode
     *            Whether FreeMarker directives must start with a #
     *
     * @Deprecated This is an internal API of FreeMarker; will be removed in 2.4.
     */
    public FMParser(Template template, Reader reader, boolean strictSyntaxMode, boolean stripWhitespace) {
        this(template, reader, strictSyntaxMode, stripWhitespace, Configuration.AUTO_DETECT_TAG_SYNTAX);
    }

    /**
     * @Deprecated This is an internal API of FreeMarker; will be changed in 2.4.
     */
    public FMParser(Template template, Reader reader, boolean strictSyntaxMode, boolean stripWhitespace, int tagSyntax) {
        this(template, reader, strictSyntaxMode, stripWhitespace, tagSyntax,
                Configuration.PARSED_DEFAULT_INCOMPATIBLE_ENHANCEMENTS);
    }

    /**
     * @Deprecated This is an internal API of FreeMarker; will be changed in 2.4.
     */
    public FMParser(Template template, Reader reader, boolean strictSyntaxMode, boolean stripWhitespace,
            int tagSyntax, int incompatibleImprovements) {
        this(template, reader, strictSyntaxMode, stripWhitespace,
                tagSyntax, Configuration.AUTO_DETECT_NAMING_CONVENTION, incompatibleImprovements);
    }

    /**
     * @Deprecated This is an internal API of FreeMarker; will be changed in 2.4.
     */
    public FMParser(String template) {
        this(dummyTemplate(),
                new StringReader(template), true, true);
    }

    private static Template dummyTemplate() {
        try {
            return new Template(null, new StringReader(""), Configuration.getDefaultConfiguration());
        } catch (IOException e) {
            throw new RuntimeException("Failed to create dummy template", e);
        }
    }

    /**
     * @Deprecated This is an internal API of FreeMarker; will be changed in 2.4.
     */
    public FMParser(Template template, Reader reader, boolean strictSyntaxMode, boolean whitespaceStripping,
            int tagSyntax, int namingConvention, int incompatibleImprovements) {
        this(template, reader,
                new LegacyConstructorParserConfiguration(
                        strictSyntaxMode, whitespaceStripping,
                        tagSyntax, LEGACY_INTERPOLATION_SYNTAX, namingConvention,
                        template != null ? template.getParserConfiguration().getAutoEscapingPolicy()
                                : Configuration.ENABLE_IF_DEFAULT_AUTO_ESCAPING_POLICY,
                        template != null ? template.getParserConfiguration().getOutputFormat()
                                : null,
                        template != null ? template.getParserConfiguration().getRecognizeStandardFileExtensions()
                                : null,
                        template != null ? template.getParserConfiguration().getTabSize()
                                : null,
                        new Version(incompatibleImprovements),
                        template != null ? template.getArithmeticEngine() : null));
    }

    /**
     * @Deprecated This is an internal API of FreeMarker; don't call it from outside FreeMarker.
     * 
     * @since 2.3.24
     */
    public FMParser(Template template, Reader reader, ParserConfiguration pCfg) {
        this(template, true, readerToTokenManager(reader, pCfg), pCfg);
    }

    private static FMParserTokenManager readerToTokenManager(Reader reader, ParserConfiguration pCfg) {
        SimpleCharStream simpleCharStream = new SimpleCharStream(reader, 1, 1);
        simpleCharStream.setTabSize(pCfg.getTabSize());
        return new FMParserTokenManager(simpleCharStream);
    }

    /**
     * @Deprecated This is an internal API of FreeMarker; don't call it from outside FreeMarker.
     * 
     * @since 2.3.24
     */
    public FMParser(Template template, boolean newTemplate, FMParserTokenManager tkMan, ParserConfiguration pCfg) {
        this(tkMan);

        NullArgumentException.check(pCfg);
        this.pCfg = pCfg;

        NullArgumentException.check(template);
        this.template = template;

        // Hack due to legacy public constructors (removed in 2.4):
        if (pCfg instanceof LegacyConstructorParserConfiguration) {
            LegacyConstructorParserConfiguration lpCfg = (LegacyConstructorParserConfiguration) pCfg;
            lpCfg.setArithmeticEngineIfNotSet(template.getArithmeticEngine());
            lpCfg.setAutoEscapingPolicyIfNotSet(template.getConfiguration().getAutoEscapingPolicy());
            lpCfg.setOutputFormatIfNotSet(template.getOutputFormat());
            lpCfg.setRecognizeStandardFileExtensionsIfNotSet(
                    template.getParserConfiguration().getRecognizeStandardFileExtensions());
            lpCfg.setTabSizeIfNotSet(
                    template.getParserConfiguration().getTabSize());
        }

        int incompatibleImprovements = pCfg.getIncompatibleImprovements().intValue();
        token_source.incompatibleImprovements = incompatibleImprovements;
        this.incompatibleImprovements = incompatibleImprovements;

        {
            OutputFormat outputFormatFromExt;
            if (!pCfg.getRecognizeStandardFileExtensions()
                    || (outputFormatFromExt = getFormatFromStdFileExt()) == null) {
                autoEscapingPolicy = pCfg.getAutoEscapingPolicy();
                outputFormat = pCfg.getOutputFormat();
            } else {
                // Override it
                autoEscapingPolicy = Configuration.ENABLE_IF_DEFAULT_AUTO_ESCAPING_POLICY;
                outputFormat = outputFormatFromExt;
            }
        }
        recalculateAutoEscapingField();

        token_source.setParser(this);

        token_source.strictSyntaxMode = pCfg.getStrictSyntaxMode();

        int tagSyntax = pCfg.getTagSyntax();
        switch (tagSyntax) {
        case Configuration.AUTO_DETECT_TAG_SYNTAX:
            token_source.autodetectTagSyntax = true;
            break;
        case Configuration.ANGLE_BRACKET_TAG_SYNTAX:
            token_source.squBracTagSyntax = false;
            break;
        case Configuration.SQUARE_BRACKET_TAG_SYNTAX:
            token_source.squBracTagSyntax = true;
            break;
        default:
            throw new IllegalArgumentException("Illegal argument for tagSyntax: " + tagSyntax);
        }

        token_source.interpolationSyntax = pCfg.getInterpolationSyntax();

        int namingConvention = pCfg.getNamingConvention();
        switch (namingConvention) {
        case Configuration.AUTO_DETECT_NAMING_CONVENTION:
        case Configuration.CAMEL_CASE_NAMING_CONVENTION:
        case Configuration.LEGACY_NAMING_CONVENTION:
            token_source.initialNamingConvention = namingConvention;
            token_source.namingConvention = namingConvention;
            break;
        default:
            throw new IllegalArgumentException("Illegal argument for namingConvention: " + namingConvention);
        }

        this.stripWhitespace = pCfg.getWhitespaceStripping();

        // If this is a Template under construction, we do the below.
        // If this is just the enclosing Template for ?eval or such, we must not modify it.
        if (newTemplate) {
            _TemplateAPI.setAutoEscaping(template, autoEscaping);
            _TemplateAPI.setOutputFormat(template, outputFormat);
        }
    }
    
    void setupStringLiteralMode(FMParser parentParser, OutputFormat outputFormat) {
        FMParserTokenManager parentTokenSource = parentParser.token_source;
         
        token_source.initialNamingConvention = parentTokenSource.initialNamingConvention;
        token_source.namingConvention = parentTokenSource.namingConvention;
        token_source.namingConventionEstabilisher = parentTokenSource.namingConventionEstabilisher;
        token_source.SwitchTo(NO_DIRECTIVE);
        
        this.outputFormat = outputFormat;
        recalculateAutoEscapingField();                                
        if (incompatibleImprovements < _TemplateAPI.VERSION_INT_2_3_24) {
            // Emulate bug, where the string literal parser haven't inherited the IcI:
            incompatibleImprovements = _TemplateAPI.VERSION_INT_2_3_0;
        }
        
        // So that loop variable built-ins, like ?index, works inside the interpolations in the string literal:
        iteratorBlockContexts = parentParser.iteratorBlockContexts;
    }

    void tearDownStringLiteralMode(FMParser parentParser) {
        // If the naming convention was established inside the string literal, it's inherited by the parent:
        FMParserTokenManager parentTokenSource = parentParser.token_source; 
        parentTokenSource.namingConvention = token_source.namingConvention;
        parentTokenSource.namingConventionEstabilisher = token_source.namingConventionEstabilisher;
    }
    
    /**
     * Used when we need to recreate the source code from the AST (such as for the FM2 to FM3 converter).
     */
    void setPreventStrippings(boolean preventStrippings) {
        this.preventStrippings = preventStrippings;
    }

    private OutputFormat getFormatFromStdFileExt() {
        String sourceName = template.getSourceName();
        if (sourceName == null) {
            return null; // Not possible anyway...
        }

        int ln = sourceName.length();
        if (ln < 5) return null;

        char c = sourceName.charAt(ln - 5);
        if (c != '.') return null;

        c = sourceName.charAt(ln - 4);
        if (c != 'f' && c != 'F') return null;

        c = sourceName.charAt(ln - 3);
        if (c != 't' && c != 'T') return null;

        c = sourceName.charAt(ln - 2);
        if (c != 'l' && c != 'L') return null;

        c = sourceName.charAt(ln - 1);
        try {
            // Note: We get the output formats by name, so that custom overrides take effect.
            if (c == 'h' || c == 'H') {
                return template.getConfiguration().getOutputFormat(HTMLOutputFormat.INSTANCE.getName());
                }
            if (c == 'x' || c == 'X') {
                return template.getConfiguration().getOutputFormat(XMLOutputFormat.INSTANCE.getName());
            }
        } catch (UnregisteredOutputFormatException e) {
            throw new BugException("Unregistered std format", e);
        }
        return null;
    }
    
    /**
     * Updates the {@link #autoEscaping} field based on the {@link #autoEscapingPolicy} and {@link #outputFormat} fields.
     */
    private void recalculateAutoEscapingField() {
        if (outputFormat instanceof MarkupOutputFormat) {
            if (autoEscapingPolicy == Configuration.ENABLE_IF_DEFAULT_AUTO_ESCAPING_POLICY) {
                autoEscaping = ((MarkupOutputFormat) outputFormat).isAutoEscapedByDefault();
            } else if (autoEscapingPolicy == Configuration.ENABLE_IF_SUPPORTED_AUTO_ESCAPING_POLICY) {
                autoEscaping = true;
            } else if (autoEscapingPolicy == Configuration.DISABLE_AUTO_ESCAPING_POLICY) {
                autoEscaping = false;
            } else {
                throw new IllegalStateException("Unhandled autoEscaping enum: " + autoEscapingPolicy);
            }
        } else {
            autoEscaping = false;
        }
    }
    
    MarkupOutputFormat getMarkupOutputFormat() {
        return outputFormat instanceof MarkupOutputFormat ? (MarkupOutputFormat) outputFormat : null;
    }

    /**
     * Don't use it, unless you are developing FreeMarker itself.
     */
    public int _getLastTagSyntax() {
        return token_source.squBracTagSyntax
                ? Configuration.SQUARE_BRACKET_TAG_SYNTAX
                : Configuration.ANGLE_BRACKET_TAG_SYNTAX;
    }
    
    /**
     * Don't use it, unless you are developing FreeMarker itself.
     * The naming convention used by this template; if it couldn't be detected so far, it will be the most probable one.
     * This could be used for formatting error messages, but not for anything serious.
     */
    public int _getLastNamingConvention() {
        return token_source.namingConvention;
    }

    /**
     * Throw an exception if the expression passed in is a String Literal
     */
    private void notStringLiteral(Expression exp, String expected) throws ParseException {
        if (exp instanceof StringLiteral) {
            throw new ParseException(
                    "Found string literal: " + exp + ". Expecting: " + expected,
                    exp);
        }
    }

    /**
     * Throw an exception if the expression passed in is a Number Literal
     */
    private void notNumberLiteral(Expression exp, String expected) throws ParseException {
        if (exp instanceof NumberLiteral) {
            throw new ParseException(
                    "Found number literal: " + exp.getCanonicalForm() + ". Expecting " + expected,
                    exp);
        }
    }

    /**
     * Throw an exception if the expression passed in is a boolean Literal
     */
    private void notBooleanLiteral(Expression exp, String expected) throws ParseException {
        if (exp instanceof BooleanLiteral) {
            throw new ParseException("Found: " + exp.getCanonicalForm() + ". Expecting " + expected, exp);
        }
    }

    /**
     * Throw an exception if the expression passed in is a Hash Literal
     */
    private void notHashLiteral(Expression exp, String expected) throws ParseException {
        if (exp instanceof HashLiteral) {
            throw new ParseException(
                    "Found hash literal: " + exp.getCanonicalForm() + ". Expecting " + expected,
                    exp);
        }
    }

    /**
     * Throw an exception if the expression passed in is a List Literal
     */
    private void notListLiteral(Expression exp, String expected)
            throws ParseException
    {
        if (exp instanceof ListLiteral) {
            throw new ParseException(
                    "Found list literal: " + exp.getCanonicalForm() + ". Expecting " + expected,
                    exp);
        }
    }

    /**
     * Throw an exception if the expression passed in is a literal other than of the numerical type
     */
    private void numberLiteralOnly(Expression exp) throws ParseException {
        notStringLiteral(exp, "number");
        notListLiteral(exp, "number");
        notHashLiteral(exp, "number");
        notBooleanLiteral(exp, "number");
    }

    /**
     * Throw an exception if the expression passed in is not a string.
     */
    private void stringLiteralOnly(Expression exp) throws ParseException {
        notNumberLiteral(exp, "string");
        notListLiteral(exp, "string");
        notHashLiteral(exp, "string");
        notBooleanLiteral(exp, "string");
    }

    /**
     * Throw an exception if the expression passed in is a literal other than of the boolean type
     */
    private void booleanLiteralOnly(Expression exp) throws ParseException {
        notStringLiteral(exp, "boolean (true/false)");
        notListLiteral(exp, "boolean (true/false)");
        notHashLiteral(exp, "boolean (true/false)");
        notNumberLiteral(exp, "boolean (true/false)");
    }

    private Expression escapedExpression(Expression exp) {
        if (!escapes.isEmpty()) {
            return ((EscapeBlock) escapes.getFirst()).doEscape(exp);
        } else {
            return exp;
        }
    }

    private boolean getBoolean(Expression exp, boolean legacyCompat) throws ParseException {
        TemplateModel tm = null;
        try {
            tm = exp.eval(null);
        } catch (Exception e) {
            throw new ParseException(e.getMessage()
                    + "\nCould not evaluate expression: "
                    + exp.getCanonicalForm(),
                    exp,
                    e);
        }
        if (tm instanceof TemplateBooleanModel) {
            try {
                return ((TemplateBooleanModel) tm).getAsBoolean();
            } catch (TemplateModelException tme) {
            }
        }
        if (legacyCompat && tm instanceof TemplateScalarModel) {
            try {
                return StringUtil.getYesNo(((TemplateScalarModel) tm).getAsString());
            } catch (Exception e) {
                throw new ParseException(e.getMessage()
                        + "\nExpecting boolean (true/false), found: " + exp.getCanonicalForm(),
                        exp);
            }
        }
        throw new ParseException("Expecting boolean (true/false) parameter", exp);
    }
    
    void checkCurrentOutputFormatCanEscape(Token start) throws ParseException {
        if (!(outputFormat instanceof MarkupOutputFormat)) {
            throw new ParseException("The current output format can't do escaping: " + outputFormat,
                    template, start);
        }
    }    
    
    private ParserIteratorBlockContext pushIteratorBlockContext() {
        if (iteratorBlockContexts == null) {
            iteratorBlockContexts = new ArrayList(4);
        }
        ParserIteratorBlockContext newCtx = new ParserIteratorBlockContext();
        iteratorBlockContexts.add(newCtx);
        return newCtx;
    }
    
    private void popIteratorBlockContext() {
        iteratorBlockContexts.remove(iteratorBlockContexts.size() - 1);
    }
    
    private ParserIteratorBlockContext peekIteratorBlockContext() {
        int size = iteratorBlockContexts != null ? iteratorBlockContexts.size() : 0;
        return size != 0 ? (ParserIteratorBlockContext) iteratorBlockContexts.get(size - 1) : null; 
    }
    
    private void checkLoopVariableBuiltInLHO(String loopVarName, Expression lhoExp, Token biName)
            throws ParseException {
        int size = iteratorBlockContexts != null ? iteratorBlockContexts.size() : 0;
        for (int i = size - 1; i >= 0; i--) {
            ParserIteratorBlockContext ctx = iteratorBlockContexts.get(i);
            if (loopVarName.equals(ctx.loopVarName) || loopVarName.equals(ctx.loopVar2Name)) {
                if (ctx.kind == ITERATOR_BLOCK_KIND_USER_DIRECTIVE) {
			        throw new ParseException(
			                "The left hand operand of ?" + biName.image
			                + " can't be the loop variable of an user defined directive: "
			                +  loopVarName,
			                lhoExp);
                }
                return;  // success
            }
        }
        throw new ParseException(
                "The left hand operand of ?" + biName.image + " must be a loop variable, "
                + "but there's no loop variable in scope with this name: " + loopVarName,
                lhoExp);
    }
    
	private String forEachDirectiveSymbol() {    
	    // [2.4] Use camel case as the default
	    return token_source.namingConvention == Configuration.CAMEL_CASE_NAMING_CONVENTION ? "#forEach" : "#foreach";
	}
    
}

PARSER_END(FMParser)

/**
 * The lexer portion defines 5 lexical states:
 * DEFAULT, FM_EXPRESSION, IN_PAREN, NO_PARSE, and EXPRESSION_COMMENT.
 * The DEFAULT state is when you are parsing
 * text but are not inside a FreeMarker expression.
 * FM_EXPRESSION is the state you are in
 * when the parser wants a FreeMarker expression.
 * IN_PAREN is almost identical really. The difference
 * is that you are in this state when you are within
 * FreeMarker expression and also within (...).
 * This is a necessary subtlety because the
 * ">" and ">=" symbols can only be used
 * within parentheses because otherwise, it would
 * be ambiguous with the end of a directive.
 * So, for example, you enter the FM_EXPRESSION state
 * right after a ${ and leave it after the matching }.
 * Or, you enter the FM_EXPRESSION state right after
 * an ""
 * that ends the if directive,
 * you go back to DEFAULT lexical state.
 * If, within the FM_EXPRESSION state, you enter a
 * parenthetical expression, you enter the IN_PAREN
 * state.
 * Note that whitespace is ignored in the
 * FM_EXPRESSION and IN_PAREN states
 * but is passed through to the parser as PCDATA in the DEFAULT state.
 * NO_PARSE and EXPRESSION_COMMENT are extremely simple
 * lexical states. NO_PARSE is when you are in a comment
 * block and EXPRESSION_COMMENT is when you are in a comment
 * that is within an FTL expression.
 */
TOKEN_MGR_DECLS:
{

    private static final String PLANNED_DIRECTIVE_HINT
            = "(If you have seen this directive in use elsewhere, this was a planned directive, "
                + "so maybe you need to upgrade FreeMarker.)";

    /**
     * The noparseTag is set when we enter a block of text that the parser more or less ignores. These are  and
     * . This variable tells us what the closing tag should be, and when we hit that, we resume parsing. Note
     * that with this scheme,  and  tags cannot nest recursively, but it is not clear how important
     * that is.
     */
    String noparseTag;

    private FMParser parser;
    private int postInterpolationLexState = -1;
    /**
     * Keeps track of how deeply nested we have the hash literals. This is necessary since we need to be able to
     * distinguish the } used to close a hash literal and the one used to close a ${
     */
    private int curlyBracketNesting;
    private int parenthesisNesting;
    private int bracketNesting;
    private boolean inFTLHeader;
    boolean strictSyntaxMode,
            squBracTagSyntax,
            autodetectTagSyntax,
            tagSyntaxEstablished,
            inInvocation;
    int interpolationSyntax;
    int initialNamingConvention;
    int namingConvention;
    Token namingConventionEstabilisher;
    int incompatibleImprovements;

    void setParser(FMParser parser) {
        this.parser = parser;
    }

    // This method checks if we are in a strict mode where all
    // FreeMarker directives must start with <#. It also handles
    // tag syntax detection. If you update this logic, take a look
    // at the UNKNOWN_DIRECTIVE token too.
    private void handleTagSyntaxAndSwitch(Token tok, int tokenNamingConvention, int newLexState) {
        final String image = tok.image;
        
        // Non-strict syntax (deprecated) only supports legacy naming convention.
        // We didn't push this on the tokenizer because it made it slow, so we filter here.
        if (!strictSyntaxMode
                && (tokenNamingConvention == Configuration.CAMEL_CASE_NAMING_CONVENTION)
                && !isStrictTag(image)) {
            tok.kind = STATIC_TEXT_NON_WS;
            return;
        }
        
        char firstChar = image.charAt(0);
        if (autodetectTagSyntax && !tagSyntaxEstablished) {
            squBracTagSyntax = (firstChar == '[');
        }
        if ((firstChar == '[' && !squBracTagSyntax) || (firstChar == '<' && squBracTagSyntax)) {
            tok.kind = STATIC_TEXT_NON_WS;
            return;
        }
        
        if (!strictSyntaxMode) {
            // Legacy feature (or bug?): Tag syntax never gets estabilished in non-strict mode.
            // We do establish the naming convention though.
            checkNamingConvention(tok, tokenNamingConvention);
            SwitchTo(newLexState);
            return;
        }
        
        // For square bracket tags there's no non-strict token, so we are sure that it's an FTL tag.
        // But if it's an angle bracket tag, we have to check if it's just static text or and FTL tag, because the
        // tokenizer will emit the same kind of token for both.
        if (!squBracTagSyntax && !isStrictTag(image)) {
            tok.kind = STATIC_TEXT_NON_WS;
            return;
        }
        
        // We only get here if this is a strict FTL tag.
        tagSyntaxEstablished = true;
        
        if (incompatibleImprovements >= _TemplateAPI.VERSION_INT_2_3_28
                || interpolationSyntax == SQUARE_BRACKET_INTERPOLATION_SYNTAX) {
	        // For END_xxx tags, as they can't contain expressions, the whole tag is a single token. So this is the only
	        // chance to check if we got something inconsistent like `') {
	            if (!squBracTagSyntax && lastChar != '>' || squBracTagSyntax && lastChar != ']') {
		            throw new TokenMgrError(
		                    "The tag shouldn't end with \""+ lastChar + "\".",
		                    TokenMgrError.LEXICAL_ERROR,
		                    tok.beginLine, tok.beginColumn,
		                    tok.endLine, tok.endColumn);
                }
            } // if end-tag
        }
        
        checkNamingConvention(tok, tokenNamingConvention);
        
        SwitchTo(newLexState);
    }

    void checkNamingConvention(Token tok) {
        checkNamingConvention(tok, _CoreStringUtils.getIdentifierNamingConvention(tok.image)); 
    }
    
    void checkNamingConvention(Token tok, int tokenNamingConvention) {
        if (tokenNamingConvention != Configuration.AUTO_DETECT_NAMING_CONVENTION) {
	        if (namingConvention == Configuration.AUTO_DETECT_NAMING_CONVENTION) {
	            namingConvention = tokenNamingConvention;
	            namingConventionEstabilisher = tok;
	        } else if (namingConvention != tokenNamingConvention) {
                throw newNameConventionMismatchException(tok);
	        }
        }
    }
    
    private TokenMgrError newNameConventionMismatchException(Token tok) {
        return new TokenMgrError(
                "Naming convention mismatch. "
                + "Identifiers that are part of the template language (not the user specified ones) "
                + (initialNamingConvention == Configuration.AUTO_DETECT_NAMING_CONVENTION
                    ? "must consistently use the same naming convention within the same template. This template uses "
                    : "must use the configured naming convention, which is the ")
                + (namingConvention == Configuration.CAMEL_CASE_NAMING_CONVENTION
                            ? "camel case naming convention (like: exampleName) "
                            : (namingConvention == Configuration.LEGACY_NAMING_CONVENTION
                                    ? "legacy naming convention (directive (tag) names are like examplename, " 
                                      + "everything else is like example_name) "
                                    : "??? (internal error)"
                                    ))
                + (namingConventionEstabilisher != null
                        ? "estabilished by auto-detection at "
                            + _MessageUtil.formatPosition(
                                    namingConventionEstabilisher.beginLine, namingConventionEstabilisher.beginColumn)
                            + " by token " + StringUtil.jQuote(namingConventionEstabilisher.image.trim())
                        : "")
                + ", but the problematic token, " + StringUtil.jQuote(tok.image.trim())
                + ", uses a different convention.",
                TokenMgrError.LEXICAL_ERROR,
                tok.beginLine, tok.beginColumn, tok.endLine, tok.endColumn);
    }

    /**
     * Used for tags whose name isn't affected by naming convention.
     */
    private void handleTagSyntaxAndSwitch(Token tok, int newLexState) {
        handleTagSyntaxAndSwitch(tok, Configuration.AUTO_DETECT_NAMING_CONVENTION, newLexState);
    }
    
    private boolean isStrictTag(String image) {
        return image.length() > 2 && (image.charAt(1) == '#' || image.charAt(2) == '#');
    }
    
    /**
     * Detects the naming convention used, both in start- and end-tag tokens.
     *
     * @param charIdxInName
     *         The index of the deciding character relatively to the first letter of the name.
     */
    private static int getTagNamingConvention(Token tok, int charIdxInName) {
        return _CoreStringUtils.isUpperUSASCII(getTagNameCharAt(tok, charIdxInName))
                ? Configuration.CAMEL_CASE_NAMING_CONVENTION : Configuration.LEGACY_NAMING_CONVENTION;
    }

    static char getTagNameCharAt(Token tok, int charIdxInName) {
        final String image = tok.image;
        
        // Skip tag delimiter:
        int idx = 0;
        for (;;) {
            final char c = image.charAt(idx);
            if (c != '<' && c != '[' && c != '/' && c != '#') {
                break;
            }
            idx++;
        }

        return image.charAt(idx + charIdxInName);
    }

    private void unifiedCall(Token tok) {
        char firstChar = tok.image.charAt(0);
        if (autodetectTagSyntax && !tagSyntaxEstablished) {
            squBracTagSyntax = (firstChar == '[');
        }
        if (squBracTagSyntax && firstChar == '<') {
            tok.kind = STATIC_TEXT_NON_WS;
            return;
        }
        if (!squBracTagSyntax && firstChar == '[') {
            tok.kind = STATIC_TEXT_NON_WS;
            return;
        }
        tagSyntaxEstablished = true;
        SwitchTo(NO_SPACE_EXPRESSION);
    }

    private void unifiedCallEnd(Token tok) {
        char firstChar = tok.image.charAt(0);
        if (squBracTagSyntax && firstChar == '<') {
            tok.kind = STATIC_TEXT_NON_WS;
            return;
        }
        if (!squBracTagSyntax && firstChar == '[') {
            tok.kind = STATIC_TEXT_NON_WS;
            return;
        }
    }

    private void startInterpolation(Token tok) {
        if ( 
                interpolationSyntax == LEGACY_INTERPOLATION_SYNTAX
                    && tok.kind == SQUARE_BRACKET_INTERPOLATION_OPENING
                || interpolationSyntax == DOLLAR_INTERPOLATION_SYNTAX
                    && tok.kind != DOLLAR_INTERPOLATION_OPENING
                || interpolationSyntax == SQUARE_BRACKET_INTERPOLATION_SYNTAX
                    && tok.kind != SQUARE_BRACKET_INTERPOLATION_OPENING) {
            tok.kind = STATIC_TEXT_NON_WS;
            return;
        }
        
        if (postInterpolationLexState != -1) {
            // This certainly never occurs, as starting an interpolation in expression mode fails earlier.
            char c = tok.image.charAt(0);
            throw new TokenMgrError(
                    "You can't start an interpolation (" + tok.image + "..."
                    + (interpolationSyntax == SQUARE_BRACKET_INTERPOLATION_SYNTAX ? "]" : "}")
                    + ") here as you are inside another interpolation.)",
                    TokenMgrError.LEXICAL_ERROR,
                    tok.beginLine, tok.beginColumn,
                    tok.endLine, tok.endColumn);
        }
        postInterpolationLexState = curLexState;
        SwitchTo(FM_EXPRESSION);
    }

    private void endInterpolation(Token closingTk) {
        SwitchTo(postInterpolationLexState);
        postInterpolationLexState = -1;
    }
    
    private TokenMgrError newUnexpectedClosingTokenException(Token closingTk) {
            return new TokenMgrError(
                    "You can't have an \"" + closingTk.image + "\" here, as there's nothing open that it could close.",
                    TokenMgrError.LEXICAL_ERROR,
                    closingTk.beginLine, closingTk.beginColumn,
                    closingTk.endLine, closingTk.endColumn);
    }

    private void eatNewline() {
        int charsRead = 0;
        try {
            while (true) {
                char c = input_stream.readChar();
                ++charsRead;
                if (!Character.isWhitespace(c)) {
                    input_stream.backup(charsRead);
                    return;
                } else if (c == '\r') {
                    char next = input_stream.readChar();
                    ++charsRead;
                    if (next != '\n') {
                        input_stream.backup(1);
                    }
                    return;
                } else if (c == '\n') {
                    return;
                }
            }
        } catch (IOException ioe) {
            input_stream.backup(charsRead);
        }
    }

    private void ftlHeader(Token matchedToken) {
        if (!tagSyntaxEstablished) {
            squBracTagSyntax = matchedToken.image.charAt(0) == '[';
            tagSyntaxEstablished = true;
            autodetectTagSyntax = false;
        }
        String img = matchedToken.image;
        char firstChar = img.charAt(0);
        char lastChar = img.charAt(img.length() - 1);
        if ((firstChar == '[' && !squBracTagSyntax) || (firstChar == '<' && squBracTagSyntax)) {
            matchedToken.kind = STATIC_TEXT_NON_WS;
        }
        if (matchedToken.kind != STATIC_TEXT_NON_WS) {
            if (lastChar != '>' && lastChar != ']') {
                SwitchTo(FM_EXPRESSION);
                inFTLHeader = true;
            } else {
                eatNewline();
            }
        }
    }
}

TOKEN:
{
    <#BLANK : " " | "\t" | "\n" | "\r">
    |
    <#START_TAG : "<" | "<#" | "[#">
    |
    <#END_TAG : "
    |
    <#CLOSE_TAG1 : ()* (">" | "]")>
    |
    <#CLOSE_TAG2 : ()* ("/")? (">" | "]")>
    |
    /*
     * ATTENTION: Update _CoreAPI.*_BUILT_IN_DIRECTIVE_NAMES if you add new directives!
     */
     "attempt" > { handleTagSyntaxAndSwitch(matchedToken, DEFAULT); }
    |
     "recover" > { handleTagSyntaxAndSwitch(matchedToken, DEFAULT); } 
    |
     "if" > { handleTagSyntaxAndSwitch(matchedToken, FM_EXPRESSION); }
    |
     "else" ("i" | "I") "f" > {
        handleTagSyntaxAndSwitch(matchedToken, getTagNamingConvention(matchedToken, 4), FM_EXPRESSION);
    }
    |
     "list" > { handleTagSyntaxAndSwitch(matchedToken, FM_EXPRESSION); }
    |
     "items" ()+  > { handleTagSyntaxAndSwitch(matchedToken, FM_EXPRESSION); }
    |
     "sep" >
    |
     "for" ("e" | "E") "ach" > {
        handleTagSyntaxAndSwitch(matchedToken, getTagNamingConvention(matchedToken, 3), FM_EXPRESSION);
    }
    |
     "switch" > { handleTagSyntaxAndSwitch(matchedToken, FM_EXPRESSION); }
    |
     "case" > { handleTagSyntaxAndSwitch(matchedToken, FM_EXPRESSION); }
    |
     "assign" > { handleTagSyntaxAndSwitch(matchedToken, FM_EXPRESSION); }
    |
     "global" > { handleTagSyntaxAndSwitch(matchedToken, FM_EXPRESSION); }
    |
     "local" > { handleTagSyntaxAndSwitch(matchedToken, FM_EXPRESSION); }
    |
    <_INCLUDE :  "include" > { handleTagSyntaxAndSwitch(matchedToken, FM_EXPRESSION); }
    |
     "import" > { handleTagSyntaxAndSwitch(matchedToken, FM_EXPRESSION); }
    |
     "function" > { handleTagSyntaxAndSwitch(matchedToken, FM_EXPRESSION); }
    |
     "macro" > { handleTagSyntaxAndSwitch(matchedToken, FM_EXPRESSION); }
    |
     "transform" > { handleTagSyntaxAndSwitch(matchedToken, FM_EXPRESSION); }
    |
     "visit" > { handleTagSyntaxAndSwitch(matchedToken, FM_EXPRESSION); }
    |
     "stop" > { handleTagSyntaxAndSwitch(matchedToken, FM_EXPRESSION); }
    |
     "return" > { handleTagSyntaxAndSwitch(matchedToken, FM_EXPRESSION); }
    |
     "call" > { handleTagSyntaxAndSwitch(matchedToken, FM_EXPRESSION); }
    |
     "setting" > { handleTagSyntaxAndSwitch(matchedToken, FM_EXPRESSION); }
    |
     "output" ("f"|"F") "ormat" > {
        handleTagSyntaxAndSwitch(matchedToken, getTagNamingConvention(matchedToken, 6), FM_EXPRESSION);
    }
    |
     "auto" ("e"|"E") "sc" > {
        handleTagSyntaxAndSwitch(matchedToken, getTagNamingConvention(matchedToken, 4), DEFAULT);
    }
    |
     "no" ("autoe"|"AutoE") "sc" > {
        handleTagSyntaxAndSwitch(matchedToken, getTagNamingConvention(matchedToken, 2), DEFAULT);
    }
    |
     "compress" > { handleTagSyntaxAndSwitch(matchedToken, DEFAULT); }
    |
     "comment" > {
        handleTagSyntaxAndSwitch(matchedToken, NO_PARSE); noparseTag = "comment";
    }
    |
     { noparseTag = "-->"; handleTagSyntaxAndSwitch(matchedToken, NO_PARSE); }
    |
     "no" ("p" | "P") "arse" > {
        int tagNamingConvention = getTagNamingConvention(matchedToken, 2);
        handleTagSyntaxAndSwitch(matchedToken, tagNamingConvention, NO_PARSE);
        noparseTag = tagNamingConvention == Configuration.CAMEL_CASE_NAMING_CONVENTION ? "noParse" : "noparse";
    }
    |
     "if" > { handleTagSyntaxAndSwitch(matchedToken, DEFAULT); }
    |
     "list" > { handleTagSyntaxAndSwitch(matchedToken, DEFAULT); }
    |
     "items" > { handleTagSyntaxAndSwitch(matchedToken, DEFAULT); }
    |
     "sep" > { handleTagSyntaxAndSwitch(matchedToken, DEFAULT); }
    |
     "recover" > { handleTagSyntaxAndSwitch(matchedToken, DEFAULT); }
    |
     "attempt" > { handleTagSyntaxAndSwitch(matchedToken, DEFAULT); }
    |
     "for" ("e" | "E") "ach" > {
        handleTagSyntaxAndSwitch(matchedToken, getTagNamingConvention(matchedToken, 3), DEFAULT);
    }
    |
     "local" > { handleTagSyntaxAndSwitch(matchedToken, DEFAULT); }
    |
     "global" > { handleTagSyntaxAndSwitch(matchedToken, DEFAULT); }
    |
     "assign" > { handleTagSyntaxAndSwitch(matchedToken, DEFAULT); }
    |
     "function" > { handleTagSyntaxAndSwitch(matchedToken, DEFAULT); }
    |
     "macro" > { handleTagSyntaxAndSwitch(matchedToken, DEFAULT); }
    |
     "output" ("f" | "F") "ormat" > {
        handleTagSyntaxAndSwitch(matchedToken, getTagNamingConvention(matchedToken, 6), DEFAULT);
    }
    |
     "auto" ("e" | "E") "sc" > {
        handleTagSyntaxAndSwitch(matchedToken, getTagNamingConvention(matchedToken, 4), DEFAULT);
    }
    |
     "no" ("autoe"|"AutoE") "sc" > {
        handleTagSyntaxAndSwitch(matchedToken, getTagNamingConvention(matchedToken, 2), DEFAULT);
    }
    |
     "compress" > { handleTagSyntaxAndSwitch(matchedToken, DEFAULT); }
    |
     "transform" > { handleTagSyntaxAndSwitch(matchedToken, DEFAULT); }
    |
     "switch" > { handleTagSyntaxAndSwitch(matchedToken, DEFAULT); }
    |
     "else" > { handleTagSyntaxAndSwitch(matchedToken, DEFAULT); }
    |
     "break" > { handleTagSyntaxAndSwitch(matchedToken, DEFAULT); }
    |
     "continue" > { handleTagSyntaxAndSwitch(matchedToken, DEFAULT); }
    |
     "return" > { handleTagSyntaxAndSwitch(matchedToken, DEFAULT); }
    |
     "stop" > { handleTagSyntaxAndSwitch(matchedToken, DEFAULT); }
    |
     "flush" > { handleTagSyntaxAndSwitch(matchedToken, DEFAULT); }
    |
     "t" > { handleTagSyntaxAndSwitch(matchedToken, DEFAULT); }
    |
     "lt" > { handleTagSyntaxAndSwitch(matchedToken, DEFAULT); }
    |
     "rt" > { handleTagSyntaxAndSwitch(matchedToken, DEFAULT); }
    |
     "nt" > { handleTagSyntaxAndSwitch(matchedToken, DEFAULT); }
    |
     "default" > { handleTagSyntaxAndSwitch(matchedToken, DEFAULT); }
    |
     "nested" > { handleTagSyntaxAndSwitch(matchedToken, DEFAULT); }
    |
     "nested" > { handleTagSyntaxAndSwitch(matchedToken, FM_EXPRESSION); }
    |
     "recurse" > { handleTagSyntaxAndSwitch(matchedToken, DEFAULT); }
    |
     "recurse" > { handleTagSyntaxAndSwitch(matchedToken, FM_EXPRESSION); }
    |
     "fallback" > { handleTagSyntaxAndSwitch(matchedToken, DEFAULT); }
    |
     "escape" > { handleTagSyntaxAndSwitch(matchedToken, FM_EXPRESSION); }
    |
     "escape" > { handleTagSyntaxAndSwitch(matchedToken, DEFAULT); }
    |
     "no" ("e" | "E") "scape" > {
        handleTagSyntaxAndSwitch(matchedToken, getTagNamingConvention(matchedToken, 2), DEFAULT);
    }
    |
     "no" ("e" | "E") "scape" > {
        handleTagSyntaxAndSwitch(matchedToken, getTagNamingConvention(matchedToken, 2), DEFAULT);
    }
    |
     { unifiedCall(matchedToken); }
    |
    ) (".")*)? > { unifiedCallEnd(matchedToken); }
    |
    > { ftlHeader(matchedToken); }
    |
    " | "]")> { ftlHeader(matchedToken); }
    |
    /*
     * ATTENTION: Update _CoreAPI.*_BUILT_IN_DIRECTIVE_NAMES if you add new directives!
     */
    
    {
        if (!tagSyntaxEstablished && incompatibleImprovements < _TemplateAPI.VERSION_INT_2_3_19) {
            matchedToken.kind = STATIC_TEXT_NON_WS;
        } else {
            char firstChar = matchedToken.image.charAt(0);

            if (!tagSyntaxEstablished && autodetectTagSyntax) {
                squBracTagSyntax = (firstChar == '[');
                tagSyntaxEstablished = true;
            }

            if (firstChar == '<' && squBracTagSyntax) {
                matchedToken.kind = STATIC_TEXT_NON_WS;
            } else if (firstChar == '[' && !squBracTagSyntax) {
                matchedToken.kind = STATIC_TEXT_NON_WS;
            } else if (strictSyntaxMode) {
                String dn = matchedToken.image;
                int index = dn.indexOf('#');
                dn = dn.substring(index + 1);

                // Until the tokenizer/parser is reworked, we have this quirk where something like <#list>
                // doesn't match any directive starter tokens, because that token requires whitespace after the
                // name as it should be followed by parameters. For now we work this around so we don't report
                // unknown directive:
                if (_CoreAPI.ALL_BUILT_IN_DIRECTIVE_NAMES.contains(dn)) {
                    throw new TokenMgrError(
                            "#" + dn + " is an existing directive, but the tag is malformed. " 
                            + " (See FreeMarker Manual / Directive Reference.)",
                            TokenMgrError.LEXICAL_ERROR,
                            matchedToken.beginLine, matchedToken.beginColumn + 1,
                            matchedToken.endLine, matchedToken.endColumn);
                }

                String tip = null;
                if (dn.equals("set") || dn.equals("var")) {
                    tip = "Use #assign or #local or #global, depending on the intented scope "
                          + "(#assign is template-scope). " + PLANNED_DIRECTIVE_HINT;
                } else if (dn.equals("else_if") || dn.equals("elif")) {
                	tip = "Use #elseif.";
                } else if (dn.equals("no_escape")) {
                	tip = "Use #noescape instead.";
                } else if (dn.equals("method")) {
                	tip = "Use #function instead.";
                } else if (dn.equals("head") || dn.equals("template") || dn.equals("fm")) {
                	tip = "You may meant #ftl.";
                } else if (dn.equals("try") || dn.equals("atempt")) {
                	tip = "You may meant #attempt.";
                } else if (dn.equals("for") || dn.equals("each") || dn.equals("iterate") || dn.equals("iterator")) {
                    tip = "You may meant #list (http://freemarker.org/docs/ref_directive_list.html).";
                } else if (dn.equals("prefix")) {
                    tip = "You may meant #import. " + PLANNED_DIRECTIVE_HINT;
                } else if (dn.equals("item") || dn.equals("row") || dn.equals("rows")) {
                    tip = "You may meant #items.";
                } else if (dn.equals("separator") || dn.equals("separate") || dn.equals("separ")) {
                    tip = "You may meant #sep.";
                } else {
                    tip = "Help (latest version): http://freemarker.org/docs/ref_directive_alphaidx.html; "
                            + "you're using FreeMarker " + Configuration.getVersion() + ".";
                }
                throw new TokenMgrError(
                        "Unknown directive: #" + dn + (tip != null ? ". " + tip : ""),
                        TokenMgrError.LEXICAL_ERROR,
                        matchedToken.beginLine, matchedToken.beginColumn + 1,
                        matchedToken.endLine, matchedToken.endColumn);
            }
        }
    }
}

 TOKEN :
{
    
    |
    
    |
     // to handle a lone dollar sign or "<" or "# or <@ with whitespace after"
    |
     { startInterpolation(matchedToken); }
    |
     { startInterpolation(matchedToken); }
    |
    
    |
    < ("<" | "[") ("#" | "!") "--"> : EXPRESSION_COMMENT
}

 SKIP:
{
    < (~["-", ">", "]"])+ >
    |
    < ">">
    |
    < "]">
    |
    < "-">
    |
    < "-->" | "--]">
    {
        if (parenthesisNesting > 0) SwitchTo(IN_PAREN);
        else if (inInvocation) SwitchTo(NAMED_PARAMETER_EXPRESSION);
        else SwitchTo(FM_EXPRESSION);
    }
}

 TOKEN :
{
    <#ESCAPED_CHAR :
        "\\"
        (
            ("n" | "t" | "r" | "f" | "b" | "g" | "l" | "a" | "\\" | "'" | "\"" | "{" | "=")
            |
            ("x" ["0"-"9", "A"-"F", "a"-"f"])
        )
    >
    | 
    )*
            "\""
        )
        |
        (
            "'"
            ((~["'", "\\"]) | )*
            "'"
        )
    >
    |
    
    |
    
    |
    
    |
    
    |
     "." >
    |
    
    |
    
    |
    
    |
    
    |
    
    |
    
    |
    
    |
    
    |
    
    |
    
    |
    
    |
    
    |
    
    |
    
    |
    
    |
    
    |
    
    |
    
    |
    
    |
    
    |
    
    |
    
    |
    
    |
    
    |
    
    {
        ++bracketNesting;
    }
    |
    
    {
        if (bracketNesting > 0) {
            --bracketNesting;
        } else if (interpolationSyntax == SQUARE_BRACKET_INTERPOLATION_SYNTAX && postInterpolationLexState != -1) {
            endInterpolation(matchedToken);
        } else {
            // There's a legacy glitch where you can always close a tag with `]`, like `<#if x]`. We have to keep that
            // working for backward compatibility, hence we don't always throw at !squBracTagSyntax:
            if (!squBracTagSyntax
                    && (incompatibleImprovements >= _TemplateAPI.VERSION_INT_2_3_28
                            || interpolationSyntax == SQUARE_BRACKET_INTERPOLATION_SYNTAX)
                    || postInterpolationLexState != -1 /* We're in an interpolation => We aren't in a tag */) {
                throw newUnexpectedClosingTokenException(matchedToken);
            }
            
            // Close tag, either legally or to emulate legacy glitch:
            matchedToken.kind = DIRECTIVE_END;
            if (inFTLHeader) {
                eatNewline();
                inFTLHeader = false;
            }
            SwitchTo(DEFAULT);
        }
    }
    |
    
    {
        ++parenthesisNesting;
        if (parenthesisNesting == 1) SwitchTo(IN_PAREN);
    }
    |
    
    {
        --parenthesisNesting;
        if (parenthesisNesting == 0) {
            if (inInvocation) SwitchTo(NAMED_PARAMETER_EXPRESSION);
            else SwitchTo(FM_EXPRESSION);
        }
    }
    |
    
    {
        ++curlyBracketNesting;
    }
    |
    
    {
        if (curlyBracketNesting > 0) {
            --curlyBracketNesting;
        } else if (interpolationSyntax != SQUARE_BRACKET_INTERPOLATION_SYNTAX && postInterpolationLexState != -1) {
            endInterpolation(matchedToken);
        } else {
            throw newUnexpectedClosingTokenException(matchedToken);
        }
    }
    |
    
    |
    
    |
    
    |
     (|)*> {
        // Remove backslashes from Token.image:
        final String s = matchedToken.image;
        if (s.indexOf('\\') != -1) {
            final int srcLn = s.length(); 
            final char[] newS = new char[srcLn - 1];
            int dstIdx = 0;
            for (int srcIdx = 0; srcIdx < srcLn; srcIdx++) {
                final char c = s.charAt(srcIdx);
                if (c != '\\') {
                    newS[dstIdx++] = c;
                }
            }
            matchedToken.image = new String(newS, 0, dstIdx);
        }
    }
    |
    
    |
    <#ESCAPED_ID_CHAR: "\\" ("-" | "." | ":")>
    |
    <#ID_START_CHAR: |>
    |
    <#ASCII_DIGIT: ["0" - "9"]>
}

 TOKEN :
{
    ">
    {
        if (inFTLHeader) {
	        eatNewline();
	        inFTLHeader = false;
        }
        if (squBracTagSyntax || postInterpolationLexState != -1 /* We are in an interpolation */) {
            matchedToken.kind = NATURAL_GT;
        } else {
            SwitchTo(DEFAULT);
        }
    }
    |
    " | "/]">
    {
        if (tagSyntaxEstablished && (incompatibleImprovements >= _TemplateAPI.VERSION_INT_2_3_28
                || interpolationSyntax == SQUARE_BRACKET_INTERPOLATION_SYNTAX)) {
            String image = matchedToken.image;
            char lastChar = image.charAt(image.length() - 1);
            if (!squBracTagSyntax && lastChar != '>' || squBracTagSyntax && lastChar != ']') {
                throw new TokenMgrError(
                        "The tag shouldn't end with \"" + lastChar + "\".",
                        TokenMgrError.LEXICAL_ERROR,
                        matchedToken.beginLine, matchedToken.beginColumn,
                        matchedToken.endLine, matchedToken.endColumn);
            }
        }
    
        if (inFTLHeader) {
	        eatNewline();
	        inFTLHeader = false;
        }
        SwitchTo(DEFAULT);
    }
}

 TOKEN :
{
    ">
    |
    =">
}

 TOKEN :
{
     : FM_EXPRESSION
}

 TOKEN :
{
     : FM_EXPRESSION
}

 TOKEN :
{
    " | "--]">
    {
        if (noparseTag.equals("-->")) {
            boolean squareBracket = matchedToken.image.endsWith("]");
            if ((squBracTagSyntax && squareBracket) || (!squBracTagSyntax && !squareBracket)) {
                matchedToken.image = matchedToken.image + ";"; 
                SwitchTo(DEFAULT);
            }
        }
    }
    |
    " | "]")
    >
    {
        StringTokenizer st = new StringTokenizer(image.toString(), " \t\n\r<>[]/#", false);
        if (st.nextToken().equals(noparseTag)) {
            matchedToken.image = matchedToken.image + ";"; 
            SwitchTo(DEFAULT);
        }
    }
    |
    
    |
    
}

// Now the actual parsing code, starting
// with the productions for FreeMarker's
// expression syntax.

/**
 * This is the same as OrExpression, since
 * the OR is the operator with the lowest
 * precedence.
 */
Expression Expression() :
{
    Expression exp;
}
{
    exp = OrExpression()
    {
        return exp;
    }
}

/**
 * Should be called HighestPrecedenceExpression.
 * Deals with the operators that have the highest precedence. Also deals with `exp!default` and `exp!`, due to parser
 * tricks needed because of the last.
 */
Expression PrimaryExpression() :
{
    Expression exp;
}
{
    exp = AtomicExpression()
    (
        exp = DotVariable(exp)
        |
        exp = DynamicKey(exp)
        |
        exp = MethodArgs(exp)
        |
        exp = BuiltIn(exp)
        |
        exp = DefaultTo(exp)
        |
        exp = Exists(exp)
    )*
    {
        return exp;
    }
}

/**
 * Lowest level expression, a literal, a variable,
 * or a possibly more complex expression bounded
 * by parentheses.
 */
Expression AtomicExpression() :
{
    Expression exp;
}
{
    (
        exp = NumberLiteral()
        |   
        exp = HashLiteral()
        |   
        exp = StringLiteral(true)
        |   
        exp = BooleanLiteral()
        |   
        exp = ListLiteral()
        |   
        exp = Identifier()
        |   
        exp = Parenthesis()
        |   
        exp = BuiltinVariable()
    )
    {
        return exp;
    }
}

Expression Parenthesis() :
{
    Expression exp, result;
    Token start, end;
}
{
    start = 
    exp = Expression()
    end = 
    {
        result = new ParentheticalExpression(exp);
        result.setLocation(template, start, end);
        return result;
    }
}

/**
 * Should be called UnaryPrefixExpression.
 * A primary expression preceded by zero or more unary prefix operators.
 */
Expression UnaryExpression() :
{
    Expression exp, result;
    boolean haveNot = false;
    Token t = null, start = null;
}
{
    (
        result = UnaryPlusMinusExpression()
        |
        result = NotExpression()
        |
        result = PrimaryExpression()
    )
    {
        return result;
    }
}

Expression NotExpression() : 
{
    Token t;
    Expression exp, result = null;
    ArrayList nots = new ArrayList();
}
{
    (
        t =  { nots.add(t); }
    )+
    exp = PrimaryExpression()
    {
        for (int i = 0; i < nots.size(); i++) {
            result = new NotExpression(exp);
            Token tok = (Token) nots.get(nots.size() -i -1);
            result.setLocation(template, tok, exp);
            exp = result;
        }
        return result;
    }
}

Expression UnaryPlusMinusExpression() :
{
    Expression exp, result;
    boolean isMinus = false;
    Token t;
}
{
    (
        t = 
        |
        t =  { isMinus = true; }
    )
    exp = PrimaryExpression()
    {
        result = new UnaryPlusMinusExpression(exp, isMinus);  
        result.setLocation(template, t, exp);
        return result;
    }
}

Expression AdditiveExpression() :
{
    Expression lhs, rhs, result;
    boolean plus;
}
{
    lhs = MultiplicativeExpression() { result = lhs; }
    (
        LOOKAHEAD(|)
        (
            (
                 { plus = true; }
                |
                 { plus = false; }
            )
        )
        rhs = MultiplicativeExpression()
        {
            if (plus) {
	            // plus is treated separately, since it is also
	            // used for concatenation.
                result = new AddConcatExpression(lhs, rhs);
            } else {
                numberLiteralOnly(lhs);
                numberLiteralOnly(rhs);
                result = new ArithmeticExpression(lhs, rhs, ArithmeticExpression.TYPE_SUBSTRACTION);
            }
            result.setLocation(template, lhs, rhs);
            lhs = result;
        }
    )*
    {
        return result;
    }
}

/**
 * A unary prefix expression followed by zero or more
 * unary prefix expressions with operators in between.
 */
Expression MultiplicativeExpression() :
{
    Expression lhs, rhs, result;
    int operation = ArithmeticExpression.TYPE_MULTIPLICATION;
}
{
    lhs = UnaryExpression() { result = lhs; }
    (
        LOOKAHEAD(||)
        (
            (
                 { operation = ArithmeticExpression.TYPE_MULTIPLICATION; }
                |
                 { operation = ArithmeticExpression.TYPE_DIVISION; }
                |
                 {operation = ArithmeticExpression.TYPE_MODULO; }
            )
        )
        rhs = UnaryExpression()
        {
            numberLiteralOnly(lhs);
            numberLiteralOnly(rhs);
            result = new ArithmeticExpression(lhs, rhs, operation);
            result.setLocation(template, lhs, rhs);
            lhs = result;
        }
    )*
    {
        return result;
    }
}


Expression EqualityExpression() :
{
    Expression lhs, rhs, result;
    Token t;
}
{
    lhs = RelationalExpression() { result = lhs; }
    [
        LOOKAHEAD(||)
        (
            t =  
            |
            t =  
            |
            t = 
        )
        rhs = RelationalExpression()
        {
	        notHashLiteral(lhs, "string");
	        notHashLiteral(rhs, "string");
	        notListLiteral(lhs, "string");
	        notListLiteral(rhs, "string");
	        result = new ComparisonExpression(lhs, rhs, t.image);
	        result.setLocation(template, lhs, rhs);
        }
    ]
    {
        return result;
    }
}

Expression RelationalExpression() :
{
    Expression lhs, rhs, result;
    Token t;
}
{
    lhs = RangeExpression() { result = lhs; }
    [
        LOOKAHEAD(||||||)
        (
            t = 
            |
            t = 
            |
            t = 
            |
            t = 
            |
            t = 
            |
            t = 
        )
        rhs = RangeExpression()
        {
            notHashLiteral(lhs, "number");
            notHashLiteral(rhs, "number");
            notListLiteral(lhs, "number");
            notListLiteral(rhs, "number");
            notStringLiteral(lhs, "number");
            notStringLiteral(rhs, "number");
            result = new ComparisonExpression(lhs, rhs, t.image);
            result.setLocation(template, lhs, rhs);
        }
    ]
    {
        return result;
    }
}

Expression RangeExpression() :
{
    Expression lhs, rhs = null, result;
    int endType;
    Token dotDot = null;
}
{
    lhs = AdditiveExpression() { result = lhs; }
    [
        LOOKAHEAD(1)  // To suppress warning
        (
            (
                (
                     { endType = Range.END_EXCLUSIVE; }
                    |
                     { endType = Range.END_SIZE_LIMITED; }
                )
                rhs = AdditiveExpression()
            )
            | 
            (
                dotDot =  { endType = Range.END_UNBOUND; }
                [
                    LOOKAHEAD(AdditiveExpression())
                    rhs = AdditiveExpression()
                    {
                        endType = Range.END_INCLUSIVE;
                    }
                ]
            )
        )
        {
            numberLiteralOnly(lhs);
            if (rhs != null) {
                numberLiteralOnly(rhs);
            }
           
            Range range = new Range(lhs, rhs, endType);
            if (rhs != null) {
                range.setLocation(template, lhs, rhs);
            } else {
                range.setLocation(template, lhs, dotDot);
            }
            result = range;
        }
    ]
    {
        return result;
    }
}




Expression AndExpression() :
{
    Expression lhs, rhs, result;
}
{
    lhs = EqualityExpression() { result = lhs; }
    (
        LOOKAHEAD()
        
        rhs = EqualityExpression()
        {
            booleanLiteralOnly(lhs);
            booleanLiteralOnly(rhs);
            result = new AndExpression(lhs, rhs);
            result.setLocation(template, lhs, rhs);
            lhs = result;
        }
    )*
    {
        return result;
    }
}

Expression OrExpression() :
{
    Expression lhs, rhs, result;
}
{
    lhs = AndExpression() { result = lhs; }
    (
        LOOKAHEAD()
        
        rhs = AndExpression()
        {
            booleanLiteralOnly(lhs);
            booleanLiteralOnly(rhs);
            result = new OrExpression(lhs, rhs);
            result.setLocation(template, lhs, rhs);
            lhs = result;
        }
    )*
    {
        return result;
    }
}

ListLiteral ListLiteral() :
{
    ArrayList values = new ArrayList();
    Token begin, end;
}
{
    begin = 
    values = PositionalArgs()
    end = 
    {
        ListLiteral result = new ListLiteral(values);
        result.setLocation(template, begin, end);
        return result;
    }
}

Expression NumberLiteral() :
{
    Token op = null, t;
}
{
    (
        t = 
        |
        t = 
    )
    {
        String s = t.image;
        Expression result = new NumberLiteral(pCfg.getArithmeticEngine().toNumber(s));
        Token startToken = (op != null) ? op : t;
        result.setLocation(template, startToken, t);
        return result;
    }
}

Identifier Identifier() :
{
    Token t;
}
{
    t = 
    {
        Identifier id = new Identifier(t.image);
        id.setLocation(template, t, t);
        return id;
    }
}

Expression IdentifierOrStringLiteral() :
{
    Expression exp;
}
{
    (
        exp = Identifier()
        |
        exp = StringLiteral(false)
    )
    {
        return exp;
    }   
}

BuiltinVariable BuiltinVariable() :
{
    Token dot, name;
}
{
    dot = 
    name = 
    {
        BuiltinVariable result = null;
        token_source.checkNamingConvention(name);

        TemplateModel parseTimeValue;
        String nameStr = name.image;
        if (nameStr.equals(BuiltinVariable.OUTPUT_FORMAT) || nameStr.equals(BuiltinVariable.OUTPUT_FORMAT_CC)) {
            parseTimeValue = new SimpleScalar(outputFormat.getName());
        } else if (nameStr.equals(BuiltinVariable.AUTO_ESC) || nameStr.equals(BuiltinVariable.AUTO_ESC_CC)) {
            parseTimeValue = autoEscaping ? TemplateBooleanModel.TRUE : TemplateBooleanModel.FALSE;
        } else {
            parseTimeValue = null;
        }
        
        result = new BuiltinVariable(name, token_source, parseTimeValue);
        
        result.setLocation(template, dot, name);
        return result;
    }
}

Expression DefaultTo(Expression exp) :
{
    Expression rhs = null;
    Token t;
}
{
    (
        t = 
        |
        (
            t = 
            [
                LOOKAHEAD(Expression())
                rhs = Expression()
            ]
        )
    )
    {
        DefaultToExpression result = new DefaultToExpression(exp, rhs);
        if (rhs == null) {
            //  contains the whitespace after the `!`, so we have to use the t.beginXxx:
            result.setLocation(template, exp.beginColumn, exp.beginLine, t.beginColumn, t.beginLine);
        } else {
            result.setLocation(template, exp, rhs);
        }
        return result;
    }
}

Expression Exists(Expression exp) :
{
    Token t;
}
{
    t = 
    {
        ExistsExpression result = new ExistsExpression(exp);
        result.setLocation(template, exp, t);
        return result;
    }
}

Expression BuiltIn(Expression lhoExp) :
{
    Token t = null;
    BuiltIn result;
    ArrayList/**/ args = null;
    Token openParen;
    Token closeParen;
}
{
    
    t = 
    {
        token_source.checkNamingConvention(t);
        result = BuiltIn.newBuiltIn(incompatibleImprovements, lhoExp, t, token_source);
        result.setLocation(template, lhoExp, t);
        
        if (!(result instanceof SpecialBuiltIn)) {
            return result;
        }

        if (result instanceof BuiltInForLoopVariable) {
            if (!(lhoExp instanceof Identifier)) {
                throw new ParseException(
                        "Expression used as the left hand operand of ?" + t.image
                        + " must be a simple loop variable name.", lhoExp);
            }
            String loopVarName = ((Identifier) lhoExp).getName();
            checkLoopVariableBuiltInLHO(loopVarName, lhoExp, t);
            ((BuiltInForLoopVariable) result).bindToLoopVariable(loopVarName);
            
            return result;
        }
        
        if (result instanceof BuiltInBannedWhenAutoEscaping) {
	        if (outputFormat instanceof MarkupOutputFormat && autoEscaping) {
	            throw new ParseException(
	                    "Using ?" + t.image + " (legacy escaping) is not allowed when auto-escaping is on with "
	                    + "a markup output format (" + outputFormat.getName() + "), to avoid double-escaping mistakes.",
	                    template, t);
	        }
            
            return result;
        }

        if (result instanceof MarkupOutputFormatBoundBuiltIn) {
            if (!(outputFormat instanceof MarkupOutputFormat)) {
                throw new ParseException(
                        "?" + t.image + " can't be used here, as the current output format isn't a markup (escaping) "
                        + "format: " + outputFormat, template, t);
            }
            ((MarkupOutputFormatBoundBuiltIn) result).bindToMarkupOutputFormat((MarkupOutputFormat) outputFormat);
            
            return result;
        }

        if (result instanceof OutputFormatBoundBuiltIn) {
            ((OutputFormatBoundBuiltIn) result).bindToOutputFormat(outputFormat, autoEscapingPolicy);
            
            return result;
        }
    }
    [
        LOOKAHEAD({ result instanceof BuiltInWithParseTimeParameters  })
        openParen = 
        args = PositionalArgs()
        closeParen =  {
            result.setLocation(template, lhoExp, closeParen);
            ((BuiltInWithParseTimeParameters) result).bindToParameters(args, openParen, closeParen);
            
            return result;
        }
    ]
    {
        // Should have already return-ed
        throw new AssertionError("Unhandled " + SpecialBuiltIn.class.getName() + " subclass: " + result.getClass());
    }
}


/**
 * production for when a key is specified by  + keyname
 */
Expression DotVariable(Expression exp) :
{
    Token t;
}
{
        
        (
            t =  | t =  | t =  
            |
            (
                t = 
                |
                t = 
                |
                t = 
                |
                t = 
                |
                t = 
                |
                t = 
                |
                t = 
                |
                t = 
                |
                t = 
            )
            {
                if (!Character.isLetter(t.image.charAt(0))) {
                    throw new ParseException(t.image + " is not a valid identifier.", template, t);
                }
            }
        )
        {
            notListLiteral(exp, "hash");
            notStringLiteral(exp, "hash");
            notBooleanLiteral(exp, "hash");
            Dot dot = new Dot(exp, t.image);
            dot.setLocation(template, exp, t);
            return dot;
        }
}

/**
 * production for when the key is specified
 * in brackets.
 */
Expression DynamicKey(Expression exp) :
{
    Expression arg;
    Token t;
}
{
    
    arg = Expression()
    t = 
    {
        notBooleanLiteral(exp, "list or hash");
        notNumberLiteral(exp, "list or hash");
        DynamicKeyName dkn = new DynamicKeyName(exp, arg);
        dkn.setLocation(template, exp, t);
        return dkn;
    }
}

/**
 * production for an arglist part of a method invocation.
 */
MethodCall MethodArgs(Expression exp) :
{
        ArrayList args = new ArrayList();
        Token end;
}
{
        
        args = PositionalArgs()
        end = 
        {
            args.trimToSize();
            MethodCall result = new MethodCall(exp, args);
            result.setLocation(template, exp, end);
            return result;
        }
}

StringLiteral StringLiteral(boolean interpolate) :
{
    Token t;
    boolean raw = false;
}
{
    (
        t = 
        |
        t =  { raw = true; }
    )
    {
        String s;
        // Get rid of the quotes.
        if (raw) {
            s = t.image.substring(2, t.image.length() -1);
        } else {
	        try {
	            s = StringUtil.FTLStringLiteralDec(t.image.substring(1, t.image.length() -1));
	        } catch (ParseException pe) {
	            pe.lineNumber = t.beginLine;
	            pe.columnNumber = t.beginColumn;
	            pe.endLineNumber = t.endLine;
	            pe.endColumnNumber = t.endColumn;
	            throw pe;
	        }
        }
        StringLiteral result = new StringLiteral(s);
        result.setLocation(template, t, t);
        if (interpolate && !raw) {
            // TODO: This logic is broken. It can't handle literals that contains both ${...} and $\{...}. 
            int interpolationSyntax = pCfg.getInterpolationSyntax();
            if ((interpolationSyntax == LEGACY_INTERPOLATION_SYNTAX
                    || interpolationSyntax == DOLLAR_INTERPOLATION_SYNTAX)
	                    && t.image.indexOf("${") != -1
	                || interpolationSyntax == LEGACY_INTERPOLATION_SYNTAX
	                    && t.image.indexOf("#{") != -1
	                || interpolationSyntax == SQUARE_BRACKET_INTERPOLATION_SYNTAX
	                    && t.image.indexOf("[=") != -1) {
                result.parseValue(this, outputFormat);
            }
        }
        return result;
    }
}

Expression BooleanLiteral() :
{
    Token t;
    Expression result;
}
{
    (
        t =  { result = new BooleanLiteral(false); }
        |
        t =  { result = new BooleanLiteral(true); }
    )
    {
        result.setLocation(template, t, t);
        return result;
    }
}


HashLiteral HashLiteral() :
{
    Token begin, end;
    Expression key, value;
    ArrayList keys = new ArrayList();
    ArrayList values = new ArrayList();
}
{
    begin = 
    [
        key = Expression()
        (|)
        value = Expression()
        {
            stringLiteralOnly(key);
            keys.add(key);
            values.add(value);
        }
        (
            
            key = Expression()
            (|)
            value = Expression()
            {
                stringLiteralOnly(key);
                keys.add(key);
                values.add(value);
            }
        )*
    ]
    end = 
    {
        HashLiteral result = new HashLiteral(keys, values);
        result.setLocation(template, begin, end);
        return result;
    }
}

/**
 * A production representing the ${...} or [=...] that outputs a variable; should be called NormalInterpolation.
 */
DollarVariable StringOutput() :
{
    Expression exp;
    Token begin, end;
}
{
    (
	    (
	        begin = 
	        exp = Expression()
	        end = 
	    )
	    |
	    (
	        begin = 
	        exp = Expression()
	        end = 
	    )
    )
    {
        notHashLiteral(exp, NonStringException.STRING_COERCABLE_TYPES_DESC);
        notListLiteral(exp, NonStringException.STRING_COERCABLE_TYPES_DESC);
                    
        DollarVariable result = new DollarVariable(
                exp, escapedExpression(exp),
                outputFormat,
                autoEscaping);
        result.setLocation(template, begin, end);
        return result;
    }
}

/** Should be called NumericalInterpolation */
NumericalOutput NumericalOutput() :
{
    Expression exp;
    Token fmt = null, begin, end;
}
{
    begin = 
    exp = Expression() { numberLiteralOnly(exp); }
    [
        
        fmt = 
    ]
    end = 
    {
        MarkupOutputFormat autoEscOF = autoEscaping && outputFormat instanceof MarkupOutputFormat
                ? (MarkupOutputFormat) outputFormat : null;
    
        NumericalOutput result;
        if (fmt != null) {
            int minFrac = -1;  // -1 indicates that the value has not been set
            int maxFrac = -1;

            StringTokenizer st = new StringTokenizer(fmt.image, "mM", true);
            char type = '-';
            while (st.hasMoreTokens()) {
                String token = st.nextToken();
                try {
	                if (type != '-') {
	                    switch (type) {
	                    case 'm':
	                        if (minFrac != -1) throw new ParseException("Invalid formatting string", template, fmt);
	                        minFrac = Integer.parseInt(token);
	                        break;
	                    case 'M':
	                        if (maxFrac != -1) throw new ParseException("Invalid formatting string", template, fmt);
	                        maxFrac = Integer.parseInt(token);
	                        break;
	                    default:
	                        throw new ParseException("Invalid formatting string", template, fmt);
	                    }
	                    type = '-';
	                } else if (token.equals("m")) {
	                    type = 'm';
	                } else if (token.equals("M")) {
	                    type = 'M';
	                } else {
	                    throw new ParseException();
	                }
                } catch (ParseException e) {
                	throw new ParseException("Invalid format specifier " + fmt.image, template, fmt);
                } catch (NumberFormatException e) {
                	throw new ParseException("Invalid number in the format specifier " + fmt.image, template, fmt);
                }
            }

            if (maxFrac == -1) {
	            if (minFrac == -1) {
	                throw new ParseException(
	                		"Invalid format specification, at least one of m and M must be specified!", template, fmt);
	            }
            	maxFrac = minFrac;
            } else if (minFrac == -1) {
            	minFrac = 0;
            }
            if (minFrac > maxFrac) {
            	throw new ParseException(
            			"Invalid format specification, min cannot be greater than max!", template, fmt);
            }
            if (minFrac > 50 || maxFrac > 50) {// sanity check
                throw new ParseException("Cannot specify more than 50 fraction digits", template, fmt);
            }
            result = new NumericalOutput(exp, minFrac, maxFrac, autoEscOF);
        } else {  // if format != null
            result = new NumericalOutput(exp, autoEscOF);
        }
        result.setLocation(template, begin, end);
        return result;
    }
}

TemplateElement If() :
{
    Token start, end, t;
    Expression condition;
    TemplateElements children;
    IfBlock ifBlock;
    ConditionalBlock cblock;
}
{
    start = 
    condition = Expression()
    end = 
    children = MixedContentElements()
    {
        cblock = new ConditionalBlock(condition, children, ConditionalBlock.TYPE_IF);
        cblock.setLocation(template, start, end, children);
        ifBlock = new IfBlock(cblock);
    }
    (
        t = 
        condition = Expression()
        end = LooseDirectiveEnd()
        children = MixedContentElements()
        {
            cblock = new ConditionalBlock(condition, children, ConditionalBlock.TYPE_ELSE_IF);
            cblock.setLocation(template, t, end, children);
            ifBlock.addBlock(cblock);
        }
    )*
    [
            t = 
            children = MixedContentElements()
            {
                cblock = new ConditionalBlock(null, children, ConditionalBlock.TYPE_ELSE);
                cblock.setLocation(template, t, t, children);
                ifBlock.addBlock(cblock);
            }
    ]
    end = 
    {
        ifBlock.setLocation(template, start, end);
        return ifBlock;
    }
}

AttemptBlock Attempt() :
{
    Token start, end;
    TemplateElements children;
    RecoveryBlock recoveryBlock;
}
{
    start = 
    children = MixedContentElements()
    recoveryBlock = Recover()
    (
        end = 
        |
        end = 
    )
    {
        AttemptBlock result = new AttemptBlock(children, recoveryBlock);
        result.setLocation(template, start, end);
        return result;
    }
}

RecoveryBlock Recover() : 
{
    Token start;
    TemplateElements children;
}
{
    start = 
    children = MixedContentElements()
    {
        RecoveryBlock result = new RecoveryBlock(children);
        result.setLocation(template, start, start, children);
        return result;
    }
}

TemplateElement List() :
{
    Expression exp;
    Token loopVar = null, loopVar2 = null, start, end;
    TemplateElements childrendBeforeElse;
    ElseOfList elseOfList = null;
    ParserIteratorBlockContext iterCtx;
}
{
    start = 
    exp = Expression()
    [
        
        loopVar = 
        [
            
            loopVar2 = 
        ]
    ]
    
    {
        iterCtx = pushIteratorBlockContext();
        if (loopVar != null) {
            iterCtx.loopVarName = loopVar.image;
            breakableDirectiveNesting++;
            continuableDirectiveNesting++;
            if (loopVar2 != null) {
                iterCtx.loopVar2Name = loopVar2.image;
                iterCtx.hashListing = true;
                if (iterCtx.loopVar2Name.equals(iterCtx.loopVarName)) {
                    throw new ParseException(
                            "The key and value loop variable names must differ, but both were: " + iterCtx.loopVarName,
                            template, start);
                }
            }
        }
    }
    
    childrendBeforeElse = MixedContentElements()
    {
        if (loopVar != null) {
            breakableDirectiveNesting--;
            continuableDirectiveNesting--;
        } else if (iterCtx.kind != ITERATOR_BLOCK_KIND_ITEMS) {
            throw new ParseException(
                    "#list must have either \"as loopVar\" parameter or nested #items that belongs to it.",
                    template, start);
        }
        popIteratorBlockContext();
    }
    
    [
        elseOfList = ElseOfList()
    ]
    
    end = 
    {
        IteratorBlock list = new IteratorBlock(
                exp,
                loopVar != null ? loopVar.image : null,  // null when we have a nested #items
                loopVar2 != null ? loopVar2.image : null,
                childrendBeforeElse, iterCtx.hashListing, false);
        list.setLocation(template, start, end);

        TemplateElement result;
        if (elseOfList == null) {
            result = list;
        } else {
            result = new ListElseContainer(list, elseOfList);
            result.setLocation(template, start, end);
        }
        return result;
    }
}

ElseOfList ElseOfList() :
{
    Token start;
    TemplateElements children;
}
{
        start = 
        children = MixedContentElements()
        {
            ElseOfList result = new ElseOfList(children);
	        result.setLocation(template, start, start, children);
	        return result;
        }
}

IteratorBlock ForEach() :
{
    Expression exp;
    Token loopVar, start, end;
    TemplateElements children;
}
{
    start = 
    loopVar = 
    
    exp = Expression()
    
    {
        ParserIteratorBlockContext iterCtx = pushIteratorBlockContext();
        iterCtx.loopVarName = loopVar.image;
        iterCtx.kind = ITERATOR_BLOCK_KIND_FOREACH;
        breakableDirectiveNesting++;
        continuableDirectiveNesting++;
    }
    
    children = MixedContentElements()
    
    end = 
    {
        breakableDirectiveNesting--;
        continuableDirectiveNesting--;
        popIteratorBlockContext();
                
        IteratorBlock result = new IteratorBlock(exp, loopVar.image, null, children, false, true);
        result.setLocation(template, start, end);
        return result;
    }
}

Items Items() :
{
    Token loopVar, loopVar2 = null, start, end;
    TemplateElements children;
    ParserIteratorBlockContext iterCtx;
}
{
    start = 
    loopVar = 
    [
        
        loopVar2 = 
    ]
    
    {
        iterCtx = peekIteratorBlockContext();
        if (iterCtx == null) {
            throw new ParseException("#items must be inside a #list block.", template, start);
        }
        if (iterCtx.loopVarName != null) {
            String msg;
	        if (iterCtx.kind == ITERATOR_BLOCK_KIND_FOREACH) {
	            msg = forEachDirectiveSymbol() + " doesn't support nested #items.";
	        } else if (iterCtx.kind == ITERATOR_BLOCK_KIND_ITEMS) {
                msg = "Can't nest #items into each other when they belong to the same #list.";
	        } else {
	            msg = "The parent #list of the #items must not have \"as loopVar\" parameter.";
            }
            throw new ParseException(msg, template, start);
        }
        iterCtx.kind = ITERATOR_BLOCK_KIND_ITEMS;
        iterCtx.loopVarName = loopVar.image;
        if (loopVar2 != null) {
            iterCtx.loopVar2Name = loopVar2.image;
            iterCtx.hashListing = true;
            if (iterCtx.loopVar2Name.equals(iterCtx.loopVarName)) {
                throw new ParseException(
                        "The key and value loop variable names must differ, but both were: " + iterCtx.loopVarName,
                        template, start);
            }
        }
    
        breakableDirectiveNesting++;
        continuableDirectiveNesting++;
    }
    
    children = MixedContentElements()
    
    end = 
    {
        breakableDirectiveNesting--;
        continuableDirectiveNesting--;
        iterCtx.loopVarName = null;
        iterCtx.loopVar2Name = null;
        
        Items result = new Items(loopVar.image, loopVar2 != null ? loopVar2.image : null, children);
        result.setLocation(template, start, end);
        return result;
    }
}

Sep Sep() :
{
    Token loopVar, start, end = null;
    TemplateElements children;
}
{
    start = 
    {
        if (peekIteratorBlockContext() == null) {
            throw new ParseException(
                    "#sep must be inside a #list (or " + forEachDirectiveSymbol() + ") block.",
                    template, start);
        }
    }
    children = MixedContentElements()
    [
        LOOKAHEAD(1)
        end = 
    ]
    {
        Sep result = new Sep(children);
        if (end != null) {
            result.setLocation(template, start, end);
        } else {
            result.setLocation(template, start, start, children);
        }
        return result;
    }
}

VisitNode Visit() :
{
    Token start, end;
    Expression targetNode, namespaces = null;
}
{
    start = 
    targetNode = Expression()
    [
        
        namespaces = Expression()
    ]
    end = LooseDirectiveEnd()
    {
        VisitNode result = new VisitNode(targetNode, namespaces);
        result.setLocation(template, start, end);
        return result;
    }
}

RecurseNode Recurse() :
{
    Token start, end = null;
    Expression node = null, namespaces = null;
}
{
    (
        start = 
        |
        (
            start = 
            [
                node = Expression()
            ]
            [
                
                namespaces = Expression()
            ]
            end = LooseDirectiveEnd()
        )
    )
    {
        if (end == null) end = start;
        RecurseNode result = new RecurseNode(node, namespaces);
        result.setLocation(template, start, end);
        return result;
    }
}

FallbackInstruction FallBack() :
{
    Token tok;
}
{
    tok = 
    {
        if (!inMacro) {
            throw new ParseException("Cannot fall back outside a macro.", template, tok);
        }
        FallbackInstruction result = new FallbackInstruction();
        result.setLocation(template, tok, tok);
        return result;
    }
}

/**
 * Production used to break out of a loop or a switch block.
 */
BreakInstruction Break() :
{
    Token start;
}
{
    start = 
    {
        if (breakableDirectiveNesting < 1) {
            throw new ParseException(start.image + " must be nested inside a directive that supports it: " 
                    + " #list with \"as\", #items, #switch (or the deprecated " + forEachDirectiveSymbol() + ")",
                    template, start);
        }
        BreakInstruction result = new BreakInstruction();
        result.setLocation(template, start, start);
        return result;
    }
}

/**
 * Production used to skip an iteration in a loop.
 */
ContinueInstruction Continue() :
{
    Token start;
}
{
    start = 
    {
        if (continuableDirectiveNesting < 1) {
            throw new ParseException(start.image + " must be nested inside a directive that supports it: " 
                    + " #list with \"as\", #items (or the deprecated " + forEachDirectiveSymbol() + ")",
                    template, start);
        }
        ContinueInstruction result = new ContinueInstruction();
        result.setLocation(template, start, start);
        return result;
    }
}

/**
 * Production used to jump out of a macro.
 * The stop instruction terminates the rendering of the template.
 */
ReturnInstruction Return() :
{
    Token start, end = null;
    Expression exp = null;
}
{
    (
        start =  { end = start; }
        |
        start =  exp = Expression() end = LooseDirectiveEnd()
    )
    {
        if (inMacro) {
            if (exp != null) {
            	throw new ParseException("A macro cannot return a value", template, start);
            }
        } else if (inFunction) {
            if (exp == null) {
            	throw new ParseException("A function must return a value", template, start);
            }
        } else {
            if (exp == null) {
            	throw new ParseException(
            			"A return instruction can only occur inside a macro or function", template, start);
            }
        }
        ReturnInstruction result = new ReturnInstruction(exp);
        result.setLocation(template, start, end);
        return result;
    }
}

StopInstruction Stop() :
{
    Token start = null;
    Expression exp = null;
}
{
    (
        start = 
        |
        start =  exp = Expression() LooseDirectiveEnd()
    )
    {
        StopInstruction result = new StopInstruction(exp);
        result.setLocation(template, start, start);
        return result;
    }
}

TemplateElement Nested() :
{
    Token t, end;
    ArrayList bodyParameters;
    BodyInstruction result = null;
}
{
    (
        (
            t = 
            {
                result = new BodyInstruction(null);
                result.setLocation(template, t, t);
            }
        )
        |
        (
            t = 
            bodyParameters = PositionalArgs()
            end = LooseDirectiveEnd()
            {
                result = new BodyInstruction(bodyParameters);
                result.setLocation(template, t, end);
            }
        )
    )
    {
        if (!inMacro) {
            throw new ParseException("Cannot use a " + t.image + " instruction outside a macro.", template, t);
        }
        return result;
    }
}

TemplateElement Flush() :
{
    Token t;
}
{
    t = 
    {
        FlushInstruction result = new FlushInstruction();
        result.setLocation(template, t, t);
        return result;
    }
}

TemplateElement Trim() :
{
    Token t;
    TrimInstruction result = null;
}
{
    (
        t =  { result = new TrimInstruction(true, true); }
        |
        t =  { result = new TrimInstruction(true, false); }
        |
        t =  { result = new TrimInstruction(false, true); }
        |
        t =  { result = new TrimInstruction(false, false); }
    )
    {
        result.setLocation(template, t, t);
        return result;
    }
}


TemplateElement Assign() :
{
    Token start, end;
    int scope;
    Token id = null;
    Token equalsOp;
    Expression nameExp, exp, nsExp = null;
    String varName;
    ArrayList assignments = new ArrayList();
    Assignment ass;
    TemplateElements children;
}
{
    (
        start =  { scope = Assignment.NAMESPACE; }
        |
        start =  { scope = Assignment.GLOBAL; }
        |
        start =  { scope = Assignment.LOCAL; }
        {
            scope = Assignment.LOCAL;
            if (!inMacro && !inFunction) {
                throw new ParseException("Local variable assigned outside a macro.", template, start);
            }
        }
    )
    nameExp = IdentifierOrStringLiteral()
    {
        varName = (nameExp instanceof StringLiteral)
                ? ((StringLiteral) nameExp).getAsString()
                : ((Identifier) nameExp).getName();
    }
    (
    	(
            (
	    	    (
			        (|||||)
			        {
			           equalsOp = token;
			        }
			        exp = Expression()
		        )
		        |
		        (
	                (|)
	                {
	                   equalsOp = token;
	                   exp = null;
	                }
		        )
	        )
	        {
	            ass = new Assignment(varName, equalsOp.kind, exp, scope);
                if (exp != null) {
                   ass.setLocation(template, nameExp, exp);
                } else {
                   ass.setLocation(template, nameExp, equalsOp);
                }
	            assignments.add(ass);
	        }
	        (
	            LOOKAHEAD(
	               []
	               (|)
	               (|||||
	                       ||)
	            )
	            []
	            nameExp = IdentifierOrStringLiteral()
	            {
	                varName = (nameExp instanceof StringLiteral)
	                		? ((StringLiteral) nameExp).getAsString()
	                		: ((Identifier) nameExp).getName();
	            }
	            (
	                (
	                    (|||||)
	                    {
	                       equalsOp = token;
	                    }
	                    exp = Expression()
	                )
	                |
	                (
	                    (|)
	                    {
	                       equalsOp = token;
	                       exp = null;
	                    }
	                )
	            )
	            {
	                ass = new Assignment(varName, equalsOp.kind, exp, scope);
	                if (exp != null) {
	                   ass.setLocation(template, nameExp, exp);
	                } else {
                       ass.setLocation(template, nameExp, equalsOp);
	                }
	                assignments.add(ass);
	            } 
	        )*
	        [
	            id = 
	            nsExp = Expression()
	            {
	                if (scope != Assignment.NAMESPACE) {
	                	throw new ParseException("Cannot assign to namespace here.", template, id);
                	}
	            }
	        ]
	        end = LooseDirectiveEnd()
	        {
                if (assignments.size() == 1) {
                    Assignment a = (Assignment) assignments.get(0);
                    a.setNamespaceExp(nsExp);
                    a.setLocation(template, start, end);
                    return a;
                } else {
		            AssignmentInstruction ai = new AssignmentInstruction(scope);
		            for (int i = 0; i< assignments.size(); i++) {
		                ai.addAssignment((Assignment) assignments.get(i));
		            }
		            ai.setNamespaceExp(nsExp);
		            ai.setLocation(template, start, end);
		            return ai;
	            }
	        }
	    )
	    |
	    (
	        [
	            id = 
	            nsExp = Expression()
	            {
	                if (scope != Assignment.NAMESPACE) {
	                	throw new ParseException("Cannot assign to namespace here.", template, id);
	            	}
	            }
	        ]
	        
	        children = MixedContentElements()
	        (
	            end = 
	            {
	            	if (scope != Assignment.LOCAL) {
	            		throw new ParseException("Mismatched assignment tags.", template, end);
	        		}
	        	}
	            |
	            end = 
	            {
	            	if (scope != Assignment.NAMESPACE) {
	            		throw new ParseException("Mismatched assignment tags.", template, end);
	        		}
	        	}
	            |
	            end = 
	            {
	            	if (scope != Assignment.GLOBAL) throw new ParseException(
	            			"Mismatched assignment tags", template, end);
            	}
	        )
	        {
	            BlockAssignment ba = new BlockAssignment(
	                   children, varName, scope, nsExp,
	                   getMarkupOutputFormat());
	            ba.setLocation(template, start, end);
	            return ba;
	        }
	    )
    )
}

Include Include() :
{
    Expression nameExp;
    Token att, start, end;
    Expression exp, parseExp = null, encodingExp = null, ignoreMissingExp = null;
}
{
    start = <_INCLUDE>
    nameExp = Expression()
    []
    (
        att = 
        
        exp = Expression()
        {
            String attString = att.image;
            if (attString.equalsIgnoreCase("parse")) {
        	    parseExp = exp;
            } else if (attString.equalsIgnoreCase("encoding")) {
            	encodingExp = exp;
            } else if (attString.equalsIgnoreCase("ignore_missing") || attString.equals("ignoreMissing")) {
                token_source.checkNamingConvention(att);
            	ignoreMissingExp = exp;
            } else {
                String correctedName = attString.equals("ignoreMissing") ? "ignore_missing" : null;
                throw new ParseException(
                		"Unsupported named #include parameter: \"" + attString + "\". Supported parameters are: "
                		+ "\"parse\", \"encoding\", \"ignore_missing\"."
                		+ (correctedName == null
                		      ? ""
                		      : " Supporting camelCase parameter names is planned for FreeMarker 2.4.0; "
	                              + "check if an update is available, and if it indeed supports camel "
	                              + "case."),
                		template, att);
            }
        }
    )*
    end = LooseDirectiveEnd()
    {
        Include result = new Include(template, nameExp, encodingExp, parseExp, ignoreMissingExp);
        result.setLocation(template, start, end);
        return result;
    }
}

LibraryLoad Import() :
{
    Token start, end, ns;
    Expression nameExp;
}
{
    start = 
    nameExp = Expression()
    
    ns = 
    end = LooseDirectiveEnd()
    {
        LibraryLoad result = new LibraryLoad(template, nameExp, ns.image);
        result.setLocation(template, start, end);
        template.addImport(result);
        return result;
    }
}

Macro Macro() :
{
    Token arg, start, end;
    Expression nameExp;
    String name;
    ArrayList argNames = new ArrayList();
    HashMap args = new HashMap();
    ArrayList defNames = new ArrayList();
    Expression defValue = null;
    List lastIteratorBlockContexts;
    int lastBreakableDirectiveNesting;
    int lastContiunableDirectiveNesting;
    TemplateElements children;
    boolean isFunction = false, hasDefaults = false;
    boolean isCatchAll = false;
    String catchAll = null;
}
{
    (
        start = 
        |
        start =  { isFunction = true; }
    )
    {
        if (inMacro || inFunction) {
            throw new ParseException("Macro or function definitions can't be nested into each other.", template, start);
        }
        if (isFunction) inFunction = true; else inMacro = true;
    }
    nameExp = IdentifierOrStringLiteral()
    {
        name = (nameExp instanceof StringLiteral)
                ? ((StringLiteral) nameExp).getAsString()
                : ((Identifier) nameExp).getName();
    }
    []
    (
        arg =  { defValue = null; }
        [
             { isCatchAll = true; }
        ]
        [
            
            defValue = Expression()
            {
                defNames.add(arg.image);
                hasDefaults = true;
            }
        ]
        []
        {
            if (catchAll != null) {
                throw new ParseException(
                "There may only be one \"catch-all\" parameter in a macro declaration, and it must be the last parameter.",
                template, arg);
            }
            if (isCatchAll) {
                if (defValue != null) {
                    throw new ParseException(
                    "\"Catch-all\" macro parameter may not have a default value.",
                    template, arg);
                }
                catchAll = arg.image;
            } else {
                argNames.add(arg.image);
                if (hasDefaults && defValue == null) {
                    throw new ParseException(
		                    "In a macro declaration, parameters without a default value "
		                    + "must all occur before the parameters with default values.",
                    template, arg);
                }
                args.put(arg.image, defValue);
            }
        }
    )*
    []
    
    {
        // To prevent parser check loopholes like <#list ...><#macro ...><#break>.
        lastIteratorBlockContexts = iteratorBlockContexts;
        iteratorBlockContexts = null;
        if (incompatibleImprovements >= _TemplateAPI.VERSION_INT_2_3_23) {
	        lastBreakableDirectiveNesting = breakableDirectiveNesting;
	        lastContiunableDirectiveNesting = continuableDirectiveNesting;
	        breakableDirectiveNesting = 0; 
	        continuableDirectiveNesting = 0;
        } else {
            lastBreakableDirectiveNesting = 0; // Just to prevent uninitialized local variable error later
            lastContiunableDirectiveNesting = 0;
        }
    }
    children = MixedContentElements()
    (
        end = 
        {
        	if (isFunction) throw new ParseException("Expected function end tag here.", template, end);
    	}
        |
        end = 
        {
    		if (!isFunction) throw new ParseException("Expected macro end tag here.", template, end);
    	}
    )
    {
        iteratorBlockContexts = lastIteratorBlockContexts;
        if (incompatibleImprovements >= _TemplateAPI.VERSION_INT_2_3_23) {
            breakableDirectiveNesting = lastBreakableDirectiveNesting;
            continuableDirectiveNesting = lastContiunableDirectiveNesting;
        }
        
        inMacro = inFunction = false;
        Macro result = new Macro(name, argNames, args, catchAll, isFunction, children);
        result.setLocation(template, start, end);
        template.addMacro(result);
        return result;
    }
}

CompressedBlock Compress() :
{
    TemplateElements children;
    Token start, end;
}
{
    start = 
    children = MixedContentElements()
    end = 
    {
        CompressedBlock cb = new CompressedBlock(children);
        cb.setLocation(template, start, end);
        return cb;
    }
}

TemplateElement UnifiedMacroTransform() :
{
    Token start = null, end, t;
    HashMap namedArgs = null;
    ArrayList positionalArgs = null, bodyParameters = null;
    Expression startTagNameExp;
    TemplateElements children;
    Expression exp;
    int pushedCtxCount = 0;
}
{
    start = 
    exp = Expression()
    {
        if (exp instanceof Identifier || (exp instanceof Dot && ((Dot) exp).onlyHasIdentifiers())) {
            startTagNameExp = exp;
        } else {
            startTagNameExp = null;
        }
    }
    []
    (
        LOOKAHEAD()
        namedArgs = NamedArgs()
        |
        positionalArgs = PositionalArgs()
    )
    [
        
        { bodyParameters = new ArrayList(4); }
        [
            [] t =  { bodyParameters.add(t.image); }
            (
                [] 
                [] t =  {bodyParameters.add(t.image); }
            )*
        ]
    ]
    (
        end =  { children = TemplateElements.EMPTY; }
        |
        (
             {
                if (bodyParameters != null && iteratorBlockContexts != null && !iteratorBlockContexts.isEmpty()) {
                    // It's possible that we shadow a #list/#items loop variable, in which case that must be noted.
                    int ctxsLen = iteratorBlockContexts.size();
                    int bodyParsLen = bodyParameters.size();
	                for (int bodyParIdx = 0; bodyParIdx < bodyParsLen; bodyParIdx++) {
                        String bodyParName = (String) bodyParameters.get(bodyParIdx);
                        walkCtxSack: for (int ctxIdx = ctxsLen - 1; ctxIdx >= 0; ctxIdx--) {
                            ParserIteratorBlockContext ctx
                                    = (ParserIteratorBlockContext) iteratorBlockContexts.get(ctxIdx);
                            if (ctx.loopVarName != null && ctx.loopVarName.equals(bodyParName)) {
                                // If it wasn't already shadowed, shadow it:
                                if (ctx.kind != ITERATOR_BLOCK_KIND_USER_DIRECTIVE) {
                                    ParserIteratorBlockContext shadowingCtx = pushIteratorBlockContext();
                                    shadowingCtx.loopVarName = bodyParName;
                                    shadowingCtx.kind = ITERATOR_BLOCK_KIND_USER_DIRECTIVE;
                                    pushedCtxCount++;
                                }
                                break walkCtxSack;
                            }
                        }
                   }
                }
            }
            children = MixedContentElements()
            end = 
            {
                for (int i = 0; i < pushedCtxCount; i++) {
                    popIteratorBlockContext();
                }
            
                String endTagName = end.image.substring(3, end.image.length() - 1).trim();
                if (endTagName.length() > 0) {
                    if (startTagNameExp == null) {
                        throw new ParseException("Expecting ", template, end);
                    } else {
                        String startTagName = startTagNameExp.getCanonicalForm();
                        if (!endTagName.equals(startTagName)) {
                            throw new ParseException("Expecting  or ", template, end);
                        }
                    }
                }
            }
        )
    )
    {
        TemplateElement result = (positionalArgs != null)
        		? new UnifiedCall(exp, positionalArgs, children, bodyParameters)
	            : new UnifiedCall(exp, namedArgs, children, bodyParameters);
        result.setLocation(template, start, end);
        return result;
    }
}

TemplateElement Call() :
{
    Token start, end, id;
    HashMap namedArgs = null;
    ArrayList positionalArgs = null;
    Identifier macroName= null;
}
{
    start = 
    id =  {
        macroName = new Identifier(id.image);
        macroName.setLocation(template, id, id);
    }
    (
        LOOKAHEAD()
        namedArgs = NamedArgs()
        |
        (
            [
            LOOKAHEAD()
                
            ]
            positionalArgs = PositionalArgs()
            []
        )
    )
    end = LooseDirectiveEnd()
    {
        UnifiedCall result = null;
        if (positionalArgs != null) {
            result = new UnifiedCall(macroName, positionalArgs, TemplateElements.EMPTY, null);
        } else {
            result = new UnifiedCall(macroName, namedArgs, TemplateElements.EMPTY, null);
        }
        result.legacySyntax = true;
        result.setLocation(template, start, end);
        return result;
    }
}

HashMap NamedArgs() :
{
    HashMap result = new HashMap();
    Token t;
    Expression exp;
}
{
    (
        t = 
        
        {
            token_source.SwitchTo(token_source.NAMED_PARAMETER_EXPRESSION);
            token_source.inInvocation = true;
        }             
        exp = Expression()
        {
            result.put(t.image, exp);
        }
    )+
    {
        token_source.inInvocation = false;
        return result;
    }
}

ArrayList PositionalArgs() :
{
    ArrayList result = new ArrayList();
    Expression arg;
}
{
    [
        arg = Expression() { result.add(arg); }
        (
            []
            arg = Expression() { result.add(arg); }
        )*
    ]
    {
        return result;
    }
}


Comment Comment() :
{
    Token start, end;
    StringBuilder buf = new StringBuilder();
}
{
    (
        start = 
        |
        start = 
    )
    end = UnparsedContent(start, buf)
    {
        Comment result = new Comment(buf.toString());
        result.setLocation(template, start, end);
        return result;
    }
}

TextBlock NoParse() :
{
    Token start, end;
    StringBuilder buf = new StringBuilder();
}
{
    start = 
    end = UnparsedContent(start, buf)
    {
        TextBlock result = new TextBlock(buf.toString(), true);
        result.setLocation(template, start, end);
        return result;
    }
}

TransformBlock Transform() :
{
    Token start, end, argName;
    Expression exp, argExp;
    TemplateElements children = null;
    HashMap args = null;
}
{
    start = 
    exp = Expression()
    []
    (
        argName = 
        
        argExp = Expression()
        {
            if (args == null) args = new HashMap();
            args.put(argName.image, argExp);
        }
    )*
    (
        end = 
        |
        (
            
            children = MixedContentElements()
            end = 
        )
    )
    {
        TransformBlock result = new TransformBlock(exp, args, children);
        result.setLocation(template, start, end);
        return result;
    }
}

SwitchBlock Switch() :
{
    SwitchBlock switchBlock;
    MixedContent ignoredSectionBeforeFirstCase = null;
    Case caseIns;
    Expression switchExp;
    Token start, end;
    boolean defaultFound = false;
}
{
    (
	    start = 
	    switchExp = Expression()
	    
        [ ignoredSectionBeforeFirstCase = WhitespaceAndComments() ]
    )
    {
        breakableDirectiveNesting++;
        switchBlock = new SwitchBlock(switchExp, ignoredSectionBeforeFirstCase);
    }
    [
	    (
	        caseIns = Case()
	        {
	            if (caseIns.condition == null) {
	                if (defaultFound) {
	                    throw new ParseException(
	                    "You can only have one default case in a switch statement", template, start);
	                }
	                defaultFound = true;
	            }
	            switchBlock.addCase(caseIns);
	        }
	    )+
	    []
    ]
    end = 
    {
        breakableDirectiveNesting--;
        switchBlock.setLocation(template, start, end);
        return switchBlock;
    }
}

Case Case() :
{
    Expression exp;
    TemplateElements children;
    Token start;
}
{
    (
        start =  exp = Expression() 
        |
        start =  { exp = null; }
    )
    children = MixedContentElements()
    {
        Case result = new Case(exp, children);
        result.setLocation(template, start, start, children);
        return result;
    }
}

EscapeBlock Escape() :
{
    Token variable, start, end;
    Expression escapeExpr;
    TemplateElements children;
}
{
    start = 
    {
        if (outputFormat instanceof MarkupOutputFormat && autoEscaping) {
            throw new ParseException(
                    "Using the \"escape\" directive (legacy escaping) is not allowed when auto-escaping is on with "
                    + "a markup output format (" + outputFormat.getName()
                    + "), to avoid confusion and double-escaping mistakes.",
                    template, start);
        }
    }
    variable = 
    
    escapeExpr = Expression()
    
    {
        EscapeBlock result = new EscapeBlock(variable.image, escapeExpr, escapedExpression(escapeExpr));
        escapes.addFirst(result);
    }
    children = MixedContentElements()
    {
        result.setContent(children);
        escapes.removeFirst();
    }
    end = 
    {
        result.setLocation(template, start, end);
        return result;
    }
}

NoEscapeBlock NoEscape() :
{
    Token start, end;
    TemplateElements children;
}
{
    start = 
    {
        if (escapes.isEmpty()) {
            throw new ParseException("#noescape with no matching #escape encountered.", template, start);
        }
        Object escape = escapes.removeFirst();
    }
    children = MixedContentElements()
    end = 
    {
        escapes.addFirst(escape);
        NoEscapeBlock result = new NoEscapeBlock(children);
        result.setLocation(template, start, end);
        return result;
    }
}

OutputFormatBlock OutputFormat() :
{
    Token start, end;
    Expression paramExp;
    TemplateElements children;
    OutputFormat lastOutputFormat;
}
{
    start = 
    paramExp = Expression()
    
    {
        if (!paramExp.isLiteral()) {
            throw new ParseException(
                    "Parameter expression must be parse-time evaluable (constant): "
                    + paramExp.getCanonicalForm(),
                    paramExp);
        }
    
        TemplateModel paramTM;
        try {
            paramTM = paramExp.eval(null);
        } catch (Exception e) {
            throw new ParseException(
                    "Could not evaluate expression (on parse-time): " + paramExp.getCanonicalForm()
                    + "\nUnderlying cause: " +  e,
                    paramExp, e);
        }
        String paramStr;
        if (paramTM instanceof TemplateScalarModel) {
            try {
                paramStr = ((TemplateScalarModel) paramTM).getAsString();
            } catch (TemplateModelException e) {
	            throw new ParseException(
	                    "Could not evaluate expression (on parse-time): " + paramExp.getCanonicalForm()
	                    + "\nUnderlying cause: " +  e,
	                    paramExp, e);
            }
        } else {
            throw new ParseException(
                    "Parameter must be a string, but was: " + ClassUtil.getFTLTypeDescription(paramTM),
                    paramExp);
        }
        
        lastOutputFormat = outputFormat;
        try { 
            if (paramStr.startsWith("{")) {
                if (!paramStr.endsWith("}")) {
                    throw new ParseException("Output format name that starts with '{' must end with '}': " + paramStr,
                            template, start);
                }
                OutputFormat innerOutputFormat = template.getConfiguration().getOutputFormat(
                        paramStr.substring(1, paramStr.length() - 1));
                if (!(innerOutputFormat instanceof MarkupOutputFormat)) {
                    throw new ParseException(
                            "The output format inside the {...} must be a markup format, but was: "
                            + innerOutputFormat,
                            template, start);
                }
                if (!(outputFormat instanceof MarkupOutputFormat)) {
                    throw new ParseException(
                            "The current output format must be a markup format when using {...}, but was: "
                            + outputFormat,
                            template, start);
                }
                outputFormat = new CombinedMarkupOutputFormat(
                        (MarkupOutputFormat) outputFormat, (MarkupOutputFormat) innerOutputFormat);
            } else {
                outputFormat = template.getConfiguration().getOutputFormat(paramStr);
            }
            recalculateAutoEscapingField();
        } catch (IllegalArgumentException e) {
            throw new ParseException("Invalid format name: " + e.getMessage(), template, start, e.getCause());
        } catch (UnregisteredOutputFormatException e) {
            throw new ParseException(e.getMessage(), template, start, e.getCause());
        }
    }
    children = MixedContentElements()
    end = 
    {
        OutputFormatBlock result = new OutputFormatBlock(children, paramExp);
        result.setLocation(template, start, end);
        
        outputFormat = lastOutputFormat;
        recalculateAutoEscapingField();         
        return result;
    }
}

AutoEscBlock AutoEsc() :
{
    Token start, end;
    TemplateElements children;
    int lastAutoEscapingPolicy;
}
{
    start = 
    {
        checkCurrentOutputFormatCanEscape(start);
        lastAutoEscapingPolicy = autoEscapingPolicy;
        autoEscapingPolicy = Configuration.ENABLE_IF_SUPPORTED_AUTO_ESCAPING_POLICY;
        recalculateAutoEscapingField();
    }
    children = MixedContentElements()
    end = 
    {
        AutoEscBlock result = new AutoEscBlock(children);
        result.setLocation(template, start, end);
        
        autoEscapingPolicy = lastAutoEscapingPolicy; 
        recalculateAutoEscapingField();
        return result;
    }
}

NoAutoEscBlock NoAutoEsc() :
{
    Token start, end;
    TemplateElements children;
    int lastAutoEscapingPolicy;
}
{
    start = 
    {
        lastAutoEscapingPolicy = autoEscapingPolicy;
        autoEscapingPolicy = Configuration.DISABLE_AUTO_ESCAPING_POLICY;
        recalculateAutoEscapingField();
    }
    children = MixedContentElements()
    end = 
    {
        NoAutoEscBlock result = new NoAutoEscBlock(children);
        result.setLocation(template, start, end);
        
        autoEscapingPolicy = lastAutoEscapingPolicy;
        recalculateAutoEscapingField(); 
        return result;
    }
}

/**
 * Production to terminate potentially empty elements. Either a ">" or "/>"
 */
Token LooseDirectiveEnd() :
{
    Token t;
}
{
    (
        t = 
        |
        t = 
    )
    {
        return t;
    }
}

PropertySetting Setting() :
{
    Token start, end, key;
    Expression value;
}
{
    start = 
    key = 
    
    value = Expression()
    end = LooseDirectiveEnd()
    {
        token_source.checkNamingConvention(key);
        PropertySetting result = new PropertySetting(key, token_source, value, template.getConfiguration());
        result.setLocation(template, start, end);
        return result;
    }
}

/**
 * A production for FreeMarker directives.
 */
TemplateElement FreemarkerDirective() :
{
    TemplateElement tp;
}
{
    // Note that this doesn't include elements like "else", "recover", etc., because those indicate the end
    // of the MixedContentElements of "if", "attempt", etc.
    (
        tp = If()
        |
        tp = List()
        |
        tp = ForEach()
        |
        tp = Assign()
        |
        tp = Include()
        |
        tp = Import()
        |
        tp = Macro()
        |
        tp = Compress()
        |
        tp = UnifiedMacroTransform()
        |
        tp = Items()
        |
        tp = Sep()
        |
        tp = Call()
        |
        tp = Comment()
        |
        tp = NoParse()
        |
        tp = Transform()
        |
        tp = Switch()
        |
        tp = Setting()
        |
        tp = Break()
        |
        tp = Continue()
        |
        tp = Return()
        |
        tp = Stop()
        |
        tp = Flush()
        |
        tp = Trim()
        |
        tp = Nested()
        |
        tp = Escape()
        |
        tp = NoEscape()
        |
        tp = Visit()
        |
        tp = Recurse()
        |
        tp = FallBack()
        |
        tp = Attempt()
        |
        tp = OutputFormat()
        |
        tp = AutoEsc()
        |
        tp = NoAutoEsc()
    )
    {
        return tp;
    }
}

/**
 * Production for a block of raw text
 * i.e. text that contains no
 * FreeMarker directives.
 */
TextBlock PCData() :
{
    StringBuilder buf = new StringBuilder();
    Token t = null, start = null, prevToken = null;
}
{
    (
        (
            t = 
            |
            t = 
            |
            t = 
        )
        {
            buf.append(t.image);
            if (start == null) start = t;
            if (prevToken != null) prevToken.next = null;
            prevToken = t;
        }
    )+
    {
        if (stripText && mixedContentNesting == 1 && !preventStrippings) return null;

        TextBlock result = new TextBlock(buf.toString(), false);
        result.setLocation(template, start, t);
        return result;
    }
}

TextBlock WhitespaceText() :
{
    Token t = null, start = null;
}
{
    t = 
    {
        if (stripText && mixedContentNesting == 1 && !preventStrippings) return null;

        TextBlock result = new TextBlock(t.image, false);
        result.setLocation(template, t, t);
        return result;
    }
}

/**
 * Production for dealing with unparsed content,
 * i.e. what is inside a comment or noparse tag.
 * It returns the ending token. The content
 * of the tag is put in buf.
 */
Token UnparsedContent(Token start, StringBuilder buf) :
{
    Token t;
}
{
    (
        (t =  | t =  | t =  | t = )
        {
            buf.append(t.image);
        }
    )+
    {
        buf.setLength(buf.length() - t.image.length());
        if (!t.image.endsWith(";")
                && _TemplateAPI.getTemplateLanguageVersionAsInt(template) >= _TemplateAPI.VERSION_INT_2_3_21) {
            throw new ParseException("Unclosed \"" + start.image + "\"", template, start);
        }
        return t;
    }
}

TemplateElements MixedContentElements() :
{
    TemplateElement[] childBuffer = null;
    int childCount = 0;
    TemplateElement elem;
    mixedContentNesting++;
}
{
    (
        LOOKAHEAD(1) // Just tells javacc that we know what we're doing.
        (
            elem = PCData()
            |
            elem = StringOutput()
            |
            elem = NumericalOutput()
            |
            elem = FreemarkerDirective()
        )
        {
            // Note: elem == null when it's was top-level PCData removed by stripText
            if (elem != null) {
	            childCount++;
	            if (childBuffer == null) {
	                childBuffer = new TemplateElement[16]; 
	            } else if (childBuffer.length < childCount) {
	                TemplateElement[] newChildBuffer = new TemplateElement[childCount * 2];
	                for (int i = 0; i < childBuffer.length; i++) {
	                    newChildBuffer[i] = childBuffer[i];
	                }
	                childBuffer = newChildBuffer;
	            }
	            childBuffer[childCount - 1] = elem;
            }
        }
    )*
    {
        mixedContentNesting--;
        return childBuffer != null ? new TemplateElements(childBuffer, childCount) : TemplateElements.EMPTY;
    }
}

/**
 * Not used anymore; kept for backward compatibility.
 *
 * @deprecated Use {@link #MixedContentElements} instead.
 */
MixedContent MixedContent() :
{
    MixedContent mixedContent = new MixedContent();
    TemplateElement elem, begin = null;
    mixedContentNesting++;
}
{
    (
        LOOKAHEAD(1) // Just tells javacc that we know what we're doing.
        (
            elem = PCData()
            |
            elem = StringOutput()
            |
            elem = NumericalOutput()
            |
            elem = FreemarkerDirective()
        )
        {
            if (begin == null) {
                begin = elem;
            }
            mixedContent.addElement(elem);
        }
    )+
    {
        mixedContentNesting--;
        mixedContent.setLocation(template, begin, elem);
        return mixedContent;
    }
}

/**
 * Not used anymore; kept for backward compatibility.
 *
 * 

A production for a block of optional content. * Returns an empty Text block if there is no * content. * * @deprecated Use {@link #MixedContentElements} instead. */ TemplateElement OptionalBlock() : { TemplateElement tp = null; } { [ LOOKAHEAD(1) // has no effect but to get rid of a spurious warning. tp = MixedContent() ] { return tp != null ? tp : new TextBlock(CollectionUtils.EMPTY_CHAR_ARRAY, false); } } /** * A production freemarker text that may contain * ${...} and #{...} but no directives. */ TemplateElement FreeMarkerText() : { MixedContent nodes = new MixedContent(); TemplateElement elem, begin = null; } { ( ( elem = PCData() | elem = StringOutput() | elem = NumericalOutput() ) { if (begin == null) { begin = elem; } nodes.addChild(elem); } )+ { nodes.setLocation(template, begin, elem); return nodes; } } /** * To be used between tags that in theory has nothing between, such between #switch and the first #case. */ MixedContent WhitespaceAndComments() : { MixedContent nodes = new MixedContent(); TemplateElement elem, begin = null; } { ( ( elem = WhitespaceText() | elem = Comment() ) { if (elem != null) { // not removed by stripText if (begin == null) { begin = elem; } nodes.addChild(elem); } } )+ { if (begin == null // Was is removed by stripText? // Nodes here won't be ever executed anyway, but whitespace stripping should still remove the // lonely TextBlock from the AST, as that's purely source code formatting. If it's not lonely, then // there must be a comment, in which case the generic whitespace stripping algorithm will kick in. || stripWhitespace && !preventStrippings && nodes.getChildCount() == 1 && nodes.getChild(0) instanceof TextBlock) { return null; } nodes.setLocation(template, begin, elem); return nodes; } } void HeaderElement() : { Token key; Expression exp = null; Token autoEscRequester = null; } { [] ( | ( ( key = exp = Expression() { token_source.checkNamingConvention(key); String ks = key.image; TemplateModel value = null; try { value = exp.eval(null); } catch (Exception e) { throw new ParseException( "Could not evaluate expression (on parse-time): " + exp.getCanonicalForm() + " \nUnderlying cause: " + e, exp, e); } String vs = null; if (value instanceof TemplateScalarModel) { try { vs = ((TemplateScalarModel) exp).getAsString(); } catch (TemplateModelException tme) {} } if (template != null) { if (ks.equalsIgnoreCase("encoding")) { if (vs == null) { throw new ParseException("Expected a string constant for \"" + ks + "\".", exp); } String encoding = template.getEncoding(); if (encoding != null && !encoding.equalsIgnoreCase(vs)) { throw new Template.WrongEncodingException(vs, encoding); } } else if (ks.equalsIgnoreCase("STRIP_WHITESPACE") || ks.equals("stripWhitespace")) { this.stripWhitespace = getBoolean(exp, true); } else if (ks.equalsIgnoreCase("STRIP_TEXT") || ks.equals("stripText")) { this.stripText = getBoolean(exp, true); } else if (ks.equalsIgnoreCase("STRICT_SYNTAX") || ks.equals("strictSyntax")) { this.token_source.strictSyntaxMode = getBoolean(exp, true); } else if (ks.equalsIgnoreCase("auto_esc") || ks.equals("autoEsc")) { if (getBoolean(exp, false)) { autoEscRequester = key; autoEscapingPolicy = Configuration.ENABLE_IF_SUPPORTED_AUTO_ESCAPING_POLICY; } else { autoEscapingPolicy = Configuration.DISABLE_AUTO_ESCAPING_POLICY; } recalculateAutoEscapingField(); _TemplateAPI.setAutoEscaping(template, autoEscaping); } else if (ks.equalsIgnoreCase("output_format") || ks.equals("outputFormat")) { if (vs == null) { throw new ParseException("Expected a string constant for \"" + ks + "\".", exp); } try { outputFormat = template.getConfiguration().getOutputFormat(vs); } catch (IllegalArgumentException e) { throw new ParseException("Invalid format name: " + e.getMessage(), exp, e.getCause()); } catch (UnregisteredOutputFormatException e) { throw new ParseException(e.getMessage(), exp, e.getCause()); } recalculateAutoEscapingField(); _TemplateAPI.setOutputFormat(template, outputFormat); _TemplateAPI.setAutoEscaping(template, autoEscaping); } else if (ks.equalsIgnoreCase("ns_prefixes") || ks.equals("nsPrefixes")) { if (!(value instanceof TemplateHashModelEx)) { throw new ParseException("Expecting a hash of prefixes to namespace URI's.", exp); } TemplateHashModelEx prefixMap = (TemplateHashModelEx) value; try { TemplateCollectionModel keys = prefixMap.keys(); for (TemplateModelIterator it = keys.iterator(); it.hasNext();) { String prefix = ((TemplateScalarModel) it.next()).getAsString(); TemplateModel valueModel = prefixMap.get(prefix); if (!(valueModel instanceof TemplateScalarModel)) { throw new ParseException("Non-string value in prefix to namespace hash.", exp); } String nsURI = ((TemplateScalarModel) valueModel).getAsString(); try { template.addPrefixNSMapping(prefix, nsURI); } catch (IllegalArgumentException iae) { throw new ParseException(iae.getMessage(), exp); } } } catch (TemplateModelException tme) { } } else if (ks.equalsIgnoreCase("attributes")) { if (!(value instanceof TemplateHashModelEx)) { throw new ParseException("Expecting a hash of attribute names to values.", exp); } TemplateHashModelEx attributeMap = (TemplateHashModelEx) value; try { TemplateCollectionModel keys = attributeMap.keys(); for (TemplateModelIterator it = keys.iterator(); it.hasNext();) { String attName = ((TemplateScalarModel) it.next()).getAsString(); Object attValue = DeepUnwrap.unwrap(attributeMap.get(attName)); template.setCustomAttribute(attName, attValue); } } catch (TemplateModelException tme) { } } else { String correctName; if (ks.equals("charset")) { correctName = "encoding"; } else if (ks.equals("xmlns")) { // [2.4] If camel case will be the default, update this correctName = token_source.namingConvention == Configuration.CAMEL_CASE_NAMING_CONVENTION ? "nsPrefixes" : "ns_prefixes"; } else if (ks.equals("auto_escape") || ks.equals("auto_escaping") || ks.equals("autoesc")) { correctName = "auto_esc"; } else if (ks.equals("autoEscape") || ks.equals("autoEscaping")) { correctName = "autoEsc"; } else { correctName = null; } throw new ParseException( "Unknown FTL header parameter: " + key.image + (correctName == null ? "" : ". You may meant: " + correctName), template, key); } } } )* ) { if (autoEscRequester != null) { checkCurrentOutputFormatCanEscape(autoEscRequester); } } LooseDirectiveEnd() ) } Map ParamList() : { Identifier id; Expression exp; Map result = new HashMap(); } { ( id = Identifier() exp = Expression() { result.put(id.toString(), exp); } [] )+ { return result; } } /** * Parses the already un-escaped content of a string literal (input must not include the quotation marks). * * @return A {@link List} of {@link String}-s and {@link Interpolation}-s. */ List StaticTextAndInterpolations() : { Token t; Interpolation interpolation; StringBuilder staticTextCollector = null; ArrayList parts = new ArrayList(); } { ( ( t = | t = | t = ) { String s = t.image; if (s.length() != 0) { if (staticTextCollector == null) { staticTextCollector = new StringBuilder(t.image); } else { staticTextCollector.append(t.image); } } } | ( LOOKAHEAD(|) ( interpolation = StringOutput() ) | LOOKAHEAD() ( interpolation = NumericalOutput() ) ) { if (staticTextCollector != null) { parts.add(staticTextCollector.toString()); staticTextCollector.setLength(0); } parts.add(interpolation); } )* { if (staticTextCollector != null && staticTextCollector.length() != 0) { parts.add(staticTextCollector.toString()); } parts.trimToSize(); return parts; } } /** * Root production to be used when parsing * an entire file. */ TemplateElement Root() : { TemplateElements children; } { [ LOOKAHEAD([](|)) HeaderElement() ] children = MixedContentElements() { TemplateElement root = children.asSingleElement(); root.setFieldsForRootElement(); if (!preventStrippings) { root = root.postParseCleanup(stripWhitespace); } // The cleanup result is possibly an element from deeper: root.setFieldsForRootElement(); return root; } }