org.fife.ui.rsyntaxtextarea.modes.LuaTokenMaker.flex Maven / Gradle / Ivy

Show more of this group Show more artifacts with this name
Show all versions of rsyntaxtextarea Show documentation
RSyntaxTextArea is the syntax highlighting text editor for Swing applications. Features include syntax highlighting for 40+ languages, code folding, code completion, regex find and replace, macros, code templates, undo/redo, line numbering and bracket matching.
There is a newer version: 3.5.1
Show newest version
/*
 * 07/14/2006
 *
 * LuaTokenMaker.java - Scanner for the Lua programming language.
 * Copyright (C) 2006 Robert Futrell
 * robert_futrell at users.sourceforge.net
 * http://fifesoft.com/rsyntaxtextarea
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA.
 */
package org.fife.ui.rsyntaxtextarea.modes;

import java.io.*;
import javax.swing.text.Segment;

import org.fife.ui.rsyntaxtextarea.*;


/**
 * Scanner for the Lua programming language.
 *
 * This implementation was created using
 * JFlex 1.4.1; however, the generated file
 * was modified for performance.  Memory allocation needs to be almost
 * completely removed to be competitive with the handwritten lexers (subclasses
 * of AbstractTokenMaker, so this class has been modified so that
 * Strings are never allocated (via yytext()), and the scanner never has to
 * worry about refilling its buffer (needlessly copying chars around).
 * We can achieve this because RText always scans exactly 1 line of tokens at a
 * time, and hands the scanner this line as an array of characters (a Segment
 * really).  Since tokens contain pointers to char arrays instead of Strings
 * holding their contents, there is no need for allocating new memory for
 * Strings.

 *
 * The actual algorithm generated for scanning has, of course, not been
 * modified.

 *
 * If you wish to regenerate this file yourself, keep in mind the following:
 * 

 *   The generated LuaTokenMaker.java file will contain two
 *       definitions of both zzRefill and yyreset.
 *       You should hand-delete the second of each definition (the ones
 *       generated by the lexer), as these generated methods modify the input
 *       buffer, which we'll never have to do.
 *   You should also change the declaration/definition of zzBuffer to NOT
 *       be initialized.  This is a needless memory allocation for us since we
 *       will be pointing the array somewhere else anyway.
 *   You should NOT call yylex() on the generated scanner
 *       directly; rather, you should use getTokenList as you would
 *       with any other TokenMaker instance.
 * 
 *
 * @author Robert Futrell
 * @version 0.4
 *
 */
%%

%public
%class LuaTokenMaker
%extends AbstractJFlexTokenMaker
%implements TokenMaker
%unicode
%type org.fife.ui.rsyntaxtextarea.Token


%{


	/**
	 * Constructor.  This must be here because JFlex does not generate a
	 * no-parameter constructor.
	 */
	public LuaTokenMaker() {
	}


	/**
	 * Adds the token specified to the current linked list of tokens.
	 *
	 * @param tokenType The token's type.
	 */
	private void addToken(int tokenType) {
		addToken(zzStartRead, zzMarkedPos-1, tokenType);
	}


	/**
	 * Adds the token specified to the current linked list of tokens.
	 *
	 * @param tokenType The token's type.
	 */
	private void addToken(int start, int end, int tokenType) {
		int so = start + offsetShift;
		addToken(zzBuffer, start,end, tokenType, so);
	}


	/**
	 * Adds the token specified to the current linked list of tokens.
	 *
	 * @param array The character array.
	 * @param start The starting offset in the array.
	 * @param end The ending offset in the array.
	 * @param tokenType The token's type.
	 * @param startOffset The offset in the document at which this token
	 *                    occurs.
	 */
	public void addToken(char[] array, int start, int end, int tokenType, int startOffset) {
		super.addToken(array, start,end, tokenType, startOffset);
		zzStartRead = zzMarkedPos;
	}


	/**
	 * Returns the text to place at the beginning and end of a
	 * line to "comment" it in a this programming language.
	 *
	 * @return The start and end strings to add to a line to "comment"
	 *         it out.
	 */
	public String[] getLineCommentStartAndEnd() {
		return new String[] { "--", null };
	}


	/**
	 * Returns the first token in the linked list of tokens generated
	 * from text.  This method must be implemented by
	 * subclasses so they can correctly implement syntax highlighting.
	 *
	 * @param text The text from which to get tokens.
	 * @param initialTokenType The token type we should start with.
	 * @param startOffset The offset into the document at which
	 *        text starts.
	 * @return The first Token in a linked list representing
	 *         the syntax highlighted text.
	 */
	public Token getTokenList(Segment text, int initialTokenType, int startOffset) {

		resetTokenList();
		this.offsetShift = -text.offset + startOffset;

		// Start off in the proper state.
		int state = Token.NULL;
		switch (initialTokenType) {
			case Token.COMMENT_MULTILINE:
				state = MLC;
				start = text.offset;
				break;
			case Token.LITERAL_STRING_DOUBLE_QUOTE:
				state = LONGSTRING;
				start = text.offset;
				break;
			default:
				state = Token.NULL;
		}

		s = text;
		try {
			yyreset(zzReader);
			yybegin(state);
			return yylex();
		} catch (IOException ioe) {
			ioe.printStackTrace();
			return new DefaultToken();
		}

	}


	/**
	 * Refills the input buffer.
	 *
	 * @return      true if EOF was reached, otherwise
	 *              false.
	 * @exception   IOException  if any I/O-Error occurs.
	 */
	private boolean zzRefill() throws java.io.IOException {
		return zzCurrentPos>=s.offset+s.count;
	}


	/**
	 * Resets the scanner to read from a new input stream.
	 * Does not close the old reader.
	 *
	 * All internal variables are reset, the old input stream 
	 * cannot be reused (internal buffer is discarded and lost).
	 * Lexical state is set to YY_INITIAL.
	 *
	 * @param reader   the new input stream 
	 */
	public final void yyreset(java.io.Reader reader) throws java.io.IOException {
		// 's' has been updated.
		zzBuffer = s.array;
		/*
		 * We replaced the line below with the two below it because zzRefill
		 * no longer "refills" the buffer (since the way we do it, it's always
		 * "full" the first time through, since it points to the segment's
		 * array).  So, we assign zzEndRead here.
		 */
		//zzStartRead = zzEndRead = s.offset;
		zzStartRead = s.offset;
		zzEndRead = zzStartRead + s.count - 1;
		zzCurrentPos = zzMarkedPos = zzPushbackPos = s.offset;
		zzLexicalState = YYINITIAL;
		zzReader = reader;
		zzAtBOL  = true;
		zzAtEOF  = false;
	}


%}

Letter					= [A-Za-z_]
Digit					= [0-9]

LineTerminator				= (\n)
WhiteSpace				= ([ \t\f])

UnclosedCharLiteral			= ([\']([^\'\n]|"\\'")*)
CharLiteral				= ({UnclosedCharLiteral}"'")
UnclosedStringLiteral		= ([\"]([^\"\n]|"\\\"")*)
StringLiteral				= ({UnclosedStringLiteral}[\"])
LongStringBegin			= ("[[")
LongStringEnd				= ("]]")

LineCommentBegin			= ("--")
MLCBegin					= ({LineCommentBegin}{LongStringBegin})

Number					= ( "."? {Digit} ({Digit}|".")* ([eE][+-]?)? ({Letter}|{Digit})* )
BooleanLiteral				= ("true"|"false")

Separator					= ([\(\)\{\}\[\]\]])
Separator2				= ([\;:,.])

ArithmeticOperator			= ("+"|"-"|"*"|"/"|"^"|"%")
RelationalOperator			= ("<"|">"|"<="|">="|"=="|"~=")
LogicalOperator			= ("and"|"or"|"not"|"#")
ConcatenationOperator		= ("..")
Elipsis					= ({ConcatenationOperator}".")
Operator					= ({ArithmeticOperator}|{RelationalOperator}|{LogicalOperator}|{ConcatenationOperator}|{Elipsis})

Identifier				= ({Letter}({Letter}|{Digit})*)


%state MLC
%state LONGSTRING
%state LINECOMMENT


%%

/* Keywords */
 "break"					{ addToken(Token.RESERVED_WORD); }
 "do"					{ addToken(Token.RESERVED_WORD); }
 "else"					{ addToken(Token.RESERVED_WORD); }
 "elseif"				{ addToken(Token.RESERVED_WORD); }
 "end"					{ addToken(Token.RESERVED_WORD); }
 "for"					{ addToken(Token.RESERVED_WORD); }
 "function"				{ addToken(Token.RESERVED_WORD); }
 "if"					{ addToken(Token.RESERVED_WORD); }
 "local"					{ addToken(Token.RESERVED_WORD); }
 "nil"					{ addToken(Token.RESERVED_WORD); }
 "repeat"				{ addToken(Token.RESERVED_WORD); }
 "return"				{ addToken(Token.RESERVED_WORD); }
 "then"					{ addToken(Token.RESERVED_WORD); }
 "until"					{ addToken(Token.RESERVED_WORD); }
 "while"					{ addToken(Token.RESERVED_WORD); }

/* Data types. */
 ""				{ addToken(Token.DATA_TYPE); }
 ""				{ addToken(Token.DATA_TYPE); }
 ""				{ addToken(Token.DATA_TYPE); }
 ""					{ addToken(Token.DATA_TYPE); }
 "NULL"					{ addToken(Token.DATA_TYPE); }

/* Functions. */
 "_G"					{ addToken(Token.FUNCTION); }
 "_VERSION"				{ addToken(Token.FUNCTION); }
 "assert"				{ addToken(Token.FUNCTION); }
 "collectgarbage"			{ addToken(Token.FUNCTION); }
 "dofile"				{ addToken(Token.FUNCTION); }
 "error"					{ addToken(Token.FUNCTION); }
 "getfenv"				{ addToken(Token.FUNCTION); }
 "getmetatable"			{ addToken(Token.FUNCTION); }
 "ipairs"				{ addToken(Token.FUNCTION); }
 "load"					{ addToken(Token.FUNCTION); }
 "loadfile"				{ addToken(Token.FUNCTION); }
 "loadstring"				{ addToken(Token.FUNCTION); }
 "module"				{ addToken(Token.FUNCTION); }
 "next"					{ addToken(Token.FUNCTION); }
 "pairs"					{ addToken(Token.FUNCTION); }
 "pcall"					{ addToken(Token.FUNCTION); }
 "print"					{ addToken(Token.FUNCTION); }
 "rawequal"				{ addToken(Token.FUNCTION); }
 "rawget"				{ addToken(Token.FUNCTION); }
 "rawset"				{ addToken(Token.FUNCTION); }
 "require"				{ addToken(Token.FUNCTION); }
 "select"				{ addToken(Token.FUNCTION); }
 "setfenv"				{ addToken(Token.FUNCTION); }
 "setmetatable"			{ addToken(Token.FUNCTION); }
 "tonumber"				{ addToken(Token.FUNCTION); }
 "tostring"				{ addToken(Token.FUNCTION); }
 "type"					{ addToken(Token.FUNCTION); }
 "unpack"				{ addToken(Token.FUNCTION); }
 "xpcall"				{ addToken(Token.FUNCTION); }

/* Booleans. */
 {BooleanLiteral}			{ addToken(Token.LITERAL_BOOLEAN); }


 {

	{LineTerminator}				{ addNullToken(); return firstToken; }

	{WhiteSpace}+					{ addToken(Token.WHITESPACE); }

	/* String/Character literals. */
	{CharLiteral}					{ addToken(Token.LITERAL_CHAR); }
	{UnclosedCharLiteral}			{ addToken(Token.ERROR_CHAR); addNullToken(); return firstToken; }
	{StringLiteral}				{ addToken(Token.LITERAL_STRING_DOUBLE_QUOTE); }
	{UnclosedStringLiteral}			{ addToken(Token.ERROR_STRING_DOUBLE); addNullToken(); return firstToken; }
	{LongStringBegin}				{ start = zzMarkedPos-2; yybegin(LONGSTRING); }

	/* Comment literals. */
	{MLCBegin}					{ start = zzMarkedPos-4; yybegin(MLC); }
	{LineCommentBegin}				{ start = zzMarkedPos-2; yybegin(LINECOMMENT); }

	/* Separators. */
	{Separator}					{ addToken(Token.SEPARATOR); }
	{Separator2}					{ addToken(Token.IDENTIFIER); }

	/* Operators. */
	{Operator}					{ addToken(Token.OPERATOR); }

	/* Identifiers - Comes after Operators for "and", "not" and "or". */
	{Identifier}					{ addToken(Token.IDENTIFIER); }

	/* Numbers */
	{Number}						{ addToken(Token.LITERAL_NUMBER_FLOAT); }

	/* Ended with a line not in a string or comment. */
	<>						{ addNullToken(); return firstToken; }

	/* Catch any other (unhandled) characters. */
	.							{ addToken(Token.IDENTIFIER); }

}


 {
	[^\n\]]+					{}
	\n						{ addToken(start,zzStartRead-1, Token.COMMENT_MULTILINE); return firstToken; }
	{LongStringEnd}			{ yybegin(YYINITIAL); addToken(start,zzStartRead+1, Token.COMMENT_MULTILINE); }
	\]						{}
	<>					{ addToken(start,zzStartRead-1, Token.COMMENT_MULTILINE); return firstToken; }
}


 {
	[^\n\]]+					{}
	\n						{ addToken(start,zzStartRead-1, Token.LITERAL_STRING_DOUBLE_QUOTE); return firstToken; }
	{LongStringEnd}			{ yybegin(YYINITIAL); addToken(start,zzStartRead+1, Token.LITERAL_STRING_DOUBLE_QUOTE); }
	\]						{}
	<>					{ addToken(start,zzStartRead-1, Token.LITERAL_STRING_DOUBLE_QUOTE); return firstToken; }
}


 {
	[^\n]+					{}
	\n						{ addToken(start,zzStartRead-1, Token.COMMENT_EOL); return firstToken; }
	<>					{ addToken(start,zzStartRead-1, Token.COMMENT_EOL); return firstToken; }
}