org.fife.ui.rsyntaxtextarea.modes.LuaTokenMaker.flex Maven / Gradle / Ivy
The newest version!
/*
* 07/14/2006
*
* LuaTokenMaker.java - Scanner for the Lua programming language.
*
* This library is distributed under a modified BSD license. See the included
* RSyntaxTextArea.License.txt file for details.
*/
package org.fife.ui.rsyntaxtextarea.modes;
import java.io.*;
import javax.swing.text.Segment;
import org.fife.ui.rsyntaxtextarea.*;
/**
* Scanner for the Lua programming language.
*
* This implementation was created using
* JFlex 1.4.1; however, the generated file
* was modified for performance. Memory allocation needs to be almost
* completely removed to be competitive with the handwritten lexers (subclasses
* of AbstractTokenMaker, so this class has been modified so that
* Strings are never allocated (via yytext()), and the scanner never has to
* worry about refilling its buffer (needlessly copying chars around).
* We can achieve this because RText always scans exactly 1 line of tokens at a
* time, and hands the scanner this line as an array of characters (a Segment
* really). Since tokens contain pointers to char arrays instead of Strings
* holding their contents, there is no need for allocating new memory for
* Strings.
*
* The actual algorithm generated for scanning has, of course, not been
* modified.
*
* If you wish to regenerate this file yourself, keep in mind the following:
*
* - The generated
LuaTokenMaker.java file will contain two
* definitions of both zzRefill and yyreset.
* You should hand-delete the second of each definition (the ones
* generated by the lexer), as these generated methods modify the input
* buffer, which we'll never have to do.
* - You should also change the declaration/definition of zzBuffer to NOT
* be initialized. This is a needless memory allocation for us since we
* will be pointing the array somewhere else anyway.
* - You should NOT call
yylex() on the generated scanner
* directly; rather, you should use getTokenList as you would
* with any other TokenMaker instance.
*
*
* @author Robert Futrell
* @version 0.4
*
*/
%%
%public
%class LuaTokenMaker
%extends AbstractJFlexTokenMaker
%unicode
%type org.fife.ui.rsyntaxtextarea.Token
%{
/**
* Constructor. This must be here because JFlex does not generate a
* no-parameter constructor.
*/
public LuaTokenMaker() {
}
/**
* Adds the token specified to the current linked list of tokens.
*
* @param tokenType The token's type.
*/
private void addToken(int tokenType) {
addToken(zzStartRead, zzMarkedPos-1, tokenType);
}
/**
* Adds the token specified to the current linked list of tokens.
*
* @param tokenType The token's type.
*/
private void addToken(int start, int end, int tokenType) {
int so = start + offsetShift;
addToken(zzBuffer, start,end, tokenType, so);
}
/**
* Adds the token specified to the current linked list of tokens.
*
* @param array The character array.
* @param start The starting offset in the array.
* @param end The ending offset in the array.
* @param tokenType The token's type.
* @param startOffset The offset in the document at which this token
* occurs.
*/
@Override
public void addToken(char[] array, int start, int end, int tokenType, int startOffset) {
super.addToken(array, start,end, tokenType, startOffset);
zzStartRead = zzMarkedPos;
}
/**
* {@inheritDoc}
*/
@Override
public String[] getLineCommentStartAndEnd(int languageIndex) {
return new String[] { "--", null };
}
/**
* Returns the first token in the linked list of tokens generated
* from text. This method must be implemented by
* subclasses so they can correctly implement syntax highlighting.
*
* @param text The text from which to get tokens.
* @param initialTokenType The token type we should start with.
* @param startOffset The offset into the document at which
* text starts.
* @return The first Token in a linked list representing
* the syntax highlighted text.
*/
public Token getTokenList(Segment text, int initialTokenType, int startOffset) {
resetTokenList();
this.offsetShift = -text.offset + startOffset;
// Start off in the proper state.
int state = Token.NULL;
switch (initialTokenType) {
case Token.COMMENT_MULTILINE:
state = MLC;
start = text.offset;
break;
case Token.LITERAL_STRING_DOUBLE_QUOTE:
state = LONGSTRING;
start = text.offset;
break;
default:
state = Token.NULL;
}
s = text;
try {
yyreset(zzReader);
yybegin(state);
return yylex();
} catch (IOException ioe) {
ioe.printStackTrace();
return new TokenImpl();
}
}
/**
* Refills the input buffer.
*
* @return true if EOF was reached, otherwise
* false.
* @exception IOException if any I/O-Error occurs.
*/
private boolean zzRefill() throws java.io.IOException {
return zzCurrentPos>=s.offset+s.count;
}
/**
* Resets the scanner to read from a new input stream.
* Does not close the old reader.
*
* All internal variables are reset, the old input stream
* cannot be reused (internal buffer is discarded and lost).
* Lexical state is set to YY_INITIAL.
*
* @param reader the new input stream
*/
public final void yyreset(java.io.Reader reader) throws java.io.IOException {
// 's' has been updated.
zzBuffer = s.array;
/*
* We replaced the line below with the two below it because zzRefill
* no longer "refills" the buffer (since the way we do it, it's always
* "full" the first time through, since it points to the segment's
* array). So, we assign zzEndRead here.
*/
//zzStartRead = zzEndRead = s.offset;
zzStartRead = s.offset;
zzEndRead = zzStartRead + s.count - 1;
zzCurrentPos = zzMarkedPos = zzPushbackPos = s.offset;
zzLexicalState = YYINITIAL;
zzReader = reader;
zzAtBOL = true;
zzAtEOF = false;
}
%}
Letter = [A-Za-z_]
Digit = [0-9]
LineTerminator = (\n)
WhiteSpace = ([ \t\f])
UnclosedCharLiteral = ([\']([^\'\n]|"\\'")*)
CharLiteral = ({UnclosedCharLiteral}"'")
UnclosedStringLiteral = ([\"]([^\"\n]|"\\\"")*)
StringLiteral = ({UnclosedStringLiteral}[\"])
LongStringBegin = ("[[")
LongStringEnd = ("]]")
LineCommentBegin = ("--")
MLCBegin = ({LineCommentBegin}{LongStringBegin})
Number = ( "."? {Digit} ({Digit}|".")* ([eE][+-]?)? ({Letter}|{Digit})* )
BooleanLiteral = ("true"|"false")
Separator = ([\(\)\{\}\[\]\]])
Separator2 = ([\;:,.])
ArithmeticOperator = ("+"|"-"|"*"|"/"|"^"|"%")
RelationalOperator = ("<"|">"|"<="|">="|"=="|"~=")
LogicalOperator = ("and"|"or"|"not"|"#")
ConcatenationOperator = ("..")
Elipsis = ({ConcatenationOperator}".")
Operator = ({ArithmeticOperator}|{RelationalOperator}|{LogicalOperator}|{ConcatenationOperator}|{Elipsis})
Identifier = ({Letter}({Letter}|{Digit})*)
%state MLC
%state LONGSTRING
%state LINECOMMENT
%%
/* Keywords */
"break" { addToken(Token.RESERVED_WORD); }
"do" { addToken(Token.RESERVED_WORD); }
"else" { addToken(Token.RESERVED_WORD); }
"elseif" { addToken(Token.RESERVED_WORD); }
"end" { addToken(Token.RESERVED_WORD); }
"for" { addToken(Token.RESERVED_WORD); }
"function" { addToken(Token.RESERVED_WORD); }
"if" { addToken(Token.RESERVED_WORD); }
"local" { addToken(Token.RESERVED_WORD); }
"nil" { addToken(Token.RESERVED_WORD); }
"repeat" { addToken(Token.RESERVED_WORD); }
"return" { addToken(Token.RESERVED_WORD); }
"then" { addToken(Token.RESERVED_WORD); }
"until" { addToken(Token.RESERVED_WORD); }
"while" { addToken(Token.RESERVED_WORD); }
/* Data types. */
"" { addToken(Token.DATA_TYPE); }
"" { addToken(Token.DATA_TYPE); }
"" { addToken(Token.DATA_TYPE); }
"" { addToken(Token.DATA_TYPE); }
"NULL" { addToken(Token.DATA_TYPE); }
/* Functions. */
"_G" { addToken(Token.FUNCTION); }
"_VERSION" { addToken(Token.FUNCTION); }
"assert" { addToken(Token.FUNCTION); }
"collectgarbage" { addToken(Token.FUNCTION); }
"dofile" { addToken(Token.FUNCTION); }
"error" { addToken(Token.FUNCTION); }
"getfenv" { addToken(Token.FUNCTION); }
"getmetatable" { addToken(Token.FUNCTION); }
"ipairs" { addToken(Token.FUNCTION); }
"load" { addToken(Token.FUNCTION); }
"loadfile" { addToken(Token.FUNCTION); }
"loadstring" { addToken(Token.FUNCTION); }
"module" { addToken(Token.FUNCTION); }
"next" { addToken(Token.FUNCTION); }
"pairs" { addToken(Token.FUNCTION); }
"pcall" { addToken(Token.FUNCTION); }
"print" { addToken(Token.FUNCTION); }
"rawequal" { addToken(Token.FUNCTION); }
"rawget" { addToken(Token.FUNCTION); }
"rawset" { addToken(Token.FUNCTION); }
"require" { addToken(Token.FUNCTION); }
"select" { addToken(Token.FUNCTION); }
"setfenv" { addToken(Token.FUNCTION); }
"setmetatable" { addToken(Token.FUNCTION); }
"tonumber" { addToken(Token.FUNCTION); }
"tostring" { addToken(Token.FUNCTION); }
"type" { addToken(Token.FUNCTION); }
"unpack" { addToken(Token.FUNCTION); }
"xpcall" { addToken(Token.FUNCTION); }
/* Booleans. */
{BooleanLiteral} { addToken(Token.LITERAL_BOOLEAN); }
{
{LineTerminator} { addNullToken(); return firstToken; }
{WhiteSpace}+ { addToken(Token.WHITESPACE); }
/* String/Character literals. */
{CharLiteral} { addToken(Token.LITERAL_CHAR); }
{UnclosedCharLiteral} { addToken(Token.ERROR_CHAR); addNullToken(); return firstToken; }
{StringLiteral} { addToken(Token.LITERAL_STRING_DOUBLE_QUOTE); }
{UnclosedStringLiteral} { addToken(Token.ERROR_STRING_DOUBLE); addNullToken(); return firstToken; }
{LongStringBegin} { start = zzMarkedPos-2; yybegin(LONGSTRING); }
/* Comment literals. */
{MLCBegin} { start = zzMarkedPos-4; yybegin(MLC); }
{LineCommentBegin} { start = zzMarkedPos-2; yybegin(LINECOMMENT); }
/* Separators. */
{Separator} { addToken(Token.SEPARATOR); }
{Separator2} { addToken(Token.IDENTIFIER); }
/* Operators. */
{Operator} { addToken(Token.OPERATOR); }
/* Identifiers - Comes after Operators for "and", "not" and "or". */
{Identifier} { addToken(Token.IDENTIFIER); }
/* Numbers */
{Number} { addToken(Token.LITERAL_NUMBER_FLOAT); }
/* Ended with a line not in a string or comment. */
<> { addNullToken(); return firstToken; }
/* Catch any other (unhandled) characters. */
. { addToken(Token.IDENTIFIER); }
}
{
[^\n\]]+ {}
\n { addToken(start,zzStartRead-1, Token.COMMENT_MULTILINE); return firstToken; }
{LongStringEnd} { yybegin(YYINITIAL); addToken(start,zzStartRead+1, Token.COMMENT_MULTILINE); }
\] {}
<> { addToken(start,zzStartRead-1, Token.COMMENT_MULTILINE); return firstToken; }
}
{
[^\n\]]+ {}
\n { addToken(start,zzStartRead-1, Token.LITERAL_STRING_DOUBLE_QUOTE); return firstToken; }
{LongStringEnd} { yybegin(YYINITIAL); addToken(start,zzStartRead+1, Token.LITERAL_STRING_DOUBLE_QUOTE); }
\] {}
<> { addToken(start,zzStartRead-1, Token.LITERAL_STRING_DOUBLE_QUOTE); return firstToken; }
}
{
[^\n]+ {}
\n { addToken(start,zzStartRead-1, Token.COMMENT_EOL); return firstToken; }
<> { addToken(start,zzStartRead-1, Token.COMMENT_EOL); return firstToken; }
}