org.fife.ui.rsyntaxtextarea.Token Maven / Gradle / Ivy

/*
 * 02/21/2004
 *
 * Token.java - A token used in syntax highlighting.
 * 
 * This library is distributed under a modified BSD license.  See the included
 * RSyntaxTextArea.License.txt file for details.
 */
package org.fife.ui.rsyntaxtextarea;

import java.awt.Color;
import java.awt.Font;
import java.awt.FontMetrics;
import java.awt.Graphics2D;
import java.awt.Rectangle;
import java.awt.geom.Rectangle2D;
import javax.swing.text.TabExpander;


/**
 * A generic token that functions as a node in a linked list of syntax
 * highlighted tokens for some language.
 *
 * A Token is a piece of text representing some logical token in
 * source code for a programming language.  For example, the line of C code:

 * 
 * int i = 0;
 * 
 * would be broken into 8 Tokens: the first representing
 * int, the second whitespace, the third i, the fourth
 * whitespace, the fifth =, etc.
 *
 * @author Robert Futrell
 * @version 0.3
 */
public abstract class Token implements TokenTypes {

	/**
	 * The text this token represents.  This is implemented as a segment so we
	 * can point directly to the text in the document without having to make a
	 * copy of it.
	 */
	public char[] text;
	public int textOffset;
	public int textCount;
	
	/**
	 * The offset into the document at which this token resides.
	 */
	public int offset;

	/**
	 * The type of token this is; for example, {@link #FUNCTION}.
	 */
	public int type;

	/**
	 * Whether this token is a hyperlink.
	 */
	private boolean hyperlink;

	/**
	 * The next token in this linked list.
	 */
	private Token nextToken;

	/**
	 * Rectangle used for filling token backgrounds.
	 */
	private Rectangle2D.Float bgRect;


	/**
	 * Creates a "null token."  The token itself is not null; rather, it
	 * signifies that it is the last token in a linked list of tokens and
	 * that it is not part of a "multi-line token."
	 */
	public Token() {
		this.text = null;
		this.textOffset = -1;
		this.textCount = -1;
		this.type = NULL;
		offset = -1;
		hyperlink = false;
		nextToken = null;
		bgRect = new Rectangle2D.Float();
	}


	/**
	 * Constructor.
	 *
	 * @param line The segment from which to get the token.
	 * @param beg The first character's position in line.
	 * @param end The last character's position in line.
	 * @param startOffset The offset into the document at which this
	 *        token begins.
	 * @param type A token type listed as "generic" above.
	 */
	public Token(final char[] line, final int beg, final int end,
							final int startOffset, final int type) {
		this();
		set(line, beg,end, startOffset, type);
	}


	/**
	 * Creates this token as a deep copy of the passed-in token.
	 *
	 * @param t2 The token from which to make a copy.
	 */
	public Token(Token t2) {
		this();
		copyFrom(t2);
	}


	/**
	 * Appends HTML code for painting this token, using the given text area's
	 * color scheme.
	 *
	 * @param sb The buffer to append to.
	 * @param textArea The text area whose color scheme to use.
	 * @param fontFamily Whether to include the font family in the HTML for
	 *        this token.  You can pass false for this parameter
	 *        if, for example, you are making all your HTML be monospaced,
	 *        and don't want any crazy fonts being used in the editor to be
	 *        reflected in your HTML.
	 * @return The buffer appended to.
	 * @see #getHTMLRepresentation(RSyntaxTextArea)
	 */
	public StringBuffer appendHTMLRepresentation(StringBuffer sb,
											RSyntaxTextArea textArea,
											boolean fontFamily) {
		return appendHTMLRepresentation(sb, textArea, fontFamily, false);
	}


	/**
	 * Appends HTML code for painting this token, using the given text area's
	 * color scheme.
	 *
	 * @param sb The buffer to append to.
	 * @param textArea The text area whose color scheme to use.
	 * @param fontFamily Whether to include the font family in the HTML for
	 *        this token.  You can pass false for this parameter
	 *        if, for example, you are making all your HTML be monospaced,
	 *        and don't want any crazy fonts being used in the editor to be
	 *        reflected in your HTML.
	 * @param tabsToSpaces Whether to convert tabs into spaces.
	 * @return The buffer appended to.
	 * @see #getHTMLRepresentation(RSyntaxTextArea)
	 */
	public StringBuffer appendHTMLRepresentation(StringBuffer sb,
								RSyntaxTextArea textArea, boolean fontFamily,
								boolean tabsToSpaces) {

		SyntaxScheme colorScheme = textArea.getSyntaxScheme();
		Style scheme = colorScheme.getStyle(type);
		Font font = textArea.getFontForTokenType(type);//scheme.font;

		if (font.isBold()) sb.append("");
		if (font.isItalic()) sb.append("");
		if (scheme.underline || isHyperlink()) sb.append("");

		sb.append("");

		// NOTE: Don't use getLexeme().trim() because whitespace tokens will
		// be turned into NOTHING.
		appendHtmlLexeme(textArea, sb, tabsToSpaces);

		sb.append("");
		if (scheme.underline || isHyperlink()) sb.append("");
		if (font.isItalic()) sb.append("");
		if (font.isBold()) sb.append("");

		return sb;

	}


	/**
	 * Appends an HTML version of the lexeme of this token (i.e. no style
	 * HTML, but replacing chars such as \t, <
	 * and > with their escapes).
	 *
	 * @param textArea The text area.
	 * @param sb The buffer to append to.
	 * @param tabsToSpaces Whether to convert tabs into spaces.
	 * @return The same buffer.
	 */
	private final StringBuffer appendHtmlLexeme(RSyntaxTextArea textArea,
								StringBuffer sb, boolean tabsToSpaces) {

		boolean lastWasSpace = false;
		int i = textOffset;
		int lastI = i;
		String tabStr = null;

		while (i':
					sb.append(text, lastI, i-lastI);
					lastI = i+1;
					sb.append(">");
					lastWasSpace = false;
					break;
				default:
					lastWasSpace = false;
					break;
			}
			i++;
		}
		if (lastI=offset && pos
	 * Note that this method does NOT do any bounds checking; you can pass in
	 * a document position that does not correspond to a position in this
	 * token, and you will not receive an Exception or any other notification;
	 * it is up to the caller to ensure valid input.
	 *
	 * @param pos A position in the document that is represented by this token.
	 * @return The corresponding token position >= textOffset and
	 *         < textOffset+textCount.
	 * @see #tokenToDocument
	 */
	public int documentToToken(int pos) {
		return pos + (textOffset-offset);
	}


	/**
	 * Returns whether this token's lexeme ends with the specified characters.
	 *
	 * @param ch The characters.
	 * @return Whether this token's lexeme ends with the specified characters.
	 */
	public boolean endsWith(char[] ch) {
		if (ch==null || ch.length>textCount) {
			return false;
		}
		final int start = textOffset + textCount - ch.length;
		for (int i=0; iString of the form "#xxxxxx" good for use
	 * in HTML, representing the given color.
	 *
	 * @param color The color to get a string for.
	 * @return The HTML form of the color.  If color is
	 *         null, #000000 is returned.
	 */
	private static final String getHTMLFormatForColor(Color color) {
		if (color==null) {
			return "black";
		}
		String hexRed = Integer.toHexString(color.getRed());
		if (hexRed.length()==1)
			hexRed = "0" + hexRed;
		String hexGreen = Integer.toHexString(color.getGreen());
		if (hexGreen.length()==1)
			hexGreen = "0" + hexGreen;
		String hexBlue = Integer.toHexString(color.getBlue());
		if (hexBlue.length()==1)
			hexBlue = "0" + hexBlue;
		return "#" + hexRed + hexGreen + hexBlue;
	}


	/**
	 * Returns a String containing HTML code for painting this
	 * token, using the given text area's color scheme.
	 *
	 * @param textArea The text area whose color scheme to use.
	 * @return The HTML representation of the token.
	 * @see #appendHTMLRepresentation(StringBuffer, RSyntaxTextArea, boolean)
	 */
	public String getHTMLRepresentation(RSyntaxTextArea textArea) {
		StringBuffer buf = new StringBuffer();
		appendHTMLRepresentation(buf, textArea, true);
		return buf.toString();
	}


	/**
	 * Returns the last token in this list that is not whitespace or a
	 * comment.
	 *
	 * @return The last non-comment, non-whitespace token, or null
	 *         if there isn't one.
	 */
	public Token getLastNonCommentNonWhitespaceToken() {

		Token last = null;

		for (Token t=this; t!=null && t.isPaintable(); t=t.nextToken) {
			switch (t.type) {
				case COMMENT_DOCUMENTATION:
				case COMMENT_EOL:
				case COMMENT_MULTILINE:
				case WHITESPACE:
					break;
				default:
					last = t;
					break;
			}
		}

		return last;

	}


	/**
	 * Returns the last paintable token in this token list, or null
	 * if there is no paintable token.
	 *
	 * @return The last paintable token in this token list.
	 */
	public Token getLastPaintableToken() {
		Token t = this;
		while (t.isPaintable()) {
			if (t.nextToken==null || !t.nextToken.isPaintable()) {
				return t;
			}
			t = t.nextToken;
		}
		return null;
	}


	/**
	 * Returns the text of this token, as a string.

	 *
	 * Note that this method isn't used much by the
	 * rsyntaxtextarea package internally, as it tries to limit
	 * memory allocation.
	 *
	 * @return The text of this token.
	 */
	public String getLexeme() {
		return new String(text, textOffset, textCount);
	}


	/**
	 * Determines the offset into this token list (i.e., into the
	 * document) that covers pixel location x if the token list
	 * starts at pixel location x0
.
	 * This method will return the document position "closest" to the
	 * x-coordinate (i.e., if they click on the "right-half" of the
	 * w in awe, the caret will be placed in
	 * between the w and e; similarly, clicking on
	 * the left-half places the caret between the a and
	 * w).  This makes it useful for methods such as
	 * viewToModel found in javax.swing.text.View
	 * subclasses.

	 *
	 * This method is abstract so subclasses who paint themselves differently
	 * (i.e., {@link VisibleWhitespaceToken} is painted a tad differently than
	 * {@link DefaultToken} when rendering hints are enabled) can still return
	 * accurate results.
	 *
	 * @param textArea The text area from which the token list was derived.
	 * @param e How to expand tabs.
	 * @param x0 The pixel x-location that is the beginning of
	 *        tokenList.
	 * @param x The pixel-position for which you want to get the corresponding
	 *        offset.
	 * @return The position (in the document, NOT into the token list!) that
	 *         covers the pixel location.  If tokenList is
	 *         null or has type Token.NULL, then
	 *         -1endBeforeX-startX pixels.
	 * For example, if you're using a monospaced 8-pixel-per-character font,
	 * have the token "while" and startX is 0 and
	 * endBeforeX is 30, this method will return the
	 * document position of the "i" in "while", because the "i" ends at pixel
	 * 24, while the "l" ends at 32.  If not even the
	 * first character fits in endBeforeX-startX, the first
	 * character's position is still returned so calling methods don't go into
	 * infinite loops.
	 *
	 * @param textArea The text area in which this token is being painted.
	 * @param e How to expand tabs.
	 * @param startX The x-coordinate at which the token will be painted.  This
	 *        is needed because of tabs.
	 * @param endBeforeX The x-coordinate for which you want to find the last
	 *        character of t which comes before it.
	 * @return The last document position that will fit in the specified amount
	 *         of pixels.
	 */
	/*
	 * @see #getTokenListOffsetBeforeX
	 * FIXME:  This method does not compute correctly!  It needs to be abstract
	 * and implemented by subclasses.
	 */
	public int getOffsetBeforeX(RSyntaxTextArea textArea, TabExpander e,
							float startX, float endBeforeX) {

		FontMetrics fm = textArea.getFontMetricsForTokenType(type);
		int i = textOffset;
		int stop = i + textCount;
		float x = startX;

		while (iendBeforeX) {
				// If not even the first character fits into the space, go
				// ahead and say the first char does fit so we don't go into
				// an infinite loop.
				int intoToken = Math.max(i-textOffset, 1);
				return offset + intoToken;
			}
			i++;
		}

		// If we got here, the whole token fit in (endBeforeX-startX) pixels.
		return offset + textCount - 1;

	}


	/**
	 * Returns the width of this token given the specified parameters.
	 *
	 * @param textArea The text area in which the token is being painted.
	 * @param e Describes how to expand tabs.  This parameter cannot be
	 *        null.
	 * @param x0 The pixel-location at which the token begins.  This is needed
	 *        because of tabs.
	 * @return The width of the token, in pixels.
	 * @see #getWidthUpTo
	 */
	public float getWidth(RSyntaxTextArea textArea, TabExpander e, float x0) {
		return getWidthUpTo(textCount, textArea, e, x0);
	}


	/**
	 * Returns the width of a specified number of characters in this token.
	 * For example, for the token "while", specifying a value of 3
	 * here returns the width of the "whi" portion of the token.

	 *
	 * This method is abstract so subclasses who paint themselves differently
	 * (i.e., {@link VisibleWhitespaceToken} is painted a tad differently than
	 * {@link DefaultToken} when rendering hints are enabled) can still return
	 * accurate results.
	 *
	 * @param numChars The number of characters for which to get the width.
	 * @param textArea The text area in which the token is being painted.
	 * @param e How to expand tabs.  This value cannot be null.
	 * @param x0 The pixel-location at which this token begins.  This is needed
	 *        because of tabs.
	 * @return The width of the specified number of characters in this token.
	 * @see #getWidth
	 */
	public abstract float getWidthUpTo(int numChars, RSyntaxTextArea textArea,
										TabExpander e, float x0);


	/**
	 * Returns whether this token is of the specified type, with the specified
	 * lexeme.

	 * This method is preferred over the other overload in performance-critical
	 * code where this operation may be called frequently, since it does not
	 * involve any String allocations.
	 *
	 * @param type The type to check for.
	 * @param lexeme The lexeme to check for.
	 * @return Whether this token has that type and lexeme.
	 * @see #is(int, String)
	 */
	public boolean is(int type, char[] lexeme) {
		if (this.type==type && textCount==lexeme.length) {
			for (int i=0; i
	 * The other overload of this method is preferred over this one in
	 * performance-critical code, as this one involves a String allocation
	 * while the other does not.
	 *
	 * @param type The type to check for.
	 * @param lexeme The lexeme to check for.
	 * @return Whether this token has that type and lexeme.
	 * @see #is(int, char[])
	 */
	public boolean is(int type, String lexeme) {
		return this.type==type && textCount==lexeme.length() &&
				lexeme.equals(getLexeme());
	}


	/**
	 * Returns whether this token is a comment.
	 *
	 * @return Whether this token is a comment.
	 * @see #isWhitespace()
	 */
	public boolean isComment() {
		return type>=Token.COMMENT_EOL && type<=Token.COMMENT_MARKUP;
	}


	/**
	 * Returns whether this token is a hyperlink.
	 *
	 * @return Whether this token is a hyperlink.
	 * @see #setHyperlink(boolean)
	 */
	public boolean isHyperlink() {
		return hyperlink;
	}


	/**
	 * Returns whether this token is a {@link #SEPARATOR} representing a single
	 * left curly brace.
	 *
	 * @return Whether this token is a left curly brace.
	 * @see #isRightCurly()
	 */
	public boolean isLeftCurly() {
		return type==SEPARATOR && isSingleChar('{');
	}


	/**
	 * Returns whether this token is a {@link #SEPARATOR} representing a single
	 * right curly brace.
	 *
	 * @return Whether this token is a right curly brace.
	 * @see #isLeftCurly()
	 */
	public boolean isRightCurly() {
		return type==SEPARATOR && isSingleChar('}');
	}


	/**
	 * Returns whether or not this token is "paintable;" i.e., whether or not
	 * the type of this token is one such that it has an associated syntax
	 * style.  What this boils down to is whether the token type is greater
	 * than Token.NULL.
	 *
	 * @return Whether or not this token is paintable.
	 */
	public boolean isPaintable() {
		return type>Token.NULL;
	}


	/**
	 * Returns whether this token is the specified single character.
	 *
	 * @param ch The character to check for.
	 * @return Whether this token's lexeme is the single character specified.
	 */
	public boolean isSingleChar(char ch) {
		return textCount==1 && text[textOffset]==ch;
	}


	/**
	 * Returns whether or not this token is whitespace.
	 *
	 * @return true iff this token is whitespace.
	 * @see #isComment()
	 */
	public boolean isWhitespace() {
		return type==WHITESPACE;
	}


	/**
	 * Returns the bounding box for the specified document location.  The
	 * location must be in the specified token list; if it isn't,
	 * null is returned.
	 *
	 * @param textArea The text area from which the token list was derived.
	 * @param e How to expand tabs.
	 * @param pos The position in the document for which to get the bounding
	 *        box in the view.
	 * @param x0 The pixel x-location that is the beginning of
	 *        tokenList.
	 * @param rect The rectangle in which we'll be returning the results.  This
	 *        object is reused to keep from frequent memory allocations.
	 * @return The bounding box for the specified position in the model.
	 */
	public abstract Rectangle listOffsetToView(RSyntaxTextArea textArea,
								TabExpander e, int pos, int x0,
								Rectangle rect);


	/**
	 * Makes this token start at the specified offset into the document.
	 *
	 * @param pos The offset into the document this token should start at.
	 *        Note that this token must already contain this position; if
	 *        it doesn't, an exception is thrown.
	 * @throws IllegalArgumentException If pos is not already contained by
	 *         this token.
	 * @see #moveOffset(int)
	 */
	public void makeStartAt(int pos) {
		if (pos=(offset+textCount)) {
			throw new IllegalArgumentException("pos " + pos +
				" is not in range " + offset + "-" + (offset+textCount-1));
		}
		int shift = pos - offset;
		offset = pos;
		textOffset += shift;
		textCount -= shift;
	}


	/**
	 * Moves the starting offset of this token.
	 *
	 * @param amt The amount to move the starting offset.  This should be
	 *        between 0 and textCount, inclusive.
	 * @throws IllegalArgumentException If amt is an invalid value.
	 * @see #makeStartAt(int)
	 */
	public void moveOffset(int amt) {
		if (amt<0 || amt>textCount) {
			throw new IllegalArgumentException("amt " + amt +
					" is not in range 0-" + textCount);
		}
		offset += amt;
		textOffset += amt;
		textCount -= amt;
	}


	/**
	 * Paints this token.
	 *
	 * @param g The graphics context in which to paint.
	 * @param x The x-coordinate at which to paint.
	 * @param y The y-coordinate at which to paint.
	 * @param host The text area this token is in.
	 * @param e How to expand tabs.
	 * @return The x-coordinate representing the end of the painted text.
	 */
	public final float paint(Graphics2D g, float x, float y,
						RSyntaxTextArea host, TabExpander e) {
		return paint(g, x,y, host, e, 0);
	}


	/**
	 * Paints this token.
	 *
	 * @param g The graphics context in which to paint.
	 * @param x The x-coordinate at which to paint.
	 * @param y The y-coordinate at which to paint.
	 * @param host The text area this token is in.
	 * @param e How to expand tabs.
	 * @param clipStart The left boundary of the clip rectangle in which we're
	 *        painting.  This optimizes painting by allowing us to not paint
	 *        paint when this token is "to the left" of the clip rectangle.
	 * @return The x-coordinate representing the end of the painted text.
	 */
	public abstract float paint(Graphics2D g, float x, float y,
							RSyntaxTextArea host,
							TabExpander e, float clipStart);


	/**
	 * Paints the background of a token.
	 *
	 * @param x The x-coordinate of the token.
	 * @param y The y-coordinate of the token.
	 * @param width The width of the token (actually, the width of the part of
	 *        the token to paint).
	 * @param height The height of the token.
	 * @param g The graphics context with which to paint.
	 * @param fontAscent The ascent of the token's font.
	 * @param host The text area.
	 * @param color The color with which to paint.
	 */
	protected void paintBackground(float x, float y, float width, float height,
							Graphics2D g, int fontAscent,
							RSyntaxTextArea host, Color color) {
		// RSyntaxTextArea's bg can be null, so we must check for this.
		Color temp = host.getBackground();
		g.setXORMode(temp!=null ? temp : Color.WHITE);
		g.setColor(color);
		bgRect.setRect(x,y-fontAscent, width,height);
		g.fill(bgRect);
		g.setPaintMode();
	}

	/**
	 * Sets the value of this token to a particular segment of a document.
	 * The "next token" value is cleared.
	 *
	 * @param line The segment from which to get the token.
	 * @param beg The first character's position in line.
	 * @param end The last character's position in line.
	 * @param offset The offset into the document at which this token begins.
	 * @param type A token type listed as "generic" above.
	 */
	public void set(final char[] line, final int beg, final int end,
							final int offset, final int type) {
		this.text = line;
		this.textOffset = beg;
		this.textCount = end - beg + 1;
		this.type = type;
		this.offset = offset;
		nextToken = null;
	}


	/**
	 * Sets whether this token is a hyperlink.
	 *
	 * @param hyperlink Whether this token is a hyperlink.
	 * @see #isHyperlink()
	 */
	public void setHyperlink(boolean hyperlink) {
		this.hyperlink = hyperlink;
	}


	/**
	 * Sets the "next token" pointer of this token to point to the specified
	 * token.
	 *
	 * @param nextToken The new next token.
	 * @see #getNextToken
	 */
	public void setNextToken(Token nextToken) {
		this.nextToken = nextToken;
	}


	/**
	 * Returns the position in the document corresponding to the specified
	 * position in this token's internal char array (textOffset -
	 * textOffset+textCount-1).

	 * Note that this method does NOT do any bounds checking; you can pass in
	 * an invalid token position, and you will not receive an Exception or any
	 * other indication that the returned document position is invalid.  It is
	 * up to the user to ensure valid input.
	 *
	 * @param pos A position in the token's internal char array
	 *        (textOffset - textOffset+textCount).
	 * @return The corresponding position in the document.
	 * @see #documentToToken
	 */
	public int tokenToDocument(int pos) {
		return pos + (offset-textOffset);
	}


	/**
	 * Returns this token as a String, which is useful for
	 * debugging.
	 *
	 * @return A string describing this token.
	 */
	public String toString() {
		return "[Token: " +
			(type==Token.NULL ? "" :
				"text: '" +
					(text==null ? "" : getLexeme() + "'; " +
	       		"offset: " + offset + "; type: " + type + "; " +
		   		"isPaintable: " + isPaintable() +
		   		"; nextToken==null: " + (nextToken==null))) +
		   "]";
	}


}