com.jetbrains.python.lexer.PyStringLiteralLexer Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of python-community Show documentation
A packaging of the IntelliJ Community Edition python-community library. This is release number 1 of trunk branch 142.
The newest version!
/*
 * Copyright 2000-2014 JetBrains s.r.o.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.jetbrains.python.lexer;

import com.intellij.lexer.LexerBase;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.util.text.StringUtil;
import com.intellij.psi.StringEscapesTokenTypes;
import com.intellij.psi.tree.IElementType;
import com.jetbrains.python.PyTokenTypes;
import org.jetbrains.annotations.NotNull;

/**
 * Specialized lexer for string literals. To be used as a layer in a LayeredLexer.
 * Mostly handles escapes, differently in byte / unicode / raw strings.
 * Snatched from com.intellij.lexer.StringLiteralLexer and may inherit from it in the future.
 * Lexes the entire string, with u/b/r designator, quotes, and content, thus self-adjusts for the format.
 * User: dcheryasov
 * Date: May 13, 2009 7:35:59 PM
 */
public class PyStringLiteralLexer extends LexerBase {
  private static final Logger LOG = Logger.getInstance("#com.jetbrains.python.lexer.PyStringLiteralLexer");

  private static final short BEFORE_FIRST_QUOTE = 0; // the initial state; may last during 'u' and 'r' prefixes.
  private static final short AFTER_FIRST_QUOTE = 1;
  private static final short AFTER_LAST_QUOTE = 2;

  private CharSequence myBuffer;
  private int myStart;
  private int myEnd;
  private int myState;
  private int myLastState;
  private int myBufferEnd;
  private char myQuoteChar;

  private boolean myIsRaw;
  private boolean myIsTriple;
  private final IElementType myOriginalLiteralToken;
  private boolean mySeenEscapedSpacesOnly;


  /**
   * @param originalLiteralToken the AST node we're layering over.
   */
  public PyStringLiteralLexer(final IElementType originalLiteralToken) {
    myOriginalLiteralToken = originalLiteralToken;
  }

  public void start(@NotNull CharSequence buffer, int startOffset, int endOffset, int initialState) {
    myBuffer = buffer;
    myStart = startOffset;
    myState = initialState;
    myLastState = initialState;
    mySeenEscapedSpacesOnly = true;
    myBufferEnd = endOffset;

    // the following could be parsing steps if we wanted this info as tokens
    int i = myStart;

    i = skipEncodingPrefix(buffer, i);
    int offset = skipRawPrefix(buffer, i);
    if (offset > i) myIsRaw = true;
    i = offset;
    i = skipEncodingPrefix(buffer, i);
    offset = skipRawPrefix(buffer, i);
    if (offset > i) myIsRaw = true;
    i = offset;

    // which quote char?
    char c = buffer.charAt(i);
    assert (c == '"') || (c == '\'') : "String must be quoted by single or double quote. Found '" + c + "' in string " + buffer;
    myQuoteChar = c;

    myIsTriple = (buffer.length() > i + 2) && (buffer.charAt(i + 1) == c) && (buffer.charAt(i + 2) == c);

    // calculate myEnd at last
    myEnd = locateToken(myStart);
  }

  public static int skipRawPrefix(CharSequence text, int startOffset) {
    char c = Character.toUpperCase(text.charAt(startOffset));
    if (c == 'R') {
      startOffset++;
    }
    return startOffset;
  }

  public static int skipEncodingPrefix(CharSequence text, int startOffset) {
    char c = Character.toUpperCase(text.charAt(startOffset));
    if (c == 'U' || c == 'B' || c == 'C') {
      startOffset++;
    }
    return startOffset;
  }

  public int getState() {
    return myLastState;
  }

  public IElementType getTokenType() {
    if (myStart >= myEnd) return null;

    // skip non-escapes immediately
    if (myBuffer.charAt(myStart) != '\\' || (myIsRaw && (!isUnicodeMode() || !nextIsUnicodeEscape()))) {
      mySeenEscapedSpacesOnly = false;
      return myOriginalLiteralToken;
    }

    // from here on, only escapes
    if (myStart + 1 >= myEnd) return StringEscapesTokenTypes.INVALID_CHARACTER_ESCAPE_TOKEN; // escape ends too early
    char nextChar = myBuffer.charAt(myStart + 1);
    mySeenEscapedSpacesOnly &= nextChar == ' ';
    if ((nextChar == '\n' || nextChar == ' ' && (mySeenEscapedSpacesOnly || isTrailingSpace(myStart+2)))) {
      return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN; // escaped EOL
    }
    if (nextChar == 'u' || nextChar == 'U') {
      if (isUnicodeMode()) {
        final int width = nextChar == 'u'? 4 : 8; // is it uNNNN or Unnnnnnnn
        for(int i = myStart + 2; i < myStart + width + 2; i++) {
          if (i >= myEnd || !StringUtil.isHexDigit(myBuffer.charAt(i))) return StringEscapesTokenTypes.INVALID_UNICODE_ESCAPE_TOKEN;
        }
        return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN;
      }
      else return myOriginalLiteralToken; // b"\u1234" is just b"\\u1234", nothing gets escaped
    }

    if (nextChar == 'x') { // \xNN is allowed both in bytes and unicode.
      for(int i = myStart + 2; i < myStart + 4; i++) {
        if (i >= myEnd || !StringUtil.isHexDigit(myBuffer.charAt(i))) return StringEscapesTokenTypes.INVALID_UNICODE_ESCAPE_TOKEN;
      }
      return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN;
    }

    if (nextChar == 'N' && isUnicodeMode()) {
      int i = myStart+2;
      if (i >= myEnd || myBuffer.charAt(i) != '{') return StringEscapesTokenTypes.INVALID_UNICODE_ESCAPE_TOKEN;
      i++;
      while(i < myEnd && myBuffer.charAt(i) != '}') i++;
      if (i >= myEnd) return StringEscapesTokenTypes.INVALID_UNICODE_ESCAPE_TOKEN;
      return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN;
    }

    switch (nextChar) {
      case 'a':
      case 'b':
      case 'f':
      case 'n':
      case 'r':
      case 't':
      case 'v':
      case '\'':
      case '\"':
      case '\\':
      case '0':
      case '1':
      case '2':
      case '3':
      case '4':
      case '5':
      case '6':
      case '7': return StringEscapesTokenTypes.VALID_STRING_ESCAPE_TOKEN;
    }

    // other unrecognized escapes are just part of string, not an error
    return myOriginalLiteralToken;
  }

  private boolean nextIsUnicodeEscape() {
    if (myStart + 1 < myEnd) {
      char nextChar = myBuffer.charAt(myStart + 1);
      return nextChar == 'u' || nextChar == 'U';
    }
    return false;
  }

  private boolean isUnicodeMode() {
    return PyTokenTypes.UNICODE_NODES.contains(myOriginalLiteralToken);
  }

  // all subsequent chars are escaped spaces
  private boolean isTrailingSpace(final int start) {
    for (int i=start; i