pythonparse.Statements.scala Maven / Gradle / Ivy
The newest version!
package pythonparse
import fastparse._
import Expressions.{whitespace => _, _}
import Lexical.kw
object Statements extends Statements(0)
/**
* Python's statement grammar. This can only be used in statement-blocks,
* and is sensitive to newlines and indentation to determine nesting
*
* Manually transcribed from https://docs.python.org/2/reference/grammar.html
*/
class Statements(indent: Int){
implicit def whitespace(cfg: P[_]): P[Unit] = Lexical.wscomment(cfg)
def space[_: P] = P( CharIn(" \n") )
def NEWLINE[_: P]: P0 = P( "\n" | End )
def ENDMARKER[_: P]: P0 = P( End )
def single_input[_: P]: P[Seq[Ast.stmt]] = P(
NEWLINE.map(_ => Nil) |
simple_stmt |
compound_stmt.map(Seq(_)) ~ NEWLINE
)
def indents[_: P] = P( "\n" ~~ " ".repX(indent) )
def spaces[_: P] = P( (Lexical.nonewlinewscomment.? ~~ "\n").repX(1) )
def file_input[_: P]: P[Seq[Ast.stmt]] =
P( spaces.? ~ stmt.repX(0, spaces) ~ spaces.? ).map(_.flatten)
def eval_input[_: P]: P[Ast.expr] = P( testlist ~ NEWLINE.rep ~ ENDMARKER ).map(tuplize)
def collapse_dotted_name(name: Seq[Ast.identifier]): Ast.expr = {
name.tail.foldLeft[Ast.expr](Ast.expr.Name(name.head, Ast.expr_context.Load))(
(x, y) => Ast.expr.Attribute(x, y, Ast.expr_context.Load)
)
}
def decorator[_: P]: P[Ast.expr] = P( "@" ~/ dotted_name ~ ("(" ~ arglist ~ ")" ).? ~~ Lexical.nonewlinewscomment.? ~~ NEWLINE).map{
case (name, None) => collapse_dotted_name(name)
case (name, Some((args, (keywords, starargs, kwargs)))) =>
val x = collapse_dotted_name(name)
Ast.expr.Call(x, args, keywords, starargs, kwargs)
}
def decorators[_: P] = P( decorator.rep )
def decorated[_: P]: P[Ast.stmt] = P( decorators ~ (classdef | funcdef) ).map{case (a, b) => b(a)}
def classdef[_: P]: P[Seq[Ast.expr] => Ast.stmt.ClassDef] =
P( kw("class") ~/ NAME ~ ("(" ~ testlist.? ~ ")").?.map(_.toSeq.flatten.flatten) ~ ":" ~~ suite ).map{
case (a, b, c) => Ast.stmt.ClassDef(a, b, c, _)
}
def funcdef[_: P]: P[Seq[Ast.expr] => Ast.stmt.FunctionDef] = P( kw("def") ~/ NAME ~ parameters ~ ":" ~~ suite ).map{
case (name, args, suite) => Ast.stmt.FunctionDef(name, args, suite, _)
}
def parameters[_: P]: P[Ast.arguments] = P( "(" ~ varargslist ~ ")" )
def stmt[_: P]: P[Seq[Ast.stmt]] = P( compound_stmt.map(Seq(_)) | simple_stmt )
def simple_stmt[_: P]: P[Seq[Ast.stmt]] = P( small_stmt.rep(1, sep = ";") ~ ";".? )
def small_stmt[_: P]: P[Ast.stmt] = P(
print_stmt | del_stmt | pass_stmt | flow_stmt |
import_stmt | global_stmt | exec_stmt | assert_stmt | expr_stmt
)
def expr_stmt[_: P]: P[Ast.stmt] = {
def aug = P( testlist ~ augassign ~ (yield_expr | testlist.map(tuplize)) )
def assign = P( testlist ~ ("=" ~ (yield_expr | testlist.map(tuplize))).rep )
P(
aug.map{case (a, b, c) => Ast.stmt.AugAssign(tuplize(a), b, c) } |
assign.map{
case (a, Nil) => Ast.stmt.Expr(tuplize(a))
case (a, b) => Ast.stmt.Assign(Seq(tuplize(a)) ++ b.init, b.last)
}
)
}
def augassign[_: P]: P[Ast.operator] = P(
"+=".!.map(_ => Ast.operator.Add) |
"-=".!.map(_ => Ast.operator.Sub) |
"*=".!.map(_ => Ast.operator.Mult) |
"/=".!.map(_ => Ast.operator.Div) |
"%=".!.map(_ => Ast.operator.Mod) |
"&=".!.map(_ => Ast.operator.BitAnd) |
"|=".!.map(_ => Ast.operator.BitOr) |
"^=".!.map(_ => Ast.operator.BitXor) |
"<<=".!.map(_ => Ast.operator.LShift) |
">>=".!.map(_ => Ast.operator.RShift) |
"**=".!.map(_ => Ast.operator.Pow) |
"//=".!.map(_ => Ast.operator.FloorDiv)
)
def print_stmt[_: P]: P[Ast.stmt.Print] = {
def noDest = P( test.rep(sep = ",") ~ ",".?).map(Ast.stmt.Print(None, _, true))
def dest = P( ">>" ~ test ~ ("," ~ test).rep ~ ",".?).map{case (dest, exprs) => Ast.stmt.Print(Some(dest), exprs, true)}
P( "print" ~~ " ".rep ~~ (noDest | dest) )
}
def del_stmt[_: P] = P( kw("del") ~~ " ".rep ~~ exprlist ).map(Ast.stmt.Delete)
def pass_stmt[_: P] = P( kw("pass") ).map(_ => Ast.stmt.Pass)
def flow_stmt[_: P]: P[Ast.stmt] = P( break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt )
def break_stmt[_: P] = P( kw("break") ).map(_ => Ast.stmt.Break)
def continue_stmt[_: P] = P( kw("continue") ).map(_ => Ast.stmt.Continue)
def return_stmt[_: P] = P( kw("return") ~~ " ".rep ~~ testlist.map(tuplize).? ).map(Ast.stmt.Return)
def yield_stmt[_: P] = P( yield_expr ).map(Ast.stmt.Expr)
def raise_stmt[_: P]: P[Ast.stmt.Raise] = P( kw("raise") ~~ " ".rep ~~test.? ~ ("," ~ test).? ~ ("," ~ test).? ).map(Ast.stmt.Raise.tupled)
def import_stmt[_: P]: P[Ast.stmt] = P( import_name | import_from )
def import_name[_: P]: P[Ast.stmt.Import] = P( kw("import") ~ dotted_as_names ).map(Ast.stmt.Import)
def import_from[_: P]: P[Ast.stmt.ImportFrom] = {
def named = P( ".".rep(1).!.? ~ dotted_name.!.map(Some(_)) )
def unNamed = P( ".".rep(1).!.map(x => (Some(x), None)) )
def star = P( "*".!.map(_ => Seq(Ast.alias(Ast.identifier("*"), None))) )
P( kw("from") ~ (named | unNamed) ~ kw("import") ~ (star | "(" ~ import_as_names ~ ")" | import_as_names) ).map{
case (dots, module, names) => Ast.stmt.ImportFrom(module.map(Ast.identifier), names, dots.map(_.length))
}
}
def import_as_name[_: P]: P[Ast.alias] = P( NAME ~ (kw("as") ~ NAME).? ).map(Ast.alias.tupled)
def dotted_as_name[_: P]: P[Ast.alias] = P( dotted_name.map(x => Ast.identifier(x.map(_.name).mkString("."))) ~ (kw("as") ~ NAME).? ).map(Ast.alias.tupled)
def import_as_names[_: P] = P( import_as_name.rep(1, ",") ~ (",").? )
def dotted_as_names[_: P] = P( dotted_as_name.rep(1, ",") )
def dotted_name[_: P] = P( NAME.rep(1, ".") )
def global_stmt[_: P]: P[Ast.stmt.Global] = P( kw("global") ~ NAME.rep(sep = ",") ).map(Ast.stmt.Global)
def exec_stmt[_: P]: P[Ast.stmt.Exec] = P( kw("exec") ~ expr ~ (kw("in") ~ test ~ ("," ~ test).?).? ).map {
case (expr, None) => Ast.stmt.Exec(expr, None, None)
case (expr, Some((globals, None))) => Ast.stmt.Exec(expr, Some(globals), None)
case (expr, Some((globals, Some(locals)))) => Ast.stmt.Exec(expr, Some(globals), Some(locals))
}
def assert_stmt[_: P]: P[Ast.stmt.Assert] = P( kw("assert") ~ test ~ ("," ~ test).? ).map(Ast.stmt.Assert.tupled)
def compound_stmt[_: P]: P[Ast.stmt] = P( if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | decorated )
def if_stmt[_: P]: P[Ast.stmt.If] = {
def firstIf = P( kw("if") ~/ test ~ ":" ~~ suite )
def elifs = P( (space_indents ~~ kw("elif") ~/ test ~ ":" ~~ suite).repX )
def lastElse = P( (space_indents ~~ kw("else") ~/ ":" ~~ suite).? )
P( firstIf ~~ elifs ~~ lastElse ).map{
case (test, body, elifs, orelse) =>
val (init :+ last) = (test, body) +: elifs
val (last_test, last_body) = last
init.foldRight(Ast.stmt.If(last_test, last_body, orelse.toSeq.flatten)){
case ((test, body), rhs) => Ast.stmt.If(test, body, Seq(rhs))
}
}
}
def space_indents[_: P] = P( spaces.repX ~~ " ".repX(indent) )
def while_stmt[_: P] = P( kw("while") ~/ test ~ ":" ~~ suite ~~ (space_indents ~~ kw("else") ~/ ":" ~~ suite).?.map(_.toSeq.flatten) ).map(Ast.stmt.While.tupled)
def for_stmt[_: P]: P[Ast.stmt.For] = P( kw("for") ~/ exprlist ~ kw("in") ~ testlist ~ ":" ~~ suite ~~ (space_indents ~ kw("else") ~/ ":" ~~ suite).? ).map {
case (itervars, generator, body, orelse) =>
Ast.stmt.For(tuplize(itervars), tuplize(generator), body, orelse.toSeq.flatten)
}
def try_stmt[_: P]: P[Ast.stmt]= {
def `try` = P( kw("try") ~/ ":" ~~ suite )
def excepts: P[Seq[Ast.excepthandler]] = P( (except_clause ~ ":" ~~ suite).map{
case (None, body) => Ast.excepthandler.ExceptHandler(None, None, body)
case (Some((x, None)), body) => Ast.excepthandler.ExceptHandler(Some(x), None, body)
case (Some((x, Some(y))), body) => Ast.excepthandler.ExceptHandler(Some(x), Some(y), body)
}.repX )
def `else` = P( space_indents ~~ kw("else") ~/ ":" ~~ suite )
def `finally` = P( space_indents ~~ kw("finally") ~/ ":" ~~ suite )
P( `try` ~~ excepts ~~ `else`.? ~~ `finally`.? ).map{
case (tryBlock, excepts, elseBlock, None) =>
Ast.stmt.TryExcept(tryBlock, excepts, elseBlock.toSeq.flatten)
case (tryBlock, Nil, None, Some(finallyBlock)) =>
Ast.stmt.TryFinally(tryBlock, finallyBlock)
case (tryBlock, excepts, elseBlock, Some(finallyBlock)) =>
Ast.stmt.TryFinally(
Seq(Ast.stmt.TryExcept(tryBlock, excepts, elseBlock.toSeq.flatten)),
finallyBlock
)
}
}
def with_stmt[_: P]: P[Ast.stmt.With] = P( kw("with") ~/ with_item.rep(1, ",")~ ":" ~~ suite ).map{
case (items, body) =>
val (last_expr, last_vars) = items.last
val inner = Ast.stmt.With(last_expr, last_vars, body)
items.init.foldRight(inner){
case ((expr, vars), body) => Ast.stmt.With(expr, vars, Seq(body))
}
}
def with_item[_: P]: P[(Ast.expr, Option[Ast.expr])] = P( test ~ (kw("as") ~ expr).? )
// NB compile.c makes sure that the default except clause is last
def except_clause[_: P] = P( space_indents ~ kw("except") ~/ (test ~ ((kw("as") | ",") ~ test).?).? )
def suite[_: P]: P[Seq[Ast.stmt]] = {
def deeper: P[Int] = {
def commentLine = P("\n" ~~ Lexical.nonewlinewscomment.?.map(_ => 0)).map((_, Some("")))
def endLine = P("\n" ~~ (" "|"\t").repX(indent + 1).!.map(_.length) ~~ Lexical.comment.!.? )
P( Lexical.nonewlinewscomment.? ~~ ( endLine | commentLine ).repX(1) ).map{
_.collectFirst{ case (s, None) => s}
}.filter(_.isDefined).map(_.get)
}
def indented = P( deeper.flatMapX{ nextIndent =>
new Statements(nextIndent).stmt.repX(1, spaces.repX(1) ~~ (" " * nextIndent | "\t" * nextIndent)).map(_.flatten)
} )
P( indented | " ".rep ~ simple_stmt )
}
}