ast.asdl_py.py Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of jython Show documentation
Show all versions of jython Show documentation
Jython is an implementation of the high-level, dynamic, object-oriented
language Python written in 100% Pure Java, and seamlessly integrated with
the Java platform. It thus allows you to run Python on any Java platform.
"""
Generate AST node definitions from an ASDL description.
"""
import sys
import os
import asdl
class ASDLVisitor(asdl.VisitorBase):
def __init__(self, stream, data):
super(ASDLVisitor, self).__init__()
self.stream = stream
self.data = data
def visitModule(self, mod, *args):
for df in mod.dfns:
self.visit(df, *args)
def visitSum(self, sum, *args):
for tp in sum.types:
self.visit(tp, *args)
def visitType(self, tp, *args):
self.visit(tp.value, *args)
def visitProduct(self, prod, *args):
for field in prod.fields:
self.visit(field, *args)
def visitConstructor(self, cons, *args):
for field in cons.fields:
self.visit(field, *args)
def visitField(self, field):
pass
def emit(self, line, level=0):
indent = " "*level
self.stream.write(indent + line + "\n")
def is_simple_sum(sum):
assert isinstance(sum, asdl.Sum)
for constructor in sum.types:
if constructor.fields:
return False
return True
class ASTNodeVisitor(ASDLVisitor):
def visitType(self, tp):
self.visit(tp.value, tp.name)
def visitSum(self, sum, base):
if is_simple_sum(sum):
self.emit("class %s(AST):" % (base,))
self.emit("")
self.emit("def to_simple_int(self, space):", 1)
self.emit("w_msg = space.wrap(\"not a valid %s\")" % (base,), 2)
self.emit("raise OperationError(space.w_TypeError, w_msg)", 2)
self.emit("")
for i, cons in enumerate(sum.types):
self.emit("class _%s(%s):" % (cons.name, base))
self.emit("")
self.emit("def to_simple_int(self, space):", 1)
self.emit("return %i" % (i + 1,), 2)
self.emit("")
for i, cons in enumerate(sum.types):
self.emit("%s = %i" % (cons.name, i + 1))
self.emit("")
self.emit("%s_to_class = [" % (base,))
for cons in sum.types:
self.emit("_%s," % (cons.name,), 1)
self.emit("]")
self.emit("")
else:
self.emit("class %s(AST):" % (base,))
self.emit("")
slots = ", ".join(repr(attr.name.value) for attr in sum.attributes)
self.emit("__slots__ = (%s)" % (slots,), 1)
self.emit("")
if sum.attributes:
args = ", ".join(attr.name.value for attr in sum.attributes)
self.emit("def __init__(self, %s):" % (args,), 1)
for attr in sum.attributes:
self.visit(attr)
self.emit("")
for cons in sum.types:
self.visit(cons, base, sum.attributes)
self.emit("")
def visitProduct(self, product, name):
self.emit("class %s(AST):" % (name,))
self.emit("")
slots = self.make_slots(product.fields)
self.emit("__slots__ = (%s)" % (slots,), 1)
self.emit("")
self.make_constructor(product.fields, product)
self.emit("")
self.emit("def walkabout(self, visitor):", 1)
self.emit("visitor.visit_%s(self)" % (name,), 2)
self.emit("")
self.make_var_syncer(product.fields, product, name)
def make_slots(self, fields):
slots = []
for field in fields:
name = repr(field.name.value)
slots.append(name)
if field.seq:
slots.append("'w_%s'" % (field.name,))
return ", ".join(slots)
def make_var_syncer(self, fields, node, name):
self.emit("def sync_app_attrs(self, space):", 1)
config = (self.data.optional_masks[node],
self.data.required_masks[node])
self.emit("if (self.initialization_state & ~%i) ^ %i:" % config, 2)
names = []
for field in fields:
if field.opt:
names.append("None")
else:
names.append(repr(field.name.value))
sub = (", ".join(names), name.value)
self.emit("missing_field(space, self.initialization_state, [%s], %r)"
% sub, 3)
self.emit("else:", 2)
# Fill in all the default fields.
doing_something = False
for field in fields:
if field.opt:
doing_something = True
flag = self.data.field_masks[field]
self.emit("if not self.initialization_state & %i:" % (flag,), 3)
default = "0" if field.type.value == "int" else "None"
self.emit("self.%s = %s" % (field.name, default), 4)
if not doing_something:
self.emit("pass", 3)
for attr in fields:
if attr.seq:
self.emit("w_list = self.w_%s" % (attr.name,), 2)
self.emit("if w_list is not None:", 2)
self.emit("list_w = space.viewiterable(w_list)", 3)
self.emit("if list_w:", 3)
unwrapper = get_unwrapper(attr.type.value, "w_obj",
self.data.simple_types)
config = (attr.name, unwrapper)
self.emit("self.%s = [%s for w_obj in list_w]" % config,
4),
self.emit("else:", 3)
self.emit("self.%s = None" % (attr.name,), 4)
if attr.type.value not in asdl.builtin_types and \
attr.type.value not in self.data.simple_types:
self.emit("if self.%s is not None:" % (attr.name,), 2)
self.emit("for node in self.%s:" % (attr.name,), 3)
self.emit("node.sync_app_attrs(space)", 4)
elif attr.type.value not in asdl.builtin_types and \
attr.type.value not in self.data.simple_types:
doing_something = True
level = 2
if attr.opt:
self.emit("if self.%s:" % (attr.name,), 2)
level += 1
self.emit("self.%s.sync_app_attrs(space)" % (attr.name,), level)
self.emit("")
def make_constructor(self, fields, node, extras=None, base=None):
if fields or extras:
arg_fields = fields + extras if extras else fields
args = ", ".join(str(field.name) for field in arg_fields)
self.emit("def __init__(self, %s):" % args, 1)
for field in fields:
self.visit(field)
if extras:
base_args = ", ".join(str(field.name) for field in extras)
self.emit("%s.__init__(self, %s)" % (base, base_args), 2)
else:
self.emit("def __init__(self):", 1)
have_everything = self.data.required_masks[node] | \
self.data.optional_masks[node]
self.emit("self.initialization_state = %i" % (have_everything,), 2)
def visitConstructor(self, cons, base, extra_attributes):
self.emit("class %s(%s):" % (cons.name, base))
self.emit("")
slots = self.make_slots(cons.fields)
self.emit("__slots__ = (%s)" % (slots,), 1)
self.emit("")
for field in self.data.cons_attributes[cons]:
subst = (field.name, self.data.field_masks[field])
self.emit("_%s_mask = %i" % subst, 1)
self.emit("")
self.make_constructor(cons.fields, cons, extra_attributes, base)
self.emit("")
self.emit("def walkabout(self, visitor):", 1)
self.emit("visitor.visit_%s(self)" % (cons.name,), 2)
self.emit("")
self.emit("def mutate_over(self, visitor):", 1)
for field in cons.fields:
if field.type.value not in asdl.builtin_types and \
field.type.value not in self.data.prod_simple:
if field.opt or field.seq:
level = 3
self.emit("if self.%s:" % (field.name,), 2)
else:
level = 2
if field.seq:
sub = (field.name,)
self.emit("visitor._mutate_sequence(self.%s)" % sub, level)
else:
sub = (field.name, field.name)
self.emit("self.%s = self.%s.mutate_over(visitor)" % sub,
level)
self.emit("return visitor.visit_%s(self)" % (cons.name,), 2)
self.emit("")
self.make_var_syncer(cons.fields + self.data.cons_attributes[cons],
cons, cons.name)
def visitField(self, field):
self.emit("self.%s = %s" % (field.name, field.name), 2)
if field.seq:
self.emit("self.w_%s = None" % (field.name,), 2)
class ASTVisitorVisitor(ASDLVisitor):
"""A meta visitor! :)"""
def visitModule(self, mod):
self.emit("class ASTVisitor(object):")
self.emit("")
self.emit("def visit_sequence(self, seq):", 1)
self.emit("for node in seq:", 2)
self.emit("node.walkabout(self)", 3)
self.emit("")
self.emit("def default_visitor(self, node):", 1)
self.emit("raise NodeVisitorNotImplemented", 2)
self.emit("")
self.emit("def _mutate_sequence(self, seq):", 1)
self.emit("for i in range(len(seq)):", 2)
self.emit("seq[i] = seq[i].mutate_over(self)", 3)
self.emit("")
super(ASTVisitorVisitor, self).visitModule(mod)
self.emit("")
def visitType(self, tp):
if not (isinstance(tp.value, asdl.Sum) and
is_simple_sum(tp.value)):
super(ASTVisitorVisitor, self).visitType(tp, tp.name)
def visitProduct(self, prod, name):
self.emit("def visit_%s(self, node):" % (name,), 1)
self.emit("return self.default_visitor(node)", 2)
def visitConstructor(self, cons, _):
self.emit("def visit_%s(self, node):" % (cons.name,), 1)
self.emit("return self.default_visitor(node)", 2)
class GenericASTVisitorVisitor(ASDLVisitor):
def visitModule(self, mod):
self.emit("class GenericASTVisitor(ASTVisitor):")
self.emit("")
super(GenericASTVisitorVisitor, self).visitModule(mod)
self.emit("")
def visitType(self, tp):
if not (isinstance(tp.value, asdl.Sum) and
is_simple_sum(tp.value)):
super(GenericASTVisitorVisitor, self).visitType(tp, tp.name)
def visitProduct(self, prod, name):
self.make_visitor(name, prod.fields)
def visitConstructor(self, cons, _):
self.make_visitor(cons.name, cons.fields)
def make_visitor(self, name, fields):
self.emit("def visit_%s(self, node):" % (name,), 1)
have_body = False
for field in fields:
if self.visitField(field):
have_body = True
if not have_body:
self.emit("pass", 2)
self.emit("")
def visitField(self, field):
if field.type.value not in asdl.builtin_types and \
field.type.value not in self.data.simple_types:
if field.seq or field.opt:
self.emit("if node.%s:" % (field.name,), 2)
level = 3
else:
level = 2
if field.seq:
template = "self.visit_sequence(node.%s)"
else:
template = "node.%s.walkabout(self)"
self.emit(template % (field.name,), level)
return True
return False
asdl_type_map = {
"int" : "int_w",
"identifier" : "str_w",
"bool" : "bool_w"
}
def get_unwrapper(tp, name, simple_types):
if tp in asdl.builtin_types:
return "space.%s(%s)" % (asdl_type_map[tp], name)
elif tp in simple_types:
return "space.interp_w(%s, %s).to_simple_int(space)" % (tp, name)
else:
return "space.interp_w(%s, %s)" % (tp, name)
# CPython lets blank AST nodes (no constructor arguments) be created
# and the attributes added later. In CPython, it is implemented by
# implementing applevel and c level AST as different structures and
# copying between them. This is hideous, so we use a slightly less
# ugly hack in PyPy. Each field has a bitmask which is set on the
# initialization_state attribute when the field type is set. When
# sync_app_attrs() is called, it's a simple matter of removing the
# optional field flags from initialization_state, and using XOR to
# test if all the required fields have been set.
class AppExposeVisitor(ASDLVisitor):
def visitType(self, tp):
super(AppExposeVisitor, self).visitType(tp, tp.name)
def visitSum(self, sum, name):
for field in sum.attributes:
self.make_property(field, name, True)
self.make_typedef(name, "AST", sum.attributes,
fields_name="_attributes")
if not is_simple_sum(sum):
super(AppExposeVisitor, self).visitSum(sum, name)
else:
for cons in sum.types:
self.make_typedef("_" + cons.name.value, name, (), cons.name,
concrete=True)
def make_typedef(self, name, base, fields, display_name=None,
fields_name="_fields", concrete=False, needs_init=False):
if display_name is None:
display_name = name
self.emit("%s.typedef = typedef.TypeDef(\"%s\"," % (name, display_name))
self.emit("%s.typedef," % (base,), 1)
comma_fields = ", ".join(repr(field.name.value) for field in fields)
self.emit("%s=_FieldsWrapper([%s])," % (fields_name, comma_fields), 1)
for field in fields:
getter = "%s_get_%s" % (name, field.name)
setter = "%s_set_%s" % (name, field.name)
config = (field.name, getter, setter, name)
self.emit("%s=typedef.GetSetProperty(%s, %s, cls=%s)," % config, 1)
# CPython lets you create instances of "abstract" AST nodes
# like ast.expr or even ast.AST. This doesn't seem to useful
# and would be a pain to implement safely, so we don't allow
# it.
if concrete:
self.emit("__new__=interp2app(get_AST_new(%s))," % (name,), 1)
if needs_init:
self.emit("__init__=interp2app(%s_init)," % (name,), 1)
self.emit(")")
self.emit("%s.typedef.acceptable_as_base_class = False" % (name,))
self.emit("")
def make_init(self, name, fields):
comma_fields = ", ".join(repr(field.name.value) for field in fields)
config = (name, comma_fields)
self.emit("_%s_field_unroller = unrolling_iterable([%s])" % config)
self.emit("def %s_init(space, w_self, args):" % (name,))
self.emit("w_self = space.descr_self_interp_w(%s, w_self)" % (name,), 1)
for field in fields:
if field.seq:
self.emit("w_self.w_%s = None" % (field.name,), 1)
self.emit("args_w, kwargs_w = args.unpack()", 1)
self.emit("if args_w:", 1)
arity = len(fields)
if arity:
self.emit("if len(args_w) != %i:" % (arity,), 2)
self.emit("w_err = space.wrap(\"%s constructor takes 0 or %i " \
"positional arguments\")" % (name, arity), 3)
self.emit("raise OperationError(space.w_TypeError, w_err)", 3)
self.emit("i = 0", 2)
self.emit("for field in _%s_field_unroller:" % (name,), 2)
self.emit("space.setattr(w_self, space.wrap(field), args_w[i])", 3)
self.emit("i += 1", 3)
else:
self.emit("w_err = space.wrap(\"%s constructor takes no " \
" arguments\")" % (name,), 2)
self.emit("raise OperationError(space.w_TypeError, w_err)", 2)
self.emit("for field, w_value in kwargs_w.iteritems():", 1)
self.emit("space.setattr(w_self, space.wrap(field), w_value)", 2)
self.emit("%s_init.unwrap_spec = [ObjSpace, W_Root, Arguments]"
% (name,))
self.emit("")
def visitConstructor(self, cons, base):
super(AppExposeVisitor, self).visitConstructor(cons, cons.name)
self.make_init(cons.name, cons.fields + self.data.cons_attributes[cons])
self.make_typedef(cons.name, base, cons.fields, concrete=True,
needs_init=True)
def visitProduct(self, product, name):
super(AppExposeVisitor, self).visitProduct(product, name)
self.make_init(name, product.fields)
self.make_typedef(name, "AST", product.fields, concrete=True,
needs_init=True)
def visitField(self, field, name):
self.make_property(field, name)
def make_property(self, field, name, different_masks=False):
func = "def %s_get_%s(space, w_self):" % (name, field.name)
self.emit(func)
if different_masks:
flag = "w_self._%s_mask" % (field.name,)
else:
flag = self.data.field_masks[field]
self.emit("if not w_self.initialization_state & %s:" % (flag,), 1)
self.emit("w_err = space.wrap(\"attribute '%s' has not been set\")" %
(field.name,), 2)
self.emit("raise OperationError(space.w_AttributeError, w_err)", 2)
if field.seq:
self.emit("if w_self.w_%s is None:" % (field.name,), 1)
self.emit("if w_self.%s is None:" % (field.name,), 2)
self.emit("w_list = space.newlist([])", 3)
self.emit("else:", 2)
if field.type.value in self.data.simple_types:
wrapper = "%s_to_class[node - 1]()" % (field.type,)
else:
wrapper = "space.wrap(node)"
self.emit("list_w = [%s for node in w_self.%s]" %
(wrapper, field.name), 3)
self.emit("w_list = space.newlist(list_w)", 3)
self.emit("w_self.w_%s = w_list" % (field.name,), 2)
self.emit("return w_self.w_%s" % (field.name,), 1)
elif field.type.value in self.data.simple_types:
config = (field.type, field.name)
self.emit("return %s_to_class[w_self.%s - 1]()" % config, 1)
elif field.type.value in ("object", "string"):
self.emit("return w_self.%s" % (field.name,), 1)
else:
self.emit("return space.wrap(w_self.%s)" % (field.name,), 1)
self.emit("")
func = "def %s_set_%s(space, w_self, w_new_value):" % (name, field.name)
self.emit(func)
if field.seq:
self.emit("w_self.w_%s = w_new_value" % (field.name,), 1)
elif field.type.value not in asdl.builtin_types:
# These are always other AST nodes.
if field.type.value in self.data.simple_types:
self.emit("obj = space.interp_w(%s, w_new_value)" % \
(field.type,), 1)
self.emit("w_self.%s = obj.to_simple_int(space)" %
(field.name,), 1)
else:
config = (field.name, field.type, repr(field.opt))
self.emit("w_self.%s = space.interp_w(%s, w_new_value, %s)" %
config, 1)
else:
level = 1
if field.opt and field.type.value != "int":
self.emit("if space.is_w(w_new_value, space.w_None):", 1)
self.emit("w_self.%s = None" % (field.name,), 2)
level += 1
self.emit("else:", 1)
if field.type.value == "object":
self.emit("w_self.%s = w_new_value" % (field.name,), level)
elif field.type.value == "string":
self.emit("if not space.is_true(space.isinstance(" \
"w_new_value, space.w_basestring)):", level)
line = "w_err = space.wrap(\"some kind of string required\")"
self.emit(line, level + 1)
self.emit("raise OperationError(space.w_TypeError, w_err)",
level + 1)
self.emit("w_self.%s = w_new_value" % (field.name,), level)
else:
space_method = asdl_type_map[field.type.value]
config = (field.name, space_method)
self.emit("w_self.%s = space.%s(w_new_value)" % config, level)
self.emit("w_self.initialization_state |= %s" % (flag,), 1)
self.emit("")
def copy_field(field):
return asdl.Field(field.type, field.name, field.seq, field.opt)
class ASDLData(object):
def __init__(self, tree):
simple_types = set()
prod_simple = set()
field_masks = {}
required_masks = {}
optional_masks = {}
cons_attributes = {}
def add_masks(fields, node):
required_mask = 0
optional_mask = 0
for i, field in enumerate(fields):
flag = 1 << i
field_masks[field] = flag
if field.opt:
optional_mask |= flag
else:
required_mask |= flag
required_masks[node] = required_mask
optional_masks[node] = optional_mask
for tp in tree.dfns:
if isinstance(tp.value, asdl.Sum):
sum = tp.value
if is_simple_sum(sum):
simple_types.add(tp.name.value)
else:
for cons in sum.types:
attrs = [copy_field(field) for field in sum.attributes]
add_masks(cons.fields + attrs, cons)
cons_attributes[cons] = attrs
else:
prod = tp.value
prod_simple.add(tp.name.value)
add_masks(prod.fields, prod)
prod_simple.update(simple_types)
self.cons_attributes = cons_attributes
self.simple_types = simple_types
self.prod_simple = prod_simple
self.field_masks = field_masks
self.required_masks = required_masks
self.optional_masks = optional_masks
HEAD = """# Generated by tools/asdl_py.py
from pypy.interpreter.baseobjspace import Wrappable, ObjSpace, W_Root
from pypy.interpreter import typedef
from pypy.interpreter.gateway import interp2app
from pypy.interpreter.argument import Arguments
from pypy.interpreter.error import OperationError
from pypy.rlib.unroll import unrolling_iterable
from pypy.tool.pairtype import extendabletype
from pypy.tool.sourcetools import func_with_new_name
class AST(Wrappable):
__slots__ = ("initialization_state",)
__metaclass__ = extendabletype
def walkabout(self, visitor):
raise AssertionError("walkabout() implementation not provided")
def mutate_over(self, visitor):
raise AssertionError("mutate_over() implementation not provided")
def sync_app_attrs(self, space):
raise NotImplementedError
class NodeVisitorNotImplemented(Exception):
pass
class _FieldsWrapper(Wrappable):
"Hack around the fact we can't store tuples on a TypeDef."
def __init__(self, fields):
self.fields = fields
def __spacebind__(self, space):
return space.newtuple([space.wrap(field) for field in self.fields])
def get_AST_new(node_class):
def generic_AST_new(space, w_type, __args__):
node = space.allocate_instance(node_class, w_type)
node.initialization_state = 0
return space.wrap(node)
generic_AST_new.unwrap_spec = [ObjSpace, W_Root, Arguments]
return func_with_new_name(generic_AST_new, "new_%s" % node_class.__name__)
AST.typedef = typedef.TypeDef("AST",
_fields=_FieldsWrapper([]),
_attributes=_FieldsWrapper([]),
)
AST.typedef.acceptable_as_base_class = False
def missing_field(space, state, required, host):
"Find which required field is missing."
for i in range(len(required)):
if not (state >> i) & 1:
missing = required[i]
if missing is not None:
err = "required attribute '%s' missing from %s"
err = err % (missing, host)
w_err = space.wrap(err)
raise OperationError(space.w_TypeError, w_err)
raise AssertionError("should not reach here")
"""
visitors = [ASTNodeVisitor, ASTVisitorVisitor, GenericASTVisitorVisitor,
AppExposeVisitor]
def main(argv):
if len(argv) == 3:
def_file, out_file = argv[1:]
elif len(argv) == 1:
print "Assuming default values of Python.asdl and ast.py"
here = os.path.dirname(__file__)
def_file = os.path.join(here, "Python.asdl")
out_file = os.path.join(here, "..", "ast.py")
else:
print >> sys.stderr, "invalid arguments"
return 2
mod = asdl.parse(def_file)
data = ASDLData(mod)
fp = open(out_file, "w")
try:
fp.write(HEAD)
for visitor in visitors:
visitor(fp, data).visit(mod)
finally:
fp.close()
if __name__ == "__main__":
sys.exit(main(sys.argv))