org.python.compiler.Module Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of jython-slim Show documentation
Show all versions of jython-slim Show documentation
Jython is an implementation of the high-level, dynamic, object-oriented
language Python written in 100% Pure Java, and seamlessly integrated with
the Java platform. It thus allows you to run Python on any Java platform.
// Copyright (c) Corporation for National Research Initiatives
package org.python.compiler;
import static org.python.core.RegistryKey.PYTHON_CPYTHON;
import static org.python.util.CodegenUtils.ci;
import static org.python.util.CodegenUtils.p;
import static org.python.util.CodegenUtils.sig;
import java.io.IOException;
import java.io.OutputStream;
import java.io.ByteArrayOutputStream;
import java.io.ObjectOutputStream;
import java.io.File;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.URL;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.Hashtable;
import java.util.Stack;
import java.util.List;
import org.objectweb.asm.Label;
import org.objectweb.asm.MethodTooLargeException;
import org.objectweb.asm.Opcodes;
import org.objectweb.asm.Type;
import org.python.antlr.ParseException;
import org.python.antlr.PythonTree;
import org.python.antlr.ast.Num;
import org.python.antlr.ast.Str;
import org.python.antlr.ast.Suite;
import org.python.antlr.base.mod;
import org.python.core.ClasspathPyImporter;
import org.python.core.CodeBootstrap;
import org.python.core.CodeFlag;
import org.python.core.CodeLoader;
import org.python.core.CompilerFlags;
import org.python.core.imp;
import org.python.core.Py;
import org.python.core.PyCode;
import org.python.core.PyBytecode;
import org.python.core.PyComplex;
import org.python.core.PyException;
import org.python.core.PyFile;
import org.python.core.PyFloat;
import org.python.core.PyFrame;
import org.python.core.PyFunctionTable;
import org.python.core.PyInteger;
import org.python.core.PyLong;
import org.python.core.PyObject;
import org.python.core.PyRunnable;
import org.python.core.PyRunnableBootstrap;
import org.python.core.PyString;
import org.python.core.PyUnicode;
import org.python.core.ThreadState;
import org.python.modules._marshal;
class PyIntegerConstant extends Constant implements ClassConstants, Opcodes {
final int value;
PyIntegerConstant(int value) {
this.value = value;
}
@Override
void get(Code c) throws IOException {
c.iconst(value); // it would be nice if we knew we didn't have to box next
c.invokestatic(p(Py.class), "newInteger", sig(PyInteger.class, Integer.TYPE));
}
@Override
void put(Code c) throws IOException {}
@Override
public int hashCode() {
return value;
}
@Override
public boolean equals(Object o) {
if (o instanceof PyIntegerConstant) {
return ((PyIntegerConstant)o).value == value;
} else {
return false;
}
}
}
class PyFloatConstant extends Constant implements ClassConstants, Opcodes {
final double value;
PyFloatConstant(double value) {
this.value = value;
}
@Override
void get(Code c) throws IOException {
c.ldc(Double.valueOf(value));
c.invokestatic(p(Py.class), "newFloat", sig(PyFloat.class, Double.TYPE));
}
@Override
void put(Code c) throws IOException {}
@Override
public int hashCode() {
return (int)value;
}
@Override
public boolean equals(Object o) {
if (o instanceof PyFloatConstant) {
// Ensure hashtable works things like for -0.0 and NaN (see java.lang.Double.equals).
PyFloatConstant pyco = (PyFloatConstant)o;
return Double.doubleToLongBits(pyco.value) == Double.doubleToLongBits(value);
} else {
return false;
}
}
}
class PyComplexConstant extends Constant implements ClassConstants, Opcodes {
final double value;
PyComplexConstant(double value) {
this.value = value;
}
@Override
void get(Code c) throws IOException {
c.ldc(Double.valueOf(value));
c.invokestatic(p(Py.class), "newImaginary", sig(PyComplex.class, Double.TYPE));
}
@Override
void put(Code c) throws IOException {}
@Override
public int hashCode() {
return (int)value;
}
@Override
public boolean equals(Object o) {
if (o instanceof PyComplexConstant) {
// Ensure hashtable works things like for -0.0 and NaN (see java.lang.Double.equals).
PyComplexConstant pyco = (PyComplexConstant)o;
return Double.doubleToLongBits(pyco.value) == Double.doubleToLongBits(value);
} else {
return false;
}
}
}
class PyStringConstant extends Constant implements ClassConstants, Opcodes {
final String value;
PyStringConstant(String value) {
this.value = value;
}
@Override
void get(Code c) throws IOException {
c.ldc(value);
c.invokestatic(p(PyString.class), "fromInterned", sig(PyString.class, String.class));
}
@Override
void put(Code c) throws IOException {}
@Override
public int hashCode() {
return value.hashCode();
}
@Override
public boolean equals(Object o) {
if (o instanceof PyStringConstant) {
return ((PyStringConstant)o).value.equals(value);
} else {
return false;
}
}
}
class PyUnicodeConstant extends Constant implements ClassConstants, Opcodes {
final String value;
PyUnicodeConstant(String value) {
this.value = value;
}
@Override
void get(Code c) throws IOException {
c.ldc(value);
c.invokestatic(p(PyUnicode.class), "fromInterned", sig(PyUnicode.class, String.class));
}
@Override
void put(Code c) throws IOException {}
@Override
public int hashCode() {
return value.hashCode();
}
@Override
public boolean equals(Object o) {
if (o instanceof PyUnicodeConstant) {
return ((PyUnicodeConstant)o).value.equals(value);
} else {
return false;
}
}
}
class PyLongConstant extends Constant implements ClassConstants, Opcodes {
final String value;
PyLongConstant(String value) {
this.value = value;
}
@Override
void get(Code c) throws IOException {
c.ldc(value);
c.invokestatic(p(Py.class), "newLong", sig(PyLong.class, String.class));
}
@Override
void put(Code c) throws IOException {}
@Override
public int hashCode() {
return value.hashCode();
}
@Override
public boolean equals(Object o) {
if (o instanceof PyLongConstant) {
return ((PyLongConstant)o).value.equals(value);
} else {
return false;
}
}
}
class PyCodeConstant extends Constant implements ClassConstants, Opcodes {
final String co_name;
final int argcount;
final List names;
final int id;
final int co_firstlineno;
final boolean arglist, keywordlist;
final String fname;
// for nested scopes
final List cellvars;
final List freevars;
final int jy_npurecell;
final int moreflags;
PyCodeConstant(mod tree, String name, boolean fast_locals, String className, boolean classBody,
boolean printResults, int firstlineno, ScopeInfo scope, CompilerFlags cflags,
Module module) throws Exception {
this.co_name = name;
this.co_firstlineno = firstlineno;
this.module = module;
// Needed so that moreflags can be final.
int _moreflags = 0;
if (scope.ac != null) {
arglist = scope.ac.arglist;
keywordlist = scope.ac.keywordlist;
argcount = scope.ac.names.size();
// Do something to add init_code to tree
// XXX: not sure we should be modifying scope.ac in a PyCodeConstant
// constructor.
if (scope.ac.init_code.size() > 0) {
scope.ac.appendInitCode((Suite)tree);
}
} else {
arglist = false;
keywordlist = false;
argcount = 0;
}
id = module.codes.size();
// Better names in the future?
if (isJavaIdentifier(name)) {
fname = name + "$" + id;
} else {
fname = "f$" + id;
}
// XXX: is fname needed at all, or should we just use "name"?
// It is needed to disambiguate functions and methods with
// same name, but in different classes. The function-fields
// and PyCode-fields that Jython generates don't use fully
// qualified names. So fname is used.
this.name = fname;
// !classdef only
if (!classBody) {
names = toNameAr(scope.names, false);
} else {
names = null;
}
cellvars = toNameAr(scope.cellvars, true);
freevars = toNameAr(scope.freevars, true);
jy_npurecell = scope.jy_npurecell;
if (CodeCompiler.checkOptimizeGlobals(fast_locals, scope)) {
_moreflags |= org.python.core.CodeFlag.CO_OPTIMIZED.flag;
}
if (scope.generator) {
_moreflags |= org.python.core.CodeFlag.CO_GENERATOR.flag;
}
if (cflags != null) {
if (cflags.isFlagSet(CodeFlag.CO_GENERATOR_ALLOWED)) {
_moreflags |= org.python.core.CodeFlag.CO_GENERATOR_ALLOWED.flag;
}
if (cflags.isFlagSet(CodeFlag.CO_FUTURE_DIVISION)) {
_moreflags |= org.python.core.CodeFlag.CO_FUTURE_DIVISION.flag;
}
}
moreflags = _moreflags;
}
// XXX: this can probably go away now that we can probably just copy the list.
private List toNameAr(List names, boolean nullok) {
int sz = names.size();
if (sz == 0 && nullok) {
return null;
}
List nameArray = new ArrayList();
nameArray.addAll(names);
return nameArray;
}
public static boolean isJavaIdentifier(String s) {
char[] chars = s.toCharArray();
if (chars.length == 0) {
return false;
}
if (!Character.isJavaIdentifierStart(chars[0])) {
return false;
}
for (int i = 1; i < chars.length; i++) {
if (!Character.isJavaIdentifierPart(chars[i])) {
return false;
}
}
return true;
}
@Override
void get(Code c) throws IOException {
c.getstatic(module.classfile.name, name, ci(PyCode.class));
}
@Override
void put(Code c) throws IOException {
module.classfile.addField(name, ci(PyCode.class), access);
c.iconst(argcount);
// Make all names
int nameArray;
if (names != null) {
nameArray = CodeCompiler.makeStrings(c, names);
} else { // classdef
nameArray = CodeCompiler.makeStrings(c, null);
}
c.aload(nameArray);
c.freeLocal(nameArray);
c.aload(1);
c.ldc(co_name);
c.iconst(co_firstlineno);
c.iconst(arglist ? 1 : 0);
c.iconst(keywordlist ? 1 : 0);
c.getstatic(module.classfile.name, "self", "L" + module.classfile.name + ";");
c.iconst(id);
if (cellvars != null) {
int strArray = CodeCompiler.makeStrings(c, cellvars);
c.aload(strArray);
c.freeLocal(strArray);
} else {
c.aconst_null();
}
if (freevars != null) {
int strArray = CodeCompiler.makeStrings(c, freevars);
c.aload(strArray);
c.freeLocal(strArray);
} else {
c.aconst_null();
}
c.iconst(jy_npurecell);
c.iconst(moreflags);
c.invokestatic(
p(Py.class),
"newCode",
sig(PyCode.class, Integer.TYPE, String[].class, String.class, String.class,
Integer.TYPE, Boolean.TYPE, Boolean.TYPE, PyFunctionTable.class,
Integer.TYPE, String[].class, String[].class, Integer.TYPE, Integer.TYPE));
c.putstatic(module.classfile.name, name, ci(PyCode.class));
}
}
class PyBytecodeConstant extends Constant implements ClassConstants, Opcodes {
PyBytecodeConstant(String name, String className, CompilerFlags cflags,
Module module) throws Exception {
super();
this.module = module;
this.name = name;
}
@Override
void get(Code c) throws IOException {
c.getstatic(module.classfile.name, name, ci(PyCode.class));
}
@Override
void put(Code c) throws IOException {
}
}
public class Module implements Opcodes, ClassConstants, CompilationContext {
ClassFile classfile;
Constant filename;
String sfilename;
Constant mainCode;
boolean linenumbers;
Future futures;
Hashtable scopes;
List codes;
long mtime;
private int setter_count = 0;
private final static int USE_SETTERS_LIMIT = 100;
private final static int MAX_SETTINGS_PER_SETTER = 4096;
/** The pool of Python Constants */
Hashtable constants;
/** Table of oversized methods represented as CPython bytecode. */
protected Hashtable oversized_methods = null;
public Module(String name, String filename, boolean linenumbers) {
this(name, filename, linenumbers, org.python.core.imp.NO_MTIME);
}
public Module(String name, String filename, boolean linenumbers, long mtime) {
this.linenumbers = linenumbers;
this.mtime = mtime;
classfile =
new ClassFile(name, p(PyFunctionTable.class), ACC_SYNCHRONIZED | ACC_PUBLIC, mtime);
constants = new Hashtable();
sfilename = filename;
if (filename != null) {
this.filename = stringConstant(filename);
} else {
this.filename = null;
}
codes = new ArrayList();
futures = new Future();
scopes = new Hashtable();
}
public Module(String name) {
this(name, name + ".py", true, org.python.core.imp.NO_MTIME);
}
private Constant findConstant(Constant c) {
Constant ret = constants.get(c);
if (ret != null) {
return ret;
}
ret = c;
c.module = this;
// More sophisticated name mappings might be nice
c.name = "_" + constants.size();
constants.put(ret, ret);
return ret;
}
Constant integerConstant(int value) {
return findConstant(new PyIntegerConstant(value));
}
Constant floatConstant(double value) {
return findConstant(new PyFloatConstant(value));
}
Constant complexConstant(double value) {
return findConstant(new PyComplexConstant(value));
}
Constant stringConstant(String value) {
return findConstant(new PyStringConstant(value));
}
Constant unicodeConstant(String value) {
return findConstant(new PyUnicodeConstant(value));
}
Constant longConstant(String value) {
return findConstant(new PyLongConstant(value));
}
Constant codeConstant(mod tree, String name, boolean fast_locals, String className,
boolean classBody, boolean printResults, int firstlineno, ScopeInfo scope,
CompilerFlags cflags) throws Exception {
return codeConstant(tree, name, fast_locals, className, null, classBody, printResults,
firstlineno, scope, cflags);
}
Constant codeConstant(mod tree, String name, boolean fast_locals, String className,
Str classDoc, boolean classBody, boolean printResults, int firstlineno,
ScopeInfo scope, CompilerFlags cflags) throws Exception {
if (oversized_methods != null && oversized_methods.containsKey(name+firstlineno)) {
// For now this only declares the field.
// PyBytecodeConstant is just a dummy to allow the caller to work properly.
// It is intentionally not added to codes, because it doesn't participate in
// FunctionTable and doesn't mess up addFunctions and addConstants this way.
PyBytecodeConstant bcode = new PyBytecodeConstant(
oversized_methods.get(name+firstlineno), className, cflags, this);
classfile.addField(bcode.name, ci(PyCode.class), ACC_PUBLIC | ACC_STATIC);
return bcode;
}
PyCodeConstant code = new PyCodeConstant(tree, name, fast_locals, className, classBody,
printResults, firstlineno, scope, cflags, this);
codes.add(code);
CodeCompiler compiler = new CodeCompiler(this, printResults);
Code c = classfile.addMethod(code.fname,
sig(PyObject.class, PyFrame.class, ThreadState.class), ACC_PUBLIC);
compiler.parse(tree, c, fast_locals, className, classDoc, classBody, scope, cflags);
return code;
}
/** This block of code writes out the various standard methods */
public void addInit() throws IOException {
Code c = classfile.addMethod("", sig(Void.TYPE, String.class), ACC_PUBLIC);
c.aload(0);
c.invokespecial(p(PyFunctionTable.class), "", sig(Void.TYPE));
addConstants(c);
}
public void addRunnable() throws IOException {
Code c = classfile.addMethod("getMain", sig(PyCode.class), ACC_PUBLIC);
mainCode.get(c);
c.areturn();
}
public void addMain() throws IOException {
Code c = classfile.addMethod("main",
sig(Void.TYPE, String[].class), ACC_PUBLIC | ACC_STATIC);
c.new_(classfile.name);
c.dup();
c.ldc(classfile.name);
c.invokespecial(classfile.name, "", sig(Void.TYPE, String.class));
c.invokevirtual(classfile.name, "getMain", sig(PyCode.class));
c.invokestatic(p(CodeLoader.class), CodeLoader.SIMPLE_FACTORY_METHOD_NAME,
sig(CodeBootstrap.class, PyCode.class));
c.aload(0);
c.invokestatic(p(Py.class), "runMain", sig(Void.TYPE, CodeBootstrap.class, String[].class));
c.return_();
}
public void addBootstrap() throws IOException {
Code c = classfile.addMethod(CodeLoader.GET_BOOTSTRAP_METHOD_NAME,
sig(CodeBootstrap.class), ACC_PUBLIC | ACC_STATIC);
c.ldc(Type.getType("L" + classfile.name + ";"));
c.invokestatic(p(PyRunnableBootstrap.class), PyRunnableBootstrap.REFLECTION_METHOD_NAME,
sig(CodeBootstrap.class, Class.class));
c.areturn();
}
void addConstants(Code c) throws IOException {
classfile.addField("self", "L" + classfile.name + ";", ACC_STATIC);
c.aload(0);
c.putstatic(classfile.name, "self", "L" + classfile.name + ";");
Enumeration e = constants.elements();
while (e.hasMoreElements()) {
Constant constant = e.nextElement();
constant.put(c);
}
for (PyCodeConstant pyc: codes) {
pyc.put(c);
}
c.return_();
}
public void addFunctions() throws IOException {
Code code = classfile.addMethod("call_function",
sig(PyObject.class, Integer.TYPE, PyFrame.class, ThreadState.class), ACC_PUBLIC);
if (!codes.isEmpty()) {
code.aload(0); // this
code.aload(2); // frame
code.aload(3); // thread state
Label def = new Label();
Label[] labels = new Label[codes.size()];
int i;
for (i = 0; i < labels.length; i++) {
labels[i] = new Label();
}
// Get index for function to call
code.iload(1);
code.tableswitch(0, labels.length - 1, def, labels);
for (i = 0; i < labels.length; i++) {
code.label(labels[i]);
code.invokevirtual(classfile.name, (codes.get(i)).fname,
sig(PyObject.class, PyFrame.class, ThreadState.class));
code.areturn();
}
code.label(def);
}
// Should probably throw internal exception here
code.aconst_null();
code.areturn();
}
public void write(OutputStream stream) throws IOException {
addInit();
addRunnable();
addMain();
addBootstrap();
addFunctions();
classfile.addInterface(p(PyRunnable.class));
if (sfilename != null) {
classfile.setSource(sfilename);
}
classfile.write(stream);
}
// Implementation of CompilationContext
@Override
public Future getFutures() {
return futures;
}
@Override
public String getFilename() {
return sfilename;
}
@Override
public ScopeInfo getScopeInfo(PythonTree node) {
return scopes.get(node);
}
@Override
public void error(String msg, boolean err, PythonTree node) throws Exception {
if (!err) {
try {
Py.warning(Py.SyntaxWarning, msg, (sfilename != null) ? sfilename : "?",
node.getLineno(), null, Py.None);
return;
} catch (PyException e) {
if (!e.match(Py.SyntaxWarning)) {
throw e;
}
}
}
throw new ParseException(msg, node);
}
public static void compile(mod node, OutputStream ostream, String name, String filename,
boolean linenumbers, boolean printResults, CompilerFlags cflags) throws Exception {
compile(node, ostream, name, filename, linenumbers, printResults, cflags,
org.python.core.imp.NO_MTIME);
}
protected static void _module_init(mod node, Module module, boolean printResults,
CompilerFlags cflags) throws Exception {
if (cflags == null) {
cflags = new CompilerFlags();
}
module.futures.preprocessFutures(node, cflags);
new ScopesCompiler(module, module.scopes).parse(node);
// Add __doc__ if it exists
Constant main = module.codeConstant(node, "", false, null, false,
printResults, 0, module.getScopeInfo(node), cflags);
module.mainCode = main;
}
// Error message formats required by loadPyBytecode
private static String TRIED_CREATE_PYC_MSG =
"\nJython tried to create a pyc-file by executing\n %s\nwhich failed because %s";
private static String LARGE_METHOD_MSG = "Module or method too large in `%s`.";
private static String PLEASE_PROVIDE_MSG =
"\n\nPlease provide a CPython 2.7 bytecode file (.pyc), e.g. run"
+ "\n python -m py_compile %s";
private static String CPYTHON_CMD_MSG =
"\n\nAlternatively, specify a CPython 2.7 command via the " //
+ PYTHON_CPYTHON + " property, e.g.:" //
+ "\n jython -D" + PYTHON_CPYTHON + "=python" //
+ "\nor (e.g. for pip) through the environment variable JYTHON_OPTS:" //
+ "\n export JYTHON_OPTS=\"-D" + PYTHON_CPYTHON
+ "=python\"\n";
private static PyBytecode loadPyBytecode(String filename, boolean try_cpython)
throws RuntimeException {
if (filename.startsWith(ClasspathPyImporter.PYCLASSPATH_PREFIX)) {
ClassLoader cld = Py.getSystemState().getClassLoader();
if (cld == null) {
cld = imp.getParentClassLoader();
}
URL py_url =
cld.getResource(filename.replace(ClasspathPyImporter.PYCLASSPATH_PREFIX, ""));
if (py_url != null) {
filename = py_url.getPath();
} else {
// Should never happen, but let's play it safe and treat this case.
throw new RuntimeException(String.format(LARGE_METHOD_MSG, filename)
+ "but couldn't resolve that filename within classpath.\n"
+ "Make sure the source file is at a proper location.");
}
}
String pyc_filename = filename + "c";
File pyc_file = new File(pyc_filename);
if (pyc_file.exists()) {
PyFile f = new PyFile(pyc_filename, "rb", 4096);
byte[] bts = f.read(8).toBytes();
int magic = (bts[1] << 8) & 0x0000FF00 | (bts[0] << 0) & 0x000000FF;
// int mtime_pyc = (bts[7]<<24) & 0xFF000000 |
// (bts[6]<<16) & 0x00FF0000 |
// (bts[5]<< 8) & 0x0000FF00 |
// (bts[4]<< 0) & 0x000000FF;
if (magic != 62211) { // check Python 2.7 bytecode
throw new RuntimeException(
String.format(LARGE_METHOD_MSG, filename) //
+ "\n'" + pyc_filename + "' is not CPython 2.7 bytecode." //
+ String.format(PLEASE_PROVIDE_MSG, filename));
}
_marshal.Unmarshaller un = new _marshal.Unmarshaller(f);
PyObject code = un.load();
f.close();
if (code instanceof PyBytecode) {
return (PyBytecode) code;
}
throw new RuntimeException(String.format(LARGE_METHOD_MSG, filename) //
+ "\n'" + pyc_filename + "' contains invalid bytecode."
+ String.format(PLEASE_PROVIDE_MSG, filename));
} else {
String CPython_command = System.getProperty(PYTHON_CPYTHON);
if (try_cpython && CPython_command != null) {
// check version...
String command_ver = CPython_command + " --version";
String command = CPython_command + " -m py_compile " + filename;
Exception exc = null;
int result = 0;
String reason;
try {
Process p = Runtime.getRuntime().exec(command_ver);
// Python 2.7 writes version to error-stream for some reason:
BufferedReader br =
new BufferedReader(new InputStreamReader(p.getErrorStream()));
String cp_version = br.readLine();
while (br.readLine() != null) {}
br.close();
if (cp_version == null) {
// Also try input-stream as fallback, just in case...
br = new BufferedReader(new InputStreamReader(p.getInputStream()));
cp_version = br.readLine();
while (br.readLine() != null) {}
br.close();
}
result = p.waitFor();
if (!cp_version.startsWith("Python 2.7.")) {
reason = cp_version + " has been provided, but 2.7.x is required.";
throw new RuntimeException(String.format(LARGE_METHOD_MSG, filename)
+ String.format(TRIED_CREATE_PYC_MSG, command, reason)
+ String.format(PLEASE_PROVIDE_MSG, filename) + CPYTHON_CMD_MSG);
}
} catch (InterruptedException | IOException e) {
exc = e;
}
if (exc == null && result == 0) {
try {
Process p = Runtime.getRuntime().exec(command);
result = p.waitFor();
if (result == 0) {
return loadPyBytecode(filename, false);
}
} catch (InterruptedException | IOException e) {
exc = e;
}
}
reason = exc != null ? "of " + exc.toString() : "of a bad return: " + result;
String exc_msg = String.format(LARGE_METHOD_MSG, filename)
+ String.format(TRIED_CREATE_PYC_MSG, command, reason)
+ String.format(PLEASE_PROVIDE_MSG, filename) + CPYTHON_CMD_MSG;
throw exc != null ? new RuntimeException(exc_msg, exc)
: new RuntimeException(exc_msg);
} else {
throw new RuntimeException(String.format(LARGE_METHOD_MSG, filename)
+ String.format(PLEASE_PROVIDE_MSG, filename) + CPYTHON_CMD_MSG);
}
}
}
private static String serializePyBytecode(PyBytecode btcode) throws java.io.IOException {
// For some reason we cannot do this using _marshal:
/*
cStringIO.StringIO buf = cStringIO.StringIO();
_marshal.Marshaller marsh = new _marshal.Marshaller(buf);
marsh.dump(largest_m_code);
String code_str = buf.getvalue().asString();
_marshal.Unmarshaller un2 = new _marshal.Unmarshaller(cStringIO.StringIO(code_str));
PyBytecode code = (PyBytecode) un2.load();
This says 'ValueError: bad marshal data'
Maybe the issue is actually with cStringIO, because bytecode-marshalling uses
bytes not directly suitable as String-values. cStringIO does not use Base64 or
something, but rather supports only string-compatible data.
*/
// so we use Java-serialization...
// serialize the object
ByteArrayOutputStream bo = new ByteArrayOutputStream();
ObjectOutputStream so = new ObjectOutputStream(bo);
so.writeObject(btcode);
so.flush();
// From Java 8 use: String code_str = Base64.getEncoder().encodeToString(bo.toByteArray());
String code_str = base64encodeToString(bo.toByteArray());
so.close();
bo.close();
return code_str;
}
/**
* Implement a simplified base64 encoding compatible with the decoding in BytecodeLoader. This
* encoder adds no '=' padding or line-breaks. equivalent to
* {@code binascii.b2a_base64(bytes).rstrip('=\n')}.
*
* @param data to encode
* @return the string encoding the data
*/
private static String base64encodeToString(byte[] data) {
final int N = data.length;
int tail = N % 3;
StringBuilder chars = new StringBuilder(((N / 3) + 1) * 4);
// Process bytes in blocks of three
int b = 0, quantum;
while (b <= N - 3) {
// Process [b:b+3]
quantum = ((data[b++] & 0xff) << 16) + ((data[b++] & 0xff) << 8) + (data[b++] & 0xff);
chars.append(base64enc[quantum >> 18]);
chars.append(base64enc[(quantum >> 12) & 0x3f]);
chars.append(base64enc[(quantum >> 6) & 0x3f]);
chars.append(base64enc[quantum & 0x3f]);
}
// Process the tail bytes
if (tail >= 1) {
quantum = ((data[b++] & 0xff) << 8);
if (tail == 2) {
quantum += data[b++] & 0xff;
}
chars.append(base64enc[quantum >> 10]);
chars.append(base64enc[(quantum >> 4) & 0x3f]);
if (tail == 2) {
chars.append(base64enc[(quantum << 2) & 0x3f]);
}
}
return chars.toString();
}
/** Look-up table for {@link #base64encodeToString(byte[])}. */
private static final char[] base64enc =
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/".toCharArray();
private static final int maxLiteral = 65535;
/**
* This method stores Base64 encoded Python byte code in one or more String literals.
*
* While Java String objects are limited only by the address range of arrays, the class file
* standard only supports literals representable in at most 65535 bytes of modified UTF-8. This
* method us used only with base64 Strings (therefore ASCII without nulls) and so each character
* occupies exactly 1 byte in the class file after encoding to UTF-8.
*
* To work within the 65535 byte limitation, the {@code code_str} is split into several literals
* with the following naming-scheme:
*
* - The marker-interface 'ContainsPyBytecode' indicates that a class contains (static final)
* literals of the following scheme:
*
- a prefix of '___' indicates a bytecode-containing string literal
*
- a number indicating the number of parts follows
*
- '0_' indicates that no splitting occurred
*
- otherwise another number follows, naming the index of the literal
*
- indexing starts at 0
*
* Examples:
*
* - {@code ___0_method1} contains bytecode for method1
*
- {@code ___2_0_method2} contains first part of method2's bytecode
*
- {@code ___2_1_method2} contains second part of method2's bytecode
*
* Note that this approach is provisional. In future, Jython might contain the bytecode directly
* as bytecode-objects. The current approach was feasible with far less complicated JVM
* bytecode-manipulation, but needs special treatment after class-loading.
*
* @param name of the method or function being generated
* @param code_str Base64 encoded CPython byte code
* @param module currently being defined as a class file
* @throws java.io.IOException
*/
private static void insert_code_str_to_classfile(String name, String code_str, Module module)
throws java.io.IOException {
if (code_str.length() <= maxLiteral) {
// This can go as a single literal
module.classfile.addFinalStringLiteral("___0_" + name, code_str);
} else {
// We need to split the code into several literals.
int splits = code_str.length() / maxLiteral;
if (code_str.length() % maxLiteral > 0) {
++splits;
}
int pos = 0, i = 0;
for (; pos + maxLiteral <= code_str.length(); ++i) {
module.classfile.addFinalStringLiteral("___" + splits + "_" + i + "_" + name,
code_str.substring(pos, pos + maxLiteral));
pos += maxLiteral;
}
if (i < splits) {
module.classfile.addFinalStringLiteral("___" + splits + "_" + i + "_" + name,
code_str.substring(pos));
}
}
}
/**
* Create and write a Python module as a Java class file.
*
* @param node AST of the module to write
* @param ostream stream onto which to write it
* @param name
* @param filename
* @param linenumbers
* @param printResults
* @param cflags
* @param mtime
* @throws Exception
*/
public static void compile(mod node, OutputStream ostream, String name, String filename,
boolean linenumbers, boolean printResults, CompilerFlags cflags, long mtime)
throws Exception {
try {
Module module = new Module(name, filename, linenumbers, mtime);
_module_init(node, module, printResults, cflags);
module.write(ostream);
} catch (MethodTooLargeException re) {
PyBytecode btcode = loadPyBytecode(filename, true);
int thresh = 22000;
/*
* No idea, how to determine at this point if a method is oversized, so we just try a
* threshold regarding Python code-length, while JVM restriction is actually about Java
* bytecode length. Anyway; given that code-lengths are strongly related, this should
* work well enough.
*/
while (true) { // Always enjoy to write a line like this :)
try {
List largest_m_codes = new ArrayList<>();
Stack buffer = new Stack<>();
// HashSet allCodes = new HashSet<>();
buffer.push(btcode);
// allCodes.add(btcode);
while (!buffer.isEmpty()) {
/*
* Probably this cannot yield cycles, so cycle-proof stuff is out-commented
* for now. (everything regarding 'allCodes')
*/
PyBytecode bcode = buffer.pop();
if (bcode.co_code.length > thresh) {
largest_m_codes.add(bcode);
} else {
/*
* If a function needs to be represented as CPython bytecode, we create
* all inner PyCode-items (classes, functions, methods) also as CPython
* bytecode implicitly, so no need to look at them individually. Maybe
* we can later optimize this such that inner methods can be
* JVM-bytecode as well (if not oversized themselves).
*/
for (PyObject item : bcode.co_consts) {
if (item instanceof PyBytecode /* && !allCodes.contains(item) */) {
PyBytecode mpbc = (PyBytecode) item;
buffer.push(mpbc);
// allCodes.add(mpbc);
}
}
}
}
Module module = new Module(name, filename, linenumbers, mtime);
module.oversized_methods = new Hashtable<>(largest_m_codes.size());
int ov_id = 0;
String name_id;
for (PyBytecode largest_m_code : largest_m_codes) {
if (!PyCodeConstant.isJavaIdentifier(largest_m_code.co_name)) {
name_id = "f$_" + ov_id++;
} else {
name_id = largest_m_code.co_name + "$_" + ov_id++;
}
if (largest_m_code.co_name.equals("")) {
/*
* In Jython's opinion module begins at line 0 (while CPython reports
* line 1)
*/
module.oversized_methods.put(largest_m_code.co_name + 0, name_id);
} else {
module.oversized_methods.put(
largest_m_code.co_name + largest_m_code.co_firstlineno,
name_id);
}
String code_str = serializePyBytecode(largest_m_code);
insert_code_str_to_classfile(name_id, code_str, module);
}
module.classfile.addInterface(p(org.python.core.ContainsPyBytecode.class));
_module_init(node, module, printResults, cflags);
module.write(ostream);
break;
} catch (MethodTooLargeException e) {
thresh -= 1000;
}
if (thresh < 10000) {
/*
* This value should be well feasible by JVM-bytecode, so something else must be
* wrong.
*/
throw new RuntimeException(
"For unknown reason, too large method code couldn't be resolved"
+ "\nby PyBytecode-approach:\n" + filename);
}
}
}
}
public void emitNum(Num node, Code code) throws Exception {
if (node.getInternalN() instanceof PyInteger) {
integerConstant(((PyInteger)node.getInternalN()).getValue()).get(code);
} else if (node.getInternalN() instanceof PyLong) {
longConstant(((PyObject)node.getInternalN()).__str__().toString()).get(code);
} else if (node.getInternalN() instanceof PyFloat) {
floatConstant(((PyFloat)node.getInternalN()).getValue()).get(code);
} else if (node.getInternalN() instanceof PyComplex) {
complexConstant(((PyComplex)node.getInternalN()).imag).get(code);
}
}
public void emitStr(Str node, Code code) throws Exception {
PyString s = (PyString)node.getInternalS();
if (s instanceof PyUnicode) {
unicodeConstant(s.asString()).get(code);
} else {
stringConstant(s.asString()).get(code);
}
}
public boolean emitPrimitiveArraySetters(java.util.List nodes, Code code)
throws Exception {
final int n = nodes.size();
if (n < USE_SETTERS_LIMIT) {
return false; // Too small to matter, so bail
}
// Only attempt if all nodes are either Num or Str, otherwise bail
boolean primitive_literals = true;
for (int i = 0; i < n; i++) {
PythonTree node = nodes.get(i);
if (!(node instanceof Num || node instanceof Str)) {
primitive_literals = false;
}
}
if (!primitive_literals) {
return false;
}
final int num_setters = (n / MAX_SETTINGS_PER_SETTER) + 1;
code.iconst(n);
code.anewarray(p(PyObject.class));
for (int i = 0; i < num_setters; i++) {
Code setter = this.classfile.addMethod("set$$" + setter_count,
sig(Void.TYPE, PyObject[].class), ACC_STATIC | ACC_PRIVATE);
for (int j = 0; (j < MAX_SETTINGS_PER_SETTER)
&& ((i * MAX_SETTINGS_PER_SETTER + j) < n); j++) {
setter.aload(0);
setter.iconst(i * MAX_SETTINGS_PER_SETTER + j);
PythonTree node = nodes.get(i * MAX_SETTINGS_PER_SETTER + j);
if (node instanceof Num) {
emitNum((Num)node, setter);
} else if (node instanceof Str) {
emitStr((Str)node, setter);
}
setter.aastore();
}
setter.return_();
code.dup();
code.invokestatic(this.classfile.name, "set$$" + setter_count,
sig(Void.TYPE, PyObject[].class));
setter_count++;
}
return true;
}
}