org.python.modules._marshal Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of jython-slim Show documentation
Show all versions of jython-slim Show documentation
Jython is an implementation of the high-level, dynamic, object-oriented
language Python written in 100% Pure Java, and seamlessly integrated with
the Java platform. It thus allows you to run Python on any Java platform.
package org.python.modules;
import org.python.core.BaseSet;
import org.python.core.BufferProtocol;
import org.python.core.ClassDictInit;
import org.python.core.Py;
import org.python.core.PyBUF;
import org.python.core.PyBuffer;
import org.python.core.PyBytecode;
import org.python.core.PyComplex;
import org.python.core.PyDictionary;
import org.python.core.PyException;
import org.python.core.PyFloat;
import org.python.core.PyFrozenSet;
import org.python.core.PyInteger;
import org.python.core.PyList;
import org.python.core.PyLong;
import org.python.core.PyObject;
import org.python.core.PySet;
import org.python.core.PyString;
import org.python.core.PyTuple;
import org.python.core.PyType;
import org.python.core.PyUnicode;
import org.python.core.Traverseproc;
import org.python.core.Visitproc;
import java.math.BigInteger;
public class _marshal implements ClassDictInit {
public static void classDictInit(PyObject dict) {
dict.__setitem__("__name__", Py.newString("_marshal"));
}
private final static char TYPE_NULL = '0';
private final static char TYPE_NONE = 'N';
private final static char TYPE_FALSE = 'F';
private final static char TYPE_TRUE = 'T';
private final static char TYPE_STOPITER = 'S';
private final static char TYPE_ELLIPSIS = '.';
private final static char TYPE_INT = 'i';
private final static char TYPE_INT64 = 'I';
private final static char TYPE_FLOAT = 'f';
private final static char TYPE_BINARY_FLOAT = 'g';
private final static char TYPE_COMPLEX = 'x';
private final static char TYPE_BINARY_COMPLEX = 'y';
private final static char TYPE_LONG = 'l';
private final static char TYPE_STRING = 's';
private final static char TYPE_INTERNED = 't';
private final static char TYPE_STRINGREF = 'R';
private final static char TYPE_TUPLE = '(';
private final static char TYPE_LIST = '[';
private final static char TYPE_DICT = '{';
private final static char TYPE_CODE = 'c';
private final static char TYPE_UNICODE = 'u';
private final static char TYPE_SET = '<';
private final static char TYPE_FROZENSET = '>';
private final static int MAX_MARSHAL_STACK_DEPTH = 2000;
private final static int CURRENT_VERSION = 2;
public static class Marshaller extends PyObject implements Traverseproc {
private final PyIOFile file;
private final int version;
public Marshaller(PyObject file) {
this(file, CURRENT_VERSION);
}
public Marshaller(PyObject file, int version) {
this.file = PyIOFileFactory.createIOFile(file);
this.version = version;
}
private boolean debug = false;
public void _debug() {
debug = true;
}
public void dump(PyObject obj) {
write_object(obj, 0);
}
private void write_byte(char c) {
if (debug) {
System.err.print("[" + (int) c + "]");
}
file.write(c);
}
private void write_string(String s) {
file.write(s);
}
private void write_strings(String[] some_strings, int depth) {
PyObject items[] = new PyObject[some_strings.length];
for (int i = 0; i < some_strings.length; i++) {
items[i] = Py.newString(some_strings[i]);
}
write_object(new PyTuple(items), depth + 1);
}
private void write_short(short x) {
write_byte((char) (x & 0xff));
write_byte((char) ((x >> 8) & 0xff));
}
private void write_int(int x) {
write_byte((char) (x & 0xff));
write_byte((char) ((x >> 8) & 0xff));
write_byte((char) ((x >> 16) & 0xff));
write_byte((char) ((x >> 24) & 0xff));
}
private void write_long64(long x) {
write_int((int) (x & 0xffffffff));
write_int((int) ((x >> 32) & 0xffffffff));
}
// writes output in 15 bit "digits"
private void write_long(BigInteger x) {
boolean negative = x.signum() < 0;
if (negative) {
x = x.negate();
}
int num_digits = (x.bitLength() + 14) / 15;
write_int(negative ? -num_digits : num_digits);
BigInteger mask = BigInteger.valueOf(0x7FFF);
for (int i = 0; i < num_digits; i++) {
write_short(x.and(mask).shortValue());
x = x.shiftRight(15);
}
}
private void write_float(PyFloat f) {
write_string(f.__repr__().toString());
}
private void write_binary_float(PyFloat f) {
write_long64(Double.doubleToLongBits(f.getValue()));
}
private void write_object(PyObject v, int depth) {
if (depth >= MAX_MARSHAL_STACK_DEPTH) {
throw Py.ValueError("Maximum marshal stack depth"); // XXX - fix this exception
} else if (v == null) {
write_byte(TYPE_NULL);
} else if (v == Py.None) {
write_byte(TYPE_NONE);
} else if (v == Py.StopIteration) {
write_byte(TYPE_STOPITER);
} else if (v == Py.Ellipsis) {
write_byte(TYPE_ELLIPSIS);
} else if (v == Py.False) {
write_byte(TYPE_FALSE);
} else if (v == Py.True) {
write_byte(TYPE_TRUE);
} else {
PyType vt = v.getType();
if (vt == PyInteger.TYPE) {
write_byte(TYPE_INT);
write_int(((PyInteger) v).asInt());
} else if (vt == PyLong.TYPE) {
write_byte(TYPE_LONG);
write_long(((PyLong) v).getValue());
} else if (vt == PyFloat.TYPE) {
if (version == CURRENT_VERSION) {
write_byte(TYPE_BINARY_FLOAT);
write_binary_float((PyFloat) v);
} else {
write_byte(TYPE_FLOAT);
write_float((PyFloat) v);
}
} else if (vt == PyComplex.TYPE) {
PyComplex x = (PyComplex) v;
if (version == CURRENT_VERSION) {
write_byte(TYPE_BINARY_COMPLEX);
write_binary_float(x.getReal());
write_binary_float(x.getImag());
} else {
write_byte(TYPE_COMPLEX);
write_float(x.getReal());
write_float(x.getImag());
}
} else if (vt == PyUnicode.TYPE) {
write_byte(TYPE_UNICODE);
String buffer = ((PyUnicode) v).encode("utf-8").toString();
write_int(buffer.length());
write_string(buffer);
} else if (vt == PyString.TYPE) {
// ignore interning
write_byte(TYPE_STRING);
write_int(v.__len__());
write_string(v.toString());
} else if (vt == PyTuple.TYPE) {
write_byte(TYPE_TUPLE);
PyTuple t = (PyTuple) v;
int n = t.__len__();
write_int(n);
for (int i = 0; i < n; i++) {
write_object(t.__getitem__(i), depth + 1);
}
} else if (vt == PyList.TYPE) {
write_byte(TYPE_LIST);
PyList list = (PyList) v;
int n = list.__len__();
write_int(n);
for (int i = 0; i < n; i++) {
write_object(list.__getitem__(i), depth + 1);
}
} else if (vt == PyDictionary.TYPE) {
write_byte(TYPE_DICT);
PyDictionary dict = (PyDictionary) v;
for (PyObject item : dict.iteritems().asIterable()) {
PyTuple pair = (PyTuple) item;
write_object(pair.__getitem__(0), depth + 1);
write_object(pair.__getitem__(1), depth + 1);
}
write_object(null, depth + 1);
} else if (vt == PySet.TYPE || vt == PyFrozenSet.TYPE) {
if (vt == PySet.TYPE) {
write_byte(TYPE_SET);
} else {
write_byte(TYPE_FROZENSET);
}
int n = v.__len__();
write_int(n);
BaseSet set = (BaseSet) v;
for (PyObject item : set.asIterable()) {
write_object(item, depth + 1);
}
} else if (vt == PyBytecode.TYPE) {
PyBytecode code = (PyBytecode) v;
write_byte(TYPE_CODE);
write_int(code.co_argcount);
write_int(code.co_nlocals);
write_int(code.co_stacksize);
write_int(code.co_flags.toBits());
write_object(Py.newString(new String(code.co_code)), depth + 1);
write_object(new PyTuple(code.co_consts), depth + 1);
write_strings(code.co_names, depth + 1);
write_strings(code.co_varnames, depth + 1);
write_strings(code.co_freevars, depth + 1);
write_strings(code.co_cellvars, depth + 1);
write_object(Py.newString(code.co_name), depth + 1);
write_int(code.co_firstlineno);
write_object(Py.newString(new String(code.co_lnotab)), depth + 1);
} else {
// Try to get a simple byte-oriented buffer
try (PyBuffer buf = ((BufferProtocol) v).getBuffer(PyBUF.SIMPLE)) {
// ... and treat those bytes as a String
write_byte(TYPE_STRING);
write_int(v.__len__());
write_string(buf.toString());
} catch (ClassCastException | PyException e) {
// Does not implement BufferProtocol (in simple byte form).
throw Py.ValueError("unmarshallable object");
}
}
}
depth--;
}
/* Traverseproc implementation */
@Override
public int traverse(Visitproc visit, Object arg) {
return file != null && file instanceof Traverseproc ?
((Traverseproc) file).traverse(visit, arg) : 0;
}
@Override
public boolean refersDirectlyTo(PyObject ob) {
return file != null && file instanceof Traverseproc ?
((Traverseproc) file).refersDirectlyTo(ob) : false;
}
}
public static class Unmarshaller extends PyObject implements Traverseproc {
private final PyIOFile file;
private final PyList strings = new PyList();
private final int version;
int depth = 0;
public Unmarshaller(PyObject file) {
this(file, CURRENT_VERSION);
}
public Unmarshaller(PyObject file, int version) {
this.file = PyIOFileFactory.createIOFile(file);
this.version = version;
}
private boolean debug = false;
public void _debug() {
debug = true;
}
public PyObject load() {
try {
PyObject obj = read_object(0);
if (obj == null) {
throw Py.TypeError("NULL object in marshal data");
}
return obj;
} catch (StringIndexOutOfBoundsException e) {
// convert from our PyIOFile abstraction to what marshal in CPython returns
// (although it's really just looking for no bombing)
throw Py.EOFError("EOF read where object expected");
}
}
private int read_byte() {
int b = file.read(1).charAt(0);
if (debug) {
System.err.print("[" + b + "]");
}
return b;
}
private String read_string(int n) {
return file.read(n);
}
private int read_short() {
int x = read_byte();
x |= read_byte() << 8;
return x;
}
private int read_int() { // cpython calls this r_long
int x = read_byte();
x |= read_byte() << 8;
x |= read_byte() << 16;
x |= read_byte() << 24;
return x;
}
private long read_long64() { // cpython calls this r_long64
long lo4 = read_int();
long hi4 = read_int();
long x = (hi4 << 32) | (lo4 & 0xFFFFFFFFL);
return x;
}
private BigInteger read_long() {
BigInteger result = BigInteger.ZERO;
boolean negative = false;
int digit = 0, size = read_int();
if (size == 0) {
return result;
} else if (size < 0) {
negative = true;
size = -size;
}
for (int i = 0; i < size; i++) {
if ((digit = read_short()) < 0) {
throw badMarshalData("digit out of range in long");
}
result = result.or(BigInteger.valueOf(digit).shiftLeft(i * 15));
}
if (digit == 0) {
throw badMarshalData("unnormalized long data");
}
return negative ? result.negate() : result;
}
private double read_float() {
int size = read_byte();
return Py.newString(read_string(size)).atof();
}
private double read_binary_float() {
return Double.longBitsToDouble(read_long64());
}
private PyObject read_object_notnull(int depth) {
PyObject v = read_object(depth);
if (v == null) {
throw badMarshalData(null);
}
return v;
}
private String[] read_strings(int depth) {
PyTuple t = (PyTuple) read_object_notnull(depth);
String some_strings[] = new String[t.__len__()];
int i = 0;
for (PyObject item : t.asIterable()) {
some_strings[i++] = item.toString().intern();
}
return some_strings;
}
private PyObject read_object(int depth) {
if (depth >= MAX_MARSHAL_STACK_DEPTH) {
throw Py.ValueError("Maximum marshal stack depth"); // XXX - fix this exception
}
int type = read_byte();
switch (type) {
case TYPE_NULL:
return null;
case TYPE_NONE:
return Py.None;
case TYPE_STOPITER:
return Py.StopIteration;
case TYPE_ELLIPSIS:
return Py.Ellipsis;
case TYPE_FALSE:
return Py.False;
case TYPE_TRUE:
return Py.True;
case TYPE_INT:
return Py.newInteger(read_int());
case TYPE_INT64:
return Py.newInteger(read_long64());
case TYPE_LONG: {
return Py.newLong(read_long());
}
case TYPE_FLOAT:
return Py.newFloat(read_float());
case TYPE_BINARY_FLOAT:
return Py.newFloat(read_binary_float());
case TYPE_COMPLEX: {
double real = read_float();
double imag = read_float();
return new PyComplex(real, imag);
}
case TYPE_BINARY_COMPLEX: {
double real = read_binary_float();
double imag = read_binary_float();
return new PyComplex(real, imag);
}
case TYPE_INTERNED:
case TYPE_STRING: {
int size = read_int();
String s = read_string(size);
if (type == TYPE_INTERNED) {
PyString pys = PyString.fromInterned(s.intern());
strings.append(pys);
return pys;
} else {
return Py.newString(s);
}
}
case TYPE_STRINGREF: {
int i = read_int();
return strings.__getitem__(i);
}
case TYPE_UNICODE: {
int n = read_int();
PyString buffer = Py.newString(read_string(n));
return buffer.decode("utf-8");
}
case TYPE_TUPLE: {
int n = read_int();
if (n < 0) {
throw badMarshalData(null);
}
PyObject items[] = new PyObject[n];
for (int i = 0; i < n; i++) {
items[i] = read_object_notnull(depth + 1);
}
return new PyTuple(items);
}
case TYPE_LIST: {
int n = read_int();
if (n < 0) {
throw badMarshalData(null);
}
PyObject items[] = new PyObject[n];
for (int i = 0; i < n; i++) {
items[i] = read_object_notnull(depth + 1);
}
return new PyList(items);
}
case TYPE_DICT: {
PyDictionary d = new PyDictionary();
while (true) {
PyObject key = read_object(depth + 1);
if (key == null) {
break;
}
PyObject value = read_object(depth + 1);
if (value != null) {
d.__setitem__(key, value);
}
}
return d;
}
case TYPE_SET:
case TYPE_FROZENSET: {
int n = read_int();
PyObject items[] = new PyObject[n];
for (int i = 0; i < n; i++) {
items[i] = read_object(depth + 1);
}
PyTuple v = new PyTuple(items);
if (type == TYPE_SET) {
return new PySet(v);
} else {
return new PyFrozenSet(v);
}
}
case TYPE_CODE: {
// XXX - support restricted execution mode? not certain if this is just legacy
int argcount = read_int();
int nlocals = read_int();
int stacksize = read_int();
int flags = read_int();
String code = read_object_notnull(depth + 1).toString();
PyObject consts[] = ((PyTuple) read_object_notnull(depth + 1)).getArray();
String names[] = read_strings(depth + 1);
String varnames[] = read_strings(depth + 1);
String freevars[] = read_strings(depth + 1);
String cellvars[] = read_strings(depth + 1);
String filename = read_object_notnull(depth + 1).toString();
String name = read_object_notnull(depth + 1).toString();
int firstlineno = read_int();
String lnotab = read_object_notnull(depth + 1).toString();
return new PyBytecode(
argcount, nlocals, stacksize, flags,
code, consts, names, varnames,
filename, name, firstlineno, lnotab,
cellvars, freevars);
}
default:
throw badMarshalData("unknown type code");
}
}
/** Helper returning "bad marshal data" or "bad marshal data ()". */
private static PyException badMarshalData(String reason) {
StringBuilder msg = (new StringBuilder(60)).append("bad marshal data");
if (reason != null) {
msg.append(" (").append(reason).append(')');
}
return Py.ValueError(msg.toString());
}
/* Traverseproc implementation */
@Override
public int traverse(Visitproc visit, Object arg) {
if (file instanceof Traverseproc) {
int retVal = ((Traverseproc) file).traverse(visit, arg);
if (retVal != 0) {
return retVal;
}
}
return visit.visit(strings, arg);
}
@Override
public boolean refersDirectlyTo(PyObject ob) {
if (ob == null) {
return false;
} else if (file != null && file instanceof Traverseproc
&& ((Traverseproc) file).refersDirectlyTo(ob)) {
return true;
} else {
return ob == strings;
}
}
}
}