src.org.python.modules._marshal Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of jython Show documentation
Show all versions of jython Show documentation
Jython is an implementation of the high-level, dynamic, object-oriented
language Python written in 100% Pure Java, and seamlessly integrated with
the Java platform. It thus allows you to run Python on any Java platform.
package org.python.modules;
import org.python.core.BaseSet;
import org.python.core.BufferProtocol;
import org.python.core.ClassDictInit;
import org.python.core.Py;
import org.python.core.PyBUF;
import org.python.core.PyBuffer;
import org.python.core.PyBytecode;
import org.python.core.PyComplex;
import org.python.core.PyDictionary;
import org.python.core.PyException;
import org.python.core.PyFloat;
import org.python.core.PyFrozenSet;
import org.python.core.PyInteger;
import org.python.core.PyList;
import org.python.core.PyLong;
import org.python.core.PyObject;
import org.python.core.PySet;
import org.python.core.PyString;
import org.python.core.PyTuple;
import org.python.core.PyType;
import org.python.core.PyUnicode;
import org.python.core.Traverseproc;
import org.python.core.Visitproc;
import java.math.BigInteger;
public class _marshal implements ClassDictInit {
public static void classDictInit(PyObject dict) {
dict.__setitem__("__name__", Py.newString("_marshal"));
}
private final static char TYPE_NULL = '0';
private final static char TYPE_NONE = 'N';
private final static char TYPE_FALSE = 'F';
private final static char TYPE_TRUE = 'T';
private final static char TYPE_STOPITER = 'S';
private final static char TYPE_ELLIPSIS = '.';
private final static char TYPE_INT = 'i';
private final static char TYPE_INT64 = 'I';
private final static char TYPE_FLOAT = 'f';
private final static char TYPE_BINARY_FLOAT = 'g';
private final static char TYPE_COMPLEX = 'x';
private final static char TYPE_BINARY_COMPLEX = 'y';
private final static char TYPE_LONG = 'l';
private final static char TYPE_STRING = 's';
private final static char TYPE_INTERNED = 't';
private final static char TYPE_STRINGREF = 'R';
private final static char TYPE_TUPLE = '(';
private final static char TYPE_LIST = '[';
private final static char TYPE_DICT = '{';
private final static char TYPE_CODE = 'c';
private final static char TYPE_UNICODE = 'u';
private final static char TYPE_SET = '<';
private final static char TYPE_FROZENSET = '>';
private final static int MAX_MARSHAL_STACK_DEPTH = 2000;
private final static int CURRENT_VERSION = 2;
public static class Marshaller extends PyObject implements Traverseproc {
private final PyIOFile file;
private final int version;
public Marshaller(PyObject file) {
this(file, CURRENT_VERSION);
}
public Marshaller(PyObject file, int version) {
this.file = PyIOFileFactory.createIOFile(file);
this.version = version;
}
private boolean debug = false;
public void _debug() {
debug = true;
}
public void dump(PyObject obj) {
write_object(obj, 0);
}
private void write_byte(char c) {
if (debug) {
System.err.print("[" + (int) c + "]");
}
file.write(c);
}
private void write_string(String s) {
file.write(s);
}
private void write_strings(String[] some_strings, int depth) {
PyObject items[] = new PyObject[some_strings.length];
for (int i = 0; i < some_strings.length; i++) {
items[i] = Py.newString(some_strings[i]);
}
write_object(new PyTuple(items), depth + 1);
}
private void write_short(short x) {
write_byte((char) (x & 0xff));
write_byte((char) ((x >> 8) & 0xff));
}
private void write_int(int x) {
write_byte((char) (x & 0xff));
write_byte((char) ((x >> 8) & 0xff));
write_byte((char) ((x >> 16) & 0xff));
write_byte((char) ((x >> 24) & 0xff));
}
private void write_long64(long x) {
write_int((int) (x & 0xffffffff));
write_int((int) ((x >> 32) & 0xffffffff));
}
// writes output in 15 bit "digits"
private void write_long(BigInteger x) {
boolean negative = x.signum() < 0;
if (negative) {
x = x.negate();
}
int num_digits = (x.bitLength() + 14) / 15;
write_int(negative ? -num_digits : num_digits);
BigInteger mask = BigInteger.valueOf(0x7FFF);
for (int i = 0; i < num_digits; i++) {
write_short(x.and(mask).shortValue());
x = x.shiftRight(15);
}
}
private void write_float(PyFloat f) {
write_string(f.__repr__().toString());
}
private void write_binary_float(PyFloat f) {
write_long64(Double.doubleToLongBits(f.getValue()));
}
private void write_object(PyObject v, int depth) {
if (depth >= MAX_MARSHAL_STACK_DEPTH) {
throw Py.ValueError("Maximum marshal stack depth"); // XXX - fix this exception
} else if (v == null) {
write_byte(TYPE_NULL);
} else if (v == Py.None) {
write_byte(TYPE_NONE);
} else if (v == Py.StopIteration) {
write_byte(TYPE_STOPITER);
} else if (v == Py.Ellipsis) {
write_byte(TYPE_ELLIPSIS);
} else if (v == Py.False) {
write_byte(TYPE_FALSE);
} else if (v == Py.True) {
write_byte(TYPE_TRUE);
} else {
PyType vt = v.getType();
if (vt == PyInteger.TYPE) {
write_byte(TYPE_INT);
write_int(((PyInteger) v).asInt());
} else if (vt == PyLong.TYPE) {
write_byte(TYPE_LONG);
write_long(((PyLong) v).getValue());
} else if (vt == PyFloat.TYPE) {
if (version == CURRENT_VERSION) {
write_byte(TYPE_BINARY_FLOAT);
write_binary_float((PyFloat) v);
} else {
write_byte(TYPE_FLOAT);
write_float((PyFloat) v);
}
} else if (vt == PyComplex.TYPE) {
PyComplex x = (PyComplex) v;
if (version == CURRENT_VERSION) {
write_byte(TYPE_BINARY_COMPLEX);
write_binary_float(x.getReal());
write_binary_float(x.getImag());
} else {
write_byte(TYPE_COMPLEX);
write_float(x.getReal());
write_float(x.getImag());
}
} else if (vt == PyUnicode.TYPE) {
write_byte(TYPE_UNICODE);
String buffer = ((PyUnicode) v).encode("utf-8").toString();
write_int(buffer.length());
write_string(buffer);
} else if (vt == PyString.TYPE) {
// ignore interning
write_byte(TYPE_STRING);
write_int(v.__len__());
write_string(v.toString());
} else if (vt == PyTuple.TYPE) {
write_byte(TYPE_TUPLE);
PyTuple t = (PyTuple) v;
int n = t.__len__();
write_int(n);
for (int i = 0; i < n; i++) {
write_object(t.__getitem__(i), depth + 1);
}
} else if (vt == PyList.TYPE) {
write_byte(TYPE_LIST);
PyList list = (PyList) v;
int n = list.__len__();
write_int(n);
for (int i = 0; i < n; i++) {
write_object(list.__getitem__(i), depth + 1);
}
} else if (vt == PyDictionary.TYPE) {
write_byte(TYPE_DICT);
PyDictionary dict = (PyDictionary) v;
for (PyObject item : dict.iteritems().asIterable()) {
PyTuple pair = (PyTuple) item;
write_object(pair.__getitem__(0), depth + 1);
write_object(pair.__getitem__(1), depth + 1);
}
write_object(null, depth + 1);
} else if (vt == PySet.TYPE || vt == PyFrozenSet.TYPE) {
if (vt == PySet.TYPE) {
write_byte(TYPE_SET);
} else {
write_byte(TYPE_FROZENSET);
}
int n = v.__len__();
write_int(n);
BaseSet set = (BaseSet) v;
for (PyObject item : set.asIterable()) {
write_object(item, depth + 1);
}
} else if (vt == PyBytecode.TYPE) {
PyBytecode code = (PyBytecode) v;
write_byte(TYPE_CODE);
write_int(code.co_argcount);
write_int(code.co_nlocals);
write_int(code.co_stacksize);
write_int(code.co_flags.toBits());
write_object(Py.newString(new String(code.co_code)), depth + 1);
write_object(new PyTuple(code.co_consts), depth + 1);
write_strings(code.co_names, depth + 1);
write_strings(code.co_varnames, depth + 1);
write_strings(code.co_freevars, depth + 1);
write_strings(code.co_cellvars, depth + 1);
write_object(Py.newString(code.co_name), depth + 1);
write_int(code.co_firstlineno);
write_object(Py.newString(new String(code.co_lnotab)), depth + 1);
} else {
// Try to get a simple byte-oriented buffer
try (PyBuffer buf = ((BufferProtocol) v).getBuffer(PyBUF.SIMPLE)) {
// ... and treat those bytes as a String
write_byte(TYPE_STRING);
write_int(v.__len__());
write_string(buf.toString());
} catch (ClassCastException | PyException e) {
// Does not implement BufferProtocol (in simple byte form).
throw Py.ValueError("unmarshallable object");
}
}
}
depth--;
}
/* Traverseproc implementation */
@Override
public int traverse(Visitproc visit, Object arg) {
return file != null && file instanceof Traverseproc ?
((Traverseproc) file).traverse(visit, arg) : 0;
}
@Override
public boolean refersDirectlyTo(PyObject ob) {
return file != null && file instanceof Traverseproc ?
((Traverseproc) file).refersDirectlyTo(ob) : false;
}
}
public static class Unmarshaller extends PyObject implements Traverseproc {
private final PyIOFile file;
private final PyList strings = new PyList();
private final int version;
int depth = 0;
public Unmarshaller(PyObject file) {
this(file, CURRENT_VERSION);
}
public Unmarshaller(PyObject file, int version) {
this.file = PyIOFileFactory.createIOFile(file);
this.version = version;
}
private boolean debug = false;
public void _debug() {
debug = true;
}
public PyObject load() {
try {
PyObject obj = read_object(0);
if (obj == null) {
throw Py.TypeError("NULL object in marshal data");
}
return obj;
} catch (StringIndexOutOfBoundsException e) {
// convert from our PyIOFile abstraction to what marshal in CPython returns
// (although it's really just looking for no bombing)
throw Py.EOFError("EOF read where object expected");
}
}
private int read_byte() {
int b = file.read(1).charAt(0);
if (debug) {
System.err.print("[" + b + "]");
}
return b;
}
private String read_string(int n) {
return file.read(n);
}
private int read_short() {
int x = read_byte();
x |= read_byte() << 8;
return x;
}
private int read_int() { // cpython calls this r_long
int x = read_byte();
x |= read_byte() << 8;
x |= read_byte() << 16;
x |= read_byte() << 24;
return x;
}
private long read_long64() { // cpython calls this r_long64
long lo4 = read_int();
long hi4 = read_int();
long x = (hi4 << 32) | (lo4 & 0xFFFFFFFFL);
return x;
}
private BigInteger read_long() {
BigInteger result = BigInteger.ZERO;
boolean negative = false;
int digit = 0, size = read_int();
if (size == 0) {
return result;
} else if (size < 0) {
negative = true;
size = -size;
}
for (int i = 0; i < size; i++) {
if ((digit = read_short()) < 0) {
throw badMarshalData("digit out of range in long");
}
result = result.or(BigInteger.valueOf(digit).shiftLeft(i * 15));
}
if (digit == 0) {
throw badMarshalData("unnormalized long data");
}
return negative ? result.negate() : result;
}
private double read_float() {
int size = read_byte();
return Py.newString(read_string(size)).atof();
}
private double read_binary_float() {
return Double.longBitsToDouble(read_long64());
}
private PyObject read_object_notnull(int depth) {
PyObject v = read_object(depth);
if (v == null) {
throw badMarshalData(null);
}
return v;
}
private String[] read_strings(int depth) {
PyTuple t = (PyTuple) read_object_notnull(depth);
String some_strings[] = new String[t.__len__()];
int i = 0;
for (PyObject item : t.asIterable()) {
some_strings[i++] = item.toString().intern();
}
return some_strings;
}
private PyObject read_object(int depth) {
if (depth >= MAX_MARSHAL_STACK_DEPTH) {
throw Py.ValueError("Maximum marshal stack depth"); // XXX - fix this exception
}
int type = read_byte();
switch (type) {
case TYPE_NULL:
return null;
case TYPE_NONE:
return Py.None;
case TYPE_STOPITER:
return Py.StopIteration;
case TYPE_ELLIPSIS:
return Py.Ellipsis;
case TYPE_FALSE:
return Py.False;
case TYPE_TRUE:
return Py.True;
case TYPE_INT:
return Py.newInteger(read_int());
case TYPE_INT64:
return Py.newInteger(read_long64());
case TYPE_LONG: {
return Py.newLong(read_long());
}
case TYPE_FLOAT:
return Py.newFloat(read_float());
case TYPE_BINARY_FLOAT:
return Py.newFloat(read_binary_float());
case TYPE_COMPLEX: {
double real = read_float();
double imag = read_float();
return new PyComplex(real, imag);
}
case TYPE_BINARY_COMPLEX: {
double real = read_binary_float();
double imag = read_binary_float();
return new PyComplex(real, imag);
}
case TYPE_INTERNED:
case TYPE_STRING: {
int size = read_int();
String s = read_string(size);
if (type == TYPE_INTERNED) {
PyString pys = PyString.fromInterned(s.intern());
strings.append(pys);
return pys;
} else {
return Py.newString(s);
}
}
case TYPE_STRINGREF: {
int i = read_int();
return strings.__getitem__(i);
}
case TYPE_UNICODE: {
int n = read_int();
PyString buffer = Py.newString(read_string(n));
return buffer.decode("utf-8");
}
case TYPE_TUPLE: {
int n = read_int();
if (n < 0) {
throw badMarshalData(null);
}
PyObject items[] = new PyObject[n];
for (int i = 0; i < n; i++) {
items[i] = read_object_notnull(depth + 1);
}
return new PyTuple(items);
}
case TYPE_LIST: {
int n = read_int();
if (n < 0) {
throw badMarshalData(null);
}
PyObject items[] = new PyObject[n];
for (int i = 0; i < n; i++) {
items[i] = read_object_notnull(depth + 1);
}
return new PyList(items);
}
case TYPE_DICT: {
PyDictionary d = new PyDictionary();
while (true) {
PyObject key = read_object(depth + 1);
if (key == null) {
break;
}
PyObject value = read_object(depth + 1);
if (value != null) {
d.__setitem__(key, value);
}
}
return d;
}
case TYPE_SET:
case TYPE_FROZENSET: {
int n = read_int();
PyObject items[] = new PyObject[n];
for (int i = 0; i < n; i++) {
items[i] = read_object(depth + 1);
}
PyTuple v = new PyTuple(items);
if (type == TYPE_SET) {
return new PySet(v);
} else {
return new PyFrozenSet(v);
}
}
case TYPE_CODE: {
// XXX - support restricted execution mode? not certain if this is just legacy
int argcount = read_int();
int nlocals = read_int();
int stacksize = read_int();
int flags = read_int();
String code = read_object_notnull(depth + 1).toString();
PyObject consts[] = ((PyTuple) read_object_notnull(depth + 1)).getArray();
String names[] = read_strings(depth + 1);
String varnames[] = read_strings(depth + 1);
String freevars[] = read_strings(depth + 1);
String cellvars[] = read_strings(depth + 1);
String filename = read_object_notnull(depth + 1).toString();
String name = read_object_notnull(depth + 1).toString();
int firstlineno = read_int();
String lnotab = read_object_notnull(depth + 1).toString();
return new PyBytecode(
argcount, nlocals, stacksize, flags,
code, consts, names, varnames,
filename, name, firstlineno, lnotab,
cellvars, freevars);
}
default:
throw badMarshalData("unknown type code");
}
}
/** Helper returning "bad marshal data" or "bad marshal data ()". */
private static PyException badMarshalData(String reason) {
StringBuilder msg = (new StringBuilder(60)).append("bad marshal data");
if (reason != null) {
msg.append(" (").append(reason).append(')');
}
return Py.ValueError(msg.toString());
}
/* Traverseproc implementation */
@Override
public int traverse(Visitproc visit, Object arg) {
if (file instanceof Traverseproc) {
int retVal = ((Traverseproc) file).traverse(visit, arg);
if (retVal != 0) {
return retVal;
}
}
return visit.visit(strings, arg);
}
@Override
public boolean refersDirectlyTo(PyObject ob) {
if (ob == null) {
return false;
} else if (file != null && file instanceof Traverseproc
&& ((Traverseproc) file).refersDirectlyTo(ob)) {
return true;
} else {
return ob == strings;
}
}
}
}