com.oracle.graal.python.compiler.Compiler Maven / Gradle / Ivy
/*
* Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* The Universal Permissive License (UPL), Version 1.0
*
* Subject to the condition set forth below, permission is hereby granted to any
* person obtaining a copy of this software, associated documentation and/or
* data (collectively the "Software"), free of charge and under any and all
* copyright rights in the Software, and any and all patent rights owned or
* freely licensable by each licensor hereunder covering either (i) the
* unmodified Software as contributed to or provided by such licensor, or (ii)
* the Larger Works (as defined below), to deal in both
*
* (a) the Software, and
*
* (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
* one is included with the Software each a "Larger Work" to which the Software
* is contributed by such licensors),
*
* without restriction, including without limitation the rights to copy, create
* derivative works of, display, perform, and distribute the Software and make,
* use, sell, offer for sale, import, export, have made, and have sold the
* Software and the Larger Work(s), and to sublicense the foregoing rights on
* either these or other terms.
*
* This license is subject to the following condition:
*
* The above copyright notice and either this complete permission notice or at a
* minimum a reference to the UPL must be included in all copies or substantial
* portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
package com.oracle.graal.python.compiler;
import static com.oracle.graal.python.compiler.OpCodes.ADD_TO_COLLECTION;
import static com.oracle.graal.python.compiler.OpCodes.ASYNCGEN_WRAP;
import static com.oracle.graal.python.compiler.OpCodes.BINARY_OP;
import static com.oracle.graal.python.compiler.OpCodes.BINARY_SUBSCR;
import static com.oracle.graal.python.compiler.OpCodes.BUILD_SLICE;
import static com.oracle.graal.python.compiler.OpCodes.CALL_COMPREHENSION;
import static com.oracle.graal.python.compiler.OpCodes.CALL_FUNCTION;
import static com.oracle.graal.python.compiler.OpCodes.CALL_FUNCTION_KW;
import static com.oracle.graal.python.compiler.OpCodes.CALL_FUNCTION_VARARGS;
import static com.oracle.graal.python.compiler.OpCodes.CALL_METHOD;
import static com.oracle.graal.python.compiler.OpCodes.CALL_METHOD_VARARGS;
import static com.oracle.graal.python.compiler.OpCodes.CLOSURE_FROM_STACK;
import static com.oracle.graal.python.compiler.OpCodes.COLLECTION_ADD_COLLECTION;
import static com.oracle.graal.python.compiler.OpCodes.COLLECTION_ADD_STACK;
import static com.oracle.graal.python.compiler.OpCodes.COLLECTION_FROM_COLLECTION;
import static com.oracle.graal.python.compiler.OpCodes.COLLECTION_FROM_STACK;
import static com.oracle.graal.python.compiler.OpCodes.COPY_DICT_WITHOUT_KEYS;
import static com.oracle.graal.python.compiler.OpCodes.DELETE_ATTR;
import static com.oracle.graal.python.compiler.OpCodes.DELETE_DEREF;
import static com.oracle.graal.python.compiler.OpCodes.DELETE_FAST;
import static com.oracle.graal.python.compiler.OpCodes.DELETE_GLOBAL;
import static com.oracle.graal.python.compiler.OpCodes.DELETE_NAME;
import static com.oracle.graal.python.compiler.OpCodes.DELETE_SUBSCR;
import static com.oracle.graal.python.compiler.OpCodes.DUP_TOP;
import static com.oracle.graal.python.compiler.OpCodes.END_ASYNC_FOR;
import static com.oracle.graal.python.compiler.OpCodes.END_EXC_HANDLER;
import static com.oracle.graal.python.compiler.OpCodes.EXIT_AWITH;
import static com.oracle.graal.python.compiler.OpCodes.EXIT_WITH;
import static com.oracle.graal.python.compiler.OpCodes.FORMAT_VALUE;
import static com.oracle.graal.python.compiler.OpCodes.FOR_ITER;
import static com.oracle.graal.python.compiler.OpCodes.FROZENSET_FROM_LIST;
import static com.oracle.graal.python.compiler.OpCodes.GET_AEXIT_CORO;
import static com.oracle.graal.python.compiler.OpCodes.GET_AITER;
import static com.oracle.graal.python.compiler.OpCodes.GET_ANEXT;
import static com.oracle.graal.python.compiler.OpCodes.GET_AWAITABLE;
import static com.oracle.graal.python.compiler.OpCodes.GET_ITER;
import static com.oracle.graal.python.compiler.OpCodes.GET_LEN;
import static com.oracle.graal.python.compiler.OpCodes.GET_YIELD_FROM_ITER;
import static com.oracle.graal.python.compiler.OpCodes.IMPORT_FROM;
import static com.oracle.graal.python.compiler.OpCodes.IMPORT_NAME;
import static com.oracle.graal.python.compiler.OpCodes.IMPORT_STAR;
import static com.oracle.graal.python.compiler.OpCodes.JUMP_BACKWARD;
import static com.oracle.graal.python.compiler.OpCodes.JUMP_FORWARD;
import static com.oracle.graal.python.compiler.OpCodes.JUMP_IF_FALSE_OR_POP;
import static com.oracle.graal.python.compiler.OpCodes.JUMP_IF_TRUE_OR_POP;
import static com.oracle.graal.python.compiler.OpCodes.KWARGS_DICT_MERGE;
import static com.oracle.graal.python.compiler.OpCodes.LOAD_ASSERTION_ERROR;
import static com.oracle.graal.python.compiler.OpCodes.LOAD_ATTR;
import static com.oracle.graal.python.compiler.OpCodes.LOAD_BIGINT;
import static com.oracle.graal.python.compiler.OpCodes.LOAD_BUILD_CLASS;
import static com.oracle.graal.python.compiler.OpCodes.LOAD_BYTE;
import static com.oracle.graal.python.compiler.OpCodes.LOAD_BYTES;
import static com.oracle.graal.python.compiler.OpCodes.LOAD_CLASSDEREF;
import static com.oracle.graal.python.compiler.OpCodes.LOAD_CLOSURE;
import static com.oracle.graal.python.compiler.OpCodes.LOAD_COMPLEX;
import static com.oracle.graal.python.compiler.OpCodes.LOAD_CONST;
import static com.oracle.graal.python.compiler.OpCodes.LOAD_CONST_COLLECTION;
import static com.oracle.graal.python.compiler.OpCodes.LOAD_DEREF;
import static com.oracle.graal.python.compiler.OpCodes.LOAD_DOUBLE;
import static com.oracle.graal.python.compiler.OpCodes.LOAD_ELLIPSIS;
import static com.oracle.graal.python.compiler.OpCodes.LOAD_FALSE;
import static com.oracle.graal.python.compiler.OpCodes.LOAD_FAST;
import static com.oracle.graal.python.compiler.OpCodes.LOAD_GLOBAL;
import static com.oracle.graal.python.compiler.OpCodes.LOAD_INT;
import static com.oracle.graal.python.compiler.OpCodes.LOAD_LONG;
import static com.oracle.graal.python.compiler.OpCodes.LOAD_METHOD;
import static com.oracle.graal.python.compiler.OpCodes.LOAD_NAME;
import static com.oracle.graal.python.compiler.OpCodes.LOAD_NONE;
import static com.oracle.graal.python.compiler.OpCodes.LOAD_STRING;
import static com.oracle.graal.python.compiler.OpCodes.LOAD_TRUE;
import static com.oracle.graal.python.compiler.OpCodes.MAKE_FUNCTION;
import static com.oracle.graal.python.compiler.OpCodes.MAKE_KEYWORD;
import static com.oracle.graal.python.compiler.OpCodes.MATCH_CLASS;
import static com.oracle.graal.python.compiler.OpCodes.MATCH_EXC_OR_JUMP;
import static com.oracle.graal.python.compiler.OpCodes.MATCH_KEYS;
import static com.oracle.graal.python.compiler.OpCodes.MATCH_MAPPING;
import static com.oracle.graal.python.compiler.OpCodes.MATCH_SEQUENCE;
import static com.oracle.graal.python.compiler.OpCodes.NOP;
import static com.oracle.graal.python.compiler.OpCodes.POP_AND_JUMP_IF_FALSE;
import static com.oracle.graal.python.compiler.OpCodes.POP_AND_JUMP_IF_TRUE;
import static com.oracle.graal.python.compiler.OpCodes.POP_EXCEPT;
import static com.oracle.graal.python.compiler.OpCodes.POP_TOP;
import static com.oracle.graal.python.compiler.OpCodes.PRINT_EXPR;
import static com.oracle.graal.python.compiler.OpCodes.PUSH_EXC_INFO;
import static com.oracle.graal.python.compiler.OpCodes.RAISE_VARARGS;
import static com.oracle.graal.python.compiler.OpCodes.RESUME_YIELD;
import static com.oracle.graal.python.compiler.OpCodes.RETURN_VALUE;
import static com.oracle.graal.python.compiler.OpCodes.ROT_N;
import static com.oracle.graal.python.compiler.OpCodes.ROT_THREE;
import static com.oracle.graal.python.compiler.OpCodes.ROT_TWO;
import static com.oracle.graal.python.compiler.OpCodes.SEND;
import static com.oracle.graal.python.compiler.OpCodes.SETUP_ANNOTATIONS;
import static com.oracle.graal.python.compiler.OpCodes.SETUP_AWITH;
import static com.oracle.graal.python.compiler.OpCodes.SETUP_WITH;
import static com.oracle.graal.python.compiler.OpCodes.STORE_ATTR;
import static com.oracle.graal.python.compiler.OpCodes.STORE_DEREF;
import static com.oracle.graal.python.compiler.OpCodes.STORE_FAST;
import static com.oracle.graal.python.compiler.OpCodes.STORE_GLOBAL;
import static com.oracle.graal.python.compiler.OpCodes.STORE_NAME;
import static com.oracle.graal.python.compiler.OpCodes.STORE_SUBSCR;
import static com.oracle.graal.python.compiler.OpCodes.THROW;
import static com.oracle.graal.python.compiler.OpCodes.TUPLE_FROM_LIST;
import static com.oracle.graal.python.compiler.OpCodes.UNARY_OP;
import static com.oracle.graal.python.compiler.OpCodes.UNPACK_EX;
import static com.oracle.graal.python.compiler.OpCodes.UNPACK_SEQUENCE;
import static com.oracle.graal.python.compiler.OpCodes.UNWRAP_EXC;
import static com.oracle.graal.python.compiler.OpCodes.YIELD_VALUE;
import static com.oracle.graal.python.nodes.StringLiterals.T_EMPTY_STRING;
import static com.oracle.graal.python.util.PythonUtils.toTruffleStringUncached;
import static com.oracle.truffle.api.CompilerDirectives.shouldNotReachHere;
import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import com.oracle.graal.python.PythonLanguage;
import com.oracle.graal.python.builtins.PythonBuiltinClassType;
import com.oracle.graal.python.builtins.objects.PNone;
import com.oracle.graal.python.compiler.OpCodes.CollectionBits;
import com.oracle.graal.python.lib.PyObjectRichCompareBool;
import com.oracle.graal.python.pegparser.AbstractParser;
import com.oracle.graal.python.pegparser.ErrorCallback;
import com.oracle.graal.python.pegparser.ErrorCallback.ErrorType;
import com.oracle.graal.python.pegparser.ErrorCallback.WarningType;
import com.oracle.graal.python.pegparser.FutureFeature;
import com.oracle.graal.python.pegparser.InputType;
import com.oracle.graal.python.pegparser.Parser;
import com.oracle.graal.python.pegparser.scope.Scope;
import com.oracle.graal.python.pegparser.scope.ScopeEnvironment;
import com.oracle.graal.python.pegparser.sst.AliasTy;
import com.oracle.graal.python.pegparser.sst.ArgTy;
import com.oracle.graal.python.pegparser.sst.ArgumentsTy;
import com.oracle.graal.python.pegparser.sst.BoolOpTy;
import com.oracle.graal.python.pegparser.sst.CmpOpTy;
import com.oracle.graal.python.pegparser.sst.ComprehensionTy;
import com.oracle.graal.python.pegparser.sst.ConstantValue;
import com.oracle.graal.python.pegparser.sst.ConstantValue.Kind;
import com.oracle.graal.python.pegparser.sst.ExceptHandlerTy;
import com.oracle.graal.python.pegparser.sst.ExprContextTy;
import com.oracle.graal.python.pegparser.sst.ExprTy;
import com.oracle.graal.python.pegparser.sst.ExprTy.Constant;
import com.oracle.graal.python.pegparser.sst.KeywordTy;
import com.oracle.graal.python.pegparser.sst.MatchCaseTy;
import com.oracle.graal.python.pegparser.sst.ModTy;
import com.oracle.graal.python.pegparser.sst.OperatorTy;
import com.oracle.graal.python.pegparser.sst.PatternTy;
import com.oracle.graal.python.pegparser.sst.SSTNode;
import com.oracle.graal.python.pegparser.sst.SSTreeVisitor;
import com.oracle.graal.python.pegparser.sst.StmtTy;
import com.oracle.graal.python.pegparser.sst.TypeIgnoreTy;
import com.oracle.graal.python.pegparser.sst.UnaryOpTy;
import com.oracle.graal.python.pegparser.sst.WithItemTy;
import com.oracle.graal.python.pegparser.tokenizer.SourceRange;
import com.oracle.graal.python.runtime.object.PythonObjectFactory;
import com.oracle.graal.python.util.PythonUtils;
import com.oracle.graal.python.util.SuppressFBWarnings;
import com.oracle.truffle.api.memory.ByteArraySupport;
import com.oracle.truffle.api.strings.TruffleString;
/**
* Compiler for bytecode interpreter.
*/
public class Compiler implements SSTreeVisitor {
public static final int BYTECODE_VERSION = 28;
private final ErrorCallback errorCallback;
ScopeEnvironment env;
EnumSet flags = EnumSet.noneOf(Flags.class);
EnumSet futureFeatures = EnumSet.noneOf(FutureFeature.class);
int futureLineno = -1;
int optimizationLevel = 0;
int nestingLevel = 0;
CompilationUnit unit;
List stack = new ArrayList<>();
private boolean interactive;
private static class PatternContext {
ArrayList stores;
boolean allowIrrefutable;
ArrayList failPop;
int onTop;
}
public enum Flags {
}
public Compiler(ErrorCallback errorCallback) {
this.errorCallback = errorCallback;
}
@SuppressWarnings("hiding")
public CompilationUnit compile(ModTy mod, EnumSet flags, int optimizationLevel, EnumSet futureFeatures) {
this.flags = flags;
if (mod instanceof ModTy.Module) {
parseFuture(((ModTy.Module) mod).body);
} else if (mod instanceof ModTy.Interactive) {
parseFuture(((ModTy.Interactive) mod).body);
}
this.futureFeatures.addAll(futureFeatures);
this.env = ScopeEnvironment.analyze(mod, errorCallback, this.futureFeatures);
this.optimizationLevel = optimizationLevel;
enterScope("", CompilationScope.Module, mod);
mod.accept(this);
CompilationUnit topUnit = unit;
exitScope();
return topUnit;
}
private void parseFuture(StmtTy[] modBody) {
if (modBody == null || modBody.length == 0) {
return;
}
boolean done = false;
int prevLine = 0;
int i = 0;
if (getDocstring(modBody) != null) {
i++;
}
for (; i < modBody.length; i++) {
StmtTy s = modBody[i];
int line = s.getSourceRange().startLine;
if (done && line > prevLine) {
return;
}
prevLine = line;
if (s instanceof StmtTy.ImportFrom) {
StmtTy.ImportFrom importFrom = (StmtTy.ImportFrom) s;
if ("__future__".equals(importFrom.module)) {
if (done) {
errorCallback.onError(ErrorType.Syntax, s.getSourceRange(), "from __future__ imports must occur at the beginning of the file");
}
parseFutureFeatures(importFrom, futureFeatures);
futureLineno = line;
} else {
done = true;
}
} else {
done = true;
}
}
}
private void parseFutureFeatures(StmtTy.ImportFrom node, EnumSet features) {
for (AliasTy alias : node.names) {
if (alias.name != null) {
switch (alias.name) {
case "nested_scopes":
case "generators":
case "division":
case "absolute_import":
case "with_statement":
case "print_function":
case "unicode_literals":
case "generator_stop":
break;
case "barry_as_FLUFL":
features.add(FutureFeature.BARRY_AS_BDFL);
break;
case "annotations":
features.add(FutureFeature.ANNOTATIONS);
break;
case "braces":
errorCallback.onError(ErrorType.Syntax, node.getSourceRange(), "not a chance");
break;
default:
errorCallback.onError(ErrorType.Syntax, node.getSourceRange(), "future feature %s is not defined", alias.name);
break;
}
}
}
}
private List quickeningStack = new ArrayList<>();
void pushOp(Instruction insn) {
if (insn.opcode == FOR_ITER) {
quickeningStack.add(insn);
return;
}
if (insn.target != null && insn.opcode.getNumberOfProducedStackItems(insn.arg, insn.followingArgs, true) > 0) {
// TODO support control flow
quickeningStack.clear();
return;
}
int consumed = insn.opcode.getNumberOfConsumedStackItems(insn.arg, insn.followingArgs, false);
int produced = insn.opcode.getNumberOfProducedStackItems(insn.arg, insn.followingArgs, false);
byte canQuickenInputTypes = insn.opcode.canQuickenInputTypes();
List inputs = null;
if (insn.opcode == STORE_SUBSCR) {
// Asymmetric, needs to be handled separately
Instruction index = popQuickeningStack();
popQuickeningStack(); // Ignore the collection, it's always object
Instruction value = popQuickeningStack();
if (index != null && (index.opcode.canQuickenOutputTypes() & QuickeningTypes.INT) != 0) {
index.quickenOutput = QuickeningTypes.INT;
if (value != null && (value.opcode.canQuickenOutputTypes() & canQuickenInputTypes) != 0) {
value.quickenOutput = canQuickenInputTypes;
insn.quickeningGeneralizeList = List.of(value, index);
} else {
insn.quickeningGeneralizeList = List.of(index);
}
}
return;
}
if (consumed > 0) {
if (canQuickenInputTypes != 0) {
inputs = new ArrayList<>(consumed);
for (int i = 0; i < consumed; i++) {
Instruction input = popQuickeningStack();
if (input == null) {
/*
* This happens when we cleared the stack after a jump or other
* not-yet-supported scenario
*/
canQuickenInputTypes = 0;
break;
}
canQuickenInputTypes &= input.opcode.canQuickenOutputTypes();
inputs.add(input);
}
} else {
for (int i = 0; i < consumed; i++) {
popQuickeningStack();
}
}
}
if (produced > 0) {
if (produced > 1) {
// TODO instructions that produce multiple values?
quickeningStack.clear();
return;
}
quickeningStack.add(insn);
}
if (canQuickenInputTypes != 0) {
insn.quickeningGeneralizeList = inputs;
}
if (canQuickenInputTypes != 0 && inputs != null) {
for (int i = 0; i < inputs.size(); i++) {
inputs.get(i).quickenOutput = canQuickenInputTypes;
}
}
}
Instruction popQuickeningStack() {
if (quickeningStack.size() == 0) {
return null;
}
return quickeningStack.remove(quickeningStack.size() - 1);
}
// helpers
private void enterScope(String name, CompilationScope scopeType, SSTNode node) {
enterScope(name, scopeType, node, 0, 0, 0, false, false);
}
private void enterScope(String name, CompilationScope scope, SSTNode node, ArgumentsTy args) {
int argc, pargc, kwargc;
boolean splat, kwSplat;
if (args == null) {
argc = pargc = kwargc = 0;
splat = kwSplat = false;
} else {
argc = args.args == null ? 0 : args.args.length;
pargc = args.posOnlyArgs == null ? 0 : args.posOnlyArgs.length;
kwargc = args.kwOnlyArgs == null ? 0 : args.kwOnlyArgs.length;
splat = args.varArg != null;
kwSplat = args.kwArg != null;
}
enterScope(name, scope, node, argc, pargc, kwargc, splat, kwSplat);
}
private void enterScope(String name, CompilationScope scopeType, SSTNode node, int argc, int pargc, int kwargc,
boolean hasSplat, boolean hasKwSplat) {
if (unit != null) {
stack.add(unit);
}
unit = new CompilationUnit(scopeType, env.lookupScope(node), name, unit, stack.size(), argc, pargc, kwargc,
hasSplat, hasKwSplat, node.getSourceRange(), futureFeatures);
nestingLevel++;
}
private void exitScope() {
nestingLevel--;
if (!stack.isEmpty()) {
unit = stack.remove(stack.size() - 1);
} else {
unit = null;
}
}
private void checkForbiddenName(String id, ExprContextTy context) {
if (context == ExprContextTy.Store) {
if (id.equals("__debug__")) {
errorCallback.onError(ErrorType.Syntax, unit.currentLocation, "cannot assign to __debug__");
}
}
if (context == ExprContextTy.Del) {
if (id.equals("__debug__")) {
errorCallback.onError(ErrorType.Syntax, unit.currentLocation, "cannot delete __debug__");
}
}
}
private boolean containsAnnotations(StmtTy[] stmts) {
if (stmts == null) {
return false;
}
for (StmtTy stmt : stmts) {
if (containsAnnotations(stmt)) {
return true;
}
}
return false;
}
private boolean containsAnnotations(StmtTy stmt) {
if (stmt instanceof StmtTy.AnnAssign) {
return true;
} else if (stmt instanceof StmtTy.For) {
return containsAnnotations(((StmtTy.For) stmt).body) || containsAnnotations(((StmtTy.For) stmt).orElse);
} else if (stmt instanceof StmtTy.While) {
return containsAnnotations(((StmtTy.While) stmt).body) || containsAnnotations(((StmtTy.While) stmt).orElse);
} else if (stmt instanceof StmtTy.If) {
return containsAnnotations(((StmtTy.If) stmt).body) || containsAnnotations(((StmtTy.If) stmt).orElse);
} else if (stmt instanceof StmtTy.With) {
return containsAnnotations(((StmtTy.With) stmt).body);
} else if (stmt instanceof StmtTy.Try) {
StmtTy.Try tryStmt = (StmtTy.Try) stmt;
if (tryStmt.handlers != null) {
for (ExceptHandlerTy h : tryStmt.handlers) {
if (containsAnnotations(((ExceptHandlerTy.ExceptHandler) h).body)) {
return true;
}
}
}
return containsAnnotations(tryStmt.body) || containsAnnotations(tryStmt.finalBody) || containsAnnotations(tryStmt.orElse);
}
return false;
}
private Void addOp(OpCodes code) {
addOp(code, 0, null, unit.currentLocation);
return null;
}
private void addOp(OpCodes code, Block target) {
addOp(code, target, null);
}
private void addConditionalJump(OpCodes code, Block target) {
int profileIndex = unit.conditionProfileCount;
unit.conditionProfileCount += 2;
/*
* Intentionally ignoring overflow in the conversion of the index. If the unit has more than
* 2^16 conditionals it most likely wouldn't compile anyway, so there's not much harm if
* there's some false sharing of profiles.
*/
byte[] followingArgs = new byte[2];
ByteArraySupport.littleEndian().putShort(followingArgs, 0, (short) profileIndex);
addOp(code, target, followingArgs);
}
private void addOp(OpCodes code, Block target, byte[] followingArgs) {
Block b = unit.currentBlock;
Instruction insn = new Instruction(code, 0, followingArgs, target, unit.currentLocation);
b.instr.add(insn);
pushOp(insn);
}
private Void addOp(OpCodes code, int arg) {
addOp(code, arg, null, unit.currentLocation);
return null;
}
private Void addOp(OpCodes code, int arg, byte[] followingArgs) {
addOp(code, arg, followingArgs, unit.currentLocation);
return null;
}
private void addOp(OpCodes code, int arg, byte[] followingArgs, SourceRange location) {
Block b = unit.currentBlock;
Instruction insn = new Instruction(code, arg, followingArgs, null, location);
b.instr.add(insn);
pushOp(insn);
}
private Void addOpName(OpCodes code, HashMap dict, String name) {
String mangled = ScopeEnvironment.mangle(unit.privateName, name);
addOpObject(code, dict, mangled);
return null;
}
private void addOpObject(OpCodes code, HashMap dict, T obj) {
int arg = addObject(dict, obj);
addOp(code, arg);
}
private void addDerefVariableOpcode(ExprContextTy ctx, int idx) {
switch (ctx) {
case Load:
addOp(unit.scope.isClass() ? LOAD_CLASSDEREF : LOAD_DEREF, idx);
break;
case Store:
addOp(STORE_DEREF, idx);
break;
case Del:
addOp(DELETE_DEREF, idx);
break;
}
}
private void addFastVariableOpcode(ExprContextTy ctx, int idx) {
switch (ctx) {
case Load:
addOp(LOAD_FAST, idx);
break;
case Store:
addOp(STORE_FAST, idx);
break;
case Del:
addOp(DELETE_FAST, idx);
break;
}
}
private void addGlobalVariableOpcode(ExprContextTy ctx, int idx) {
switch (ctx) {
case Load:
addOp(LOAD_GLOBAL, idx);
break;
case Store:
addOp(STORE_GLOBAL, idx);
break;
case Del:
addOp(DELETE_GLOBAL, idx);
break;
}
}
private void addNameVariableOpcode(ExprContextTy ctx, int idx) {
switch (ctx) {
case Load:
addOp(LOAD_NAME, idx);
break;
case Store:
addOp(STORE_NAME, idx);
break;
case Del:
addOp(DELETE_NAME, idx);
break;
}
}
private void addNameOp(String name, ExprContextTy ctx) {
checkForbiddenName(name, ctx);
String mangled = ScopeEnvironment.mangle(unit.privateName, name);
EnumSet uses = unit.scope.getUseOfName(mangled);
if (uses != null) {
if (uses.contains(Scope.DefUse.Free)) {
addDerefVariableOpcode(ctx, addObject(unit.freevars, mangled));
return;
} else if (uses.contains(Scope.DefUse.Cell)) {
addDerefVariableOpcode(ctx, addObject(unit.cellvars, mangled));
return;
} else if (uses.contains(Scope.DefUse.Local)) {
if (unit.scope.isFunction()) {
addFastVariableOpcode(ctx, addObject(unit.varnames, mangled));
return;
}
} else if (uses.contains(Scope.DefUse.GlobalImplicit)) {
if (unit.scope.isFunction()) {
addGlobalVariableOpcode(ctx, addObject(unit.names, mangled));
return;
}
} else if (uses.contains(Scope.DefUse.GlobalExplicit)) {
addGlobalVariableOpcode(ctx, addObject(unit.names, mangled));
return;
}
}
addNameVariableOpcode(ctx, addObject(unit.names, mangled));
}
private static int addObject(HashMap dict, T o) {
Integer v = dict.get(o);
if (v == null) {
v = dict.size();
dict.put(o, v);
}
return v;
}
private TruffleString getDocstring(StmtTy[] body) {
if (optimizationLevel >= 2) {
return null;
}
if (body != null && body.length > 0) {
StmtTy stmt = body[0];
if (stmt instanceof StmtTy.Expr) {
ExprTy expr = ((StmtTy.Expr) stmt).value;
if (expr instanceof ExprTy.Constant) {
ConstantValue value = ((ExprTy.Constant) expr).value;
if (value.kind == ConstantValue.Kind.RAW) {
return value.getRaw(TruffleString.class);
}
}
}
}
return null;
}
private SourceRange setLocation(SourceRange location) {
SourceRange savedLocation = unit.currentLocation;
unit.currentLocation = location;
return savedLocation;
}
private SourceRange setLocation(SSTNode node) {
return setLocation(node.getSourceRange());
}
private class Collector {
protected final int typeBits;
protected final int stackItemsPerItem;
protected int stackItems = 0;
protected boolean collectionOnStack = false;
public Collector(int typeBits) {
this.typeBits = typeBits;
stackItemsPerItem = typeBits == CollectionBits.KIND_DICT ? 2 : 1;
}
public Collector(int typeBits, int stackItems) {
this(typeBits);
this.stackItems = stackItems;
}
public void appendItem() {
stackItems += stackItemsPerItem;
if (stackItems + stackItemsPerItem > CollectionBits.KIND_MASK) {
doFlushStack();
}
}
public void flushStackIfNecessary() {
if (stackItems > 0) {
doFlushStack();
}
}
private void doFlushStack() {
assert stackItems <= CollectionBits.KIND_MASK;
if (collectionOnStack) {
addOp(COLLECTION_ADD_STACK, typeBits | stackItems);
} else {
addOp(COLLECTION_FROM_STACK, typeBits | stackItems);
}
collectionOnStack = true;
stackItems = 0;
}
public void appendCollection() {
assert stackItems == 0;
if (collectionOnStack) {
addOp(COLLECTION_ADD_COLLECTION, typeBits);
} else {
addOp(COLLECTION_FROM_COLLECTION, typeBits);
}
collectionOnStack = true;
}
public void finishCollection() {
if (stackItems > 0 || !collectionOnStack) {
doFlushStack();
}
}
public boolean isEmpty() {
return stackItems == 0 && !collectionOnStack;
}
}
private class KwargsMergingDictCollector extends Collector {
/*
* Keyword arguments get evaluated in "groups", where a group is a contiguous sequence of
* named keyword arguments or a keyword-splat. For example, in
*
* f(a=2, b=3, **kws, c=4, d=5, **more_kws)
*
* there are 4 groups. Each group is evaluated and only then merged with the existing
* keyword arguments. The merge step is where duplicate keyword arguments are checked.
*
* A call can have an arbitrarily long sequence of named keyword arguments, so we flush them
* from the stack and collect them into an intermediate dict. We cannot eagerly merge them
* with the existing keyword arguments since that could trigger a duplication check before
* all of the arguments in the group have been evaluated. For example, in
*
* f(**{'a': 2}, a=3, b=4, ..., z=function_with_side_effects())
*
* we cannot merge "a=3" with the existing keywords dict (and consequently raise a
* TypeError) until we compute "z".
*/
private boolean namedKeywordDictOnStack = false;
public KwargsMergingDictCollector(OpCodes callOp) {
super(CollectionBits.KIND_DICT);
/*
* We're making assumptions about the stack layout below this instruction to obtain the
* callable for error reporting. If we ever add more keywords call instructions, we need
* to adjust the implementation to be able to get the callable.
*/
assert callOp == CALL_FUNCTION_KW;
}
@Override
public void appendItem() {
stackItems += stackItemsPerItem;
if (stackItems + stackItemsPerItem > CollectionBits.KIND_MASK) {
collectIntoNamedKeywordDict();
}
}
@Override
public void flushStackIfNecessary() {
if (stackItems == 0 && !namedKeywordDictOnStack) {
// Nothing pending to be merged.
return;
}
if (stackItems > 0) {
collectIntoNamedKeywordDict();
}
if (collectionOnStack) {
// If there's already a collection on the stack, merge it with the dict we just
// finished creating.
addOp(KWARGS_DICT_MERGE);
}
collectionOnStack = true;
namedKeywordDictOnStack = false;
}
protected void collectIntoNamedKeywordDict() {
if (namedKeywordDictOnStack) {
// Just add the keywords to the existing dict. Since we statically check for
// duplicate named keywords, we don't need to worry about duplicates in this dict.
addOp(COLLECTION_ADD_STACK, typeBits | stackItems);
} else {
// Create a new dict.
addOp(COLLECTION_FROM_STACK, typeBits | stackItems);
namedKeywordDictOnStack = true;
}
stackItems = 0;
}
@Override
public void appendCollection() {
assert stackItems == 0;
assert !namedKeywordDictOnStack;
if (collectionOnStack) {
addOp(KWARGS_DICT_MERGE);
} else {
addOp(COLLECTION_FROM_COLLECTION, typeBits);
}
collectionOnStack = true;
}
@Override
public void finishCollection() {
flushStackIfNecessary();
}
@Override
public boolean isEmpty() {
return stackItems == 0 || !namedKeywordDictOnStack || !collectionOnStack;
}
}
private void collectIntoArray(ExprTy[] nodes, int bits, int alreadyOnStack) {
Collector collector = new Collector(bits, alreadyOnStack);
if (nodes != null) {
for (ExprTy e : nodes) {
if (e instanceof ExprTy.Starred) {
// splat
collector.flushStackIfNecessary();
((ExprTy.Starred) e).value.accept(this);
collector.appendCollection();
} else {
e.accept(this);
collector.appendItem();
}
}
}
collector.finishCollection();
}
private void collectIntoArray(ExprTy[] nodes, int bits) {
collectIntoArray(nodes, bits, 0);
}
private void collectIntoDict(ExprTy[] keys, ExprTy[] values) {
Collector collector = new Collector(CollectionBits.KIND_DICT);
if (keys != null) {
assert keys.length == values.length;
for (int i = 0; i < keys.length; i++) {
ExprTy key = keys[i];
ExprTy value = values[i];
if (key == null) {
// splat
collector.flushStackIfNecessary();
value.accept(this);
collector.appendCollection();
} else {
key.accept(this);
value.accept(this);
collector.appendItem();
}
}
}
collector.finishCollection();
}
private void validateKeywords(KeywordTy[] keywords) {
for (int i = 0; i < keywords.length; i++) {
if (keywords[i].arg != null) {
checkForbiddenName(keywords[i].arg, ExprContextTy.Store);
for (int j = i + 1; j < keywords.length; j++) {
if (keywords[i].arg.equals(keywords[j].arg)) {
errorCallback.onError(ErrorType.Syntax, unit.currentLocation, "keyword argument repeated: " + keywords[i].arg);
}
}
}
}
}
private void collectKeywords(KeywordTy[] keywords, OpCodes callOp) {
validateKeywords(keywords);
boolean hasSplat = false;
for (KeywordTy k : keywords) {
if (k.arg == null) {
hasSplat = true;
break;
}
}
if (!hasSplat) {
Collector collector = new Collector(CollectionBits.KIND_KWORDS);
for (KeywordTy k : keywords) {
k.accept(this);
collector.appendItem();
}
collector.finishCollection();
} else if (keywords.length == 1) {
// Just one splat, no need for merging
keywords[0].value.accept(this);
addOp(COLLECTION_FROM_COLLECTION, CollectionBits.KIND_KWORDS);
} else {
/*
* We need to emit bytecodes for proper keywords merging with checking for duplicate
* keys. We accumulate them in an intermediate dict.
*/
Collector collector = new KwargsMergingDictCollector(callOp);
for (KeywordTy k : keywords) {
if (k.arg == null) {
// splat
collector.flushStackIfNecessary();
k.value.accept(this);
collector.appendCollection();
} else {
addOp(LOAD_STRING, addObject(unit.constants, toTruffleStringUncached(k.arg)));
k.value.accept(this);
collector.appendItem();
}
}
collector.finishCollection();
addOp(COLLECTION_FROM_COLLECTION, CollectionBits.KIND_KWORDS);
}
}
private void makeClosure(CodeUnit code, int makeFunctionFlags) {
int newFlags = makeFunctionFlags;
if (code.freevars.length > 0) {
// add the closure
for (TruffleString tfv : code.freevars) {
String fv = tfv.toJavaStringUncached();
// special case for class scopes
int arg;
if (unit.scopeType == CompilationScope.Class && "__class__".equals(fv) || unit.scope.getUseOfName(fv).contains(Scope.DefUse.Cell)) {
arg = unit.cellvars.get(fv);
} else {
arg = unit.freevars.get(fv);
}
addOp(LOAD_CLOSURE, arg);
}
addOp(CLOSURE_FROM_STACK, code.freevars.length);
newFlags |= OpCodes.MakeFunctionFlags.HAS_CLOSURE;
}
addObject(unit.constants, code.qualname);
addOp(MAKE_FUNCTION, addObject(unit.constants, code), new byte[]{(byte) newFlags});
}
// visiting
private void visitBody(StmtTy[] stmts, boolean returnValue) {
if (stmts != null) {
if (unit.scope.isModule() && stmts.length > 0) {
/*
* Set current line number to the line number of first statement. This way line
* number for SETUP_ANNOTATIONS will always coincide with the line number of first
* "real" statement in module. If body is empty, then lineno will be set later in
* assemble.
*/
setLocation(stmts[0]);
}
if (containsAnnotations(stmts)) {
addOp(SETUP_ANNOTATIONS);
}
if (stmts.length > 0) {
int i = 0;
TruffleString docstring = getDocstring(stmts);
if (docstring != null) {
i++;
StmtTy.Expr stmt = (StmtTy.Expr) stmts[0];
stmt.value.accept(this);
addNameOp("__doc__", ExprContextTy.Store);
}
for (; i < stmts.length - 1; i++) {
stmts[i].accept(this);
}
/*
* To support interop eval we need to return the value of the last statement even if
* we're in file mode. Also used when parsing with arguments.
*/
StmtTy lastStatement = stmts[stmts.length - 1];
if (returnValue && lastStatement instanceof StmtTy.Expr) {
ExprTy value = ((StmtTy.Expr) lastStatement).value;
value.accept(this);
setLocation(value);
addOp(RETURN_VALUE);
return;
} else {
lastStatement.accept(this);
}
}
}
if (returnValue) {
addOp(LOAD_NONE);
addOp(RETURN_VALUE);
}
}
@Override
public Void visit(AliasTy node) {
addOp(LOAD_BYTE, 0);
addOp(LOAD_CONST, addObject(unit.constants, PythonUtils.EMPTY_TRUFFLESTRING_ARRAY));
addOpName(IMPORT_NAME, unit.names, node.name);
if (node.asName != null) {
int dotIdx = node.name.indexOf('.');
if (dotIdx >= 0) {
while (true) {
int pos = dotIdx + 1;
dotIdx = node.name.indexOf('.', pos);
int end = dotIdx >= 0 ? dotIdx : node.name.length();
String attr = node.name.substring(pos, end);
addOpObject(IMPORT_FROM, unit.names, attr);
if (dotIdx < 0) {
break;
}
addOp(ROT_TWO);
addOp(POP_TOP);
}
addNameOp(node.asName, ExprContextTy.Store);
addOp(POP_TOP);
} else {
addNameOp(node.asName, ExprContextTy.Store);
}
} else {
int dotIdx = node.name.indexOf('.');
if (dotIdx >= 0) {
addNameOp(node.name.substring(0, dotIdx), ExprContextTy.Store);
} else {
addNameOp(node.name, ExprContextTy.Store);
}
}
return null;
}
@Override
public Void visit(ArgTy node) {
throw new IllegalStateException("Should not be visited");
}
@Override
public Void visit(ArgumentsTy node) {
throw new IllegalStateException("Should not be visited");
}
@Override
public Void visit(ComprehensionTy node) {
throw new IllegalStateException("Should not be visited");
}
@Override
public Void visit(ExprTy.Attribute node) {
SourceRange savedLocation = setLocation(node);
try {
node.value.accept(this);
switch (node.context) {
case Store:
checkForbiddenName(node.attr, node.context);
return addOpName(STORE_ATTR, unit.names, node.attr);
case Del:
return addOpName(DELETE_ATTR, unit.names, node.attr);
case Load:
default:
return addOpName(LOAD_ATTR, unit.names, node.attr);
}
} finally {
setLocation(savedLocation);
}
}
@Override
public Void visit(ExprTy.Await node) {
// TODO if !IS_TOP_LEVEL_AWAIT
// TODO handle await in comprehension correctly (currently, it is always allowed)
if (!unit.scope.isFunction()) {
errorCallback.onError(ErrorType.Syntax, unit.currentLocation, "'await' outside function");
}
if (unit.scopeType != CompilationScope.AsyncFunction && unit.scopeType != CompilationScope.Comprehension) {
errorCallback.onError(ErrorType.Syntax, unit.currentLocation, "'await' outside async function");
}
SourceRange savedLocation = setLocation(node);
try {
node.value.accept(this);
addOp(GET_AWAITABLE);
addOp(LOAD_NONE);
addYieldFrom();
return null;
} finally {
setLocation(savedLocation);
}
}
@Override
public Void visit(ExprTy.BinOp node) {
SourceRange savedLocation = setLocation(node);
try {
node.left.accept(this);
node.right.accept(this);
switch (node.op) {
case Add:
addOp(BINARY_OP, BinaryOps.ADD.ordinal());
break;
case Sub:
addOp(BINARY_OP, BinaryOps.SUB.ordinal());
break;
case Mult:
addOp(BINARY_OP, BinaryOps.MUL.ordinal());
break;
case MatMult:
addOp(BINARY_OP, BinaryOps.MATMUL.ordinal());
break;
case Div:
addOp(BINARY_OP, BinaryOps.TRUEDIV.ordinal());
break;
case Mod:
addOp(BINARY_OP, BinaryOps.MOD.ordinal());
break;
case Pow:
addOp(BINARY_OP, BinaryOps.POW.ordinal());
break;
case LShift:
addOp(BINARY_OP, BinaryOps.LSHIFT.ordinal());
break;
case RShift:
addOp(BINARY_OP, BinaryOps.RSHIFT.ordinal());
break;
case BitOr:
addOp(BINARY_OP, BinaryOps.OR.ordinal());
break;
case BitXor:
addOp(BINARY_OP, BinaryOps.XOR.ordinal());
break;
case BitAnd:
addOp(BINARY_OP, BinaryOps.AND.ordinal());
break;
case FloorDiv:
addOp(BINARY_OP, BinaryOps.FLOORDIV.ordinal());
break;
default:
throw new IllegalStateException("Unknown binary operation " + node.op);
}
return null;
} finally {
setLocation(savedLocation);
}
}
@Override
public Void visit(ExprTy.BoolOp node) {
SourceRange savedLocation = setLocation(node);
try {
Block end = new Block();
ExprTy[] values = node.values;
OpCodes op;
if (node.op == BoolOpTy.And) {
op = JUMP_IF_FALSE_OR_POP;
} else {
op = JUMP_IF_TRUE_OR_POP;
}
for (int i = 0; i < values.length - 1; i++) {
ExprTy v = values[i];
v.accept(this);
addConditionalJump(op, end);
}
values[values.length - 1].accept(this);
unit.useNextBlock(end);
return null;
} finally {
setLocation(savedLocation);
}
}
private static boolean isAttributeLoad(ExprTy node) {
return node instanceof ExprTy.Attribute && ((ExprTy.Attribute) node).context == ExprContextTy.Load;
}
private static boolean hasOnlyPlainArgs(ExprTy[] args, KeywordTy[] keywords) {
if (keywords.length > 0) {
return false;
}
for (ExprTy arg : args) {
if (arg instanceof ExprTy.Starred) {
return false;
}
}
return true;
}
@Override
public Void visit(ExprTy.Call node) {
SourceRange savedLocation = setLocation(node);
checkCaller(node.func);
try {
// n.b.: we do things completely different from python for calls
OpCodes op = CALL_FUNCTION_VARARGS;
int opArg = 0;
boolean shortCall;
final ExprTy func = node.func;
final ExprTy[] args = node.args;
final KeywordTy[] keywords = node.keywords;
if (isAttributeLoad(func) && keywords.length == 0) {
((ExprTy.Attribute) func).value.accept(this);
op = CALL_METHOD_VARARGS;
String mangled = ScopeEnvironment.mangle(unit.privateName, ((ExprTy.Attribute) func).attr);
opArg = addObject(unit.names, mangled);
shortCall = args.length <= 3;
} else {
func.accept(this);
shortCall = args.length <= 4;
}
if (hasOnlyPlainArgs(args, keywords) && shortCall) {
if (op == CALL_METHOD_VARARGS) {
addOp(LOAD_METHOD, opArg);
op = CALL_METHOD;
} else {
op = CALL_FUNCTION;
}
opArg = args.length;
// fast calls without extra arguments array
visitSequence(args);
return addOp(op, opArg);
} else {
return callHelper(op, opArg, 0, args, keywords);
}
} finally {
setLocation(savedLocation);
}
}
private Void callHelper(OpCodes op, int opArg, int alreadyOnStack, ExprTy[] args, KeywordTy[] keywords) {
collectIntoArray(args, CollectionBits.KIND_OBJECT, op == CALL_METHOD_VARARGS ? 1 + alreadyOnStack : alreadyOnStack);
if (keywords.length > 0) {
assert op == CALL_FUNCTION_VARARGS;
collectKeywords(keywords, CALL_FUNCTION_KW);
return addOp(CALL_FUNCTION_KW);
} else {
return addOp(op, opArg);
}
}
private void addCompareOp(CmpOpTy op) {
switch (op) {
case Eq:
addOp(BINARY_OP, BinaryOps.EQ.ordinal());
break;
case NotEq:
addOp(BINARY_OP, BinaryOps.NE.ordinal());
break;
case Lt:
addOp(BINARY_OP, BinaryOps.LT.ordinal());
break;
case LtE:
addOp(BINARY_OP, BinaryOps.LE.ordinal());
break;
case Gt:
addOp(BINARY_OP, BinaryOps.GT.ordinal());
break;
case GtE:
addOp(BINARY_OP, BinaryOps.GE.ordinal());
break;
case Is:
addOp(BINARY_OP, BinaryOps.IS.ordinal());
break;
case IsNot:
addOp(BINARY_OP, BinaryOps.IS.ordinal());
addOp(UNARY_OP, UnaryOps.NOT.ordinal());
break;
case In:
addOp(BINARY_OP, BinaryOps.IN.ordinal());
break;
case NotIn:
addOp(BINARY_OP, BinaryOps.IN.ordinal());
addOp(UNARY_OP, UnaryOps.NOT.ordinal());
break;
default:
throw new IllegalStateException("Unknown comparison operation " + op);
}
}
@Override
public Void visit(ExprTy.Compare node) {
SourceRange savedLocation = setLocation(node);
checkCompare(node);
try {
node.left.accept(this);
if (node.comparators.length == 1) {
visitSequence(node.comparators);
addCompareOp(node.ops[0]);
} else {
Block cleanup = new Block();
int i;
for (i = 0; i < node.comparators.length - 1; i++) {
node.comparators[i].accept(this);
addOp(DUP_TOP);
addOp(ROT_THREE);
addCompareOp(node.ops[i]);
addConditionalJump(JUMP_IF_FALSE_OR_POP, cleanup);
}
node.comparators[i].accept(this);
addCompareOp(node.ops[i]);
Block end = new Block();
addOp(JUMP_FORWARD, end);
unit.useNextBlock(cleanup);
addOp(ROT_TWO);
addOp(POP_TOP);
unit.useNextBlock(end);
}
return null;
} finally {
setLocation(savedLocation);
}
}
@Override
public Void visit(ExprTy.Constant node) {
SourceRange savedLocation = setLocation(node);
try {
return addConstant(node.value);
} finally {
setLocation(savedLocation);
}
}
private Void addConstant(ConstantValue value) {
switch (value.kind) {
case NONE:
return addOp(LOAD_NONE);
case ELLIPSIS:
return addOp(LOAD_ELLIPSIS);
case BOOLEAN:
return addOp(value.getBoolean() ? LOAD_TRUE : LOAD_FALSE);
case LONG:
return addLoadNumber(value.getLong());
case DOUBLE:
return addOp(LOAD_DOUBLE, addObject(unit.primitiveConstants, Double.doubleToRawLongBits(value.getDouble())));
case COMPLEX:
return addOp(LOAD_COMPLEX, addObject(unit.constants, value.getComplex()));
case BIGINTEGER:
return addOp(LOAD_BIGINT, addObject(unit.constants, value.getBigInteger()));
case RAW:
return addOp(LOAD_STRING, addObject(unit.constants, value.getRaw(TruffleString.class)));
case BYTES:
return addOp(LOAD_BYTES, addObject(unit.constants, value.getBytes()));
case TUPLE:
addConstantList(value.getTupleElements());
return addOp(TUPLE_FROM_LIST);
case FROZENSET:
addConstantList(value.getFrozensetElements());
return addOp(FROZENSET_FROM_LIST);
default:
throw new IllegalStateException("Unknown constant kind " + value.kind);
}
}
private void addConstantList(ConstantValue[] values) {
Collector collector = new Collector(CollectionBits.KIND_LIST, 0);
for (ConstantValue v : values) {
addConstant(v);
collector.appendItem();
}
collector.finishCollection();
}
private Void addLoadNumber(long value) {
if (value == (byte) value) {
return addOp(LOAD_BYTE, (byte) value);
} else if (value == (int) value) {
return addOp(LOAD_INT, addObject(unit.primitiveConstants, value));
} else {
return addOp(LOAD_LONG, addObject(unit.primitiveConstants, value));
}
}
@Override
public Void visit(ExprTy.Dict node) {
SourceRange savedLocation = setLocation(node);
try {
collectIntoDict(node.keys, node.values);
return null;
} finally {
setLocation(savedLocation);
}
}
@Override
public Void visit(ExprTy.DictComp node) {
return visitComprehension(node, "", node.generators, node.key, node.value, ComprehensionType.DICT);
}
@Override
public Void visit(ExprTy.FormattedValue node) {
SourceRange savedLocation = setLocation(node);
try {
node.value.accept(this);
int oparg;
switch (node.conversion) {
case 's':
oparg = FormatOptions.FVC_STR;
break;
case 'r':
oparg = FormatOptions.FVC_REPR;
break;
case 'a':
oparg = FormatOptions.FVC_ASCII;
break;
case -1:
oparg = FormatOptions.FVC_NONE;
break;
default:
errorCallback.onError(ErrorType.System, node.getSourceRange(), "Unrecognized conversion character %d", node.conversion);
throw shouldNotReachHere("Error callback did not throw an exception");
}
if (node.formatSpec != null) {
node.formatSpec.accept(this);
oparg |= FormatOptions.FVS_HAVE_SPEC;
}
addOp(FORMAT_VALUE, oparg);
return null;
} finally {
setLocation(savedLocation);
}
}
@Override
public Void visit(ExprTy.GeneratorExp node) {
return visitComprehension(node, "", node.generators, node.element, null, ComprehensionType.GENEXPR);
}
@Override
public Void visit(ExprTy.IfExp node) {
SourceRange savedLocation = setLocation(node);
try {
Block end = new Block();
Block next = new Block();
jumpIf(node.test, next, false);
node.body.accept(this);
addOp(JUMP_FORWARD, end);
unit.useNextBlock(next);
node.orElse.accept(this);
unit.useNextBlock(end);
return null;
} finally {
setLocation(savedLocation);
}
}
@Override
public Void visit(ExprTy.JoinedStr node) {
SourceRange savedLocation = setLocation(node);
try {
// TODO add optimized op for small chains
addOp(LOAD_STRING, addObject(unit.constants, T_EMPTY_STRING));
addOpName(LOAD_METHOD, unit.names, "join");
collectIntoArray(node.values, CollectionBits.KIND_LIST);
addOp(CALL_METHOD, 1);
return null;
} finally {
setLocation(savedLocation);
}
}
@Override
public Void visit(ExprTy.Lambda node) {
SourceRange savedLocation = setLocation(node);
try {
checkForbiddenArgs(node.args);
int makeFunctionFlags = collectDefaults(node.args);
enterScope("", CompilationScope.Lambda, node, node.args);
/* Make None the first constant, so the lambda can't have a docstring. */
addObject(unit.constants, PNone.NONE);
CodeUnit code;
try {
node.body.accept(this);
addOp(RETURN_VALUE);
code = unit.assemble();
} finally {
exitScope();
}
makeClosure(code, makeFunctionFlags);
return null;
} finally {
setLocation(savedLocation);
}
}
private Void unpackInto(ExprTy[] elements) {
boolean unpack = false;
for (int i = 0; i < elements.length; i++) {
ExprTy e = elements[i];
if (e instanceof ExprTy.Starred) {
if (unpack) {
errorCallback.onError(ErrorType.Syntax, unit.currentLocation, "multiple starred expressions in assignment");
}
unpack = true;
int n = elements.length;
int countAfter = n - i - 1;
if (countAfter != (byte) countAfter) {
errorCallback.onError(ErrorType.Syntax, unit.currentLocation, "too many expressions in star-unpacking assignment");
}
addOp(UNPACK_EX, i, new byte[]{(byte) countAfter});
}
}
if (!unpack) {
addOp(UNPACK_SEQUENCE, elements.length);
}
for (ExprTy e : elements) {
if (e instanceof ExprTy.Starred) {
((ExprTy.Starred) e).value.accept(this);
} else if (e != null) {
e.accept(this);
}
}
return null;
}
@Override
public Void visit(ExprTy.List node) {
SourceRange savedLocation = setLocation(node);
try {
switch (node.context) {
case Store:
return unpackInto(node.elements);
case Load:
boolean emittedConstant = tryCollectConstantCollection(node.elements, CollectionBits.KIND_LIST);
if (emittedConstant) {
return null;
}
collectIntoArray(node.elements, CollectionBits.KIND_LIST);
return null;
case Del:
default:
return visitSequence(node.elements);
}
} finally {
setLocation(savedLocation);
}
}
private enum ComprehensionType {
LIST(CollectionBits.KIND_LIST),
SET(CollectionBits.KIND_SET),
DICT(CollectionBits.KIND_DICT),
GENEXPR(-1);
public final int typeBits;
ComprehensionType(int typeBits) {
this.typeBits = typeBits;
}
}
@Override
public Void visit(ExprTy.ListComp node) {
return visitComprehension(node, "", node.generators, node.element, null, ComprehensionType.LIST);
}
private Void visitComprehension(ExprTy node, String name, ComprehensionTy[] generators, ExprTy element, ExprTy value, ComprehensionType type) {
/*
* Create an inner anonymous function to run the comprehension. It takes the outermost
* iterator as an argument and returns the accumulated sequence
*/
SourceRange savedLocation = setLocation(node);
try {
enterScope(name, CompilationScope.Comprehension, node, 1, 0, 0, false, false);
if (type != ComprehensionType.GENEXPR) {
// The result accumulator, empty at the beginning
addOp(COLLECTION_FROM_STACK, type.typeBits);
}
visitComprehensionGenerator(generators, 0, element, value, type);
if (type != ComprehensionType.GENEXPR) {
addOp(RETURN_VALUE);
}
CodeUnit code = unit.assemble();
exitScope();
makeClosure(code, 0);
generators[0].iter.accept(this);
if (generators[0].isAsync) {
addOp(GET_AITER);
} else {
addOp(GET_ITER);
}
addOp(CALL_COMPREHENSION);
// a genexpr will create an asyncgen, which we cannot await
if (type != ComprehensionType.GENEXPR) {
for (ComprehensionTy gen : generators) {
// if we have a non-genexpr async comprehension, the call will produce a
// coroutine which we need to await
if (gen.isAsync) {
addOp(GET_AWAITABLE);
addOp(LOAD_NONE);
addYieldFrom();
break;
}
}
}
return null;
} finally {
setLocation(savedLocation);
}
}
private void visitComprehensionGenerator(ComprehensionTy[] generators, int i, ExprTy element, ExprTy value, ComprehensionType type) {
ComprehensionTy gen = generators[i];
if (i == 0) {
/* The iterator is the function argument for the outermost generator */
addOp(LOAD_FAST, 0);
} else {
/* Create the iterator for nested iteration */
gen.iter.accept(this);
if (gen.isAsync) {
addOp(GET_AITER);
} else {
addOp(GET_ITER);
}
}
Block start = new Block();
Block ifCleanup = new Block();
Block anchor = new Block();
Block asyncForTry = gen.isAsync ? new Block() : null;
Block asyncForExcept = gen.isAsync ? new Block() : null;
Block asyncForBody = gen.isAsync ? new Block() : null;
unit.useNextBlock(start);
if (gen.isAsync) {
addOp(DUP_TOP);
unit.useNextBlock(asyncForTry);
unit.pushBlock(new BlockInfo.AsyncForLoopExit(asyncForTry, asyncForExcept));
addOp(GET_ANEXT);
addOp(LOAD_NONE);
addYieldFrom();
unit.popBlock();
unit.useNextBlock(asyncForBody);
gen.target.accept(this);
} else {
addOp(FOR_ITER, anchor);
gen.target.accept(this);
}
for (ExprTy ifExpr : gen.ifs) {
jumpIf(ifExpr, ifCleanup, false);
}
if (i + 1 < generators.length) {
visitComprehensionGenerator(generators, i + 1, element, value, type);
}
if (i == generators.length - 1) {
/* The last generator produces the resulting element to be appended/yielded */
element.accept(this);
int collectionStackDepth = generators.length + 1;
if (value != null) {
value.accept(this);
collectionStackDepth++;
}
if (type == ComprehensionType.GENEXPR) {
if (generators[i].isAsync) {
addOp(ASYNCGEN_WRAP);
}
addOp(YIELD_VALUE);
addOp(RESUME_YIELD);
addOp(POP_TOP);
} else {
/*
* There is an iterator for every generator on the stack. We need to append to the
* collection that's below them
*/
if (collectionStackDepth > CollectionBits.KIND_MASK) {
errorCallback.onError(ErrorType.Syntax, unit.currentLocation, "too many levels of nested comprehensions");
}
addOp(ADD_TO_COLLECTION, collectionStackDepth | type.typeBits);
}
}
unit.useNextBlock(ifCleanup);
addOp(JUMP_BACKWARD, start);
if (gen.isAsync) {
unit.useNextBlock(asyncForExcept);
addOp(END_ASYNC_FOR);
}
unit.useNextBlock(anchor);
}
@Override
public Void visit(ExprTy.Name node) {
SourceRange savedLocation = setLocation(node);
try {
addNameOp(node.id, node.context);
return null;
} finally {
setLocation(savedLocation);
}
}
@Override
public Void visit(ExprTy.NamedExpr node) {
SourceRange savedLocation = setLocation(node);
try {
node.value.accept(this);
addOp(DUP_TOP);
node.target.accept(this);
return null;
} finally {
setLocation(savedLocation);
}
}
@Override
public Void visit(ExprTy.Set node) {
SourceRange savedLocation = setLocation(node);
try {
collectIntoArray(node.elements, CollectionBits.KIND_SET);
return null;
} finally {
setLocation(savedLocation);
}
}
@Override
public Void visit(ExprTy.SetComp node) {
return visitComprehension(node, "", node.generators, node.element, null, ComprehensionType.SET);
}
@Override
public Void visit(ExprTy.Slice node) {
SourceRange savedLocation = setLocation(node);
try {
int n = 2;
if (node.lower != null) {
node.lower.accept(this);
} else {
addOp(LOAD_NONE);
}
if (node.upper != null) {
node.upper.accept(this);
} else {
addOp(LOAD_NONE);
}
if (node.step != null) {
node.step.accept(this);
n++;
}
addOp(BUILD_SLICE, n);
return null;
} finally {
setLocation(savedLocation);
}
}
@Override
public Void visit(ExprTy.Starred node) {
// Valid occurrences are handled by other visitors
if (node.context == ExprContextTy.Store) {
errorCallback.onError(ErrorType.Syntax, unit.currentLocation, "starred assignment target must be in a list or tuple");
} else {
errorCallback.onError(ErrorType.Syntax, unit.currentLocation, "can't use starred expression here");
}
return null;
}
@Override
public Void visit(ExprTy.Subscript node) {
SourceRange savedLocation = setLocation(node);
if (node.context == ExprContextTy.Load) {
checkSubscripter(node.value);
checkIndex(node.value, node.slice);
}
try {
node.value.accept(this);
node.slice.accept(this);
switch (node.context) {
case Load:
return addOp(BINARY_SUBSCR);
case Store:
return addOp(STORE_SUBSCR);
case Del:
default:
return addOp(DELETE_SUBSCR);
}
} finally {
setLocation(savedLocation);
}
}
@Override
public Void visit(ExprTy.Tuple node) {
SourceRange savedLocation = setLocation(node);
try {
switch (node.context) {
case Store:
return unpackInto(node.elements);
case Load:
/*
* We don't have mutation operations for tuples, so if we cannot construct the
* tuple within a single instruction, we construct a list and convert it to a
* tuple.
*/
boolean useList = false;
if (node.elements != null) {
if (node.elements.length > CollectionBits.KIND_MASK) {
useList = true;
} else {
for (ExprTy e : node.elements) {
if (e instanceof ExprTy.Starred) {
useList = true;
break;
}
}
}
}
boolean emittedConstant = tryCollectConstantCollection(node.elements, CollectionBits.KIND_TUPLE);
if (emittedConstant) {
return null;
}
if (!useList) {
collectIntoArray(node.elements, CollectionBits.KIND_TUPLE);
} else {
collectIntoArray(node.elements, CollectionBits.KIND_LIST);
addOp(TUPLE_FROM_LIST);
}
return null;
case Del:
default:
return visitSequence(node.elements);
}
} finally {
setLocation(savedLocation);
}
}
private boolean tryCollectConstantCollection(ExprTy[] elements, int collectionKind) {
/*
* We try to store the whole tuple as a Java array constant when all the elements are
* constant and context-independent.
*/
if (elements == null || elements.length == 0) {
return false;
}
int constantType = -1;
List © 2015 - 2025 Weber Informatics LLC | Privacy Policy