[GR-13199] Fix a more complex case for the scope translator

PullRequest: graalpython/342

Author: timfel

graalpython/com.oracle.graal.python.test/src/tests/test_generators.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2018, Oracle and/or its affiliates.
+# Copyright (c) 2018, 2019, Oracle and/or its affiliates.
 # Copyright (C) 1996-2017 Python Software Foundation
 #
 # Licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
@@ -178,3 +178,46 @@ def func(*args):
     gen = (x for x in range(10))
     assert func(*lst) == set(lst)
     assert func(*gen) == set(lst)
+
+
+def test_generator_cell_confusion():
+    # tfel: this demonstrates various errors we can get when we get confused
+    # about the generator scopes and their parents for eager iterator
+    # evaluation. In fact, all of these should work
+    def unbound_local_l2():
+        l1 = []
+        l2 = []
+        return [
+            link for link in (
+                (url for url in l1),
+                (url for url in l2)
+            )
+        ]
+
+    assert len(unbound_local_l2()) == 2
+
+    def assertion_error_getting_closure_from_locals():
+        l1 = []
+        l2 = []
+        l3 = []
+        return [
+            link for link in (
+                (url for url in l1),
+                (url for url in l2),
+                (url for url in l3),
+            )
+        ]
+
+    assert len(assertion_error_getting_closure_from_locals()) == 3
+
+    def illegal_state_expected_cell_got_list():
+        l11, l1 = [], []
+        l22, l2 = [], []
+        return [
+            link for link in (
+                (url for url in l1),
+                (url for url in l2),
+            )
+        ]
+
+    assert len(illegal_state_expected_cell_got_list()) == 2

graalpython/com.oracle.graal.python.test/src/tests/test_parser.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0
@@ -88,3 +88,13 @@ def test_lambda_no_args_with_nested_lambdas():
 
 def test_byte_numeric_escapes():
     assert eval('b"PK\\005\\006\\00\\11\\22\\08"') == b'PK\x05\x06\x00\t\x12\x008'
+
+
+def test_decorator_cell():
+    foo = lambda x: "just a string, not %s" % x.__name__
+    def run_me():
+        @foo
+        def func():
+            pass
+        return func
+    assert run_me() == "just a string, not func", run_me()

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/generator/GeneratorBuiltins.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2018, Oracle and/or its affiliates.
+ * Copyright (c) 2017, 2019, Oracle and/or its affiliates.
  * Copyright (c) 2014, Regents of the University of California
  *
  * All rights reserved.
@@ -27,6 +27,7 @@
 
 import static com.oracle.graal.python.nodes.SpecialMethodNames.__ITER__;
 import static com.oracle.graal.python.nodes.SpecialMethodNames.__NEXT__;
+import static com.oracle.graal.python.nodes.SpecialMethodNames.__REPR__;
 import static com.oracle.graal.python.runtime.exception.PythonErrorType.StopIteration;
 import static com.oracle.graal.python.runtime.exception.PythonErrorType.TypeError;
 
@@ -52,6 +53,7 @@
 import com.oracle.truffle.api.CallTarget;
 import com.oracle.truffle.api.RootCallTarget;
 import com.oracle.truffle.api.Truffle;
+import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
 import com.oracle.truffle.api.dsl.Cached;
 import com.oracle.truffle.api.dsl.GenerateNodeFactory;
 import com.oracle.truffle.api.dsl.NodeFactory;
@@ -220,4 +222,14 @@ Object getCode(PGenerator self,
             return code;
         }
     }
+
+    @Builtin(name = __REPR__, fixedNumOfPositionalArgs = 1)
+    @GenerateNodeFactory
+    abstract static class ReprNode extends PythonUnaryBuiltinNode {
+        @Specialization
+        @TruffleBoundary
+        String repr(PGenerator self) {
+            return self.toString();
+        }
+    }
 }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/PythonTreeTranslator.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2018, Oracle and/or its affiliates.
+ * Copyright (c) 2017, 2019, Oracle and/or its affiliates.
  * Copyright (c) 2013, Regents of the University of California
  *
  * All rights reserved.
@@ -228,31 +228,31 @@ private SourceSection deriveSourceSection(RuleNode node) {
 
     @Override
     public Object visitFile_input(Python3Parser.File_inputContext ctx) {
-        environment.enterScope(ctx.scope);
+        environment.pushScope(ctx.scope);
         ExpressionNode file = asExpression(super.visitFile_input(ctx));
         deriveSourceSection(ctx, file);
-        environment.leaveScope();
+        environment.popScope();
         return factory.createModuleRoot(name, file, ctx.scope.getFrameDescriptor());
     }
 
     @Override
     public Object visitEval_input(Python3Parser.Eval_inputContext ctx) {
-        environment.enterScope(ctx.scope);
+        environment.pushScope(ctx.scope);
         ExpressionNode node = (ExpressionNode) super.visitEval_input(ctx);
         deriveSourceSection(ctx, node);
         StatementNode evalReturn = factory.createFrameReturn(factory.createWriteLocal(node, environment.getReturnSlot()));
         ReturnTargetNode returnTarget = new ReturnTargetNode(evalReturn, factory.createReadLocal(environment.getReturnSlot()));
         FunctionRootNode functionRoot = factory.createFunctionRoot(node.getSourceSection(), name, false, ctx.scope.getFrameDescriptor(), returnTarget, environment.getExecutionCellSlots());
-        environment.leaveScope();
+        environment.popScope();
         return functionRoot;
     }
 
     @Override
     public Object visitSingle_input(Python3Parser.Single_inputContext ctx) {
-        environment.enterScope(ctx.scope);
+        environment.pushScope(ctx.scope);
         ExpressionNode body = asExpression(super.visitSingle_input(ctx));
         deriveSourceSection(ctx, body);
-        environment.leaveScope();
+        environment.popScope();
         if (isInlineMode) {
             return body;
         } else {
@@ -468,7 +468,7 @@ private ExpressionNode createComprehensionExpression(ParserRuleContext ctx, Pyth
 
     private ExpressionNode createComprehensionExpression(ParserRuleContext ctx, Python3Parser.Comp_forContext compctx, Function<ParserRuleContext, ExpressionNode> getBlock) {
         try {
-            environment.enterScope(compctx.scope);
+            environment.pushScope(compctx.scope);
             ExpressionNode block = getBlock.apply(ctx);
             ExpressionNode yield = factory.createYield(block, environment.getReturnSlot());
             yield.assignSourceSection(block.getSourceSection());
@@ -482,13 +482,13 @@ private ExpressionNode createComprehensionExpression(ParserRuleContext ctx, Pyth
             genExprDef.assignSourceSection(srcSection);
             return genExprDef;
         } finally {
-            environment.leaveScope();
+            environment.popScope();
         }
     }
 
     private GeneratorExpressionNode createGeneratorExpressionDefinition(ExpressionNode body, int lineNum) {
         FrameDescriptor fd = environment.getCurrentFrame();
-        String generatorName = source.getName() + ":generator_exp:" + lineNum;
+        String generatorName = environment.getCurrentScope().getParent().getScopeId() + ".<locals>.<genexp>:" + source.getName() + ":" + lineNum;
         FunctionRootNode funcRoot = factory.createFunctionRoot(body.getSourceSection(), generatorName, true, fd, body, environment.getExecutionCellSlots());
         GeneratorTranslator gtran = new GeneratorTranslator(funcRoot, true);
         RootCallTarget callTarget = gtran.translate();
@@ -1440,7 +1440,7 @@ public Object visitFuncdef(Python3Parser.FuncdefContext ctx) {
             }
         }
 
-        environment.enterScope(ctx.scope);
+        environment.pushScope(ctx.scope);
         environment.setDefaultArgumentNodes(defaultArgs);
 
         /**
@@ -1491,7 +1491,7 @@ public Object visitFuncdef(Python3Parser.FuncdefContext ctx) {
         } else {
             funcDef = new FunctionDefinitionNode(funcName, enclosingClassName, doc, arity, defaults, ct, environment.getDefinitionCellSlots(), environment.getExecutionCellSlots());
         }
-        environment.leaveScope();
+        environment.popScope();
 
         ReadNode funcVar = environment.findVariable(funcName);
         return funcVar.makeWriteNode(funcDef);
@@ -1624,7 +1624,7 @@ private PNode createLambda(ParserRuleContext ctx, VarargslistContext varargslist
         }
 
         String funcname = "anonymous";
-        environment.enterScope(scope);
+        environment.pushScope(scope);
         environment.setDefaultArgumentNodes(defaultArgs);
 
         /**
@@ -1667,7 +1667,7 @@ private PNode createLambda(ParserRuleContext ctx, VarargslistContext varargslist
         } else {
             funcDef = new FunctionDefinitionNode(funcname, null, null, arity, defaults, ct, environment.getDefinitionCellSlots(), environment.getExecutionCellSlots());
         }
-        environment.leaveScope();
+        environment.popScope();
 
         return funcDef;
     }
@@ -1730,14 +1730,14 @@ public Object visitClassdef(Python3Parser.ClassdefContext ctx) {
         ExpressionNode[] splatArguments = new ExpressionNode[2];
         visitCallArglist(ctx.arglist(), argumentNodes, keywords, splatArguments);
 
-        environment.enterScope(ctx.scope);
+        environment.pushScope(ctx.scope);
 
         ExpressionNode body = asClassBody(ctx.suite().accept(this), qualName);
         ClassBodyRootNode classBodyRoot = factory.createClassBodyRoot(deriveSourceSection(ctx), className, environment.getCurrentFrame(), body, environment.getExecutionCellSlots());
         RootCallTarget ct = Truffle.getRuntime().createCallTarget(classBodyRoot);
         FunctionDefinitionNode funcDef = new FunctionDefinitionNode(className, null, null, Arity.createOneArgumentWithVarKwArgs(className),
                         factory.createBlock(), ct, environment.getDefinitionCellSlots(), environment.getExecutionCellSlots());
-        environment.leaveScope();
+        environment.popScope();
 
         argumentNodes.add(0, factory.createStringLiteral(className));
         argumentNodes.add(0, funcDef);
@@ -1855,11 +1855,11 @@ private StatementNode createGeneratorExpression(Python3Parser.Comp_forContext co
         // TODO: async
         ScopeInfo old = null;
         if (iteratorInParentScope) {
-            old = environment.pushCurentScope();
+            old = environment.popScope();
         }
         ExpressionNode iterator = (ExpressionNode) comp_for.or_test().accept(this);
         if (iteratorInParentScope) {
-            environment.popCurrentScope(old);
+            environment.pushScope(old);
         }
 
         StatementNode targets = assigns.translate(comp_for.exprlist());

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/ScopeInfo.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2018, Oracle and/or its affiliates.
+ * Copyright (c) 2017, 2019, Oracle and/or its affiliates.
  * Copyright (c) 2013, Regents of the University of California
  *
  * All rights reserved.
@@ -81,8 +81,6 @@ public enum ScopeKind {
     private List<ExpressionNode> defaultArgumentNodes;
     private ReadDefaultArgumentNode[] defaultArgumentReads;
 
-    private int loopCount = 0;
-
     public ScopeInfo(String scopeId, ScopeKind kind, FrameDescriptor frameDescriptor, ScopeInfo parent) {
         this.scopeId = scopeId;
         this.scopeKind = kind;
@@ -96,18 +94,6 @@ public ScopeInfo(String scopeId, ScopeKind kind, FrameDescriptor frameDescriptor
         }
     }
 
-    public void incLoopCount() {
-        loopCount++;
-    }
-
-    public int getLoopCount() {
-        return loopCount;
-    }
-
-    public void resetLoopCount() {
-        this.loopCount = 0;
-    }
-
     public ScopeInfo getFirstChildScope() {
         return firstChildScope;
     }