Extend AST python module. Add idea for scope detection.

Author: windelbouwman

compiler/src/symboltable.rs

@@ -204,8 +204,25 @@ impl SymbolTableAnalyzer {
                 // all is well
             }
             SymbolScope::Unknown => {
-                if symbol.is_assigned {
+                // Try hard to figure out what the scope of this symbol is.
+
+                if symbol.is_assigned || symbol.is_parameter {
                     symbol.scope = SymbolScope::Local;
+                } else {
+                    // TODO: comment this out and make it work properly:
+                    /*
+                    let found_in_outer_scope = self
+                        .tables
+                        .iter()
+                        .any(|t| t.symbols.contains_key(&symbol.name));
+                    if found_in_outer_scope {
+                        // Symbol is in some outer scope.
+
+                    } else {
+                        // Well, it must be a global then :)
+                        // symbol.scope = SymbolScope::Global;
+                    }
+                    */
                 }
             }
         }
@@ -710,7 +727,10 @@ impl SymbolTableBuilder {
                     });
                 }
             }
-            SymbolUsage::Parameter | SymbolUsage::Assigned => {
+            SymbolUsage::Parameter => {
+                symbol.is_parameter = true;
+            }
+            SymbolUsage::Assigned => {
                 symbol.is_assigned = true;
             }
             SymbolUsage::Global => {

vm/src/scope.rs

@@ -138,11 +138,8 @@ impl NameProtocol for Scope {
             }
         }
 
-        if let Some(value) = self.globals.get_item_option(name, vm).unwrap() {
-            return Some(value);
-        }
-
-        vm.get_attribute(vm.builtins.clone(), name).ok()
+        // Fall back to loading a global after all scopes have been searched!
+        self.load_global(vm, name)
     }
 
     #[cfg_attr(feature = "flame-it", flame("Scope"))]
@@ -174,7 +171,11 @@ impl NameProtocol for Scope {
 
     #[cfg_attr(feature = "flame-it", flame("Scope"))]
     fn load_global(&self, vm: &VirtualMachine, name: &str) -> Option<PyObjectRef> {
-        self.globals.get_item_option(name, vm).unwrap()
+        if let Some(value) = self.globals.get_item_option(name, vm).unwrap() {
+            Some(value)
+        } else {
+            vm.get_attribute(vm.builtins.clone(), name).ok()
+        }
     }
 
     fn store_global(&self, vm: &VirtualMachine, name: &str, value: PyObjectRef) {

vm/src/stdlib/ast.rs

@@ -252,7 +252,7 @@ fn optional_statements_to_ast(
     let statements = if let Some(statements) = statements {
         statements_to_ast(vm, statements)?.into_object()
     } else {
-        vm.ctx.none()
+        vm.ctx.new_list(vec![])
     };
     Ok(statements)
 }
@@ -283,6 +283,17 @@ fn make_string_list(vm: &VirtualMachine, names: &[String]) -> PyObjectRef {
     )
 }
 
+fn optional_expressions_to_ast(
+    vm: &VirtualMachine,
+    expressions: &[Option<ast::Expression>],
+) -> PyResult<PyListRef> {
+    let py_expression_nodes: PyResult<_> = expressions
+        .iter()
+        .map(|expression| Ok(optional_expression_to_ast(vm, expression)?))
+        .collect();
+    Ok(vm.ctx.new_list(py_expression_nodes?).downcast().unwrap())
+}
+
 fn optional_expression_to_ast(vm: &VirtualMachine, value: &Option<ast::Expression>) -> PyResult {
     let value = if let Some(value) = value {
         expression_to_ast(vm, value)?.into_object()
@@ -526,8 +537,23 @@ fn operator_string(op: &ast::Operator) -> String {
 }
 
 fn parameters_to_ast(vm: &VirtualMachine, args: &ast::Parameters) -> PyResult<AstNodeRef> {
-    let args = map_ast(parameter_to_ast, vm, &args.args)?;
-    Ok(node!(vm, arguments, { args => args }))
+    Ok(node!(vm, arguments, {
+        args => map_ast(parameter_to_ast, vm, &args.args)?,
+        vararg => vararg_to_ast(vm, &args.vararg)?,
+        kwonlyargs => map_ast(parameter_to_ast, vm, &args.kwonlyargs)?,
+        kw_defaults => optional_expressions_to_ast(vm, &args.kw_defaults)?,
+        kwarg => vararg_to_ast(vm, &args.kwarg)?,
+        defaults => expressions_to_ast(vm, &args.defaults)?
+    }))
+}
+
+fn vararg_to_ast(vm: &VirtualMachine, vararg: &ast::Varargs) -> PyResult {
+    let py_node = match vararg {
+        ast::Varargs::None => vm.get_none(),
+        ast::Varargs::Unnamed => vm.get_none(),
+        ast::Varargs::Named(parameter) => parameter_to_ast(vm, parameter)?.into_object(),
+    };
+    Ok(py_node)
 }
 
 fn parameter_to_ast(vm: &VirtualMachine, parameter: &ast::Parameter) -> PyResult<AstNodeRef> {
@@ -537,10 +563,15 @@ fn parameter_to_ast(vm: &VirtualMachine, parameter: &ast::Parameter) -> PyResult
         vm.ctx.none()
     };
 
-    Ok(node!(vm, arg, {
+    let py_node = node!(vm, arg, {
         arg => vm.ctx.new_str(parameter.arg.to_string()),
         annotation => py_annotation
-    }))
+    });
+
+    let lineno = vm.ctx.new_int(parameter.location.row());
+    vm.set_attr(py_node.as_object(), "lineno", lineno)?;
+
+    Ok(py_node)
 }
 
 fn optional_string_to_py_obj(vm: &VirtualMachine, name: &Option<String>) -> PyObjectRef {