Merge pull request RustPython#1215 from RustPython/syntax-fixes

Add variable annotation syntax.

Author: windelbouwman

compiler/src/compile.rs

@@ -368,33 +368,7 @@ impl<O: OutputStream> Compiler<O> {
                 }
                 self.set_label(end_label);
             }
-            While { test, body, orelse } => {
-                let start_label = self.new_label();
-                let else_label = self.new_label();
-                let end_label = self.new_label();
-                self.emit(Instruction::SetupLoop {
-                    start: start_label,
-                    end: end_label,
-                });
-
-                self.set_label(start_label);
-
-                self.compile_test(test, None, Some(else_label), EvalContext::Statement)?;
-
-                let was_in_loop = self.in_loop;
-                self.in_loop = true;
-                self.compile_statements(body)?;
-                self.in_loop = was_in_loop;
-                self.emit(Instruction::Jump {
-                    target: start_label,
-                });
-                self.set_label(else_label);
-                self.emit(Instruction::PopBlock);
-                if let Some(orelse) = orelse {
-                    self.compile_statements(orelse)?;
-                }
-                self.set_label(end_label);
-            }
+            While { test, body, orelse } => self.compile_while(test, body, orelse)?,
             With {
                 is_async,
                 items,
@@ -563,6 +537,11 @@ impl<O: OutputStream> Compiler<O> {
                 self.compile_op(op, true);
                 self.compile_store(target)?;
             }
+            AnnAssign {
+                target,
+                annotation,
+                value,
+            } => self.compile_annotated_assign(target, annotation, value)?,
             Delete { targets } => {
                 for target in targets {
                     self.compile_delete(target)?;
@@ -1011,6 +990,40 @@ impl<O: OutputStream> Compiler<O> {
         }
     }
 
+    fn compile_while(
+        &mut self,
+        test: &ast::Expression,
+        body: &[ast::Statement],
+        orelse: &Option<Vec<ast::Statement>>,
+    ) -> Result<(), CompileError> {
+        let start_label = self.new_label();
+        let else_label = self.new_label();
+        let end_label = self.new_label();
+        self.emit(Instruction::SetupLoop {
+            start: start_label,
+            end: end_label,
+        });
+
+        self.set_label(start_label);
+
+        self.compile_test(test, None, Some(else_label), EvalContext::Statement)?;
+
+        let was_in_loop = self.in_loop;
+        self.in_loop = true;
+        self.compile_statements(body)?;
+        self.in_loop = was_in_loop;
+        self.emit(Instruction::Jump {
+            target: start_label,
+        });
+        self.set_label(else_label);
+        self.emit(Instruction::PopBlock);
+        if let Some(orelse) = orelse {
+            self.compile_statements(orelse)?;
+        }
+        self.set_label(end_label);
+        Ok(())
+    }
+
     fn compile_for(
         &mut self,
         target: &ast::Expression,
@@ -1129,6 +1142,39 @@ impl<O: OutputStream> Compiler<O> {
         Ok(())
     }
 
+    fn compile_annotated_assign(
+        &mut self,
+        target: &ast::Expression,
+        annotation: &ast::Expression,
+        value: &Option<ast::Expression>,
+    ) -> Result<(), CompileError> {
+        if let Some(value) = value {
+            self.compile_expression(value)?;
+            self.compile_store(target)?;
+        }
+
+        // Compile annotation:
+        self.compile_expression(annotation)?;
+
+        if let ast::ExpressionType::Identifier { name } = &target.node {
+            // Store as dict entry in __annotations__ dict:
+            self.emit(Instruction::LoadName {
+                name: String::from("__annotations__"),
+                scope: bytecode::NameScope::Local,
+            });
+            self.emit(Instruction::LoadConst {
+                value: bytecode::Constant::String {
+                    value: name.to_string(),
+                },
+            });
+            self.emit(Instruction::StoreSubscript);
+        } else {
+            // Drop annotation if not assigned to simple identifier.
+            self.emit(Instruction::Pop);
+        }
+        Ok(())
+    }
+
     fn compile_store(&mut self, target: &ast::Expression) -> Result<(), CompileError> {
         match &target.node {
             ast::ExpressionType::Identifier { name } => {
@@ -1272,6 +1318,8 @@ impl<O: OutputStream> Compiler<O> {
 
     fn compile_expression(&mut self, expression: &ast::Expression) -> Result<(), CompileError> {
         trace!("Compiling {:?}", expression);
+        self.set_source_location(&expression.location);
+
         use ast::ExpressionType::*;
         match &expression.node {
             Call {
@@ -1664,6 +1712,10 @@ impl<O: OutputStream> Compiler<O> {
 
         let mut loop_labels = vec![];
         for generator in generators {
+            if generator.is_async {
+                unimplemented!("async for comprehensions");
+            }
+
             if loop_labels.is_empty() {
                 // Load iterator onto stack (passed as first argument):
                 self.emit(Instruction::LoadName {

compiler/src/symboltable.rs

@@ -335,6 +335,17 @@ impl SymbolTableBuilder {
                 self.scan_expression(target)?;
                 self.scan_expression(value)?;
             }
+            AnnAssign {
+                target,
+                annotation,
+                value,
+            } => {
+                self.scan_expression(target)?;
+                self.scan_expression(annotation)?;
+                if let Some(value) = value {
+                    self.scan_expression(value)?;
+                }
+            }
             With { items, body, .. } => {
                 for item in items {
                     self.scan_expression(&item.context_expr)?;

examples/parse_folder.rs

@@ -14,7 +14,8 @@ extern crate log;
 use clap::{App, Arg};
 
 use rustpython_parser::{ast, parser};
-use std::path::Path;
+use std::path::{Path, PathBuf};
+use std::time::Instant;
 
 fn main() {
     env_logger::init();
@@ -32,14 +33,21 @@ fn main() {
     let folder = Path::new(matches.value_of("folder").unwrap());
     if folder.exists() && folder.is_dir() {
         println!("Parsing folder of python code: {:?}", folder);
-        let res = parse_folder(&folder).unwrap();
-        println!("Processed {:?} files", res.len());
+        let t1 = Instant::now();
+        let parsed_files = parse_folder(&folder).unwrap();
+        let t2 = Instant::now();
+        let results = ScanResult {
+            t1,
+            t2,
+            parsed_files,
+        };
+        statistics(results);
     } else {
         println!("{:?} is not a folder.", folder);
     }
 }
 
-fn parse_folder(path: &Path) -> std::io::Result<Vec<ast::Program>> {
+fn parse_folder(path: &Path) -> std::io::Result<Vec<ParsedFile>> {
     let mut res = vec![];
     info!("Parsing folder of python code: {:?}", path);
     for entry in path.read_dir()? {
@@ -49,22 +57,66 @@ fn parse_folder(path: &Path) -> std::io::Result<Vec<ast::Program>> {
 
         let path = entry.path();
         if metadata.is_dir() {
-            let x = parse_folder(&path)?;
-            res.extend(x);
+            res.extend(parse_folder(&path)?);
         }
 
         if metadata.is_file() && path.extension().and_then(|s| s.to_str()) == Some("py") {
-            match parse_python_file(&path) {
-                Ok(x) => res.push(x),
+            let result = parse_python_file(&path);
+            match &result {
+                Ok(_) => {}
                 Err(y) => error!("Erreur in file {:?} {:?}", path, y),
             }
+            res.push(ParsedFile {
+                filename: Box::new(path),
+                result,
+            });
         }
     }
     Ok(res)
 }
 
-fn parse_python_file(filename: &Path) -> Result<ast::Program, String> {
+fn parse_python_file(filename: &Path) -> ParseResult {
     info!("Parsing file {:?}", filename);
     let source = std::fs::read_to_string(filename).map_err(|e| e.to_string())?;
     parser::parse_program(&source).map_err(|e| e.to_string())
 }
+
+fn statistics(results: ScanResult) {
+    // println!("Processed {:?} files", res.len());
+    println!("Scanned a total of {} files", results.parsed_files.len());
+    let total = results.parsed_files.len();
+    let failed = results
+        .parsed_files
+        .iter()
+        .filter(|p| p.result.is_err())
+        .count();
+    let passed = results
+        .parsed_files
+        .iter()
+        .filter(|p| p.result.is_ok())
+        .count();
+    println!("Passed: {} Failed: {} Total: {}", passed, failed, total);
+    println!(
+        "That is {} % success rate.",
+        (passed as f64 * 100.0) / total as f64
+    );
+    let duration = results.t2 - results.t1;
+    println!("Total time spend: {:?}", duration);
+    println!(
+        "File processing rate: {} files/second",
+        (total * 1_000_000) as f64 / duration.as_micros() as f64
+    );
+}
+
+struct ScanResult {
+    t1: Instant,
+    t2: Instant,
+    parsed_files: Vec<ParsedFile>,
+}
+
+struct ParsedFile {
+    filename: Box<PathBuf>,
+    result: ParseResult,
+}
+
+type ParseResult = Result<ast::Program, String>;

parser/src/ast.rs

@@ -74,6 +74,11 @@ pub enum StatementType {
         op: Operator,
         value: Box<Expression>,
     },
+    AnnAssign {
+        target: Box<Expression>,
+        annotation: Box<Expression>,
+        value: Option<Expression>,
+    },
     Expression {
         expression: Expression,
     },
@@ -313,6 +318,7 @@ pub struct Comprehension {
     pub target: Expression,
     pub iter: Expression,
     pub ifs: Vec<Expression>,
+    pub is_async: bool,
 }
 
 #[derive(Debug, PartialEq)]

parser/src/lexer.rs

@@ -396,7 +396,7 @@ where
 
             let value = f64::from_str(&value_text).unwrap();
             // Parse trailing 'j':
-            if self.chr0 == Some('j') {
+            if self.chr0 == Some('j') || self.chr0 == Some('J') {
                 self.next_char();
                 let end_pos = self.get_pos();
                 Ok((
@@ -413,7 +413,7 @@ where
             }
         } else {
             // Parse trailing 'j':
-            if self.chr0 == Some('j') {
+            if self.chr0 == Some('j') || self.chr0 == Some('J') {
                 self.next_char();
                 let end_pos = self.get_pos();
                 let imag = f64::from_str(&value_text).unwrap();

parser/src/parser.rs

@@ -376,7 +376,7 @@ mod tests {
         assert_eq!(
             parse_ast,
             ast::Expression {
-                location: ast::Location::new(1, 2),
+                location: ast::Location::new(1, 1),
                 node: ast::ExpressionType::Comprehension {
                     kind: Box::new(ast::ComprehensionKind::List {
                         element: mk_ident("x", 1, 2),
@@ -386,6 +386,7 @@ mod tests {
                         target: mk_ident("y", 1, 8),
                         iter: mk_ident("z", 1, 13),
                         ifs: vec![],
+                        is_async: false,
                     }],
                 }
             }
@@ -399,7 +400,7 @@ mod tests {
         assert_eq!(
             parse_ast,
             ast::Expression {
-                location: ast::Location::new(1, 2),
+                location: ast::Location::new(1, 1),
                 node: ast::ExpressionType::Comprehension {
                     kind: Box::new(ast::ComprehensionKind::List {
                         element: mk_ident("x", 1, 2)
@@ -415,6 +416,7 @@ mod tests {
                             },
                             iter: mk_ident("z", 1, 17),
                             ifs: vec![],
+                            is_async: false,
                         },
                         ast::Comprehension {
                             location: ast::Location::new(1, 19),
@@ -436,6 +438,7 @@ mod tests {
                                     },
                                 },
                             ],
+                            is_async: false,
                         }
                     ],
                 }