Add parser support for async for loop. Add yield expression inside parentheses.

Author: windelbouwman

compiler/src/compile.rs

@@ -1702,6 +1702,10 @@ impl Compiler {
 
         let mut loop_labels = vec![];
         for generator in generators {
+            if generator.is_async {
+                unimplemented!("async for comprehensions");
+            }
+
             if loop_labels.is_empty() {
                 // Load iterator onto stack (passed as first argument):
                 self.emit(Instruction::LoadName {

examples/parse_folder.rs

@@ -14,7 +14,8 @@ extern crate log;
 use clap::{App, Arg};
 
 use rustpython_parser::{ast, parser};
-use std::path::Path;
+use std::path::{Path, PathBuf};
+use std::time::Instant;
 
 fn main() {
     env_logger::init();
@@ -32,14 +33,21 @@ fn main() {
     let folder = Path::new(matches.value_of("folder").unwrap());
     if folder.exists() && folder.is_dir() {
         println!("Parsing folder of python code: {:?}", folder);
-        let res = parse_folder(&folder).unwrap();
-        println!("Processed {:?} files", res.len());
+        let t1 = Instant::now();
+        let parsed_files = parse_folder(&folder).unwrap();
+        let t2 = Instant::now();
+        let results = ScanResult {
+            t1,
+            t2,
+            parsed_files,
+        };
+        statistics(results);
     } else {
         println!("{:?} is not a folder.", folder);
     }
 }
 
-fn parse_folder(path: &Path) -> std::io::Result<Vec<ast::Program>> {
+fn parse_folder(path: &Path) -> std::io::Result<Vec<ParsedFile>> {
     let mut res = vec![];
     info!("Parsing folder of python code: {:?}", path);
     for entry in path.read_dir()? {
@@ -49,22 +57,66 @@ fn parse_folder(path: &Path) -> std::io::Result<Vec<ast::Program>> {
 
         let path = entry.path();
         if metadata.is_dir() {
-            let x = parse_folder(&path)?;
-            res.extend(x);
+            res.extend(parse_folder(&path)?);
         }
 
         if metadata.is_file() && path.extension().and_then(|s| s.to_str()) == Some("py") {
-            match parse_python_file(&path) {
-                Ok(x) => res.push(x),
+            let result = parse_python_file(&path);
+            match &result {
+                Ok(_) => {}
                 Err(y) => error!("Erreur in file {:?} {:?}", path, y),
             }
+            res.push(ParsedFile {
+                filename: Box::new(path),
+                result,
+            });
         }
     }
     Ok(res)
 }
 
-fn parse_python_file(filename: &Path) -> Result<ast::Program, String> {
+fn parse_python_file(filename: &Path) -> ParseResult {
     info!("Parsing file {:?}", filename);
     let source = std::fs::read_to_string(filename).map_err(|e| e.to_string())?;
     parser::parse_program(&source).map_err(|e| e.to_string())
 }
+
+fn statistics(results: ScanResult) {
+    // println!("Processed {:?} files", res.len());
+    println!("Scanned a total of {} files", results.parsed_files.len());
+    let total = results.parsed_files.len();
+    let failed = results
+        .parsed_files
+        .iter()
+        .filter(|p| p.result.is_err())
+        .count();
+    let passed = results
+        .parsed_files
+        .iter()
+        .filter(|p| p.result.is_ok())
+        .count();
+    println!("Passed: {} Failed: {} Total: {}", passed, failed, total);
+    println!(
+        "That is {} % success rate.",
+        (passed as f64 * 100.0) / total as f64
+    );
+    let duration = results.t2 - results.t1;
+    println!("Total time spend: {:?}", duration);
+    println!(
+        "File processing rate: {} files/second",
+        (total * 1_000_000) as f64 / duration.as_micros() as f64
+    );
+}
+
+struct ScanResult {
+    t1: Instant,
+    t2: Instant,
+    parsed_files: Vec<ParsedFile>,
+}
+
+struct ParsedFile {
+    filename: Box<PathBuf>,
+    result: ParseResult,
+}
+
+type ParseResult = Result<ast::Program, String>;

parser/src/ast.rs

@@ -318,6 +318,7 @@ pub struct Comprehension {
     pub target: Expression,
     pub iter: Expression,
     pub ifs: Vec<Expression>,
+    pub is_async: bool,
 }
 
 #[derive(Debug, PartialEq)]

parser/src/parser.rs

@@ -386,6 +386,7 @@ mod tests {
                         target: mk_ident("y", 1, 8),
                         iter: mk_ident("z", 1, 13),
                         ifs: vec![],
+                        is_async: false,
                     }],
                 }
             }
@@ -415,6 +416,7 @@ mod tests {
                             },
                             iter: mk_ident("z", 1, 17),
                             ifs: vec![],
+                            is_async: false,
                         },
                         ast::Comprehension {
                             location: ast::Location::new(1, 19),
@@ -436,6 +438,7 @@ mod tests {
                                     },
                                 },
                             ],
+                            is_async: false,
                         }
                     ],
                 }

parser/src/python.lalrpop

@@ -908,6 +908,7 @@ Atom: ast::Expression = {
              node: ast::ExpressionType::Tuple { elements: Vec::new() }
         })
     },
+    "(" <e:YieldExpr> ")" => e,
     <location:@L> "(" <element:Test> <generators:CompFor> ")" => {
         ast::Expression {
             location,
@@ -1021,8 +1022,9 @@ StarExpr: ast::Expression = {
 CompFor: Vec<ast::Comprehension> = <c:SingleForComprehension+> => c;
 
 SingleForComprehension: ast::Comprehension = {
-    <location:@L> "for" <target:ExpressionList> "in" <iter:OrTest> <ifs:ComprehensionIf*> => {
-        ast::Comprehension { location, target, iter, ifs }
+    <location:@L> <is_async:"async"?> "for" <target:ExpressionList> "in" <iter:OrTest> <ifs:ComprehensionIf*> => {
+        let is_async = is_async.is_some();
+        ast::Comprehension { location, target, iter, ifs, is_async }
     }
 };
 

tests/snippets/generators.py

@@ -64,3 +64,26 @@ def catch_exception():
 
 with assertRaises(KeyError):
     assert g.throw(KeyError, KeyError(), None) == 2
+
+
+r = []
+def p(a, b, c):
+    # print(a, b, c)
+    r.append(a)
+    r.append(b)
+    r.append(c)
+
+
+def g5():
+    p('a', (yield 2), (yield 5))
+    yield 99
+
+g = g5()
+g.send(None)
+g.send(66)
+# g.send(88)
+l = list(g)
+# print(r)
+# print(l)
+assert l == [99]
+assert r == ['a', 66, None]

vm/src/stdlib/ast.rs

@@ -579,6 +579,7 @@ fn comprehension_to_ast(
         target => expression_to_ast(vm, &comprehension.target)?,
         iter => expression_to_ast(vm, &comprehension.iter)?,
         ifs => expressions_to_ast(vm, &comprehension.ifs)?,
+        is_async => vm.new_bool(comprehension.is_async),
     }))
 }