Merge remote-tracking branch 'upstream/master' into fstrings

Author: OddCoincidence

parser/src/error.rs

@@ -0,0 +1,79 @@
+//! Define internal parse error types
+//! The goal is to provide a matching and a safe error API, maksing errors from LALR
+extern crate lalrpop_util;
+use self::lalrpop_util::ParseError as InnerError;
+
+use lexer::{LexicalError, Location};
+use token::Tok;
+
+use std::error::Error;
+use std::fmt;
+
+// A token of type `Tok` was observed, with a span given by the two Location values
+type TokSpan = (Location, Tok, Location);
+
+/// Represents an error during parsing
+#[derive(Debug, PartialEq)]
+pub enum ParseError {
+    /// Parser encountered an unexpected end of input
+    EOF(Option<Location>),
+    /// Parser encountered an extra token
+    ExtraToken(TokSpan),
+    /// Parser encountered an invalid token
+    InvalidToken(Location),
+    /// Parser encountered an unexpected token
+    UnrecognizedToken(TokSpan, Vec<String>),
+    /// Maps to `User` type from `lalrpop-util`
+    Other,
+}
+
+/// Convert `lalrpop_util::ParseError` to our internal type
+impl From<InnerError<Location, Tok, LexicalError>> for ParseError {
+    fn from(err: InnerError<Location, Tok, LexicalError>) -> Self {
+        match err {
+            // TODO: Are there cases where this isn't an EOF?
+            InnerError::InvalidToken { location } => ParseError::EOF(Some(location)),
+            InnerError::ExtraToken { token } => ParseError::ExtraToken(token),
+            // Inner field is a unit-like enum `LexicalError::StringError` with no useful info
+            InnerError::User { .. } => ParseError::Other,
+            InnerError::UnrecognizedToken { token, expected } => {
+                match token {
+                    Some(tok) => ParseError::UnrecognizedToken(tok, expected),
+                    // EOF was observed when it was unexpected
+                    None => ParseError::EOF(None),
+                }
+            }
+        }
+    }
+}
+
+impl fmt::Display for ParseError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match *self {
+            ParseError::EOF(ref location) => {
+                if let Some(l) = location {
+                    write!(f, "Got unexpected EOF at: {:?}", l)
+                } else {
+                    write!(f, "Got unexpected EOF")
+                }
+            }
+            ParseError::ExtraToken(ref t_span) => {
+                write!(f, "Got extraneous token: {:?} at: {:?}", t_span.1, t_span.0)
+            }
+            ParseError::InvalidToken(ref location) => {
+                write!(f, "Got invalid token at: {:?}", location)
+            }
+            ParseError::UnrecognizedToken(ref t_span, _) => {
+                write!(f, "Got unexpected token: {:?} at {:?}", t_span.1, t_span.0)
+            }
+            // This is user defined, it probably means a more useful error should have been given upstream.
+            ParseError::Other => write!(f, "Got unsupported token(s)"),
+        }
+    }
+}
+
+impl Error for ParseError {
+    fn source(&self) -> Option<&(dyn Error + 'static)> {
+        None
+    }
+}

parser/src/lib.rs

@@ -5,10 +5,9 @@ extern crate num_bigint;
 extern crate num_traits;
 
 pub mod ast;
+pub mod error;
 pub mod lexer;
 pub mod parser;
 #[cfg_attr(rustfmt, rustfmt_skip)]
 mod python;
 pub mod token;
-
-pub use self::parser::parse;

parser/src/parser.rs

@@ -1,55 +1,28 @@
 extern crate lalrpop_util;
 
-use std::error::Error;
-use std::fs::File;
-use std::io::Read;
 use std::iter;
 use std::mem;
-use std::path::Path;
-
-use self::lalrpop_util::ParseError;
 
 use super::ast;
+use super::error::ParseError;
 use super::lexer;
 use super::python;
 use super::token;
 
-pub fn read_file(filename: &Path) -> Result<String, String> {
-    info!("Loading file {:?}", filename);
-    match File::open(&filename) {
-        Ok(mut file) => {
-            let mut s = String::new();
-
-            match file.read_to_string(&mut s) {
-                Err(why) => Err(String::from("Reading file failed: ") + why.description()),
-                Ok(_) => Ok(s),
-            }
-        }
-        Err(why) => Err(String::from("Opening file failed: ") + why.description()),
-    }
-}
-
 /*
  * Parse python code.
  * Grammar may be inspired by antlr grammar for python:
  * https://github.com/antlr/grammars-v4/tree/master/python3
  */
 
-pub fn parse(filename: &Path) -> Result<ast::Program, String> {
-    info!("Parsing: {}", filename.display());
-    let txt = read_file(filename)?;
-    debug!("Read contents of file: {}", txt);
-    parse_program(&txt)
-}
-
 macro_rules! do_lalr_parsing {
     ($input: expr, $pat: ident, $tok: ident) => {{
         let lxr = lexer::make_tokenizer($input);
         let marker_token = (Default::default(), token::Tok::$tok, Default::default());
         let tokenizer = iter::once(Ok(marker_token)).chain(lxr);
 
         match python::TopParser::new().parse(tokenizer) {
-            Err(why) => Err(format!("{:?}", why)),
+            Err(err) => Err(ParseError::from(err)),
             Ok(top) => {
                 if let ast::Top::$pat(x) = top {
                     Ok(x)
@@ -61,11 +34,11 @@ macro_rules! do_lalr_parsing {
     }};
 }
 
-pub fn parse_program(source: &str) -> Result<ast::Program, String> {
+pub fn parse_program(source: &str) -> Result<ast::Program, ParseError> {
     do_lalr_parsing!(source, Program, StartProgram)
 }
 
-pub fn parse_statement(source: &str) -> Result<ast::LocatedStatement, String> {
+pub fn parse_statement(source: &str) -> Result<ast::LocatedStatement, ParseError> {
     do_lalr_parsing!(source, Statement, StartStatement)
 }
 
@@ -91,21 +64,22 @@ pub fn parse_statement(source: &str) -> Result<ast::LocatedStatement, String> {
 ///     expr);
 ///
 /// ```
-pub fn parse_expression(source: &str) -> Result<ast::Expression, String> {
+pub fn parse_expression(source: &str) -> Result<ast::Expression, ParseError> {
     do_lalr_parsing!(source, Expression, StartExpression)
 }
 
+// TODO: consolidate these with ParseError
 pub enum FStringError {
     UnclosedLbrace,
     UnopenedRbrace,
     InvalidExpression,
 }
 
-impl From<FStringError> for ParseError<lexer::Location, token::Tok, lexer::LexicalError> {
+impl From<FStringError>
+    for lalrpop_util::ParseError<lexer::Location, token::Tok, lexer::LexicalError>
+{
     fn from(_err: FStringError) -> Self {
-        // TODO: we should have our own top-level ParseError to properly propagate f-string (and
-        // other) syntax errors
-        ParseError::User {
+        lalrpop_util::ParseError::User {
             error: lexer::LexicalError::StringError,
         }
     }
@@ -196,7 +170,6 @@ mod tests {
     #[test]
     fn test_parse_empty() {
         let parse_ast = parse_program(&String::from("\n"));
-
         assert_eq!(parse_ast, Ok(ast::Program { statements: vec![] }))
     }