Add bytes literal support.

Author: windelbouwman

parser/src/ast.rs

@@ -191,6 +191,9 @@ pub enum Expression {
     String {
         value: String,
     },
+    Bytes {
+        value: Vec<u8>,
+    },
     Identifier {
         name: String,
     },

parser/src/lexer.rs

@@ -232,7 +232,7 @@ where
         let mut saw_b = false;
         let mut saw_r = false;
         let mut saw_u = false;
-        let saw_f = false;
+        let mut saw_f = false;
         loop {
             // Detect r"", f"", b"" and u""
             // TODO: handle f-strings
@@ -242,10 +242,12 @@ where
                 && (self.chr0 == Some('u') || self.chr0 == Some('U'))
             {
                 saw_u = true;
-            } else if !(saw_r || saw_u || saw_f)
-                && (self.chr0 == Some('r') || self.chr0 == Some('R'))
-            {
+            } else if !(saw_r || saw_u) && (self.chr0 == Some('r') || self.chr0 == Some('R')) {
                 saw_r = true;
+            } else if !(saw_b || saw_u || saw_f)
+                && (self.chr0 == Some('f') || self.chr0 == Some('F'))
+            {
+                saw_f = true;
             } else {
                 break;
             }
@@ -313,7 +315,7 @@ where
 
     fn lex_string(
         &mut self,
-        _is_bytes: bool,
+        is_bytes: bool,
         is_raw: bool,
         _is_unicode: bool,
         _is_fstring: bool,
@@ -400,13 +402,17 @@ where
         }
         let end_pos = self.get_pos();
 
-        return Ok((
-            start_pos,
+        let tok = if is_bytes {
+            Tok::Bytes {
+                value: string_content.as_bytes().to_vec(),
+            }
+        } else {
             Tok::String {
                 value: string_content,
-            },
-            end_pos,
-        ));
+            }
+        };
+
+        return Ok((start_pos, tok, end_pos));
     }
 
     fn is_char(&self) -> bool {
@@ -921,10 +927,10 @@ mod tests {
             tokens,
             vec![
                 Tok::String {
-                    value: "\\\\".to_string()
+                    value: "\\\\".to_string(),
                 },
                 Tok::String {
-                    value: "\\".to_string()
+                    value: "\\".to_string(),
                 }
             ]
         );

parser/src/python.lalrpop

@@ -768,7 +768,7 @@ SliceOp: ast::Expression = {
 }
 
 Atom: ast::Expression = {
-    <s:String> => ast::Expression::String { value: s },
+    StringConstant,
     <n:Number> => ast::Expression::Number { value: n },
     <i:Identifier> => ast::Expression::Identifier { name: i },
     "[" <e:TestListComp?> "]" => {
@@ -972,9 +972,14 @@ Number: ast::Number = {
   }
 };
 
-String: String = {
+StringConstant: ast::Expression = {
     <s:string+> => {
-        s.join("")
+        let glued = s.join("");
+        ast::Expression::String { value: glued }
+    },
+    <s:bytes+> => {
+        let glued = s.into_iter().flatten().collect::<Vec<u8>>();
+        ast::Expression::Bytes { value: glued }
     },
 };
 
@@ -1067,6 +1072,7 @@ extern {
         "None" => lexer::Tok::None,
         number => lexer::Tok::Number { value: <String> },
         string => lexer::Tok::String { value: <String> },
+        bytes => lexer::Tok::Bytes { value: <Vec<u8>> },
         name => lexer::Tok::Name { name: <String> },
         "\n" => lexer::Tok::Newline,
         ";" => lexer::Tok::Semi,

parser/src/token.rs

@@ -4,6 +4,7 @@ pub enum Tok {
     Name { name: String },
     Number { value: String },
     String { value: String },
+    Bytes { value: Vec<u8> },
     Newline,
     Indent,
     Dedent,

tests/snippets/strings.py

@@ -10,6 +10,7 @@
 assert len(""" " \" """) == 5
 
 assert type("") is str
+assert type(b"") is bytes
 
 assert str(1) == "1"
 assert str(2.1) == "2.1"

vm/src/bytecode.rs

@@ -199,6 +199,7 @@ pub enum Constant {
     Float { value: f64 },
     Boolean { value: bool },
     String { value: String },
+    Bytes { value: Vec<u8> },
     Code { code: CodeObject },
     Tuple { elements: Vec<Constant> },
     None,

vm/src/compile.rs

@@ -932,6 +932,13 @@ impl Compiler {
                     },
                 });
             }
+            ast::Expression::Bytes { value } => {
+                self.emit(Instruction::LoadConst {
+                    value: bytecode::Constant::Bytes {
+                        value: value.clone(),
+                    },
+                });
+            }
             ast::Expression::Identifier { name } => {
                 self.emit(Instruction::LoadName {
                     name: name.to_string(),

vm/src/frame.rs

@@ -1075,6 +1075,7 @@ impl Frame {
             bytecode::Constant::Integer { ref value } => vm.ctx.new_int(*value),
             bytecode::Constant::Float { ref value } => vm.ctx.new_float(*value),
             bytecode::Constant::String { ref value } => vm.new_str(value.clone()),
+            bytecode::Constant::Bytes { ref value } => vm.ctx.new_bytes(value.clone()),
             bytecode::Constant::Boolean { ref value } => vm.new_bool(value.clone()),
             bytecode::Constant::Code { ref code } => {
                 PyObject::new(PyObjectKind::Code { code: code.clone() }, vm.get_type())

vm/src/stdlib/ast.rs

@@ -543,6 +543,11 @@ fn expression_to_ast(ctx: &PyContext, expression: &ast::Expression) -> PyObjectR
             node.set_attr("s", ctx.new_str(value.clone()));
             node
         }
+        ast::Expression::Bytes { value } => {
+            let node = create_node(ctx, "Bytes");
+            node.set_attr("s", ctx.new_bytes(value.clone()));
+            node
+        }
     };
 
     // TODO: retrieve correct lineno: