JS: Parse mustache-style tags as expressions

Author: asgerf

javascript/extractor/src/com/semmle/jcorn/Options.java

@@ -39,7 +39,7 @@ public void call(
         Position endLoc);
   }
 
-  private boolean allowHashBang, allowReturnOutsideFunction, allowImportExportEverywhere;
+  private boolean allowHashBang, allowReturnOutsideFunction, allowImportExportEverywhere, allowGeneratedCodeExprs;
   private boolean preserveParens, mozExtensions, jscript, esnext, v8Extensions, e4x;
   private int ecmaVersion;
   private AllowReserved allowReserved;
@@ -58,6 +58,7 @@ public Options() {
     this.allowReserved = AllowReserved.YES;
     this.allowReturnOutsideFunction = false;
     this.allowImportExportEverywhere = false;
+    this.allowGeneratedCodeExprs = true;
     this.allowHashBang = false;
     this.onToken = null;
     this.onComment = null;
@@ -75,6 +76,7 @@ public Options(Options that) {
     this.allowHashBang = that.allowHashBang;
     this.allowReturnOutsideFunction = that.allowReturnOutsideFunction;
     this.allowImportExportEverywhere = that.allowImportExportEverywhere;
+    this.allowGeneratedCodeExprs = that.allowGeneratedCodeExprs;
     this.preserveParens = that.preserveParens;
     this.mozExtensions = that.mozExtensions;
     this.jscript = that.jscript;
@@ -104,6 +106,10 @@ public boolean allowImportExportEverywhere() {
     return allowImportExportEverywhere;
   }
 
+  public boolean allowGeneratedCodeExprs() {
+    return allowGeneratedCodeExprs;
+  }
+
   public boolean preserveParens() {
     return preserveParens;
   }

javascript/extractor/src/com/semmle/jcorn/Parser.java

@@ -52,6 +52,7 @@
 import com.semmle.js.ast.ForStatement;
 import com.semmle.js.ast.FunctionDeclaration;
 import com.semmle.js.ast.FunctionExpression;
+import com.semmle.js.ast.GeneratedCodeExpr;
 import com.semmle.js.ast.IFunction;
 import com.semmle.js.ast.INode;
 import com.semmle.js.ast.IPattern;
@@ -537,7 +538,7 @@ private Token readToken_question() { // '?'
         }
         return this.finishOp(TokenType.questionquestion, 2);
       }
-      
+
     }
     return this.finishOp(TokenType.question, 1);
   }
@@ -1929,10 +1930,16 @@ MethodDefinition.Kind getMethodKind() {
   // Parse an object literal or binding pattern.
   protected Expression parseObj(boolean isPattern, DestructuringErrors refDestructuringErrors) {
     Position startLoc = this.startLoc;
+    if (!isPattern && options.allowGeneratedCodeExprs() && charAt(pos) == '{') {
+      // Parse mustache-style placeholder expression: {{ ... }} or {{{ ... }}}
+      return charAt(pos + 1) == '{'
+          ? parseGeneratedCodeExpr(startLoc, "{{{", "}}}")
+          : parseGeneratedCodeExpr(startLoc, "{{", "}}");
+    }
     boolean first = true;
     Map<String, PropInfo> propHash = new LinkedHashMap<>();
     List<Property> properties = new ArrayList<Property>();
-    this.next();
+    this.next(); // skip '{'
     while (!this.eat(TokenType.braceR)) {
       if (!first) {
         this.expect(TokenType.comma);
@@ -1949,6 +1956,42 @@ protected Expression parseObj(boolean isPattern, DestructuringErrors refDestruct
     return this.finishNode(node);
   }
 
+  /** Emit a token ranging from the current position until <code>endOfToken</code>. */
+  private Token generateTokenEndingAt(int endOfToken, TokenType tokenType) {
+    this.lastTokEnd = this.end;
+    this.lastTokStart = this.start;
+    this.lastTokEndLoc = this.endLoc;
+    this.lastTokStartLoc = this.startLoc;
+    this.start = this.pos;
+    this.startLoc = this.curPosition();
+    this.pos = endOfToken;
+    return finishToken(tokenType);
+  }
+
+  /** Parse a generated expression. The current token refers to the opening delimiter. */
+  protected Expression parseGeneratedCodeExpr(Position startLoc, String openingDelimiter, String closingDelimiter) {
+    // Emit a token for what's left of the opening delimiter, if there are any remaining characters
+    int startOfBody = startLoc.getOffset() + openingDelimiter.length();
+    if (this.pos != startOfBody) {
+      this.generateTokenEndingAt(startOfBody, TokenType.generatedCodeDelimiter);
+    }
+
+    // Emit a token for the generated code body
+    int endOfBody = this.input.indexOf(closingDelimiter, startOfBody);
+    if (endOfBody == -1) {
+    	this.unexpected(startLoc);
+    }
+    Token bodyToken = this.generateTokenEndingAt(endOfBody, TokenType.generatedCodeExpr);
+
+    // Emit a token for the closing delimiter
+    this.generateTokenEndingAt(endOfBody + closingDelimiter.length(), TokenType.generatedCodeDelimiter);
+
+    this.next(); // produce lookahead token
+
+    return finishNode(new GeneratedCodeExpr(new SourceLocation(startLoc), openingDelimiter, closingDelimiter,
+        bodyToken.getValue()));
+  }
+
   protected Property parseProperty(
       boolean isPattern,
       DestructuringErrors refDestructuringErrors,

javascript/extractor/src/com/semmle/jcorn/TokenType.java

@@ -89,6 +89,8 @@ public void updateContext(Parser parser, TokenType prevType) {
       arrow = new TokenType(new Properties("=>").beforeExpr()),
       template = new TokenType(new Properties("template")),
       invalidTemplate = new TokenType(new Properties("invalidTemplate")),
+      generatedCodeExpr = new TokenType(new Properties("generatedCodeExpr")),
+      generatedCodeDelimiter = new TokenType(new Properties("generatedCodeDelimiter")),
       ellipsis = new TokenType(new Properties("...").beforeExpr()),
       backQuote =
           new TokenType(new Properties("`").startsExpr()) {

javascript/extractor/src/com/semmle/js/ast/DefaultVisitor.java

@@ -777,4 +777,9 @@ public R visit(XMLQualifiedIdentifier nd, C c) {
   public R visit(XMLDotDotExpression nd, C c) {
     return visit((Expression) nd, c);
   }
+
+  @Override
+  public R visit(GeneratedCodeExpr nd, C c) {
+    return visit((Expression) nd, c);
+  }
 }

javascript/extractor/src/com/semmle/js/ast/GeneratedCodeExpr.java

@@ -0,0 +1,45 @@
+package com.semmle.js.ast;
+
+/**
+ * A placeholder for generated code, speculatively parsed as a primary expression.
+ *
+ * <p>For example, in this snippet,
+ *
+ * <pre>
+ * let data = {{user_data}};
+ * </pre>
+ *
+ * the expression <code>{{user_data}}</code> is assumed to be filled in by a templating engine so
+ * that it can be parsed as an expression, and a <code>GeneratedCodeExpr</code> is thus created to
+ * represent it.
+ */
+public class GeneratedCodeExpr extends Expression {
+  private String openingDelimiter;
+  private String closingDelimiter;
+  private String body;
+
+  public GeneratedCodeExpr(
+      SourceLocation loc, String openingDelimiter, String closingDelimiter, String body) {
+    super("GeneratedCodeExpr", loc);
+    this.openingDelimiter = openingDelimiter;
+    this.closingDelimiter = closingDelimiter;
+    this.body = body;
+  }
+
+  public String getOpeningDelimiter() {
+    return openingDelimiter;
+  }
+
+  public String getClosingDelimiter() {
+    return closingDelimiter;
+  }
+
+  public String getBody() {
+    return body;
+  }
+
+  @Override
+  public <C, R> R accept(Visitor<C, R> v, C c) {
+    return v.visit(this, c);
+  }
+}

javascript/extractor/src/com/semmle/js/ast/NodeCopier.java

@@ -894,4 +894,9 @@ public INode visit(XMLQualifiedIdentifier nd, Void c) {
   public INode visit(XMLDotDotExpression nd, Void c) {
     return new XMLDotDotExpression(visit(nd.getLoc()), copy(nd.getLeft()), copy(nd.getRight()));
   }
+
+  @Override
+  public INode visit(GeneratedCodeExpr nd, Void c) {
+  return new GeneratedCodeExpr(visit(nd.getLoc()), nd.getOpeningDelimiter(), nd.getClosingDelimiter(), nd.getBody());
+  }
 }

javascript/extractor/src/com/semmle/js/ast/Visitor.java

@@ -313,4 +313,6 @@ public interface Visitor<C, R> {
   public R visit(XMLQualifiedIdentifier nd, C c);
 
   public R visit(XMLDotDotExpression nd, C c);
+
+  public R visit(GeneratedCodeExpr generatedCodeExpr, C c);
 }

javascript/extractor/src/com/semmle/js/extractor/ASTExtractor.java

@@ -45,6 +45,7 @@
 import com.semmle.js.ast.ForStatement;
 import com.semmle.js.ast.FunctionDeclaration;
 import com.semmle.js.ast.FunctionExpression;
+import com.semmle.js.ast.GeneratedCodeExpr;
 import com.semmle.js.ast.IFunction;
 import com.semmle.js.ast.INode;
 import com.semmle.js.ast.IPattern;
@@ -1162,7 +1163,7 @@ public Label visit(Property nd, Context c) {
       if (!nd.isComputed() && "template".equals(tryGetIdentifierName(nd.getKey()))) {
         extractStringValueAsHtml(nd.getValue(), valueLabel);
       }
-      
+
       return propkey;
     }
 
@@ -2220,6 +2221,13 @@ public Label visit(AngularPipeRef nd, Context c) {
       visit(nd.getIdentifier(), key, 0, IdContext.LABEL);
       return key;
     }
+
+    @Override
+    public Label visit(GeneratedCodeExpr nd, Context c) {
+        Label key = super.visit(nd, c);
+        trapwriter.addTuple("generated_code_expr_info", key, nd.getOpeningDelimiter(), nd.getClosingDelimiter(), nd.getBody());
+        return key;
+    }
   }
 
   public List<ParseError> extract(

javascript/extractor/src/com/semmle/js/extractor/ExprKinds.java

@@ -149,6 +149,7 @@ public class ExprKinds {
     exprKinds.put("ExternalModuleReference", 98);
     exprKinds.put("NonNullAssertion", 105);
     exprKinds.put("AngularPipeRef", 119);
+    exprKinds.put("GeneratedCodeExpr", 120);
   }
 
   private static final Map<IdContext, Integer> idKinds =

javascript/extractor/tests/generatedcode/input/tst.js

@@ -0,0 +1,9 @@
+let data1 = {{ user_data1 }};
+let data2 = {{{ user_data2 }}};
+if ({{something}}) {}
+foo({{bar}}, {{baz}});
+
+{{not_generated_code}} // parse as block
+{{ if (not_generated_code) { } }} // parse as block
+if (1 == 2) {{not_generated_code}} // parse as block
+let string = "{{ not_generated_code }}"; // parse as string literal