Give token an index and throw unexpected token from a common place

src/ol/expression/lexer.js

@@ -82,13 +82,15 @@ ol.expression.TokenType = {
   NULL_LITERAL: 'Null',
   NUMERIC_LITERAL: 'Numeric',
   PUNCTUATOR: 'Punctuator',
-  STRING_LITERAL: 'String'
+  STRING_LITERAL: 'String',
+  UNKNOWN: 'Unknown'
 };
 
 
 /**
  * @typedef {{type: (ol.expression.TokenType),
- *            value: (string|number|boolean|null)}}
+ *            value: (string|number|boolean|null),
+ *            index: (number)}}
  */
 ol.expression.Token;
 
@@ -140,11 +142,14 @@ ol.expression.Lexer = function(source) {
  */
 ol.expression.Lexer.prototype.expect = function(value) {
   var match = this.match(value);
-  this.skip();
   if (!match) {
-    throw new Error('Unexpected token at index ' + this.index_ +
+    this.throwUnexpected({
-        ': ' + this.getCurrentChar_());
+      type: ol.expression.TokenType.UNKNOWN,
+      value: this.getCurrentChar_(),
+      index: this.index_
+    });
   }
+  this.skip();
 };
 
 
@@ -388,7 +393,8 @@ ol.expression.Lexer.prototype.next = function() {
   if (this.index_ >= this.length_) {
     return {
       type: ol.expression.TokenType.EOF,
-      value: null
+      value: null,
+      index: this.index_
     };
   }
 
@@ -450,6 +456,7 @@ ol.expression.Lexer.prototype.peek = function() {
  */
 ol.expression.Lexer.prototype.scanHexLiteral_ = function(code) {
   var str = '';
+  var start = this.index_;
 
   while (this.index_ < this.length_) {
     if (!this.isHexDigit_(code)) {
@@ -460,21 +467,20 @@ ol.expression.Lexer.prototype.scanHexLiteral_ = function(code) {
     code = this.getCurrentCharCode_();
   }
 
-  if (str.length === 0) {
+  if (str.length === 0 || this.isIdentifierStart_(code)) {
-    throw new Error('Unexpected token at index ' + this.index_ +
+    this.throwUnexpected({
-        ': ' + String.fromCharCode(code));
+      type: ol.expression.TokenType.UNKNOWN,
-  }
+      value: String.fromCharCode(code),
-
+      index: this.index_
-  if (this.isIdentifierStart_(code)) {
+    });
-    throw new Error('Unexpected token at index ' + this.index_ +
-        ': ' + String.fromCharCode(code));
   }
 
   goog.asserts.assert(!isNaN(parseInt('0x' + str, 16)), 'Valid hex: ' + str);
 
   return {
     type: ol.expression.TokenType.NUMERIC_LITERAL,
-    value: parseInt('0x' + str, 16)
+    value: parseInt('0x' + str, 16),
+    index: start
   };
 };
 
@@ -518,7 +524,8 @@ ol.expression.Lexer.prototype.scanIdentifier_ = function(code) {
 
   return {
     type: type,
-    value: id
+    value: id,
+    index: start
   };
 };
 
@@ -538,6 +545,7 @@ ol.expression.Lexer.prototype.scanNumericLiteral_ = function(code) {
 
   // start assembling numeric string
   var str = '';
+  var start = this.index_;
 
   if (code !== ol.expression.Char.DOT) {
 
@@ -559,8 +567,11 @@ ol.expression.Lexer.prototype.scanNumericLiteral_ = function(code) {
 
       // numbers like 09 not allowed
       if (this.isDecimalDigit_(nextCode)) {
-        throw new Error('Unexpected token at index ' + this.index_ +
+        this.throwUnexpected({
-            ': ' + String.fromCharCode(nextCode));
+          type: ol.expression.TokenType.UNKNOWN,
+          value: String.fromCharCode(nextCode),
+          index: this.index_
+        });
       }
     }
 
@@ -601,8 +612,11 @@ ol.expression.Lexer.prototype.scanNumericLiteral_ = function(code) {
     }
 
     if (!this.isDecimalDigit_(code)) {
-      throw new Error('Unexpected token at index ' + this.index_ +
+      this.throwUnexpected({
-          ': ' + String.fromCharCode(code));
+        type: ol.expression.TokenType.UNKNOWN,
+        value: String.fromCharCode(code),
+        index: this.index_
+      });
     }
 
     // scan all decimal chars (TODO: unduplicate this)
@@ -614,15 +628,19 @@ ol.expression.Lexer.prototype.scanNumericLiteral_ = function(code) {
   }
 
   if (this.isIdentifierStart_(code)) {
-    throw new Error('Unexpected token at index ' + this.index_ +
+    this.throwUnexpected({
-        ': ' + String.fromCharCode(code));
+      type: ol.expression.TokenType.UNKNOWN,
+      value: String.fromCharCode(code),
+      index: this.index_
+    });
   }
 
   goog.asserts.assert(!isNaN(parseFloat(str)), 'Valid number: ' + str);
 
   return {
     type: ol.expression.TokenType.NUMERIC_LITERAL,
-    value: parseFloat(str)
+    value: parseFloat(str),
+    index: start
   };
 
 };
@@ -639,6 +657,7 @@ ol.expression.Lexer.prototype.scanOctalLiteral_ = function(code) {
   goog.asserts.assert(this.isOctalDigit_(code));
 
   var str = '0' + String.fromCharCode(code);
+  var start = this.index_;
   this.increment_(1);
 
   while (this.index_ < this.length_) {
@@ -653,15 +672,19 @@ ol.expression.Lexer.prototype.scanOctalLiteral_ = function(code) {
   code = this.getCurrentCharCode_();
   if (this.isIdentifierStart_(code) ||
       this.isDecimalDigit_(code)) {
-    throw new Error('Unexpected token at index ' + (this.index_ - 1) +
+    this.throwUnexpected({
-        ': ' + String.fromCharCode(code));
+      type: ol.expression.TokenType.UNKNOWN,
+      value: String.fromCharCode(code),
+      index: this.index_ - 1
+    });
   }
 
   goog.asserts.assert(!isNaN(parseInt(str, 8)), 'Valid octal: ' + str);
 
   return {
     type: ol.expression.TokenType.NUMERIC_LITERAL,
-    value: parseInt(str, 8)
+    value: parseInt(str, 8),
+    index: start
   };
 };
 
@@ -674,6 +697,7 @@ ol.expression.Lexer.prototype.scanOctalLiteral_ = function(code) {
  * @private
  */
 ol.expression.Lexer.prototype.scanPunctuator_ = function(code) {
+  var start = this.index_;
 
   // single char punctuation that also doesn't start longer punctuation
   // (we disallow assignment, so no += etc.)
@@ -691,7 +715,8 @@ ol.expression.Lexer.prototype.scanPunctuator_ = function(code) {
     this.increment_(1);
     return {
       type: ol.expression.TokenType.PUNCTUATOR,
-      value: String.fromCharCode(code)
+      value: String.fromCharCode(code),
+      index: start
     };
   }
 
@@ -709,13 +734,15 @@ ol.expression.Lexer.prototype.scanPunctuator_ = function(code) {
         this.increment_(1);
         return {
           type: ol.expression.TokenType.PUNCTUATOR,
-          value: String.fromCharCode(code) + '=='
+          value: String.fromCharCode(code) + '==',
+          index: start
         };
       } else {
         // != or ==
         return {
           type: ol.expression.TokenType.PUNCTUATOR,
-          value: String.fromCharCode(code) + '='
+          value: String.fromCharCode(code) + '=',
+          index: start
         };
       }
     }
@@ -725,7 +752,8 @@ ol.expression.Lexer.prototype.scanPunctuator_ = function(code) {
       this.increment_(2);
       return {
         type: ol.expression.TokenType.PUNCTUATOR,
-        value: String.fromCharCode(code) + '='
+        value: String.fromCharCode(code) + '=',
+        index: start
       };
     }
   }
@@ -739,7 +767,8 @@ ol.expression.Lexer.prototype.scanPunctuator_ = function(code) {
     var str = String.fromCharCode(code);
     return {
       type: ol.expression.TokenType.PUNCTUATOR,
-      value: str + str
+      value: str + str,
+      index: start
     };
   }
 
@@ -756,12 +785,24 @@ ol.expression.Lexer.prototype.scanPunctuator_ = function(code) {
     this.increment_(1);
     return {
       type: ol.expression.TokenType.PUNCTUATOR,
-      value: String.fromCharCode(code)
+      value: String.fromCharCode(code),
+      index: start
     };
   }
 
-  throw new Error('Unexpected token at index ' + (this.index_ - 1) +
+  this.throwUnexpected({
-      ': ' + String.fromCharCode(code));
+    type: ol.expression.TokenType.UNKNOWN,
+    value: String.fromCharCode(code),
+    index: this.index_
+  });
+
+  // This code is unreachable, but the compiler complains with
+  // JSC_MISSING_RETURN_STATEMENT without it.
+  return {
+    type: ol.expression.TokenType.UNKNOWN,
+    value: '',
+    index: 0
+  };
 };
 
 
@@ -780,6 +821,7 @@ ol.expression.Lexer.prototype.scanStringLiteral_ = function(quote) {
   this.increment_(1);
 
   var str = '';
+  var start = this.index_;
   var code;
   while (this.index_ < this.length_) {
     code = this.getCurrentCharCode_();
@@ -798,12 +840,14 @@ ol.expression.Lexer.prototype.scanStringLiteral_ = function(quote) {
   }
 
   if (quote !== 0) {
-    throw new Error('Unterminated string literal');
+    // unterminated string literal
+    this.throwUnexpected(this.peek());
   }
 
   return {
     type: ol.expression.TokenType.STRING_LITERAL,
-    value: str
+    value: str,
+    index: start
   };
 };
 
@@ -833,3 +877,15 @@ ol.expression.Lexer.prototype.skipWhitespace_ = function() {
   }
   return code;
 };
+
+
+/**
+ * Throw an error about an unexpected token.
+ * @param {ol.expression.Token} token The unexpected token.
+ */
+ol.expression.Lexer.prototype.throwUnexpected = function(token) {
+  var error = new Error('Unexpected token at ' + token.index + ' : ' +
+      token.value);
+  error.token = token;
+  throw error;
+};

src/ol/expression/parser.js

@@ -148,13 +148,13 @@ ol.expression.Parser.prototype.createBinaryExpression_ = function(operator,
 /**
  * Create a call expression.
  *
- * @param {ol.expression.Identifier} expr Identifier expression for function.
+ * @param {ol.expression.Expression} callee Expression for function.
  * @param {Array.<ol.expression.Expression>} args Arguments array.
  * @return {ol.expression.Call} Call expression.
  * @private
  */
-ol.expression.Parser.prototype.createCallExpression_ = function(expr, args) {
+ol.expression.Parser.prototype.createCallExpression_ = function(callee, args) {
-  return new ol.expression.Call(expr, args);
+  return new ol.expression.Call(callee, args);
 };
 
 
@@ -186,14 +186,14 @@ ol.expression.Parser.prototype.createLiteral_ = function(value) {
  * Create a member expression.
  *
  * // TODO: make exp {ol.expression.Member|ol.expression.Identifier}
- * @param {ol.expression.Expression} expr Expression.
+ * @param {ol.expression.Expression} object Expression.
  * @param {ol.expression.Identifier} property Member name.
  * @return {ol.expression.Member} The member expression.
  * @private
  */
-ol.expression.Parser.prototype.createMemberExpression_ = function(expr,
+ol.expression.Parser.prototype.createMemberExpression_ = function(object,
     property) {
-  return new ol.expression.Member(expr, property);
+  return new ol.expression.Member(object, property);
 };
 
 
@@ -201,13 +201,13 @@ ol.expression.Parser.prototype.createMemberExpression_ = function(expr,
  * Create a unary expression.
  *
  * @param {string} op Operator.
- * @param {ol.expression.Expression} expr Expression.
+ * @param {ol.expression.Expression} argument Expression.
  * @return {ol.expression.Not} The logical not of the input expression.
  * @private
  */
-ol.expression.Parser.prototype.createUnaryExpression_ = function(op, expr) {
+ol.expression.Parser.prototype.createUnaryExpression_ = function(op, argument) {
   goog.asserts.assert(op === '!');
-  return new ol.expression.Not(expr);
+  return new ol.expression.Not(argument);
 };
 
 
@@ -222,8 +222,7 @@ ol.expression.Parser.prototype.parse = function(source) {
   var expr = this.parseExpression_(lexer);
   var token = lexer.peek();
   if (token.type !== ol.expression.TokenType.EOF) {
-    // TODO: token.index
+    lexer.throwUnexpected(token);
-    throw new Error('Unexpected token: ' + token.value);
   }
   return expr;
 };
@@ -353,9 +352,8 @@ ol.expression.Parser.prototype.parseLeftHandSideExpression_ = function(lexer) {
   if (token.value === '(') {
     // only allow calls on identifiers (e.g. `foo()` not `foo.bar()`)
     if (!(expr instanceof ol.expression.Identifier)) {
-      // TODO: token.index
       // TODO: more helpful error messages for restricted syntax
-      throw new Error('Unexpected token: (');
+      lexer.throwUnexpected(token);
     }
     var args = this.parseArguments_(lexer);
     expr = this.createCallExpression_(expr, args);
@@ -387,8 +385,7 @@ ol.expression.Parser.prototype.parseNonComputedMember_ = function(lexer) {
       token.type !== ol.expression.TokenType.KEYWORD &&
       token.type !== ol.expression.TokenType.BOOLEAN_LITERAL &&
       token.type !== ol.expression.TokenType.NULL_LITERAL) {
-    // TODO: token.index
+    lexer.throwUnexpected(token);
-    throw new Error('Unexpected token: ' + token.value);
   }
 
   return this.createIdentifier_(String(token.value));
@@ -423,8 +420,10 @@ ol.expression.Parser.prototype.parsePrimaryExpression_ = function(lexer) {
   } else if (type === ol.expression.TokenType.NULL_LITERAL) {
     expr = this.createLiteral_(null);
   } else {
-    throw new Error('Unexpected token: ' + token.value);
+    lexer.throwUnexpected(token);
   }
+  // the compiler doesn't recognize that we have covered all cases here
+  goog.asserts.assert(goog.isDef(expr));
   return expr;
 };
 

test/spec/ol/expression/expression.test.js

@@ -26,7 +26,13 @@ describe('ol.expression.parse', function() {
     it('throws on invalid identifier expressions', function() {
       expect(function() {
         ol.expression.parse('3foo');
-      }).throwException();
+      }).throwException(function(err) {
+        expect(err).to.be.an(Error);
+        var token = err.token;
+        expect(token).not.to.be(undefined);
+        expect(token.value).to.be('f');
+        expect(token.index).to.be(1);
+      });
     });
 
     it('parses string literal expressions', function() {
@@ -38,7 +44,13 @@ describe('ol.expression.parse', function() {
     it('throws on unterminated string', function() {
       expect(function() {
         ol.expression.parse('"foo');
-      }).throwException();
+      }).throwException(function(err) {
+        expect(err).to.be.an(Error);
+        var token = err.token;
+        expect(token).not.to.be(undefined);
+        expect(token.type).to.be(ol.expression.TokenType.EOF);
+        expect(token.index).to.be(4);
+      });
     });
 
     it('parses numeric literal expressions', function() {
@@ -50,7 +62,13 @@ describe('ol.expression.parse', function() {
     it('throws on invalid number', function() {
       expect(function() {
         ol.expression.parse('.42eX');
-      }).throwException();
+      }).throwException(function(err) {
+        expect(err).to.be.an(Error);
+        var token = err.token;
+        expect(token).not.to.be(undefined);
+        expect(token.value).to.be('X');
+        expect(token.index).to.be(4);
+      });
     });
 
     it('parses boolean literal expressions', function() {
@@ -87,7 +105,13 @@ describe('ol.expression.parse', function() {
     it('throws on invalid member expression', function() {
       expect(function() {
         ol.expression.parse('foo.4bar');
-      }).throwException();
+      }).throwException(function(err) {
+        expect(err).to.be.an(Error);
+        var token = err.token;
+        expect(token).not.to.be(undefined);
+        expect(token.value).to.be('b');
+        expect(token.index).to.be(5);
+      });
     });
 
     it('parses call expressions with literal arguments', function() {
@@ -106,7 +130,13 @@ describe('ol.expression.parse', function() {
     it('throws on calls with unterminated arguments', function() {
       expect(function() {
         ol.expression.parse('foo(42,)');
-      }).throwException();
+      }).throwException(function(err) {
+        expect(err).to.be.an(Error);
+        var token = err.token;
+        expect(token).not.to.be(undefined);
+        expect(token.value).to.be(')');
+        expect(token.index).to.be(7);
+      });
     });
 
   });
@@ -263,7 +293,13 @@ describe('ol.expression.parse', function() {
     it('throws for invalid spacing with <=', function() {
       expect(function() {
         ol.expression.parse(' foo< = 10 ');
-      }).throwException();
+      }).throwException(function(err) {
+        expect(err).to.be.an(Error);
+        var token = err.token;
+        expect(token).not.to.be(undefined);
+        expect(token.value).to.be('=');
+        expect(token.index).to.be(6);
+      });
     });
 
     it('parses >= operator', function() {
@@ -285,7 +321,13 @@ describe('ol.expression.parse', function() {
     it('throws for invalid spacing with >=', function() {
       expect(function() {
         ol.expression.parse(' 10 > =foo ');
-      }).throwException();
+      }).throwException(function(err) {
+        expect(err).to.be.an(Error);
+        var token = err.token;
+        expect(token).not.to.be(undefined);
+        expect(token.value).to.be('=');
+        expect(token.index).to.be(6);
+      });
     });
 
   });
@@ -312,7 +354,13 @@ describe('ol.expression.parse', function() {
     it('throws for invalid spacing with ==', function() {
       expect(function() {
         ol.expression.parse(' 10 = =foo ');
-      }).throwException();
+      }).throwException(function(err) {
+        expect(err).to.be.an(Error);
+        var token = err.token;
+        expect(token).not.to.be(undefined);
+        expect(token.value).to.be('=');
+        expect(token.index).to.be(4);
+      });
     });
 
     it('parses != operator', function() {
@@ -334,7 +382,13 @@ describe('ol.expression.parse', function() {
     it('throws for invalid spacing with !=', function() {
       expect(function() {
         ol.expression.parse(' 10! =foo ');
-      }).throwException();
+      }).throwException(function(err) {
+        expect(err).to.be.an(Error);
+        var token = err.token;
+        expect(token).not.to.be(undefined);
+        expect(token.value).to.be('!');
+        expect(token.index).to.be(3);
+      });
     });
 
     it('parses === operator', function() {
@@ -356,7 +410,13 @@ describe('ol.expression.parse', function() {
     it('throws for invalid spacing with ===', function() {
       expect(function() {
         ol.expression.parse(' 10 = == foo ');
-      }).throwException();
+      }).throwException(function(err) {
+        expect(err).to.be.an(Error);
+        var token = err.token;
+        expect(token).not.to.be(undefined);
+        expect(token.value).to.be('=');
+        expect(token.index).to.be(4);
+      });
     });
 
     it('parses !== operator', function() {
@@ -378,7 +438,13 @@ describe('ol.expression.parse', function() {
     it('throws for invalid spacing with !==', function() {
       expect(function() {
         ol.expression.parse(' 10 != = foo ');
-      }).throwException();
+      }).throwException(function(err) {
+        expect(err).to.be.an(Error);
+        var token = err.token;
+        expect(token).not.to.be(undefined);
+        expect(token.value).to.be('=');
+        expect(token.index).to.be(7);
+      });
     });
   });
 
@@ -411,7 +477,13 @@ describe('ol.expression.parse', function() {
     it('throws for invalid spacing with &&', function() {
       expect(function() {
         ol.expression.parse('true & & false');
-      }).throwException();
+      }).throwException(function(err) {
+        expect(err).to.be.an(Error);
+        var token = err.token;
+        expect(token).not.to.be(undefined);
+        expect(token.value).to.be('&');
+        expect(token.index).to.be(5);
+      });
     });
 
     it('parses || operator', function() {
@@ -435,7 +507,13 @@ describe('ol.expression.parse', function() {
     it('throws for invalid spacing with ||', function() {
       expect(function() {
         ol.expression.parse('true | | false');
-      }).throwException();
+      }).throwException(function(err) {
+        expect(err).to.be.an(Error);
+        var token = err.token;
+        expect(token).not.to.be(undefined);
+        expect(token.value).to.be('|');
+        expect(token.index).to.be(5);
+      });
     });
 
   });
@@ -468,3 +546,4 @@ goog.require('ol.expression.Logical');
 goog.require('ol.expression.Math');
 goog.require('ol.expression.Member');
 goog.require('ol.expression.Not');
+goog.require('ol.expression.TokenType');

test/spec/ol/expression/lexer.test.js

@@ -378,7 +378,13 @@ describe('ol.expression.Lexer', function() {
     it('throws on unterminated double quote', function() {
       expect(function() {
         scan('"never \'ending\' string');
-      }).to.throwException();
+      }).to.throwException(function(err) {
+        expect(err).to.be.an(Error);
+        var token = err.token;
+        expect(token).not.to.be(undefined);
+        expect(token.type).to.be(ol.expression.TokenType.EOF);
+        expect(token.index).to.be(22);
+      });
     });
 
     it('parses single quoted string', function() {
@@ -426,7 +432,13 @@ describe('ol.expression.Lexer', function() {
     it('throws on unterminated single quote', function() {
       expect(function() {
         scan('\'never "ending" string');
-      }).to.throwException();
+      }).to.throwException(function(err) {
+        expect(err).to.be.an(Error);
+        var token = err.token;
+        expect(token).not.to.be(undefined);
+        expect(token.type).to.be(ol.expression.TokenType.EOF);
+        expect(token.index).to.be(22);
+      });
     });
 
   });