Give token an index and throw unexpected token from a common place

2013-06-12 14:52:31 -06:00
parent bb1b0cba95
commit 62eb0dd72b
4 changed files with 213 additions and 67 deletions
--- a/src/ol/expression/lexer.js
+++ b/src/ol/expression/lexer.js
@@ -82,13 +82,15 @@ ol.expression.TokenType = {
  NULL_LITERAL: 'Null',
  NUMERIC_LITERAL: 'Numeric',
  PUNCTUATOR: 'Punctuator',
-  STRING_LITERAL: 'String'
+  STRING_LITERAL: 'String',
+  UNKNOWN: 'Unknown'
 };


 /**
 * @typedef {{type: (ol.expression.TokenType),
- *            value: (string|number|boolean|null)}}
+ *            value: (string|number|boolean|null),
+ *            index: (number)}}
 */
 ol.expression.Token;

@@ -140,11 +142,14 @@ ol.expression.Lexer = function(source) {
 */
 ol.expression.Lexer.prototype.expect = function(value) {
  var match = this.match(value);
-  this.skip();
  if (!match) {
-    throw new Error('Unexpected token at index ' + this.index_ +
-        ': ' + this.getCurrentChar_());
+    this.throwUnexpected({
+      type: ol.expression.TokenType.UNKNOWN,
+      value: this.getCurrentChar_(),
+      index: this.index_
+    });
  }
+  this.skip();
 };


@@ -388,7 +393,8 @@ ol.expression.Lexer.prototype.next = function() {
  if (this.index_ >= this.length_) {
    return {
      type: ol.expression.TokenType.EOF,
-      value: null
+      value: null,
+      index: this.index_
    };
  }

@@ -450,6 +456,7 @@ ol.expression.Lexer.prototype.peek = function() {
 */
 ol.expression.Lexer.prototype.scanHexLiteral_ = function(code) {
  var str = '';
+  var start = this.index_;

  while (this.index_ < this.length_) {
    if (!this.isHexDigit_(code)) {
@@ -460,21 +467,20 @@ ol.expression.Lexer.prototype.scanHexLiteral_ = function(code) {
    code = this.getCurrentCharCode_();
  }

-  if (str.length === 0) {
-    throw new Error('Unexpected token at index ' + this.index_ +
-        ': ' + String.fromCharCode(code));
-  }
-
-  if (this.isIdentifierStart_(code)) {
-    throw new Error('Unexpected token at index ' + this.index_ +
-        ': ' + String.fromCharCode(code));
+  if (str.length === 0 || this.isIdentifierStart_(code)) {
+    this.throwUnexpected({
+      type: ol.expression.TokenType.UNKNOWN,
+      value: String.fromCharCode(code),
+      index: this.index_
+    });
  }

  goog.asserts.assert(!isNaN(parseInt('0x' + str, 16)), 'Valid hex: ' + str);

  return {
    type: ol.expression.TokenType.NUMERIC_LITERAL,
-    value: parseInt('0x' + str, 16)
+    value: parseInt('0x' + str, 16),
+    index: start
  };
 };

@@ -518,7 +524,8 @@ ol.expression.Lexer.prototype.scanIdentifier_ = function(code) {

  return {
    type: type,
-    value: id
+    value: id,
+    index: start
  };
 };

@@ -538,6 +545,7 @@ ol.expression.Lexer.prototype.scanNumericLiteral_ = function(code) {

  // start assembling numeric string
  var str = '';
+  var start = this.index_;

  if (code !== ol.expression.Char.DOT) {

@@ -559,8 +567,11 @@ ol.expression.Lexer.prototype.scanNumericLiteral_ = function(code) {

      // numbers like 09 not allowed
      if (this.isDecimalDigit_(nextCode)) {
-        throw new Error('Unexpected token at index ' + this.index_ +
-            ': ' + String.fromCharCode(nextCode));
+        this.throwUnexpected({
+          type: ol.expression.TokenType.UNKNOWN,
+          value: String.fromCharCode(nextCode),
+          index: this.index_
+        });
      }
    }

@@ -601,8 +612,11 @@ ol.expression.Lexer.prototype.scanNumericLiteral_ = function(code) {
    }

    if (!this.isDecimalDigit_(code)) {
-      throw new Error('Unexpected token at index ' + this.index_ +
-          ': ' + String.fromCharCode(code));
+      this.throwUnexpected({
+        type: ol.expression.TokenType.UNKNOWN,
+        value: String.fromCharCode(code),
+        index: this.index_
+      });
    }

    // scan all decimal chars (TODO: unduplicate this)
@@ -614,15 +628,19 @@ ol.expression.Lexer.prototype.scanNumericLiteral_ = function(code) {
  }

  if (this.isIdentifierStart_(code)) {
-    throw new Error('Unexpected token at index ' + this.index_ +
-        ': ' + String.fromCharCode(code));
+    this.throwUnexpected({
+      type: ol.expression.TokenType.UNKNOWN,
+      value: String.fromCharCode(code),
+      index: this.index_
+    });
  }

  goog.asserts.assert(!isNaN(parseFloat(str)), 'Valid number: ' + str);

  return {
    type: ol.expression.TokenType.NUMERIC_LITERAL,
-    value: parseFloat(str)
+    value: parseFloat(str),
+    index: start
  };

 };
@@ -639,6 +657,7 @@ ol.expression.Lexer.prototype.scanOctalLiteral_ = function(code) {
  goog.asserts.assert(this.isOctalDigit_(code));

  var str = '0' + String.fromCharCode(code);
+  var start = this.index_;
  this.increment_(1);

  while (this.index_ < this.length_) {
@@ -653,15 +672,19 @@ ol.expression.Lexer.prototype.scanOctalLiteral_ = function(code) {
  code = this.getCurrentCharCode_();
  if (this.isIdentifierStart_(code) ||
      this.isDecimalDigit_(code)) {
-    throw new Error('Unexpected token at index ' + (this.index_ - 1) +
-        ': ' + String.fromCharCode(code));
+    this.throwUnexpected({
+      type: ol.expression.TokenType.UNKNOWN,
+      value: String.fromCharCode(code),
+      index: this.index_ - 1
+    });
  }

  goog.asserts.assert(!isNaN(parseInt(str, 8)), 'Valid octal: ' + str);

  return {
    type: ol.expression.TokenType.NUMERIC_LITERAL,
-    value: parseInt(str, 8)
+    value: parseInt(str, 8),
+    index: start
  };
 };

@@ -674,6 +697,7 @@ ol.expression.Lexer.prototype.scanOctalLiteral_ = function(code) {
 * @private
 */
 ol.expression.Lexer.prototype.scanPunctuator_ = function(code) {
+  var start = this.index_;

  // single char punctuation that also doesn't start longer punctuation
  // (we disallow assignment, so no += etc.)
@@ -691,7 +715,8 @@ ol.expression.Lexer.prototype.scanPunctuator_ = function(code) {
    this.increment_(1);
    return {
      type: ol.expression.TokenType.PUNCTUATOR,
-      value: String.fromCharCode(code)
+      value: String.fromCharCode(code),
+      index: start
    };
  }

@@ -709,13 +734,15 @@ ol.expression.Lexer.prototype.scanPunctuator_ = function(code) {
        this.increment_(1);
        return {
          type: ol.expression.TokenType.PUNCTUATOR,
-          value: String.fromCharCode(code) + '=='
+          value: String.fromCharCode(code) + '==',
+          index: start
        };
      } else {
        // != or ==
        return {
          type: ol.expression.TokenType.PUNCTUATOR,
-          value: String.fromCharCode(code) + '='
+          value: String.fromCharCode(code) + '=',
+          index: start
        };
      }
    }
@@ -725,7 +752,8 @@ ol.expression.Lexer.prototype.scanPunctuator_ = function(code) {
      this.increment_(2);
      return {
        type: ol.expression.TokenType.PUNCTUATOR,
-        value: String.fromCharCode(code) + '='
+        value: String.fromCharCode(code) + '=',
+        index: start
      };
    }
  }
@@ -739,7 +767,8 @@ ol.expression.Lexer.prototype.scanPunctuator_ = function(code) {
    var str = String.fromCharCode(code);
    return {
      type: ol.expression.TokenType.PUNCTUATOR,
-      value: str + str
+      value: str + str,
+      index: start
    };
  }

@@ -756,12 +785,24 @@ ol.expression.Lexer.prototype.scanPunctuator_ = function(code) {
    this.increment_(1);
    return {
      type: ol.expression.TokenType.PUNCTUATOR,
-      value: String.fromCharCode(code)
+      value: String.fromCharCode(code),
+      index: start
    };
  }

-  throw new Error('Unexpected token at index ' + (this.index_ - 1) +
-      ': ' + String.fromCharCode(code));
+  this.throwUnexpected({
+    type: ol.expression.TokenType.UNKNOWN,
+    value: String.fromCharCode(code),
+    index: this.index_
+  });
+
+  // This code is unreachable, but the compiler complains with
+  // JSC_MISSING_RETURN_STATEMENT without it.
+  return {
+    type: ol.expression.TokenType.UNKNOWN,
+    value: '',
+    index: 0
+  };
 };


@@ -780,6 +821,7 @@ ol.expression.Lexer.prototype.scanStringLiteral_ = function(quote) {
  this.increment_(1);

  var str = '';
+  var start = this.index_;
  var code;
  while (this.index_ < this.length_) {
    code = this.getCurrentCharCode_();
@@ -798,12 +840,14 @@ ol.expression.Lexer.prototype.scanStringLiteral_ = function(quote) {
  }

  if (quote !== 0) {
-    throw new Error('Unterminated string literal');
+    // unterminated string literal
+    this.throwUnexpected(this.peek());
  }

  return {
    type: ol.expression.TokenType.STRING_LITERAL,
-    value: str
+    value: str,
+    index: start
  };
 };

@@ -833,3 +877,15 @@ ol.expression.Lexer.prototype.skipWhitespace_ = function() {
  }
  return code;
 };
+
+
+/**
+ * Throw an error about an unexpected token.
+ * @param {ol.expression.Token} token The unexpected token.
+ */
+ol.expression.Lexer.prototype.throwUnexpected = function(token) {
+  var error = new Error('Unexpected token at ' + token.index + ' : ' +
+      token.value);
+  error.token = token;
+  throw error;
+};
--- a/src/ol/expression/parser.js
+++ b/src/ol/expression/parser.js
@@ -148,13 +148,13 @@ ol.expression.Parser.prototype.createBinaryExpression_ = function(operator,
 /**
 * Create a call expression.
 *
- * @param {ol.expression.Identifier} expr Identifier expression for function.
+ * @param {ol.expression.Expression} callee Expression for function.
 * @param {Array.<ol.expression.Expression>} args Arguments array.
 * @return {ol.expression.Call} Call expression.
 * @private
 */
-ol.expression.Parser.prototype.createCallExpression_ = function(expr, args) {
-  return new ol.expression.Call(expr, args);
+ol.expression.Parser.prototype.createCallExpression_ = function(callee, args) {
+  return new ol.expression.Call(callee, args);
 };


@@ -186,14 +186,14 @@ ol.expression.Parser.prototype.createLiteral_ = function(value) {
 * Create a member expression.
 *
 * // TODO: make exp {ol.expression.Member|ol.expression.Identifier}
- * @param {ol.expression.Expression} expr Expression.
+ * @param {ol.expression.Expression} object Expression.
 * @param {ol.expression.Identifier} property Member name.
 * @return {ol.expression.Member} The member expression.
 * @private
 */
-ol.expression.Parser.prototype.createMemberExpression_ = function(expr,
+ol.expression.Parser.prototype.createMemberExpression_ = function(object,
    property) {
-  return new ol.expression.Member(expr, property);
+  return new ol.expression.Member(object, property);
 };


@@ -201,13 +201,13 @@ ol.expression.Parser.prototype.createMemberExpression_ = function(expr,
 * Create a unary expression.
 *
 * @param {string} op Operator.
- * @param {ol.expression.Expression} expr Expression.
+ * @param {ol.expression.Expression} argument Expression.
 * @return {ol.expression.Not} The logical not of the input expression.
 * @private
 */
-ol.expression.Parser.prototype.createUnaryExpression_ = function(op, expr) {
+ol.expression.Parser.prototype.createUnaryExpression_ = function(op, argument) {
  goog.asserts.assert(op === '!');
-  return new ol.expression.Not(expr);
+  return new ol.expression.Not(argument);
 };


@@ -222,8 +222,7 @@ ol.expression.Parser.prototype.parse = function(source) {
  var expr = this.parseExpression_(lexer);
  var token = lexer.peek();
  if (token.type !== ol.expression.TokenType.EOF) {
-    // TODO: token.index
-    throw new Error('Unexpected token: ' + token.value);
+    lexer.throwUnexpected(token);
  }
  return expr;
 };
@@ -353,9 +352,8 @@ ol.expression.Parser.prototype.parseLeftHandSideExpression_ = function(lexer) {
  if (token.value === '(') {
    // only allow calls on identifiers (e.g. `foo()` not `foo.bar()`)
    if (!(expr instanceof ol.expression.Identifier)) {
-      // TODO: token.index
      // TODO: more helpful error messages for restricted syntax
-      throw new Error('Unexpected token: (');
+      lexer.throwUnexpected(token);
    }
    var args = this.parseArguments_(lexer);
    expr = this.createCallExpression_(expr, args);
@@ -387,8 +385,7 @@ ol.expression.Parser.prototype.parseNonComputedMember_ = function(lexer) {
      token.type !== ol.expression.TokenType.KEYWORD &&
      token.type !== ol.expression.TokenType.BOOLEAN_LITERAL &&
      token.type !== ol.expression.TokenType.NULL_LITERAL) {
-    // TODO: token.index
-    throw new Error('Unexpected token: ' + token.value);
+    lexer.throwUnexpected(token);
  }

  return this.createIdentifier_(String(token.value));
@@ -423,8 +420,10 @@ ol.expression.Parser.prototype.parsePrimaryExpression_ = function(lexer) {
  } else if (type === ol.expression.TokenType.NULL_LITERAL) {
    expr = this.createLiteral_(null);
  } else {
-    throw new Error('Unexpected token: ' + token.value);
+    lexer.throwUnexpected(token);
  }
+  // the compiler doesn't recognize that we have covered all cases here
+  goog.asserts.assert(goog.isDef(expr));
  return expr;
 };

--- a/test/spec/ol/expression/expression.test.js
+++ b/test/spec/ol/expression/expression.test.js
@@ -26,7 +26,13 @@ describe('ol.expression.parse', function() {
    it('throws on invalid identifier expressions', function() {
      expect(function() {
        ol.expression.parse('3foo');
-      }).throwException();
+      }).throwException(function(err) {
+        expect(err).to.be.an(Error);
+        var token = err.token;
+        expect(token).not.to.be(undefined);
+        expect(token.value).to.be('f');
+        expect(token.index).to.be(1);
+      });
    });

    it('parses string literal expressions', function() {
@@ -38,7 +44,13 @@ describe('ol.expression.parse', function() {
    it('throws on unterminated string', function() {
      expect(function() {
        ol.expression.parse('"foo');
-      }).throwException();
+      }).throwException(function(err) {
+        expect(err).to.be.an(Error);
+        var token = err.token;
+        expect(token).not.to.be(undefined);
+        expect(token.type).to.be(ol.expression.TokenType.EOF);
+        expect(token.index).to.be(4);
+      });
    });

    it('parses numeric literal expressions', function() {
@@ -50,7 +62,13 @@ describe('ol.expression.parse', function() {
    it('throws on invalid number', function() {
      expect(function() {
        ol.expression.parse('.42eX');
-      }).throwException();
+      }).throwException(function(err) {
+        expect(err).to.be.an(Error);
+        var token = err.token;
+        expect(token).not.to.be(undefined);
+        expect(token.value).to.be('X');
+        expect(token.index).to.be(4);
+      });
    });

    it('parses boolean literal expressions', function() {
@@ -87,7 +105,13 @@ describe('ol.expression.parse', function() {
    it('throws on invalid member expression', function() {
      expect(function() {
        ol.expression.parse('foo.4bar');
-      }).throwException();
+      }).throwException(function(err) {
+        expect(err).to.be.an(Error);
+        var token = err.token;
+        expect(token).not.to.be(undefined);
+        expect(token.value).to.be('b');
+        expect(token.index).to.be(5);
+      });
    });

    it('parses call expressions with literal arguments', function() {
@@ -106,7 +130,13 @@ describe('ol.expression.parse', function() {
    it('throws on calls with unterminated arguments', function() {
      expect(function() {
        ol.expression.parse('foo(42,)');
-      }).throwException();
+      }).throwException(function(err) {
+        expect(err).to.be.an(Error);
+        var token = err.token;
+        expect(token).not.to.be(undefined);
+        expect(token.value).to.be(')');
+        expect(token.index).to.be(7);
+      });
    });

  });
@@ -263,7 +293,13 @@ describe('ol.expression.parse', function() {
    it('throws for invalid spacing with <=', function() {
      expect(function() {
        ol.expression.parse(' foo< = 10 ');
-      }).throwException();
+      }).throwException(function(err) {
+        expect(err).to.be.an(Error);
+        var token = err.token;
+        expect(token).not.to.be(undefined);
+        expect(token.value).to.be('=');
+        expect(token.index).to.be(6);
+      });
    });

    it('parses >= operator', function() {
@@ -285,7 +321,13 @@ describe('ol.expression.parse', function() {
    it('throws for invalid spacing with >=', function() {
      expect(function() {
        ol.expression.parse(' 10 > =foo ');
-      }).throwException();
+      }).throwException(function(err) {
+        expect(err).to.be.an(Error);
+        var token = err.token;
+        expect(token).not.to.be(undefined);
+        expect(token.value).to.be('=');
+        expect(token.index).to.be(6);
+      });
    });

  });
@@ -312,7 +354,13 @@ describe('ol.expression.parse', function() {
    it('throws for invalid spacing with ==', function() {
      expect(function() {
        ol.expression.parse(' 10 = =foo ');
-      }).throwException();
+      }).throwException(function(err) {
+        expect(err).to.be.an(Error);
+        var token = err.token;
+        expect(token).not.to.be(undefined);
+        expect(token.value).to.be('=');
+        expect(token.index).to.be(4);
+      });
    });

    it('parses != operator', function() {
@@ -334,7 +382,13 @@ describe('ol.expression.parse', function() {
    it('throws for invalid spacing with !=', function() {
      expect(function() {
        ol.expression.parse(' 10! =foo ');
-      }).throwException();
+      }).throwException(function(err) {
+        expect(err).to.be.an(Error);
+        var token = err.token;
+        expect(token).not.to.be(undefined);
+        expect(token.value).to.be('!');
+        expect(token.index).to.be(3);
+      });
    });

    it('parses === operator', function() {
@@ -356,7 +410,13 @@ describe('ol.expression.parse', function() {
    it('throws for invalid spacing with ===', function() {
      expect(function() {
        ol.expression.parse(' 10 = == foo ');
-      }).throwException();
+      }).throwException(function(err) {
+        expect(err).to.be.an(Error);
+        var token = err.token;
+        expect(token).not.to.be(undefined);
+        expect(token.value).to.be('=');
+        expect(token.index).to.be(4);
+      });
    });

    it('parses !== operator', function() {
@@ -378,7 +438,13 @@ describe('ol.expression.parse', function() {
    it('throws for invalid spacing with !==', function() {
      expect(function() {
        ol.expression.parse(' 10 != = foo ');
-      }).throwException();
+      }).throwException(function(err) {
+        expect(err).to.be.an(Error);
+        var token = err.token;
+        expect(token).not.to.be(undefined);
+        expect(token.value).to.be('=');
+        expect(token.index).to.be(7);
+      });
    });
  });

@@ -411,7 +477,13 @@ describe('ol.expression.parse', function() {
    it('throws for invalid spacing with &&', function() {
      expect(function() {
        ol.expression.parse('true & & false');
-      }).throwException();
+      }).throwException(function(err) {
+        expect(err).to.be.an(Error);
+        var token = err.token;
+        expect(token).not.to.be(undefined);
+        expect(token.value).to.be('&');
+        expect(token.index).to.be(5);
+      });
    });

    it('parses || operator', function() {
@@ -435,7 +507,13 @@ describe('ol.expression.parse', function() {
    it('throws for invalid spacing with ||', function() {
      expect(function() {
        ol.expression.parse('true | | false');
-      }).throwException();
+      }).throwException(function(err) {
+        expect(err).to.be.an(Error);
+        var token = err.token;
+        expect(token).not.to.be(undefined);
+        expect(token.value).to.be('|');
+        expect(token.index).to.be(5);
+      });
    });

  });
@@ -468,3 +546,4 @@ goog.require('ol.expression.Logical');
 goog.require('ol.expression.Math');
 goog.require('ol.expression.Member');
 goog.require('ol.expression.Not');
+goog.require('ol.expression.TokenType');
--- a/test/spec/ol/expression/lexer.test.js
+++ b/test/spec/ol/expression/lexer.test.js
@@ -378,7 +378,13 @@ describe('ol.expression.Lexer', function() {
    it('throws on unterminated double quote', function() {
      expect(function() {
        scan('"never \'ending\' string');
-      }).to.throwException();
+      }).to.throwException(function(err) {
+        expect(err).to.be.an(Error);
+        var token = err.token;
+        expect(token).not.to.be(undefined);
+        expect(token.type).to.be(ol.expression.TokenType.EOF);
+        expect(token.index).to.be(22);
+      });
    });

    it('parses single quoted string', function() {
@@ -426,7 +432,13 @@ describe('ol.expression.Lexer', function() {
    it('throws on unterminated single quote', function() {
      expect(function() {
        scan('\'never "ending" string');
-      }).to.throwException();
+      }).to.throwException(function(err) {
+        expect(err).to.be.an(Error);
+        var token = err.token;
+        expect(token).not.to.be(undefined);
+        expect(token.type).to.be(ol.expression.TokenType.EOF);
+        expect(token.index).to.be(22);
+      });
    });

  });