Correct index for tokens

src/ol/expression/lexer.js

@@ -458,7 +458,7 @@ ol.expression.Lexer.prototype.peek = function() {
  */
 ol.expression.Lexer.prototype.scanHexLiteral_ = function(code) {
   var str = '';
-  var start = this.index_;
+  var start = this.index_ - 2;
 
   while (this.index_ < this.length_) {
     if (!this.isHexDigit_(code)) {
@@ -659,7 +659,7 @@ ol.expression.Lexer.prototype.scanOctalLiteral_ = function(code) {
   goog.asserts.assert(this.isOctalDigit_(code));
 
   var str = '0' + String.fromCharCode(code);
-  var start = this.index_;
+  var start = this.index_ - 1;
   this.increment_(1);
 
   while (this.index_ < this.length_) {
@@ -677,7 +677,7 @@ ol.expression.Lexer.prototype.scanOctalLiteral_ = function(code) {
     throw new ol.expression.UnexpectedToken({
       type: ol.expression.TokenType.UNKNOWN,
       value: String.fromCharCode(code),
-      index: this.index_ - 1
+      index: this.index_
     });
   }
 
@@ -812,10 +812,10 @@ ol.expression.Lexer.prototype.scanStringLiteral_ = function(quote) {
       quote === ol.expression.Char.DOUBLE_QUOTE,
       'Strings must start with a quote: ' + String.fromCharCode(quote));
 
+  var start = this.index_;
   this.increment_(1);
 
   var str = '';
-  var start = this.index_;
   var code;
   while (this.index_ < this.length_) {
     code = this.getCurrentCharCode_();

test/spec/ol/expression/lexer.test.js

@@ -97,6 +97,7 @@ describe('ol.expression.Lexer', function() {
       var lexer = new ol.expression.Lexer(source);
       var token = lexer.scanIdentifier_(lexer.getCurrentCharCode_());
       if (!part) {
+        expect(token.index).to.be(0);
         expect(lexer.peek().type).to.be(ol.expression.TokenType.EOF);
       }
       return token;
@@ -187,6 +188,7 @@ describe('ol.expression.Lexer', function() {
     function scan(source) {
       var lexer = new ol.expression.Lexer(source);
       var token = lexer.scanNumericLiteral_(lexer.getCurrentCharCode_());
+      expect(token.index).to.be(0);
       expect(lexer.peek().type).to.be(ol.expression.TokenType.EOF);
       return token;
     }
@@ -197,12 +199,34 @@ describe('ol.expression.Lexer', function() {
       expect(token.type).to.be(ol.expression.TokenType.NUMERIC_LITERAL);
     });
 
+    it('throws for bogus integer', function() {
+      expect(function() {
+        scan('123z');
+      }).throwException(function(err) {
+        expect(err).to.be.an(ol.expression.UnexpectedToken);
+        var token = err.token;
+        expect(token.value).to.be('z');
+        expect(token.index).to.be(3);
+      });
+    });
+
     it('works for float', function() {
       var token = scan('123.456');
       expect(token.value).to.be(123.456);
       expect(token.type).to.be(ol.expression.TokenType.NUMERIC_LITERAL);
     });
 
+    it('throws for bogus float', function() {
+      expect(function() {
+        scan('123.4x4');
+      }).throwException(function(err) {
+        expect(err).to.be.an(ol.expression.UnexpectedToken);
+        var token = err.token;
+        expect(token.value).to.be('x');
+        expect(token.index).to.be(5);
+      });
+    });
+
     it('works with exponent', function() {
       var token = scan('1.234e5');
       expect(token.value).to.be(1.234e5);
@@ -221,18 +245,53 @@ describe('ol.expression.Lexer', function() {
       expect(token.type).to.be(ol.expression.TokenType.NUMERIC_LITERAL);
     });
 
+    it('throws for bogus float', function() {
+      expect(function() {
+        scan('1.234eo4');
+      }).throwException(function(err) {
+        expect(err).to.be.an(ol.expression.UnexpectedToken);
+        var token = err.token;
+        expect(token.value).to.be('o');
+        expect(token.index).to.be(6);
+      });
+    });
+
     it('works with octals', function() {
       var token = scan('02322');
       expect(token.value).to.be(1234);
       expect(token.type).to.be(ol.expression.TokenType.NUMERIC_LITERAL);
     });
 
+    it('throws for bogus octal', function() {
+      // note that this is more strict than most es5 engines
+      expect(function() {
+        scan('02392');
+      }).throwException(function(err) {
+        expect(err).to.be.an(ol.expression.UnexpectedToken);
+        var token = err.token;
+        expect(token.value).to.be('9');
+        expect(token.index).to.be(3);
+      });
+    });
+
     it('works with hex', function() {
       var token = scan('0x4d2');
       expect(token.value).to.be(1234);
       expect(token.type).to.be(ol.expression.TokenType.NUMERIC_LITERAL);
     });
 
+    it('throws for bogus hex', function() {
+      // note that this is more strict than most es5 engines
+      expect(function() {
+        scan('0x4G');
+      }).throwException(function(err) {
+        expect(err).to.be.an(ol.expression.UnexpectedToken);
+        var token = err.token;
+        expect(token.value).to.be('G');
+        expect(token.index).to.be(3);
+      });
+    });
+
   });
 
   describe('#scanPunctuator_()', function() {
@@ -240,6 +299,7 @@ describe('ol.expression.Lexer', function() {
     function scan(source) {
       var lexer = new ol.expression.Lexer(source);
       var token = lexer.scanPunctuator_(lexer.getCurrentCharCode_());
+      expect(token.index).to.be(0);
       expect(lexer.peek().type).to.be(ol.expression.TokenType.EOF);
       return token;
     }
@@ -329,6 +389,7 @@ describe('ol.expression.Lexer', function() {
     function scan(source) {
       var lexer = new ol.expression.Lexer(source);
       var token = lexer.scanStringLiteral_(lexer.getCurrentCharCode_());
+      expect(token.index).to.be(0);
       expect(lexer.peek().type).to.be(ol.expression.TokenType.EOF);
       return token;
     }