From f272350e00d2ee01e3ff01328a883a75b82a1611 Mon Sep 17 00:00:00 2001 From: Tim Schaub Date: Sun, 9 Jun 2013 00:11:41 -0600 Subject: [PATCH] Expose next and peek methods --- src/ol/expression/lexer.js | 109 +++++++++++++------------- test/spec/ol/expression/lexer.test.js | 82 +++++++++++++++++++ 2 files changed, 138 insertions(+), 53 deletions(-) diff --git a/src/ol/expression/lexer.js b/src/ol/expression/lexer.js index 1b37a740c1..b1cefa4475 100644 --- a/src/ol/expression/lexer.js +++ b/src/ol/expression/lexer.js @@ -109,56 +109,6 @@ ol.expression.Lexer = function(source) { }; -/** - * Scan next token. - * - * @return {ol.expression.Token} Next token. - * @private - */ -ol.expression.Lexer.prototype.advance_ = function() { - if (this.index_ >= this.length_) { - return { - type: ol.expression.TokenType.EOF, - value: null - }; - } - var code = this.getCurrentCharCode_(); - - // check for common punctuation - if (code === ol.expression.Char.LEFT_PAREN || - code === ol.expression.Char.RIGHT_PAREN) { - return this.scanPunctuator_(); - } - - // check for string literal - if (code === ol.expression.Char.SINGLE_QUOTE || - code === ol.expression.Char.DOUBLE_QUOTE) { - return this.scanStringLiteral_(); - } - - // check for identifier - if (this.isIdentifierStart_(code)) { - this.scanIdentifier_(); - } - - // check dot punctuation or decimal - if (code === ol.expression.Char.DOT) { - if (this.isDecimalDigit_(this.getCharCode_(1))) { - return this.scanNumericLiteral_(); - } - return this.scanPunctuator_(); - } - - // check for numeric literal - if (this.isDecimalDigit_(code)) { - return this.scanNumericLiteral_(); - } - - // all the rest is punctuation - return this.scanPunctuator_(); -}; - - /** * Increment the current character index. * @@ -374,6 +324,56 @@ ol.expression.Lexer.prototype.getCurrentCharCode_ = function() { }; +/** + * Scan the next token. + * + * @return {ol.expression.Token} Next token. + */ +ol.expression.Lexer.prototype.next = function() { + var code = this.skipWhitespace_(); + + if (this.index_ >= this.length_) { + return { + type: ol.expression.TokenType.EOF, + value: null + }; + } + + // check for common punctuation + if (code === ol.expression.Char.LEFT_PAREN || + code === ol.expression.Char.RIGHT_PAREN) { + return this.scanPunctuator_(); + } + + // check for string literal + if (code === ol.expression.Char.SINGLE_QUOTE || + code === ol.expression.Char.DOUBLE_QUOTE) { + return this.scanStringLiteral_(); + } + + // check for identifier + if (this.isIdentifierStart_(code)) { + return this.scanIdentifier_(); + } + + // check dot punctuation or decimal + if (code === ol.expression.Char.DOT) { + if (this.isDecimalDigit_(this.getCharCode_(1))) { + return this.scanNumericLiteral_(); + } + return this.scanPunctuator_(); + } + + // check for numeric literal + if (this.isDecimalDigit_(code)) { + return this.scanNumericLiteral_(); + } + + // all the rest is punctuation + return this.scanPunctuator_(); +}; + + /** * Scan hex literal as numeric token. * @@ -743,6 +743,7 @@ ol.expression.Lexer.prototype.scanStringLiteral_ = function() { /** * Skip all whitespace. + * @return {number} The character code of the first non-whitespace character. * @private */ ol.expression.Lexer.prototype.skipWhitespace_ = function() { @@ -751,8 +752,11 @@ ol.expression.Lexer.prototype.skipWhitespace_ = function() { code = this.getCurrentCharCode_(); if (this.isWhitespace_(code)) { this.increment_(1); + } else { + break; } } + return code; }; @@ -760,11 +764,10 @@ ol.expression.Lexer.prototype.skipWhitespace_ = function() { * Peek at the next token, but don't advance the index. * * @return {ol.expression.Token} The upcoming token. - * @private */ -ol.expression.Lexer.prototype.peek_ = function() { +ol.expression.Lexer.prototype.peek = function() { var currentIndex = this.index_; - var token = this.advance_(); + var token = this.next(); this.index_ = currentIndex; return token; }; diff --git a/test/spec/ol/expression/lexer.test.js b/test/spec/ol/expression/lexer.test.js index 52eec7ae99..8673b52081 100644 --- a/test/spec/ol/expression/lexer.test.js +++ b/test/spec/ol/expression/lexer.test.js @@ -9,6 +9,88 @@ describe('ol.expression.Lexer', function() { }); }); + describe.only('#next()', function() { + + it('returns one token at a time', function() { + var source = 'foo === "bar"'; + var lexer = new ol.expression.Lexer(source); + + // scan first token + var token = lexer.next(); + expect(token.type).to.be(ol.expression.TokenType.IDENTIFIER); + expect(token.value).to.be('foo'); + + // scan second token + token = lexer.next(); + expect(token.type).to.be(ol.expression.TokenType.PUNCTUATOR); + expect(token.value).to.be('==='); + + // scan third token + token = lexer.next(); + expect(token.type).to.be(ol.expression.TokenType.STRING_LITERAL); + expect(token.value).to.be('bar'); + + // scan again + token = lexer.next(); + expect(token.type).to.be(ol.expression.TokenType.EOF); + + // and again + token = lexer.next(); + expect(token.type).to.be(ol.expression.TokenType.EOF); + }); + + }); + + describe.only('#peek()', function() { + + var lexer; + beforeEach(function() { + lexer = new ol.expression.Lexer('foo > 42 && bar == "chicken"'); + }); + + it('looks ahead without consuming token', function() { + var token = lexer.peek(); + expect(token.type).to.be(ol.expression.TokenType.IDENTIFIER); + expect(token.value).to.be('foo'); + + token = lexer.next(); + expect(token.type).to.be(ol.expression.TokenType.IDENTIFIER); + expect(token.value).to.be('foo'); + }); + + it('works after a couple scans', function() { + var token = lexer.next(); + expect(token.type).to.be(ol.expression.TokenType.IDENTIFIER); + expect(token.value).to.be('foo'); + + token = lexer.next(); + expect(token.type).to.be(ol.expression.TokenType.PUNCTUATOR); + expect(token.value).to.be('>'); + + token = lexer.peek(); + expect(token.type).to.be(ol.expression.TokenType.NUMERIC_LITERAL); + expect(token.value).to.be(42); + + token = lexer.next(); + expect(token.type).to.be(ol.expression.TokenType.NUMERIC_LITERAL); + expect(token.value).to.be(42); + }); + + it('returns the same thing when called multiple times', function() { + var token = lexer.peek(); + expect(token.type).to.be(ol.expression.TokenType.IDENTIFIER); + expect(token.value).to.be('foo'); + + for (var i = 0; i < 10; ++i) { + token = lexer.peek(); + expect(token.type).to.be(ol.expression.TokenType.IDENTIFIER); + expect(token.value).to.be('foo'); + } + }); + + }); + + describe('#scanIdentifier_()', function() { function scan(source) {