From 8e0c21eb580bbd7a659fa5088b9c145030f9b872 Mon Sep 17 00:00:00 2001 From: tsauerwein Date: Thu, 2 Apr 2015 12:11:03 +0200 Subject: [PATCH] Add test-suite using SlimerJS --- .travis.yml | 4 + build.py | 57 ++++++++--- package.json | 4 +- tasks/serve.js | 26 ++++-- tasks/test-coverage.js | 2 +- tasks/test-rendering.js | 65 +++++++++++++ tasks/test.js | 5 +- test/test-extensions.js | 104 ++++++++++++++++++++- test_rendering/README.md | 24 +++++ test_rendering/index.html | 93 ++++++++++++++++++ test_rendering/slimerjs-profile/prefs.js | 3 + test_rendering/slimerjs-profile/times.json | 3 + test_rendering/test.js | 30 ++++++ 13 files changed, 394 insertions(+), 26 deletions(-) create mode 100644 tasks/test-rendering.js create mode 100644 test_rendering/README.md create mode 100644 test_rendering/index.html create mode 100644 test_rendering/slimerjs-profile/prefs.js create mode 100755 test_rendering/slimerjs-profile/times.json create mode 100644 test_rendering/test.js diff --git a/.travis.yml b/.travis.yml index 2d6908b488..cf56d6cc8d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,13 @@ +env: + - DISPLAY=:99.0 + before_install: - "sudo pip install -r requirements.txt" - "npm install -g npm && npm install" before_script: - "rm src/ol/renderer/webgl/*shader.js" + - "sh -e /etc/init.d/xvfb start" script: "./build.py ci" diff --git a/build.py b/build.py index 615c5fb183..c27aec4304 100755 --- a/build.py +++ b/build.py @@ -145,6 +145,10 @@ SPEC = [path for path in ifind('test/spec') if path.endswith('.js')] +SPEC_RENDERING = [path + for path in ifind('test_rendering/spec') + if path.endswith('.js')] + TASKS = [path for path in ifind('tasks') if path.endswith('.js')] @@ -172,7 +176,7 @@ def report_sizes(t): virtual('default', 'build') -virtual('ci', 'lint', 'build', 'test', +virtual('ci', 'lint', 'build', 'test', 'test-rendering', 'build/examples/all.combined.js', 'check-examples', 'apidoc') @@ -231,19 +235,28 @@ for glsl_src in GLSL_SRC: shader_src_helper(glsl_src) -@target('build/test/requireall.js', SPEC) -def build_test_requireall_js(t): +def build_requires(task): requires = set() - for dependency in t.dependencies: + for dependency in task.dependencies: for line in open(dependency, 'rU'): match = re.match(r'goog\.provide\(\'(.*)\'\);', line) if match: requires.add(match.group(1)) - with open(t.name, 'wb') as f: + with open(task.name, 'wb') as f: for require in sorted(requires): f.write('goog.require(\'%s\');\n' % (require,)) +@target('build/test_requires.js', SPEC) +def build_test_requires(t): + build_requires(t) + + +@target('build/test_rendering_requires.js', SPEC_RENDERING) +def build_test_rendering_requires(t): + build_requires(t) + + virtual('build-examples', 'examples', 'build/examples/all.combined.js', EXAMPLES_COMBINED) @@ -389,7 +402,8 @@ def examples_star_combined_js(name, match): return Target(name, action=action, dependencies=dependencies) -@target('serve', 'examples', NPM_INSTALL) +@target('serve', 'examples', 'build/test_requires.js', 'build/test_rendering_requires.js', + NPM_INSTALL) def serve(t): t.run('node', 'tasks/serve.js') @@ -398,7 +412,8 @@ virtual('lint', 'build/lint-timestamp', 'build/check-requires-timestamp', 'build/check-whitespace-timestamp', 'jshint') -@target('build/lint-timestamp', SRC, EXAMPLES_SRC, SPEC, precious=True) +@target('build/lint-timestamp', SRC, EXAMPLES_SRC, SPEC, SPEC_RENDERING, + precious=True) def build_lint_src_timestamp(t): t.run('%(GJSLINT)s', '--jslint_error=all', @@ -409,8 +424,8 @@ def build_lint_src_timestamp(t): virtual('jshint', 'build/jshint-timestamp') -@target('build/jshint-timestamp', SRC, EXAMPLES_SRC, SPEC, TASKS, - NPM_INSTALL, precious=True) +@target('build/jshint-timestamp', SRC, EXAMPLES_SRC, SPEC, SPEC_RENDERING, + TASKS, NPM_INSTALL, precious=True) def build_jshint_timestamp(t): t.run(variables.JSHINT, '--verbose', t.newer(t.dependencies)) t.touch() @@ -439,7 +454,8 @@ def _strip_comments(lines): yield lineno, line -@target('build/check-requires-timestamp', SRC, EXAMPLES_SRC, SHADER_SRC, SPEC) +@target('build/check-requires-timestamp', SRC, EXAMPLES_SRC, SHADER_SRC, + SPEC, SPEC_RENDERING) def build_check_requires_timestamp(t): unused_count = 0 all_provides = set() @@ -580,7 +596,7 @@ def build_check_requires_timestamp(t): @target('build/check-whitespace-timestamp', SRC, EXAMPLES_SRC, - SPEC, JSDOC_SRC, precious=True) + SPEC, SPEC_RENDERING, JSDOC_SRC, precious=True) def build_check_whitespace_timestamp(t): CR_RE = re.compile(r'\r') LEADING_WHITESPACE_RE = re.compile(r'\s+') @@ -720,7 +736,7 @@ def check_examples(t): sys.exit(1) -@target('test', NPM_INSTALL, phony=True) +@target('test', NPM_INSTALL, 'build/test_requires.js', phony=True) def test(t): t.run('node', 'tasks/test.js') @@ -730,6 +746,16 @@ def test_coverage(t): t.run('node', 'tasks/test-coverage.js') +@target('test-rendering', 'build/test_rendering_requires.js', + NPM_INSTALL, phony=True) +def test_rendering(t): + # create a temp. profile to run the tests with WebGL + tmp_profile_dir = 'build/slimerjs-profile' + t.rm_rf(tmp_profile_dir) + t.cp_r('test_rendering/slimerjs-profile', tmp_profile_dir) + t.run('node', 'tasks/test-rendering.js') + + @target('fixme', phony=True) def find_fixme(t): regex = re.compile('FIXME|TODO') @@ -794,6 +820,7 @@ The most common targets are: CSS. This is also the default build target which runs when no target is specified. test - Runs the testsuite and displays the results. + test-rendering - Runs the rendering testsuite and displays the results. check - Runs the lint-target, builds some OpenLayers files, and then runs test. Many developers call this target often while working on the code. @@ -803,9 +830,9 @@ Other less frequently used targets are: apidoc - Builds the API-Documentation using JSDoc3. ci - Builds all examples in various modes and usually takes a long time to finish. This target calls the following - targets: lint, build, build-all, test, build-examples, - check-examples and apidoc. This is the target run on - Travis CI. + targets: lint, build, build-all, test, test-rendering, + build-examples, check-examples and apidoc. This is the + target run on Travis CI. test-coverage - Generates a test coverage report in the coverage folder. reallyclean - Remove untracked files from the repository. checkdeps - Checks whether all required development software is diff --git a/package.json b/package.json index b3ad9bb5b2..7e6510ad4a 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,9 @@ "mocha-phantomjs": "3.5.1", "phantomjs": "1.9.10", "proj4": "2.3.3", - "sinon": "1.10.3" + "sinon": "1.10.3", + "slimerjs-edge": "0.10.0-pre-2", + "resemblejs": "1.2.0" }, "ext": [ "rbush" diff --git a/tasks/serve.js b/tasks/serve.js index a7f792d51b..dfed1e5e8d 100644 --- a/tasks/serve.js +++ b/tasks/serve.js @@ -23,7 +23,10 @@ var createServer = exports.createServer = function(callback) { lib: [ 'src/**/*.js', 'build/ol.ext/*.js', - 'test/spec/**/*.test.js' + 'test/spec/**/*.test.js', + 'test_rendering/spec/**/*.test.js', + 'build/test_requires.js', + 'build/test_rendering_requires.js' ], main: 'examples/*.js' }); @@ -41,12 +44,21 @@ var createServer = exports.createServer = function(callback) { getMain: function(req) { var main; var query = url.parse(req.url, true).query; - if (query.id) { - var referer = req.headers.referer; - if (referer) { - var from = path.join(process.cwd(), - path.dirname(url.parse(referer).pathname)); - main = path.resolve(from, query.id + '.js'); + var referer = req.headers.referer; + var pathName = url.parse(referer).pathname; + if (pathName.indexOf('/test/') === 0) { + main = path.resolve( + path.join(process.cwd(), 'build'), 'test_requires.js'); + } else if (pathName.indexOf('/test_rendering/') === 0) { + main = path.resolve( + path.join(process.cwd(), 'build'), 'test_rendering_requires.js'); + } else { + if (query.id) { + if (referer) { + var from = path.join(process.cwd(), + path.dirname(url.parse(referer).pathname)); + main = path.resolve(from, query.id + '.js'); + } } } return main; diff --git a/tasks/test-coverage.js b/tasks/test-coverage.js index 5e1aab20be..8fc0061e43 100644 --- a/tasks/test-coverage.js +++ b/tasks/test-coverage.js @@ -13,7 +13,7 @@ var wrench = require('wrench'); var path = require('path'); var glob = require('glob'); -var runTestsuite = require('./test'); +var runTestsuite = require('./test').runTests; // setup some pathes var dir = path.join(__dirname, '../src'); diff --git a/tasks/test-rendering.js b/tasks/test-rendering.js new file mode 100644 index 0000000000..d440a52e52 --- /dev/null +++ b/tasks/test-rendering.js @@ -0,0 +1,65 @@ +/** + * This task starts a dev server that provides a script loader for OpenLayers + * and Closure Library and runs rendering tests in SlimerJS. + */ + +var fs = require('fs'); +var path = require('path'); +var spawn = require('child_process').spawn; + +var slimerjs = require('slimerjs-edge'); + +var serve = require('./serve'); +var listen = require('./test').listen; + + +/** + * Create the debug server and run tests. + */ +serve.createServer(function(err, server) { + if (err) { + process.stderr.write(err.message + '\n'); + process.exit(1); + } + + listen(3001, 3005, server, function(err) { + if (err) { + process.stderr.write('Server failed to start: ' + err.message + '\n'); + process.exit(1); + } + + var address = server.address(); + var url = 'http://' + address.address + ':' + address.port; + var profile = path.join(__dirname, '../build/slimerjs-profile'); + var args = [ + '-profile', + profile, + path.join(__dirname, + '../test_rendering/test.js'), + url + '/test_rendering/index.html' + ]; + + var child = spawn(slimerjs.path, args, {stdio: 'inherit'}); + child.on('exit', function(code) { + // FIXME SlimerJS has a problem with returning the correct return + // code when using a custom profile, see + // https://github.com/laurentj/slimerjs/issues/333 + // as a work-around we are currently reading the return code from + // a file created in the profile directory. + // if this issue is fixed we should use the npm package 'slimerjs' + // instead of the nightly build 'slimerjs-edge'. + var exitstatus = path.join(profile, 'exitstatus'); + fs.readFile(exitstatus, {encoding: 'utf-8'}, function(err, data) { + if (err) { + process.stderr.write( + 'Error getting the exit status of SlimerJS' + '\n'); + process.stderr.write(err); + process.exit(1); + } else { + process.exit(data); + } + }); + }); + }); + +}); diff --git a/tasks/test.js b/tasks/test.js index c9c8a96988..e644dea235 100644 --- a/tasks/test.js +++ b/tasks/test.js @@ -81,6 +81,9 @@ if (require.main === module) { }); } -module.exports = runTests; +module.exports = { + runTests: runTests, + listen: listen +}; diff --git a/test/test-extensions.js b/test/test-extensions.js index e5fd2a2364..73faf238e3 100644 --- a/test/test-extensions.js +++ b/test/test-extensions.js @@ -1,8 +1,13 @@ // FIXME remove afterLoadXml as it uses the wrong XML parser on IE9 -// helper functions for async testing +// helper functions for async testing and other utility functions. (function(global) { + /** + * The default tolerance for image comparisons. + */ + global.IMAGE_TOLERANCE = 1.5; + function afterLoad(type, path, next) { var client = new XMLHttpRequest(); client.open('GET', path, true); @@ -326,5 +331,102 @@ return this; }; + global.createMapDiv = function(width, height) { + var target = document.createElement('div'); + var style = target.style; + style.position = 'absolute'; + style.left = '-1000px'; + style.top = '-1000px'; + style.width = width + 'px'; + style.height = height + 'px'; + document.body.appendChild(target); + + return target; + }; + + global.disposeMap = function(map) { + var target = map.getTarget(); + map.setTarget(null); + goog.dispose(map); + document.body.removeChild(target); + }; + + global.assertWebGL = function(map) { + if(!ol.has.WEBGL) { + expect().fail('No WebGL support!'); + } + }; + + function resembleCanvas(canvas, referenceImage, tolerance, done) { + if (window.showMap) { + document.getElementById('debug').appendChild(canvas); + } + + resemble(referenceImage) + .compareTo(canvas.getContext('2d').getImageData( + 0, 0, canvas.width, canvas.height)) + .onComplete(function(data) { + if(!data.isSameDimensions) { + expect().fail( + 'The dimensions of the reference image and ' + + 'the test canvas are not the same.'); + } + + if (data.misMatchPercentage > tolerance) { + if (window.showDiff) { + var diffImage = new Image(); + diffImage.src = data.getImageDataUrl(); + document.getElementById('debug').appendChild(diffImage); + } + expect(data.misMatchPercentage).to.be.below(tolerance); + } + done(); + }); + }; + + function expectResembleCanvas(map, referenceImage, tolerance, done) { + map.render(); + map.on('postcompose', function(event) { + var canvas = event.context.canvas; + resembleCanvas(canvas, referenceImage, tolerance, done); + }); + }; + + function expectResembleWebGL(map, referenceImage, tolerance, done) { + map.render(); + map.on('postcompose', function(event) { + var webglCanvas = map.getTarget().children[0].children[0]; + expect(webglCanvas).to.be.a(HTMLCanvasElement); + + // draw the WebGL canvas on a new canvas, because we can not create + // a 2d context for that canvas because there is already a webgl context. + var canvas = document.createElement('canvas'); + canvas.width = webglCanvas.width; + canvas.height = webglCanvas.height; + canvas.getContext('2d').drawImage(webglCanvas, 0, 0, + webglCanvas.width, webglCanvas.height); + + resembleCanvas(canvas, referenceImage, tolerance, done); + }); + }; + + /** + * Assert that the given map resembles a reference image. + * + * @param {ol.Map} map A map using the canvas renderer. + * @param {string} referenceImage Path to the reference image. + * @param {number} tolerance The accepted mismatch tolerance. + * @param {function} done A callback to indicate that the test is done. + */ + global.expectResemble = function(map, referenceImage, tolerance, done) { + if (map.getRenderer() instanceof ol.renderer.canvas.Map) { + expectResembleCanvas(map, referenceImage, tolerance, done); + } else if (map.getRenderer() instanceof ol.renderer.webgl.Map) { + expectResembleWebGL(map, referenceImage, tolerance, done); + } else { + expect().fail( + 'resemble only works with the canvas and WebGL renderer.'); + } + }; })(this); diff --git a/test_rendering/README.md b/test_rendering/README.md new file mode 100644 index 0000000000..292d965bf3 --- /dev/null +++ b/test_rendering/README.md @@ -0,0 +1,24 @@ +# Rendering tests + +This directory contains rendering tests which compare a rendered map with a +reference image using [resemble.js](http://huddle.github.io/Resemble.js/). + +Similar to the unit tests, there are two ways to run the tests: directly in the +browser or using [SlimerJS](http://slimerjs.org/) from the command-line. + +To run the tests in the browser, make sure the development server is running +(`./build.py serve`) and open the URL +[http://localhost:3000/test_rendering/index.html](http://localhost:3000/test_rendering/index.html). + +From the command-line the tests can be run with the build target `./build.py test-rendering`. + +## Adding new tests +When creating a new test case, a reference image has to be created. By appending `?generate` +to the URL, a canvas with the rendered map will be shown on the page. Then the reference +image can simply be created with a right-click and "Save image as". + +It is recommended to only run a single test case when generating the reference image. + +## Image difference +When a test fails, an image showing the difference between the reference image and the +rendered map can be displayed by appending `?showdiff` to the URL. diff --git a/test_rendering/index.html b/test_rendering/index.html new file mode 100644 index 0000000000..b0293d06d0 --- /dev/null +++ b/test_rendering/index.html @@ -0,0 +1,93 @@ + + + + OL Rendering Test Runner + + + + + +
+ + + + + + + + + + + + + + + +
+ + diff --git a/test_rendering/slimerjs-profile/prefs.js b/test_rendering/slimerjs-profile/prefs.js new file mode 100644 index 0000000000..c2deee1837 --- /dev/null +++ b/test_rendering/slimerjs-profile/prefs.js @@ -0,0 +1,3 @@ +user_pref("webgl.force-enabled", true); +user_pref("webgl.disabled", false); +user_pref("webgl.msaa-force", true); diff --git a/test_rendering/slimerjs-profile/times.json b/test_rendering/slimerjs-profile/times.json new file mode 100755 index 0000000000..84e52a5d3e --- /dev/null +++ b/test_rendering/slimerjs-profile/times.json @@ -0,0 +1,3 @@ +{ +"created": 1427803527460 +} diff --git a/test_rendering/test.js b/test_rendering/test.js new file mode 100644 index 0000000000..8343ab26c9 --- /dev/null +++ b/test_rendering/test.js @@ -0,0 +1,30 @@ +var url = phantom.args[0]; +var page = require("webpage").create(); + +var v = slimer.geckoVersion; +console.log('Gecko: ' + v.major + '.' + v.minor + '.' + v.patch); + +page.open(url). + then(function(status){ + if (status == "success") { + page.onCallback = function(failedTests) { + if (failedTests.length > 0) { + for (var i = 0; i < failedTests.length; i++) { + var test = failedTests[i]; + console.log(test.title); + console.error(test.errorStack); + console.log(''); + } + console.error(failedTests.length + ' test(s) failed.'); + } else { + console.log('All tests passed.'); + } + page.close(); + phantom.exit(failedTests.length === 0 ? 0 : 1); + } + } else { + console.error("The tests could not be started. Is the server running?"); + page.close(); + phantom.exit(1); + } + });