Add test-suite using SlimerJS

.travis.yml

@@ -1,9 +1,13 @@
+env:
+  - DISPLAY=:99.0
+
 before_install:
   - "sudo pip install -r requirements.txt"
   - "npm install -g npm && npm install"
 
 before_script:
   - "rm src/ol/renderer/webgl/*shader.js"
+  - "sh -e /etc/init.d/xvfb start"
 
 script: "./build.py ci"
 

build.py

@@ -145,6 +145,10 @@ SPEC = [path
         for path in ifind('test/spec')
         if path.endswith('.js')]
 
+SPEC_RENDERING = [path
+        for path in ifind('test_rendering/spec')
+        if path.endswith('.js')]
+
 TASKS = [path
          for path in ifind('tasks')
          if path.endswith('.js')]
@@ -172,7 +176,7 @@ def report_sizes(t):
 virtual('default', 'build')
 
 
-virtual('ci', 'lint', 'build', 'test',
+virtual('ci', 'lint', 'build', 'test', 'test-rendering',
     'build/examples/all.combined.js', 'check-examples', 'apidoc')
 
 
@@ -231,19 +235,28 @@ for glsl_src in GLSL_SRC:
     shader_src_helper(glsl_src)
 
 
-@target('build/test/requireall.js', SPEC)
-def build_test_requireall_js(t):
+def build_requires(task):
     requires = set()
-    for dependency in t.dependencies:
+    for dependency in task.dependencies:
         for line in open(dependency, 'rU'):
             match = re.match(r'goog\.provide\(\'(.*)\'\);', line)
             if match:
                 requires.add(match.group(1))
-    with open(t.name, 'wb') as f:
+    with open(task.name, 'wb') as f:
         for require in sorted(requires):
             f.write('goog.require(\'%s\');\n' % (require,))
 
 
+@target('build/test_requires.js', SPEC)
+def build_test_requires(t):
+  build_requires(t)
+
+
+@target('build/test_rendering_requires.js', SPEC_RENDERING)
+def build_test_rendering_requires(t):
+  build_requires(t)
+
+
 virtual('build-examples', 'examples', 'build/examples/all.combined.js',
         EXAMPLES_COMBINED)
 
@@ -389,7 +402,8 @@ def examples_star_combined_js(name, match):
     return Target(name, action=action, dependencies=dependencies)
 
 
-@target('serve', 'examples', NPM_INSTALL)
+@target('serve', 'examples', 'build/test_requires.js', 'build/test_rendering_requires.js',
+        NPM_INSTALL)
 def serve(t):
     t.run('node', 'tasks/serve.js')
 
@@ -398,7 +412,8 @@ virtual('lint', 'build/lint-timestamp', 'build/check-requires-timestamp',
     'build/check-whitespace-timestamp', 'jshint')
 
 
-@target('build/lint-timestamp', SRC, EXAMPLES_SRC, SPEC, precious=True)
+@target('build/lint-timestamp', SRC, EXAMPLES_SRC, SPEC, SPEC_RENDERING,
+        precious=True)
 def build_lint_src_timestamp(t):
     t.run('%(GJSLINT)s',
           '--jslint_error=all',
@@ -409,8 +424,8 @@ def build_lint_src_timestamp(t):
 
 virtual('jshint', 'build/jshint-timestamp')
 
-@target('build/jshint-timestamp', SRC, EXAMPLES_SRC, SPEC, TASKS,
-        NPM_INSTALL, precious=True)
+@target('build/jshint-timestamp', SRC, EXAMPLES_SRC, SPEC, SPEC_RENDERING,
+        TASKS, NPM_INSTALL, precious=True)
 def build_jshint_timestamp(t):
     t.run(variables.JSHINT, '--verbose', t.newer(t.dependencies))
     t.touch()
@@ -439,7 +454,8 @@ def _strip_comments(lines):
                 yield lineno, line
 
 
-@target('build/check-requires-timestamp', SRC, EXAMPLES_SRC, SHADER_SRC, SPEC)
+@target('build/check-requires-timestamp', SRC, EXAMPLES_SRC, SHADER_SRC,
+        SPEC, SPEC_RENDERING)
 def build_check_requires_timestamp(t):
     unused_count = 0
     all_provides = set()
@@ -580,7 +596,7 @@ def build_check_requires_timestamp(t):
 
 
 @target('build/check-whitespace-timestamp', SRC, EXAMPLES_SRC,
-        SPEC, JSDOC_SRC, precious=True)
+        SPEC, SPEC_RENDERING, JSDOC_SRC, precious=True)
 def build_check_whitespace_timestamp(t):
     CR_RE = re.compile(r'\r')
     LEADING_WHITESPACE_RE = re.compile(r'\s+')
@@ -720,7 +736,7 @@ def check_examples(t):
         sys.exit(1)
 
 
-@target('test', NPM_INSTALL, phony=True)
+@target('test', NPM_INSTALL, 'build/test_requires.js', phony=True)
 def test(t):
     t.run('node', 'tasks/test.js')
 
@@ -730,6 +746,16 @@ def test_coverage(t):
     t.run('node', 'tasks/test-coverage.js')
 
 
+@target('test-rendering', 'build/test_rendering_requires.js',
+        NPM_INSTALL, phony=True)
+def test_rendering(t):
+    # create a temp. profile to run the tests with WebGL
+    tmp_profile_dir = 'build/slimerjs-profile'
+    t.rm_rf(tmp_profile_dir)
+    t.cp_r('test_rendering/slimerjs-profile', tmp_profile_dir)
+    t.run('node', 'tasks/test-rendering.js')
+
+
 @target('fixme', phony=True)
 def find_fixme(t):
     regex = re.compile('FIXME|TODO')
@@ -794,6 +820,7 @@ The most common targets are:
                      CSS. This is also the default build target which runs when
                      no target is specified.
   test             - Runs the testsuite and displays the results.
+  test-rendering   - Runs the rendering testsuite and displays the results.
   check            - Runs the lint-target, builds some OpenLayers files, and
                      then runs test. Many developers call this target often
                      while working on the code.
@@ -803,9 +830,9 @@ Other less frequently used targets are:
   apidoc           - Builds the API-Documentation using JSDoc3.
   ci               - Builds all examples in various modes and usually takes a
                      long time to finish. This target calls the following
-                     targets: lint, build, build-all, test, build-examples,
-                     check-examples and apidoc. This is the target run on
-                     Travis CI.
+                     targets: lint, build, build-all, test, test-rendering,
+                     build-examples, check-examples and apidoc. This is the
+                     target run on Travis CI.
   test-coverage    - Generates a test coverage report in the coverage folder.
   reallyclean      - Remove untracked files from the repository.
   checkdeps        - Checks whether all required development software is

package.json

@@ -53,7 +53,9 @@
     "mocha-phantomjs": "3.5.1",
     "phantomjs": "1.9.10",
     "proj4": "2.3.3",
-    "sinon": "1.10.3"
+    "sinon": "1.10.3",
+    "slimerjs-edge": "0.10.0-pre-2",
+    "resemblejs": "1.2.0"
   },
   "ext": [
     "rbush"

tasks/serve.js

@@ -23,7 +23,10 @@ var createServer = exports.createServer = function(callback) {
     lib: [
       'src/**/*.js',
       'build/ol.ext/*.js',
-      'test/spec/**/*.test.js'
+      'test/spec/**/*.test.js',
+      'test_rendering/spec/**/*.test.js',
+      'build/test_requires.js',
+      'build/test_rendering_requires.js'
     ],
     main: 'examples/*.js'
   });
@@ -41,12 +44,21 @@ var createServer = exports.createServer = function(callback) {
       getMain: function(req) {
         var main;
         var query = url.parse(req.url, true).query;
-        if (query.id) {
-          var referer = req.headers.referer;
-          if (referer) {
-            var from = path.join(process.cwd(),
-                path.dirname(url.parse(referer).pathname));
-            main = path.resolve(from, query.id + '.js');
+        var referer = req.headers.referer;
+        var pathName = url.parse(referer).pathname;
+        if (pathName.indexOf('/test/') === 0) {
+          main = path.resolve(
+            path.join(process.cwd(), 'build'), 'test_requires.js');
+        } else if (pathName.indexOf('/test_rendering/') === 0) {
+          main = path.resolve(
+            path.join(process.cwd(), 'build'), 'test_rendering_requires.js');
+        } else {
+          if (query.id) {
+            if (referer) {
+              var from = path.join(process.cwd(),
+                  path.dirname(url.parse(referer).pathname));
+              main = path.resolve(from, query.id + '.js');
+            }
           }
         }
         return main;

tasks/test-coverage.js

@@ -13,7 +13,7 @@ var wrench = require('wrench');
 var path = require('path');
 var glob = require('glob');
 
-var runTestsuite = require('./test');
+var runTestsuite = require('./test').runTests;
 
 // setup some pathes
 var dir = path.join(__dirname, '../src');

tasks/test-rendering.js

@@ -0,0 +1,65 @@
+/**
+ * This task starts a dev server that provides a script loader for OpenLayers
+ * and Closure Library and runs rendering tests in SlimerJS.
+ */
+
+var fs = require('fs');
+var path = require('path');
+var spawn = require('child_process').spawn;
+
+var slimerjs = require('slimerjs-edge');
+
+var serve = require('./serve');
+var listen = require('./test').listen;
+
+
+/**
+ * Create the debug server and run tests.
+ */
+serve.createServer(function(err, server) {
+  if (err) {
+    process.stderr.write(err.message + '\n');
+    process.exit(1);
+  }
+
+  listen(3001, 3005, server, function(err) {
+    if (err) {
+      process.stderr.write('Server failed to start: ' + err.message + '\n');
+      process.exit(1);
+    }
+
+    var address = server.address();
+    var url = 'http://' + address.address + ':' + address.port;
+    var profile = path.join(__dirname, '../build/slimerjs-profile');
+    var args = [
+      '-profile',
+      profile,
+      path.join(__dirname,
+          '../test_rendering/test.js'),
+      url + '/test_rendering/index.html'
+    ];
+
+    var child = spawn(slimerjs.path, args, {stdio: 'inherit'});
+    child.on('exit', function(code) {
+      // FIXME SlimerJS has a problem with returning the correct return
+      // code when using a custom profile, see
+      // https://github.com/laurentj/slimerjs/issues/333
+      // as a work-around we are currently reading the return code from
+      // a file created in the profile directory.
+      // if this issue is fixed we should use the npm package 'slimerjs'
+      // instead of the nightly build 'slimerjs-edge'.
+      var exitstatus = path.join(profile, 'exitstatus');
+      fs.readFile(exitstatus, {encoding: 'utf-8'}, function(err, data) {
+        if (err) {
+          process.stderr.write(
+              'Error getting the exit status of SlimerJS' + '\n');
+          process.stderr.write(err);
+          process.exit(1);
+        } else {
+          process.exit(data);
+        }
+      });
+    });
+  });
+
+});

tasks/test.js

@@ -81,6 +81,9 @@ if (require.main === module) {
   });
 }
 
-module.exports = runTests;
+module.exports = {
+  runTests: runTests,
+  listen: listen
+};
 
 

test/test-extensions.js

@@ -1,8 +1,13 @@
 // FIXME remove afterLoadXml as it uses the wrong XML parser on IE9
 
-// helper functions for async testing
+// helper functions for async testing and other utility functions.
 (function(global) {
 
+  /**
+   * The default tolerance for image comparisons.
+   */
+  global.IMAGE_TOLERANCE = 1.5;
+
   function afterLoad(type, path, next) {
     var client = new XMLHttpRequest();
     client.open('GET', path, true);
@@ -326,5 +331,102 @@
     return this;
   };
 
+  global.createMapDiv = function(width, height) {
+    var target = document.createElement('div');
+    var style = target.style;
+    style.position = 'absolute';
+    style.left = '-1000px';
+    style.top = '-1000px';
+    style.width = width + 'px';
+    style.height = height + 'px';
+    document.body.appendChild(target);
+
+    return target;
+  };
+
+  global.disposeMap = function(map) {
+    var target = map.getTarget();
+    map.setTarget(null);
+    goog.dispose(map);
+    document.body.removeChild(target);
+  };
+
+  global.assertWebGL = function(map) {
+    if(!ol.has.WEBGL) {
+      expect().fail('No WebGL support!');
+    }
+  };
+
+  function resembleCanvas(canvas, referenceImage, tolerance, done) {
+    if (window.showMap) {
+      document.getElementById('debug').appendChild(canvas);
+    }
+
+    resemble(referenceImage)
+      .compareTo(canvas.getContext('2d').getImageData(
+          0, 0, canvas.width, canvas.height))
+      .onComplete(function(data) {
+        if(!data.isSameDimensions) {
+          expect().fail(
+            'The dimensions of the reference image and ' +
+            'the test canvas are not the same.');
+        }
+
+        if (data.misMatchPercentage > tolerance) {
+          if (window.showDiff) {
+            var diffImage = new Image();
+            diffImage.src = data.getImageDataUrl();
+            document.getElementById('debug').appendChild(diffImage);
+          }
+          expect(data.misMatchPercentage).to.be.below(tolerance);
+        }
+        done();
+    });
+  };
+
+  function expectResembleCanvas(map, referenceImage, tolerance, done) {
+    map.render();
+    map.on('postcompose', function(event) {
+      var canvas = event.context.canvas;
+      resembleCanvas(canvas, referenceImage, tolerance, done);
+    });
+  };
+
+  function expectResembleWebGL(map, referenceImage, tolerance, done) {
+    map.render();
+    map.on('postcompose', function(event) {
+      var webglCanvas = map.getTarget().children[0].children[0];
+      expect(webglCanvas).to.be.a(HTMLCanvasElement);
+
+      // draw the WebGL canvas on a new canvas, because we can not create
+      // a 2d context for that canvas because there is already a webgl context.
+      var canvas = document.createElement('canvas');
+      canvas.width = webglCanvas.width;
+      canvas.height = webglCanvas.height;
+      canvas.getContext('2d').drawImage(webglCanvas, 0, 0,
+          webglCanvas.width, webglCanvas.height);
+
+      resembleCanvas(canvas, referenceImage, tolerance, done);
+    });
+  };
+
+  /**
+   * Assert that the given map resembles a reference image.
+   *
+   * @param {ol.Map} map A map using the canvas renderer.
+   * @param {string} referenceImage Path to the reference image.
+   * @param {number} tolerance The accepted mismatch tolerance.
+   * @param {function} done A callback to indicate that the test is done.
+   */
+  global.expectResemble = function(map, referenceImage, tolerance, done) {
+    if (map.getRenderer() instanceof ol.renderer.canvas.Map) {
+      expectResembleCanvas(map, referenceImage, tolerance, done);
+    } else if (map.getRenderer() instanceof ol.renderer.webgl.Map) {
+      expectResembleWebGL(map, referenceImage, tolerance, done);
+    } else {
+      expect().fail(
+        'resemble only works with the canvas and WebGL renderer.');
+    }
+  };
 
 })(this);

test_rendering/README.md

@@ -0,0 +1,24 @@
+# Rendering tests
+
+This directory contains rendering tests which compare a rendered map with a
+reference image using [resemble.js](http://huddle.github.io/Resemble.js/).
+
+Similar to the unit tests, there are two ways to run the tests: directly in the
+browser or using [SlimerJS](http://slimerjs.org/) from the command-line.
+
+To run the tests in the browser, make sure the development server is running
+(`./build.py serve`) and open the URL
+[http://localhost:3000/test_rendering/index.html](http://localhost:3000/test_rendering/index.html).
+
+From the command-line the tests can be run with the build target `./build.py test-rendering`.
+
+## Adding new tests
+When creating a new test case, a reference image has to be created. By appending `?generate`
+to the URL, a canvas with the rendered map will be shown on the page. Then the reference
+image can simply be created with a right-click and "Save image as".
+
+It is recommended to only run a single test case when generating the reference image.
+
+## Image difference
+When a test fails, an image showing the difference between the reference image and the
+rendered map can be displayed by appending `?showdiff` to the URL.

test_rendering/index.html

@@ -0,0 +1,93 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>OL Rendering Test Runner</title>
+  <meta charset="utf-8">
+  <link rel="stylesheet" type="text/css" href="../node_modules/mocha/mocha.css">
+</head>
+
+<body>
+  <div id="mocha"></div>
+
+  <script type="text/javascript" src="../node_modules/jquery/dist/jquery.min.js"></script>
+  <script type="text/javascript" src="../node_modules/expect.js/index.js"></script>
+  <script type="text/javascript" src="../node_modules/sinon/pkg/sinon.js"></script>
+  <script type="text/javascript" src="../node_modules/mocha/mocha.js"></script>
+  <script type="text/javascript" src="../node_modules/proj4/dist/proj4.js"></script>
+  <script type="text/javascript" src="../node_modules/resemblejs/resemble.js"></script>
+  <script type="text/javascript" src="../test/test-extensions.js"></script>
+  <script>
+    mocha.setup({
+      ui: 'bdd',
+      bail: false
+    });
+  </script>
+
+  <!-- This script is provided by the debug server (start with `build.py serve`) -->
+  <script type="text/javascript" src="loader.js"></script>
+
+  <script type="text/javascript">
+
+    /**
+     * The goog.dom.ViewportSizeMonitor (used in map.js) creates a global leak
+     * by setting goog.UID_PROPERTY_ on the monitored window.  In order to test
+     * that we don't have other global leaks, we preemptively set the property
+     * so Mocha can compare the global before and after our tests.
+     *
+     * In addition, calling goog.events.listen on the global object (as done
+     * in deviceorientation.js) creates a second leak by setting
+     * goog.events.LISTENER_MAP_PROP_ on the global object.
+     *
+     * We preemptively set both of these properties so Mocha can compare the
+     * global before and after tests.  The call to goog.events.listen also
+     * calls goog.getUid.
+     */
+    goog.events.listen(this, 'test', function() {});
+
+    var errors = [];
+    var runner = mocha.run(function(evt) {
+      if (window.callPhantom) {
+        window.callPhantom(errors);
+      }
+    });
+
+    window.showDiff = false;
+    if (window.location.search.indexOf('showdiff') >= 0) {
+      // show a difference image if '?showdiff' is in the URL
+      window.showDiff = true;
+    }
+
+    window.showMap = false;
+    if (window.location.search.indexOf('generate') >= 0) {
+      // show the canvas of the test map if '?generate' is in the URL
+      window.showMap = true;
+    }
+
+    if (window.console && console.log) {
+      // write stacks to the console for failed tests
+      runner.on('fail', function(test, err) {
+        if (test.duration > test._timeout) {
+          var titles = [];
+          for (var p = test; p; p = p.parent) {
+            if (p.title) {
+              titles.unshift(p.title);
+            }
+          }
+          console.log('Test timed out:', titles.join(' > '));
+        }
+        errors.push({
+          title: test.fullTitle(),
+          errorStack: test.err
+        });
+        console.log(test.fullTitle());
+        console.error(test.err.stack);
+      });
+    }
+  </script>
+  <!--
+    Tests should not depend on any specific markup and should instead create
+    whatever elements are needed (cleaning up when done).
+  -->
+  <div id="debug"></div>
+</body>
+</html>

test_rendering/slimerjs-profile/prefs.js

@@ -0,0 +1,3 @@
+user_pref("webgl.force-enabled", true);
+user_pref("webgl.disabled", false);
+user_pref("webgl.msaa-force", true);

test_rendering/slimerjs-profile/times.json

@@ -0,0 +1,3 @@
+{
+"created": 1427803527460
+}

test_rendering/test.js

@@ -0,0 +1,30 @@
+var url = phantom.args[0];
+var page = require("webpage").create();
+
+var v = slimer.geckoVersion;
+console.log('Gecko: ' + v.major + '.' + v.minor + '.' + v.patch);
+
+page.open(url).
+    then(function(status){
+        if (status == "success") {
+            page.onCallback = function(failedTests) {
+                if (failedTests.length > 0) {
+                    for (var i = 0; i < failedTests.length; i++) {
+                        var test = failedTests[i];
+                        console.log(test.title);
+                        console.error(test.errorStack);
+                        console.log('');
+                    }
+                    console.error(failedTests.length + ' test(s) failed.');
+                } else {
+                    console.log('All tests passed.');
+                }
+                page.close();
+                phantom.exit(failedTests.length === 0 ? 0 : 1);
+            }
+         } else {
+            console.error("The tests could not be started. Is the server running?");
+            page.close();
+            phantom.exit(1);
+         }
+    });