Merge pull request #3457 from marcjansen/test-coverage

Integrate istanbul for test coverage

.gitignore

@@ -5,3 +5,4 @@
 /examples/example-list.xml
 /node_modules/
 /dist/
+/coverage/

.travis.yml

@@ -6,3 +6,7 @@ before_script:
   - "rm src/ol/renderer/webgl/*shader.js"
 
 script: "./build.py ci"
+
+after_success:
+  - "npm run test-coverage"
+  - "cat coverage/lcov.info | node ./node_modules/coveralls/bin/coveralls.js"

build.py

@@ -718,6 +718,11 @@ def test(t):
     t.run('node', 'tasks/test.js')
 
 
+@target('test-coverage', NPM_INSTALL, phony=True)
+def test_coverage(t):
+    t.run('node', 'tasks/test-coverage.js')
+
+
 @target('fixme', phony=True)
 def find_fixme(t):
     regex = re.compile('FIXME|TODO')
@@ -794,6 +799,7 @@ Other less frequently used targets are:
                      targets: lint, build, build-all, test, build-examples,
                      check-examples and apidoc. This is the target run on
                      Travis CI.
+  test-coverage    - Generates a test coverage report in the coverage folder.
   reallyclean      - Remove untracked files from the repository.
   checkdeps        - Checks whether all required development software is
                      installed on your machine.

package.json

@@ -12,7 +12,8 @@
     "install": "node tasks/install.js",
     "postinstall": "closure-util update",
     "start": "node tasks/serve.js",
-    "test": "node tasks/test.js"
+    "test": "node tasks/test.js",
+    "test-coverage": "node tasks/test-coverage.js"
   },
   "main": "dist/ol.js",
   "repository": {
@@ -27,17 +28,21 @@
     "async": "0.9.0",
     "closure-util": "1.3.0",
     "fs-extra": "0.12.0",
+    "glob": "5.0.3",
     "graceful-fs": "3.0.2",
     "htmlparser2": "3.7.3",
     "jsdoc": "3.3.0-alpha9",
     "nomnom": "1.8.0",
     "rbush": "1.3.5",
     "temp": "0.8.1",
-    "walk": "2.3.4"
+    "walk": "2.3.4",
+    "wrench": "1.5.8"
   },
   "devDependencies": {
     "clean-css": "2.2.16",
+    "coveralls": "2.11.2",
     "expect.js": "0.3.1",
+    "istanbul": "0.3.13",
     "jquery": "2.1.1",
     "jshint": "2.5.6",
     "mocha": "1.21.5",

tasks/test-coverage.js

@@ -0,0 +1,176 @@
+/**
+ * This task instruments our source code with istanbul, runs the test suite
+ * on the instrumented source and collects the coverage data. It then creates
+ * test coverage reports.
+ *
+ * TODO This can be improved in style. We should possibly rewrite it and use
+ *      async.waterfall.
+ */
+
+var fs = require('fs');
+var istanbul = require('istanbul');
+var wrench = require('wrench');
+var path = require('path');
+var glob = require('glob');
+
+var runTestsuite = require('./test');
+
+// setup some pathes
+var dir = path.join(__dirname, '../src');
+var backupDir = path.join(__dirname, '../src-backup');
+var instrumentedDir = path.join(__dirname, '../src-instrumented');
+var coverageDir = path.join(__dirname, '../coverage');
+
+// The main players in the coverage generation via istanbul
+var instrumenter = new istanbul.Instrumenter();
+var reporter = new istanbul.Reporter(false, coverageDir);
+var collector = new istanbul.Collector();
+
+// General options used for the resource shuffling / directory copying
+var copyOpts = {
+  // Whether to overwrite existing directory or not
+  forceDelete: true,
+  // Whether to copy hidden Unix files or not (preceding .)
+  excludeHiddenUnix: false,
+  // If we're overwriting something and the file already exists, keep the
+  // existing
+  preserveFiles: false,
+  // Preserve the mtime and atime when copying files
+  preserveTimestamps: true,
+  // Whether to follow symlinks or not when copying files
+  inflateSymlinks: false
+};
+
+/**
+ * A small utility method printing out log messages.
+ */
+var log = function(msg){
+  process.stdout.write(msg + '\n');
+};
+
+
+/**
+ * A utility method to recursively delete a non-empty folder.
+ *
+ * See http://www.geedew.com/remove-a-directory-that-is-not-empty-in-nodejs/
+ * adjusted to use path.join
+ */
+var deleteFolderRecursive = function(p) {
+  if( fs.existsSync(p) ) {
+    fs.readdirSync(p).forEach(function(file,index){
+      var curPath = path.join(p, file);
+      if(fs.lstatSync(curPath).isDirectory()) { // recurse
+        deleteFolderRecursive(curPath);
+      } else { // delete file
+        fs.unlinkSync(curPath);
+      }
+    });
+    fs.rmdirSync(p);
+  }
+};
+
+/**
+ * Creates folders for backup and instrumentation and copies the contents of the
+ * current src folder into them.
+ */
+var setupBackupAndInstrumentationDir = function(){
+  if (!fs.existsSync(backupDir)) {
+    log('• create directory for backup of src: ' + backupDir);
+    fs.mkdirSync(backupDir);
+  }
+
+  if (!fs.existsSync(instrumentedDir)) {
+    log('• create directory for instrumented src: ' + instrumentedDir);
+    fs.mkdirSync(instrumentedDir);
+  }
+
+  log('• copy src files to backup folder');
+  wrench.copyDirSyncRecursive(dir, backupDir, copyOpts);
+  log('• copy src files to instrumentation folder');
+  wrench.copyDirSyncRecursive(dir, instrumentedDir, copyOpts);
+};
+
+/**
+ * Reverts the changes done in setupBackupAndInstrumentationDir, copies the
+ * backup over the src directory and removes the instrumentation and backup
+ * directory.
+ */
+var revertBackupAndInstrumentationDir = function(){
+  log('• copy original src back to src folder');
+  wrench.copyDirSyncRecursive(backupDir, dir, copyOpts);
+  log('• delete backup directory');
+  deleteFolderRecursive(backupDir);
+  log('• delete instrumentation directory');
+  deleteFolderRecursive(instrumentedDir);
+};
+
+
+/**
+ * Callback for when runTestsuite() has finished.
+ */
+var collectAndWriteCoverageData = function(code) {
+  log('• collect data from coverage.json');
+
+  var coverageFile = path.join(__dirname,'../coverage/coverage.json');
+  var coverageJson = JSON.parse(fs.readFileSync(coverageFile, 'utf8'));
+  collector.add(coverageJson);
+
+  reporter.addAll(['lcovonly','html']);
+
+  revertBackupAndInstrumentationDir();
+
+  log('• write report from collected data');
+  reporter.write(collector, true, function () {
+    process.exit(0);
+  });
+};
+
+/**
+ * Will instrument all JavaScript files that are passed as second parameter.
+ * This is the callback to the glob call.
+ */
+var foundAllJavaScriptSourceFiles = function(err, files) {
+  if (err) {
+    process.stderr.write(err.message + '\n');
+    process.exit(1);
+  }
+  log('• instrumenting every src file');
+  var cnt = 0;
+  files.forEach(function(file) {
+    cnt++;
+    var content = fs.readFileSync(file, 'utf-8');
+    var outfile = file.replace(/\/src\//, '/src-instrumented/');
+    var instrumented = instrumenter.instrumentSync(content, file);
+    fs.writeFileSync(outfile, instrumented);
+    if (cnt % 10 === 0) {
+      log('  • instrumented ' + cnt + ' files');
+    }
+  });
+  log('  • done. ' + cnt + ' files instrumented');
+  log('• copy instrumented src back to src folder');
+
+  wrench.copyDirSyncRecursive(instrumentedDir, dir, copyOpts);
+
+  log('• run test suite on instrumented code');
+  runTestsuite(true, collectAndWriteCoverageData);
+};
+
+/**
+ * Our main method, first it sets up certain directory, and then it starts the
+ * coverage process by gathering all JavaScript files and then instrumenting
+ * them.
+ */
+var main = function(){
+  setupBackupAndInstrumentationDir();
+  glob(dir + '/**/*.js', {}, foundAllJavaScriptSourceFiles);
+};
+
+
+
+if (require.main === module) {
+  main();
+}
+
+module.exports = main;
+
+

tasks/test.js

@@ -37,33 +37,50 @@ function listen(min, max, server, callback) {
 }
 
 
-/**
- * Create the debug server and run tests.
- */
-serve.createServer(function(err, server) {
-  if (err) {
-    process.stderr.write(err.message + '\n');
-    process.exit(1);
-  }
-
-  listen(3001, 3005, server, function(err) {
+function runTests(includeCoverage, callback) {
+  /**
+   * Create the debug server and run tests.
+   */
+  serve.createServer(function(err, server) {
     if (err) {
-      process.stderr.write('Server failed to start: ' + err.message + '\n');
+      process.stderr.write(err.message + '\n');
       process.exit(1);
     }
 
-    var address = server.address();
-    var url = 'http://' + address.address + ':' + address.port;
-    var args = [
-      path.join(__dirname,
-          '../node_modules/mocha-phantomjs/lib/mocha-phantomjs.coffee'),
-      url + '/test/index.html'
-    ];
+    listen(3001, 3005, server, function(err) {
+      if (err) {
+        process.stderr.write('Server failed to start: ' + err.message + '\n');
+        process.exit(1);
+      }
+      var address = server.address();
+      var url = 'http://' + address.address + ':' + address.port;
+      var args = [
+        path.join(
+          __dirname,
+          '../node_modules/mocha-phantomjs/lib/mocha-phantomjs.coffee'
+        ),
+        url + '/test/index.html'
+      ];
 
-    var child = spawn(phantomjs.path, args, {stdio: 'inherit'});
-    child.on('exit', function(code) {
-      process.exit(code);
+      if (includeCoverage) {
+        args.push('spec', '{"hooks": "' +
+          path.join(__dirname, '../test/phantom_hooks.js') + '"}');
+      }
+
+      var child = spawn(phantomjs.path, args, {stdio: 'inherit'});
+      child.on('exit', function(code) {
+        callback(code);
+      });
     });
   });
+}
+
+if (require.main === module) {
+  runTests(false, function(code){
+    process.exit(code);
+  });
+}
+
+module.exports = runTests;
+
 
-});

test/phantom_hooks.js

@@ -0,0 +1,15 @@
+module.exports = {
+  afterEnd: function(runner) {
+    var fs = require('fs');
+    var coverage = runner.page.evaluate(function() {
+      return window.__coverage__;
+    });
+
+    if (coverage) {
+      console.log('Writing coverage to coverage/coverage.json');
+      fs.write('coverage/coverage.json', JSON.stringify(coverage), 'w');
+    } else {
+      console.log('No coverage data generated');
+    }
+  }
+};