Merge pull request #8895 from tschaub/rendering-tests

New rendering tests
2018-11-09 17:46:32 -07:00
parent aab9ff023c 6f238a8d09
commit c43b0cb520
15 changed files with 571 additions and 3 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -2,7 +2,7 @@ version: 2
 jobs:
  build:
    docker:
-      - image: circleci/node:10-browsers
+      - image: circleci/node:latest-browsers

    working_directory: ~/repo

--- a/package.json
+++ b/package.json
@@ -12,7 +12,8 @@
  "scripts": {
    "lint": "eslint tasks test src/ol examples config",
    "pretest": "npm run lint",
-    "test": "npm run karma -- --single-run --log-level error",
+    "test-rendering": "node rendering/test.js",
+    "test": "npm run karma -- --single-run --log-level error && npm run test-rendering -- --force",
    "karma": "karma start test/karma.config.js",
    "serve-examples": "webpack-dev-server --config examples/webpack/config.js --mode development --watch",
    "build-examples": "webpack --config examples/webpack/config.js --mode production",
@@ -56,6 +57,7 @@
    "front-matter": "^3.0.0",
    "fs-extra": "^7.0.0",
    "glob": "^7.1.2",
+    "globby": "^8.0.1",
    "handlebars": "4.0.11",
    "istanbul": "0.4.5",
    "jquery": "3.3.1",
@@ -68,11 +70,14 @@
    "karma-mocha": "1.3.0",
    "karma-sourcemap-loader": "^0.3.7",
    "karma-webpack": "^4.0.0-rc.2",
+    "loglevelnext": "^3.0.0",
    "marked": "0.5.1",
    "mocha": "5.2.0",
    "mustache": "^3.0.0",
    "pixelmatch": "^4.0.2",
+    "pngjs": "^3.3.3",
    "proj4": "2.5.0",
+    "puppeteer": "^1.10.0",
    "rollup": "0.66.6",
    "sinon": "^6.0.0",
    "typescript": "^3.1.0-dev.20180905",
@@ -81,7 +86,9 @@
    "walk": "^2.3.9",
    "webpack": "4.25.1",
    "webpack-cli": "^3.0.8",
-    "webpack-dev-server": "^3.1.4"
+    "webpack-dev-middleware": "^3.4.0",
+    "webpack-dev-server": "^3.1.10",
+    "yargs": "^12.0.2"
  },
  "eslintConfig": {
    "extends": "openlayers",
--- a/rendering/.eslintrc
+++ b/rendering/.eslintrc
@@ -0,0 +1,11 @@
+{
+  "env": {
+    "node": true
+  },
+  "parserOptions": {
+    "ecmaVersion": 2017
+  },
+  "globals": {
+    "render": false
+  }
+}
--- a/rendering/.gitignore
+++ b/rendering/.gitignore
@@ -0,0 +1,2 @@
+actual.png
+pass
--- a/rendering/cases/linestring-style/expected.png
+++ b/rendering/cases/linestring-style/expected.png
--- a/rendering/cases/linestring-style/index.html
+++ b/rendering/cases/linestring-style/index.html
@@ -0,0 +1,22 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <style>
+      html, body {
+        margin: 0;
+        padding: 0;
+        width: 100%;
+        height: 100%;
+      }
+      #map {
+        width: 100%;
+        height: 100%;
+      }
+    </style>
+  </head>
+  <body>
+    <div id="map"></div>
+    <script src="main.js"></script>
+  </body>
+  </script>
+</html>
--- a/rendering/cases/linestring-style/main.js
+++ b/rendering/cases/linestring-style/main.js
@@ -0,0 +1,69 @@
+import Map from '../../../src/ol/Map.js';
+import View from '../../../src/ol/View.js';
+import Feature from '../../../src/ol/Feature.js';
+import LineString from '../../../src/ol/geom/LineString.js';
+import VectorLayer from '../../../src/ol/layer/Vector.js';
+import VectorSource from '../../../src/ol/source/Vector.js';
+import Style from '../../../src/ol/style/Style.js';
+import Stroke from '../../../src/ol/style/Stroke.js';
+
+
+const vectorSource = new VectorSource();
+let feature;
+
+feature = new Feature({
+  geometry: new LineString([[-60, 60], [45, 60]])
+});
+vectorSource.addFeature(feature);
+
+feature = new Feature({
+  geometry: new LineString([[-60, -50], [30, 10]])
+});
+feature.setStyle(new Style({
+  stroke: new Stroke({color: '#f00', width: 3})
+}));
+vectorSource.addFeature(feature);
+
+feature = new Feature({
+  geometry: new LineString([[-110, -100], [0, 100], [100, -90]])
+});
+feature.setStyle(new Style({
+  stroke: new Stroke({
+    color: 'rgba(55, 55, 55, 0.75)',
+    width: 5,
+    lineCap: 'square',
+    lineDash: [4, 8],
+    lineJoin: 'round'
+  })
+}));
+vectorSource.addFeature(feature);
+
+feature = new Feature({
+  geometry: new LineString([[-80, 80], [80, 80], [-40, -90]])
+});
+feature.setStyle([
+  new Style({
+    stroke: new Stroke({color: '#F2F211', width: 5})
+  }),
+  new Style({
+    stroke: new Stroke({color: '#292921', width: 1})
+  })
+]);
+vectorSource.addFeature(feature);
+
+
+new Map({
+  pixelRatio: 1,
+  layers: [
+    new VectorLayer({
+      source: vectorSource
+    })
+  ],
+  target: 'map',
+  view: new View({
+    center: [0, 0],
+    resolution: 1
+  })
+});
+
+render();
--- a/rendering/cases/multiple-layers/expected.png
+++ b/rendering/cases/multiple-layers/expected.png
--- a/rendering/cases/multiple-layers/index.html
+++ b/rendering/cases/multiple-layers/index.html
@@ -0,0 +1,22 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <style>
+      html, body {
+        margin: 0;
+        padding: 0;
+        width: 100%;
+        height: 100%;
+      }
+      #map {
+        width: 100%;
+        height: 100%;
+      }
+    </style>
+  </head>
+  <body>
+    <div id="map"></div>
+    <script src="main.js"></script>
+  </body>
+  </script>
+</html>
--- a/rendering/cases/multiple-layers/main.js
+++ b/rendering/cases/multiple-layers/main.js
@@ -0,0 +1,33 @@
+import Map from '../../../src/ol/Map.js';
+import View from '../../../src/ol/View.js';
+import {Vector as VectorLayer, Tile as TileLayer} from '../../../src/ol/layer.js';
+import {Vector as VectorSource, OSM} from '../../../src/ol/source.js';
+import Point from '../../../src/ol/geom/Point.js';
+import Feature from '../../../src/ol/Feature.js';
+import {fromLonLat} from '../../../src/ol/proj.js';
+
+const center = fromLonLat([-111, 45.7]);
+
+new Map({
+  layers: [
+    new TileLayer({
+      source: new OSM()
+    }),
+    new VectorLayer({
+      source: new VectorSource({
+        features: [
+          new Feature(
+            new Point(center)
+          )
+        ]
+      })
+    })
+  ],
+  target: 'map',
+  view: new View({
+    center: center,
+    zoom: 4
+  })
+});
+
+render();
--- a/rendering/cases/single-layer/expected.png
+++ b/rendering/cases/single-layer/expected.png
--- a/rendering/cases/single-layer/index.html
+++ b/rendering/cases/single-layer/index.html
@@ -0,0 +1,22 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <style>
+      html, body {
+        margin: 0;
+        padding: 0;
+        width: 100%;
+        height: 100%;
+      }
+      #map {
+        width: 100%;
+        height: 100%;
+      }
+    </style>
+  </head>
+  <body>
+    <div id="map"></div>
+    <script src="main.js"></script>
+  </body>
+  </script>
+</html>
--- a/rendering/cases/single-layer/main.js
+++ b/rendering/cases/single-layer/main.js
@@ -0,0 +1,19 @@
+import Map from '../../../src/ol/Map.js';
+import View from '../../../src/ol/View.js';
+import TileLayer from '../../../src/ol/layer/Tile.js';
+import OSM from '../../../src/ol/source/OSM.js';
+
+new Map({
+  layers: [
+    new TileLayer({
+      source: new OSM()
+    })
+  ],
+  target: 'map',
+  view: new View({
+    center: [0, 0],
+    zoom: 0
+  })
+});
+
+render();
--- a/rendering/test.js
+++ b/rendering/test.js
@@ -0,0 +1,325 @@
+#! /usr/bin/env node
+const puppeteer = require('puppeteer');
+const webpack = require('webpack');
+const config = require('./webpack.config');
+const middleware = require('webpack-dev-middleware');
+const http = require('http');
+const path = require('path');
+const png = require('pngjs');
+const fs = require('fs');
+const fse = require('fs-extra');
+const pixelmatch = require('pixelmatch');
+const yargs = require('yargs');
+const log = require('loglevelnext');
+const globby = require('globby');
+
+const compiler = webpack(Object.assign({mode: 'development'}, config));
+
+function getHref(entry) {
+  return path.dirname(entry).slice(1) + '/';
+}
+
+function notFound(req, res) {
+  return () => {
+    if (req.url === '/favicon.ico') {
+      res.writeHead(204);
+      res.end();
+      return;
+    }
+
+    const items = [];
+    for (const key in config.entry) {
+      const href = getHref(config.entry[key]);
+      items.push(`<li><a href="${href}">${href}</a></li>`);
+    }
+    const markup = `<!DOCTYPE html><body><ul>${items.join('')}</ul></body>`;
+
+    res.writeHead(404, {
+      'Content-Type': 'text/html'
+    });
+    res.end(markup);
+  };
+}
+
+function serve(options) {
+  const handler = middleware(compiler, {
+    lazy: true,
+    logger: options.log,
+    stats: 'minimal'
+  });
+
+  return new Promise((resolve, reject) => {
+    const server = http.createServer((req, res) => {
+      handler(req, res, notFound(req, res));
+    });
+
+    server.listen(options.port, options.host, err => {
+      if (err) {
+        return reject(err);
+      }
+      const address = server.address();
+      options.log.info(`test server listening http://${address.address}:${address.port}/`);
+      resolve(() => server.close());
+    });
+  });
+}
+
+function getActualScreenshotPath(entry) {
+  return path.join(__dirname, path.dirname(entry), 'actual.png');
+}
+
+function getExpectedScreenshotPath(entry) {
+  return path.join(__dirname, path.dirname(entry), 'expected.png');
+}
+
+function getPassFilePath(entry) {
+  return path.join(__dirname, path.dirname(entry), 'pass');
+}
+
+function parsePNG(filepath) {
+  return new Promise((resolve, reject) => {
+    const stream = fs.createReadStream(filepath);
+    stream.on('error', err => {
+      if (err.code === 'ENOENT') {
+        return reject(new Error(`File not found: ${filepath}`));
+      }
+      reject(err);
+    });
+
+    const image = stream.pipe(new png.PNG());
+    image.on('parsed', () => resolve(image));
+    image.on('error', reject);
+  });
+}
+
+async function match(actual, expected) {
+  const actualImage = await parsePNG(actual);
+  const expectedImage = await parsePNG(expected);
+  const width = expectedImage.width;
+  const height = expectedImage.height;
+  if (actualImage.width != width) {
+    throw new Error(`Unexpected width for ${actual}: expected ${width}, got ${actualImage.width}`);
+  }
+  if (actualImage.height != height) {
+    throw new Error(`Unexpected height for ${actual}: expected ${height}, got ${actualImage.height}`);
+  }
+  const count = pixelmatch(actualImage.data, expectedImage.data, null, width, height);
+  return count / (width * height);
+}
+
+async function assertScreenshotsMatch(entry) {
+  const actual = getActualScreenshotPath(entry);
+  const expected = getExpectedScreenshotPath(entry);
+  let mismatch, error;
+  try {
+    mismatch = await match(actual, expected);
+  } catch (err) {
+    error = err;
+  }
+  if (error) {
+    return error;
+  }
+  if (mismatch) {
+    return new Error(`${entry} mistmatch: ${mismatch}`);
+  }
+}
+
+let handleRender;
+async function exposeRender(page) {
+  await page.exposeFunction('render', () => {
+    if (!handleRender) {
+      throw new Error('No render handler set for current page');
+    }
+    handleRender();
+  });
+}
+
+async function renderPage(page, entry, options) {
+  const renderCalled = new Promise(resolve => {
+    handleRender = () => {
+      handleRender = null;
+      resolve();
+    };
+  });
+  await page.goto(`http://${options.host}:${options.port}${getHref(entry)}`, {waitUntil: 'networkidle0'});
+  await renderCalled;
+  await page.screenshot({path: getActualScreenshotPath(entry)});
+}
+
+async function touch(filepath) {
+  const fd = await fse.open(filepath, 'w');
+  await fse.close(fd);
+}
+
+async function copyActualToExpected(entry) {
+  const actual = getActualScreenshotPath(entry);
+  const expected = getExpectedScreenshotPath(entry);
+  await fse.copy(actual, expected);
+  await touch(getPassFilePath(entry));
+}
+
+async function renderEach(page, entries, options) {
+  let fail = false;
+  for (const entry of entries) {
+    await renderPage(page, entry, options);
+    if (options.fix) {
+      await copyActualToExpected(entry);
+      continue;
+    }
+    const error = await assertScreenshotsMatch(entry);
+    if (error) {
+      process.stderr.write(`${error.message}\n`);
+      fail = true;
+      continue;
+    }
+
+    await touch(getPassFilePath(entry));
+  }
+  return fail;
+}
+
+async function render(entries, options) {
+  const browser = await puppeteer.launch({
+    args: options.puppeteerArgs,
+    headless: !process.env.CI
+  });
+
+  let fail = false;
+
+  try {
+    const page = await browser.newPage();
+    page.on('error', err => {
+      options.log.error('page crash', err);
+    });
+    page.on('pageerror', err => {
+      options.log.error('uncaught exception', err);
+    });
+    page.on('console', message => {
+      const type = message.type();
+      if (options.log[type]) {
+        options.log[type](message.text());
+      }
+    });
+
+    page.setDefaultNavigationTimeout(options.timeout);
+    await exposeRender(page);
+    await page.setViewport({width: 256, height: 256});
+    fail = await renderEach(page, entries, options);
+  } finally {
+    browser.close();
+  }
+
+  if (fail) {
+    throw new Error('RENDERING TESTS FAILED');
+  }
+}
+
+async function getLatest(patterns) {
+  const stats = await globby(patterns, {stats: true});
+  let latest = 0;
+  for (const stat of stats) {
+    if (stat.mtime > latest) {
+      latest = stat.mtime;
+    }
+  }
+  return latest;
+}
+
+async function getOutdated(entries, options) {
+  const libTime = await getLatest(path.join(__dirname, '..', 'src', 'ol', '**', '*'));
+  options.log.debug('library time', libTime);
+  const outdated = [];
+  for (const entry of entries) {
+    const passPath = getPassFilePath(entry);
+    const passTime = await getLatest(passPath);
+    options.log.debug(entry, 'pass time', passTime);
+    if (passTime < libTime) {
+      outdated.push(entry);
+      continue;
+    }
+
+    const caseTime = await getLatest(path.join(__dirname, path.dirname(entry), '**', '*'));
+    options.log.debug(entry, 'case time', caseTime);
+    if (passTime < caseTime) {
+      outdated.push(entry);
+      continue;
+    }
+
+    options.log.info('skipping', entry);
+  }
+  return outdated;
+}
+
+async function main(entries, options) {
+  if (!options.force) {
+    entries = await getOutdated(entries, options);
+  }
+  if (entries.length === 0) {
+    return;
+  }
+
+  const done = await serve(options);
+  try {
+    await render(entries, options);
+  } finally {
+    if (!options.interactive) {
+      done();
+    }
+  }
+}
+
+if (require.main === module) {
+
+  const options = yargs.
+    option('fix', {
+      describe: 'Accept all screenshots as accepted',
+      default: false
+    }).
+    option('host', {
+      describe: 'The host for serving rendering cases',
+      default: '127.0.0.1'
+    }).
+    option('port', {
+      describe: 'The port for serving rendering cases',
+      type: 'number',
+      default: 3000
+    }).
+    option('timeout', {
+      describe: 'The timeout for loading pages (in milliseconds)',
+      type: 'number',
+      default: 60000
+    }).
+    option('force', {
+      describe: 'Run all tests (instead of just outdated tests)',
+      type: 'boolean',
+      default: false
+    }).
+    option('interactive', {
+      describe: 'Run all tests and keep the test server running (this option will be reworked later)',
+      type: 'boolean',
+      default: false
+    }).
+    option('log-level', {
+      describe: 'The level for logging',
+      choices: ['trace', 'debug', 'info', 'warn', 'error', 'silent'],
+      default: 'error'
+    }).
+    option('puppeteer-args', {
+      describe: 'Args of for puppeteer.launch()',
+      type: 'array',
+      default: process.env.CI ? ['--no-sandbox', '--disable-setuid-sandbox'] : []
+    }).
+    parse();
+
+  const entries = Object.keys(config.entry).map(key => config.entry[key]);
+
+  if (options.interactive) {
+    options.force = true;
+  }
+  options.log = log.create({name: 'rendering', level: options.logLevel});
+
+  main(entries, options).catch(err => {
+    options.log.error(err.message);
+    process.exit(1);
+  });
+}
--- a/rendering/webpack.config.js
+++ b/rendering/webpack.config.js
@@ -0,0 +1,36 @@
+const CopyPlugin = require('copy-webpack-plugin');
+const fs = require('fs');
+const path = require('path');
+
+const cases = path.join(__dirname, 'cases');
+
+const caseDirs = fs.readdirSync(cases);
+
+const entry = {};
+caseDirs.forEach(c => {
+  entry[`cases/${c}/main`] = `./cases/${c}/main.js`;
+});
+
+module.exports = {
+  context: __dirname,
+  target: 'web',
+  entry: entry,
+  module: {
+    rules: [{
+      use: {
+        loader: 'buble-loader'
+      },
+      test: /\.js$/,
+      include: [
+        path.join(__dirname, '..', 'src')
+      ]
+    }]
+  },
+  plugins: [
+    new CopyPlugin([
+      {from: '../src/ol/ol.css', to: 'css'},
+      {from: 'cases/**/*.html'}
+    ])
+  ],
+  devtool: 'source-map'
+};