Update package version to 2.3.0

Addresses issue #197, send SIGTERM through to node

Dockerfile

@@ -1,6 +1,12 @@
-FROM debian:stretch
+FROM node:6-stretch
 MAINTAINER Petr Sloup <petr.sloup@klokantech.com>
 
+ENV NODE_ENV="production"
+VOLUME /data
+WORKDIR /data
+EXPOSE 80
+ENTRYPOINT ["/bin/bash", "/usr/src/app/run.sh"]
+
 RUN apt-get -qq update \
 && DEBIAN_FRONTEND=noninteractive apt-get -y install \
     apt-transport-https \
@@ -15,22 +21,8 @@ RUN apt-get -qq update \
     libprotobuf-dev \
     libxxf86vm-dev \
     xvfb \
-&& echo "deb https://deb.nodesource.com/node_6.x stretch main" >> /etc/apt/sources.list.d/nodejs.list \
-&& echo "deb-src https://deb.nodesource.com/node_6.x stretch main" >> /etc/apt/sources.list.d/nodejs.list \
-&& apt-get -qq update \
-&& DEBIAN_FRONTEND=noninteractive apt-get -y --allow-unauthenticated install \
-    nodejs \
-&& rm /etc/apt/sources.list.d/nodejs.list \
 && apt-get clean
 
 RUN mkdir -p /usr/src/app
 COPY / /usr/src/app
 RUN cd /usr/src/app && npm install --production
-
-VOLUME /data
-WORKDIR /data
-
-ENV NODE_ENV="production"
-
-EXPOSE 80
-ENTRYPOINT ["/usr/src/app/run.sh"]

Dockerfile_light

@@ -1,14 +1,12 @@
 FROM node:6
 MAINTAINER Petr Sloup <petr.sloup@klokantech.com>
 
+ENV NODE_ENV="production"
+EXPOSE 80
+VOLUME /data
+WORKDIR /data
+ENTRYPOINT ["node", "/usr/src/app/", "-p", "80"]
+
 RUN mkdir -p /usr/src/app
 COPY / /usr/src/app
 RUN cd /usr/src/app && npm install --production
-
-VOLUME /data
-WORKDIR /data
-
-ENV NODE_ENV="production"
-
-EXPOSE 80
-ENTRYPOINT ["node", "/usr/src/app/", "-p", "80"]

ISSUE_TEMPLATE.md

@@ -0,0 +1,19 @@
+It is great you want to help us making TileServer GL project better!
+
+This is the right place only for a software bug report or a new software feature request.
+
+The usage and installation questions belongs to https://stackoverflow.com/questions/tagged/openmaptiles
+A guaranteed support and consulting from the core developers via https://openmaptiles.com/support/
+
+Please search this GitHub repo for similar requests before posting (check also closed tickets).
+
+When reporting a problem you have with the TileServer GL software please provide:
+
+- Clear description of the problem: What steps will lead to reproducing the error on our computer? What is exactly wrong?
+- Version of the TileServer GL software you have used
+- Version and name of the operating system you use or other details of your setup
+- Information about your used config / styles / vector tiles
+- URL / link to the specific location in a live map demo where a bug is visible is always great
+- Screenshot of the problem are cool! Drag&drop an image to the report here...
+
+We love pull requests! If you are able to code, please send us your fix or code modification via GitHub... Thanks!

README.md

@@ -9,6 +9,8 @@ Vector and raster maps with GL styles. Server side rendering by Mapbox GL Native
 
 ## Get Started
 
+Make sure you have Node.js version **6** installed (running `node -v` it should output something like `v6.11.3`).
+
 Install `tileserver-gl` with server-side raster rendering of vector tiles with npm
 
 ```bash
@@ -34,7 +36,7 @@ Alternatively, you can use the `tileserver-gl-light` package instead, which is p
 An alternative to npm to start the packed software easier is to install [Docker](http://www.docker.com/) on your computer and then run in the directory with the downloaded MBTiles the command:
 
 ```bash
-docker run -it -v $(pwd):/data -p 8080:80 klokantech/tileserver-gl
+docker run --rm -it -v $(pwd):/data -p 8080:80 klokantech/tileserver-gl
 ```
 
 This will download and start a ready to use container on your computer and the maps are going to be available in webbrowser on localhost:8080.

docs/config.rst

@@ -21,9 +21,7 @@ Example::
       ],
       "formatQuality": {
         "jpeg": 80,
-        "webp": 90,
-        "pngQuantization": false,
-        "png": 90
+        "webp": 90
       },
       "maxScaleFactor": 3,
       "maxSize": 2048,
@@ -83,8 +81,6 @@ Use ``false`` to disable the front page altogether (404).
 
 Quality of the compression of individual image formats. [0-100]
 
-The value for ``png`` is only used when ``pngQuantization`` is ``true``.
-
 ``maxScaleFactor``
 -----------
 
@@ -99,6 +95,27 @@ Maximum image side length to be allowed to be rendered (including scale factor).
 Be careful when changing this value since there are hardware limits that need to be considered.
 Default is ``2048``.
 
+``minRendererPoolSizes``
+------------------------
+
+Minimum amount of raster tile renderers per scale factor.
+The value is an array: the first element is the minimum amount of renderers for scale factor one, the second for scale factor two and so on.
+If the array has less elements than ``maxScaleFactor``, then the last element is used for all remaining scale factors as well.
+Selecting renderer pool sizes is a trade-off between memory use and speed.
+A reasonable value will depend on your hardware and your amount of styles and scale factors.
+If you have plenty of memory, you'll want to set this equal to ``maxRendererPoolSizes`` to avoid increased latency due to renderer destruction and recreation.
+If you need to conserve memory, you'll want something lower than ``maxRendererPoolSizes``, possibly allocating more renderers to scale factors that are more common.
+Default is ``[8, 4, 2]``.
+
+``maxRendererPoolSizes``
+------------------------
+
+Maximum amount of raster tile renderers per scale factor.
+The value and considerations are similar to ``minRendererPoolSizes`` above.
+If you have plenty of memory, try setting these equal to or slightly above your processor count, e.g. if you have four processors, try a value of ``[6]``.
+If you need to conserve memory, try lower values for scale factors that are less common.
+Default is ``[16, 8, 4]``.
+
 ``watermark``
 -----------
 

docs/installation.rst

@@ -7,7 +7,7 @@ Docker
 
 When running docker image, no special installation is needed -- the docker will automatically download the image if not present.
 
-Just run ``docker run -it -v $(pwd):/data -p 8080:80 klokantech/tileserver-gl``.
+Just run ``docker run --rm -it -v $(pwd):/data -p 8080:80 klokantech/tileserver-gl``.
 
 Additional options (see :doc:`/usage`) can be passed to the TileServer GL by appending them to the end of this command. You can, for example, do the following:
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "tileserver-gl",
-  "version": "2.1.0",
+  "version": "2.3.0",
   "description": "Map tile server for JSON GL styles - vector and server side generated raster tiles",
   "main": "src/main.js",
   "bin": "src/main.js",
@@ -16,31 +16,30 @@
     "node": ">=6 <7"
   },
   "scripts": {
-    "test": "mocha test/**.js"
+    "test": "mocha test/**.js --timeout 10000"
   },
   "dependencies": {
     "@mapbox/mapbox-gl-native": "3.5.4",
     "@mapbox/mbtiles": "0.9.0",
     "@mapbox/sphericalmercator": "1.0.5",
     "@mapbox/vector-tile": "1.3.0",
-    "advanced-pool": "0.3.2",
+    "advanced-pool": "0.3.3",
     "base64url": "2.0.0",
-    "canvas": "1.6.5",
+    "canvas": "1.6.6",
     "clone": "2.1.1",
     "color": "1.0.3",
-    "cors": "2.8.3",
-    "express": "4.15.3",
+    "cors": "2.8.4",
+    "express": "4.15.4",
     "glyph-pbf-composite": "0.0.2",
     "handlebars": "4.0.10",
     "http-shutdown": "^1.2.0",
     "morgan": "1.8.2",
-    "node-pngquant-native": "1.0.4",
     "nomnom": "1.8.1",
     "pbf": "3.0.5",
-    "proj4": "2.4.3",
+    "proj4": "2.4.4",
     "request": "2.81.0",
-    "sharp": "0.18.1",
-    "tileserver-gl-styles": "1.1.1"
+    "sharp": "0.18.2",
+    "tileserver-gl-styles": "1.2.0"
   },
   "devDependencies": {
     "should": "^11.2.0",

publish.js

@@ -23,7 +23,6 @@ packageJson.name += '-light';
 packageJson.description = 'Map tile server for JSON GL styles - serving vector tiles';
 delete packageJson.dependencies['canvas'];
 delete packageJson.dependencies['@mapbox/mapbox-gl-native'];
-delete packageJson.dependencies['node-pngquant-native'];
 delete packageJson.dependencies['sharp'];
 
 delete packageJson.optionalDependencies;

run.sh

@@ -1,8 +1,23 @@
 #!/bin/bash
+
+_term() {
+	echo "Caught signal, stopping gracefully"
+	kill -TERM "$child" 2>/dev/null
+}
+
+trap _term TERM
+
+
 start-stop-daemon --start --pidfile ~/xvfb.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -screen 0 1024x768x24 -ac +extension GLX +render -noreset
-sleep 1
+echo "Waiting 3 seconds for xvfb to start..."
+sleep 3
 
 export DISPLAY=:99.0
 
 cd /data
-node /usr/src/app/ -p 80 "$@"
+node /usr/src/app/ -p 80 "$@" &
+child=$!
+wait "$child"
+
+start-stop-daemon --stop --pidfile ~/xvfb.pid # stop xvfb when exiting
+rm ~/xvfb.pid

src/serve_data.js

@@ -37,7 +37,15 @@ module.exports = function(options, repo, params, id, styles) {
   var source;
   var sourceInfoPromise = new Promise(function(resolve, reject) {
     source = new mbtiles(mbtilesFile, function(err) {
+      if (err) {
+        reject(err);
+        return;
+      }
       source.getInfo(function(err, info) {
+        if (err) {
+          reject(err);
+          return;
+        }
         tileJSON['name'] = id;
         tileJSON['format'] = 'pbf';
 
@@ -50,6 +58,10 @@ module.exports = function(options, repo, params, id, styles) {
 
         Object.assign(tileJSON, params.tilejson || {});
         utils.fixTileJSONCenter(tileJSON);
+
+        if (options.dataDecoratorFunc) {
+          tileJSON = options.dataDecoratorFunc(id, 'tilejson', tileJSON);
+        }
         resolve();
       });
     });
@@ -114,6 +126,13 @@ module.exports = function(options, repo, params, id, styles) {
                 //console.log(shrinkers[style].getStats());
               }
             }
+            if (options.dataDecoratorFunc) {
+              if (isGzipped) {
+                data = zlib.unzipSync(data);
+                isGzipped = false;
+              }
+              data = options.dataDecoratorFunc(id, 'data', data, z, x, y);
+            }
           }
           if (format == 'pbf') {
             headers['Content-Type'] = 'application/x-protobuf';
@@ -165,9 +184,7 @@ module.exports = function(options, repo, params, id, styles) {
     return res.send(info);
   });
 
-  return new Promise(function(resolve, reject) {
-    sourceInfoPromise.then(function() {
-      resolve(app);
-    });
+  return sourceInfoPromise.then(function() {
+    return app;
   });
 };

src/serve_font.js

@@ -17,13 +17,19 @@ module.exports = function(options, allowedFonts) {
   var existingFonts = {};
   var fontListingPromise = new Promise(function(resolve, reject) {
     fs.readdir(options.paths.fonts, function(err, files) {
+      if (err) {
+        reject(err);
+        return;
+      }
       files.forEach(function(file) {
         fs.stat(path.join(fontPath, file), function(err, stats) {
-          if (!err) {
-            if (stats.isDirectory() &&
-                fs.existsSync(path.join(fontPath, file, '0-255.pbf'))) {
-              existingFonts[path.basename(file)] = true;
-            }
+          if (err) {
+            reject(err);
+            return;
+          }
+          if (stats.isDirectory() &&
+              fs.existsSync(path.join(fontPath, file, '0-255.pbf'))) {
+            existingFonts[path.basename(file)] = true;
           }
         });
       });
@@ -54,9 +60,7 @@ module.exports = function(options, allowedFonts) {
     );
   });
 
-  return new Promise(function(resolve, reject) {
-    fontListingPromise.then(function() {
-      resolve(app);
-    });
+  return fontListingPromise.then(function() {
+    return app;
   });
 };

src/serve_rendered.js

@@ -3,6 +3,7 @@
 var advancedPool = require('advanced-pool'),
     fs = require('fs'),
     path = require('path'),
+    url = require('url'),
     util = require('util'),
     zlib = require('zlib');
 
@@ -17,7 +18,6 @@ var Canvas = require('canvas'),
     mercator = new (require('@mapbox/sphericalmercator'))(),
     mbgl = require('@mapbox/mapbox-gl-native'),
     mbtiles = require('@mapbox/mbtiles'),
-    pngquant = require('node-pngquant-native'),
     proj4 = require('proj4'),
     request = require('request');
 
@@ -35,6 +35,68 @@ mbgl.on('message', function(e) {
   }
 });
 
+/**
+ * Lookup of sharp output formats by file extension.
+ */
+var extensionToFormat = {
+  '.jpg': 'jpeg',
+  '.jpeg': 'jpeg',
+  '.png': 'png',
+  '.webp': 'webp'
+};
+
+/**
+ * Cache of response data by sharp output format and color.  Entry for empty
+ * string is for unknown or unsupported formats.
+ */
+var cachedEmptyResponses = {
+  '': new Buffer(0)
+};
+
+/**
+ * Create an appropriate mbgl response for http errors.
+ * @param {string} format The format (a sharp format or 'pbf').
+ * @param {string} color The background color (or empty string for transparent).
+ * @param {Function} callback The mbgl callback.
+ */
+function createEmptyResponse(format, color, callback) {
+  if (!format || format === 'pbf') {
+    callback(null, {data: cachedEmptyResponses['']});
+    return;
+  }
+
+  if (format === 'jpg') {
+    format = 'jpeg';
+  }
+  if (!color) {
+    color = 'rgba(255,255,255,0)';
+  }
+
+  var cacheKey = format + ',' + color;
+  var data = cachedEmptyResponses[cacheKey];
+  if (data) {
+    callback(null, {data: data});
+    return;
+  }
+
+  // create an "empty" response image
+  var color = new Color(color);
+  var array = color.array();
+  var channels = array.length == 4 && format != 'jpeg' ? 4 : 3;
+  sharp(new Buffer(array), {
+    raw: {
+      width: 1,
+      height: 1,
+      channels: channels
+    }
+  }).toFormat(format).toBuffer(function(err, buffer, info) {
+    if (!err) {
+      cachedEmptyResponses[cacheKey] = buffer;
+    }
+    callback(null, {data: buffer});
+  });
+}
+
 module.exports = function(options, repo, params, id, dataResolver) {
   var app = express().disable('x-powered-by');
 
@@ -60,12 +122,18 @@ module.exports = function(options, repo, params, id, dataResolver) {
   var existingFonts = {};
   var fontListingPromise = new Promise(function(resolve, reject) {
     fs.readdir(options.paths.fonts, function(err, files) {
+      if (err) {
+        reject(err);
+        return;
+      }
       files.forEach(function(file) {
         fs.stat(path.join(options.paths.fonts, file), function(err, stats) {
-          if (!err) {
-            if (stats.isDirectory()) {
-              existingFonts[path.basename(file)] = true;
-            }
+          if (err) {
+            reject(err);
+            return;
+          }
+          if (stats.isDirectory()) {
+            existingFonts[path.basename(file)] = true;
           }
         });
       });
@@ -100,7 +168,9 @@ module.exports = function(options, repo, params, id, dataResolver) {
             });
           } else if (protocol == 'mbtiles') {
             var parts = req.url.split('/');
-            var source = map.sources[parts[2]];
+            var sourceId = parts[2];
+            var source = map.sources[sourceId];
+            var sourceInfo = styleJSON.sources[sourceId];
             var z = parts[3] | 0,
                 x = parts[4] | 0,
                 y = parts[5].split('.')[0] | 0,
@@ -108,22 +178,31 @@ module.exports = function(options, repo, params, id, dataResolver) {
             source.getTile(z, x, y, function(err, data, headers) {
               if (err) {
                 //console.log('MBTiles error, serving empty', err);
-                callback(null, { data: source.emptyTile });
-              } else {
-                var response = {};
-
-                if (headers['Last-Modified']) {
-                  response.modified = new Date(headers['Last-Modified']);
-                }
-
-                if (format == 'pbf') {
-                  response.data = zlib.unzipSync(data);
-                } else {
-                  response.data = data;
-                }
-
-                callback(null, response);
+                createEmptyResponse(sourceInfo.format, sourceInfo.color, callback);
+                return;
               }
+
+              var response = {};
+              if (headers['Last-Modified']) {
+                response.modified = new Date(headers['Last-Modified']);
+              }
+
+              if (format == 'pbf') {
+                try {
+                  response.data = zlib.unzipSync(data);
+                }
+                catch (err) {
+                  console.log("Skipping incorrect header for tile mbtiles://%s/%s/%s/%s.pbf", id, z, x, y);
+                }
+                if (options.dataDecoratorFunc) {
+                  response.data = options.dataDecoratorFunc(
+                    sourceId, 'data', response.data, z, x, y);
+                }
+              } else {
+                response.data = data;
+              }
+
+              callback(null, response);
             });
           } else if (protocol == 'http' || protocol == 'https') {
             request({
@@ -131,29 +210,28 @@ module.exports = function(options, repo, params, id, dataResolver) {
                 encoding: null,
                 gzip: true
             }, function(err, res, body) {
-                if (err) {
-                  //console.log('HTTP tile error', err);
-                  callback(null, { data: new Buffer(0) });
-                } else if (res.statusCode == 200) {
-                  var response = {};
-
-                  if (res.headers.modified) {
-                    response.modified = new Date(res.headers.modified);
-                  }
-                  if (res.headers.expires) {
-                    response.expires = new Date(res.headers.expires);
-                  }
-                  if (res.headers.etag) {
-                    response.etag = res.headers.etag;
-                  }
-
-                  response.data = body;
-
-                  callback(null, response);
-                } else {
-                  //console.log('HTTP error', JSON.parse(body).message);
-                  callback(null, { data: new Buffer(0) });
+                var parts = url.parse(req.url);
+                var extension = path.extname(parts.pathname).toLowerCase();
+                var format = extensionToFormat[extension] || '';
+                if (err || res.statusCode < 200 || res.statusCode >= 300) {
+                  // console.log('HTTP error', err || res.statusCode);
+                  createEmptyResponse(format, '', callback);
+                  return;
                 }
+
+                var response = {};
+                if (res.headers.modified) {
+                  response.modified = new Date(res.headers.modified);
+                }
+                if (res.headers.expires) {
+                  response.expires = new Date(res.headers.expires);
+                }
+                if (res.headers.etag) {
+                  response.etag = res.headers.etag;
+                }
+
+                response.data = body;
+                callback(null, response);
             });
           }
         }
@@ -258,26 +336,11 @@ module.exports = function(options, repo, params, id, dataResolver) {
               'mbtiles://' + name + '/{z}/{x}/{y}.' + (info.format || 'pbf')
             ];
             delete source.scheme;
-            if (source.format == 'pbf') {
-              map.sources[name].emptyTile = new Buffer(0);
-            } else {
-              var color = new Color(source.color || 'rgba(255,255,255,0)');
-              var format = source.format;
-              if (format == 'jpg') {
-                format = 'jpeg';
-              }
-              var array = color.array();
-              var channels = array.length == 4 && format != 'jpeg' ? 4 : 3;
-              sharp(new Buffer(array), {
-                raw: {
-                  width: 1,
-                  height: 1,
-                  channels: channels
-                }
-              }).toFormat(format).toBuffer(function(err, buffer, info) {
-                map.sources[name].emptyTile = buffer;
-              });
+
+            if (options.dataDecoratorFunc) {
+              source = options.dataDecoratorFunc(name, 'tilejson', source);
             }
+
             if (!attributionOverride &&
                 source.attribution && source.attribution.length > 0) {
               if (tileJSON.attribution.length > 0) {
@@ -293,18 +356,15 @@ module.exports = function(options, repo, params, id, dataResolver) {
   });
 
   var renderersReadyPromise = Promise.all(queue).then(function() {
-    // TODO: make pool sizes configurable
+    // standard and @2x tiles are much more usual -> default to larger pools
+    var minPoolSizes = options.minRendererPoolSizes || [8, 4, 2];
+    var maxPoolSizes = options.maxRendererPoolSizes || [16, 8, 4];
     for (var s = 1; s <= maxScaleFactor; s++) {
-      var minPoolSize = 2;
-
-      // standard and @2x tiles are much more usual -> create larger pools
-      if (s <= 2) {
-        minPoolSize *= 2;
-        if (s <= 1) {
-          minPoolSize *= 2;
-        }
-      }
-      map.renderers[s] = createPool(s, minPoolSize, 2 * minPoolSize);
+      var i = Math.min(minPoolSizes.length - 1, s - 1);
+      var j = Math.min(maxPoolSizes.length - 1, s - 1);
+      var minPoolSize = minPoolSizes[i];
+      var maxPoolSize = Math.max(minPoolSize, maxPoolSizes[j]);
+      map.renderers[s] = createPool(s, minPoolSize, maxPoolSize);
     }
   });
 
@@ -399,16 +459,6 @@ module.exports = function(options, repo, params, id, dataResolver) {
             return res.status(404).send('Not found');
           }
 
-          if (format == 'png') {
-            var usePngQuant =
-                (options.formatQuality || {}).pngQuantization === true;
-            if (usePngQuant) {
-              buffer = pngquant.compress(buffer, {
-                quality: [0, formatQuality || 90]
-              });
-            }
-          }
-
           res.set({
             'Last-Modified': lastModified,
             'Content-Type': 'image/' + format
@@ -701,9 +751,8 @@ module.exports = function(options, repo, params, id, dataResolver) {
     return res.send(info);
   });
 
-  return new Promise(function(resolve, reject) {
-    Promise.all([fontListingPromise, renderersReadyPromise]).then(function() {
-      resolve(app);
-    });
+  return Promise.all([fontListingPromise, renderersReadyPromise]).then(function() {
+    return app;
   });
+
 };

src/server.js

@@ -89,6 +89,12 @@ function start(opts) {
   checkPath('sprites');
   checkPath('mbtiles');
 
+  if (options.dataDecorator) {
+    try {
+      options.dataDecoratorFunc = require(path.resolve(paths.root, options.dataDecorator));
+    } catch (e) {}
+  }
+
   var data = clone(config.data || {});
 
   if (opts.cors) {
@@ -240,8 +246,9 @@ function start(opts) {
     startupPromises.push(new Promise(function(resolve, reject) {
       fs.readFile(templateFile, function(err, content) {
         if (err) {
-          console.error('Template not found:', err);
+          err = new Error('Template not found: ' + err.message);
           reject(err);
+          return;
         }
         var compiled = handlebars.compile(content.toString());
 
@@ -409,6 +416,11 @@ function start(opts) {
 module.exports = function(opts) {
   var running = start(opts);
 
+  running.startupPromise.catch(function(err) {
+    console.error(err.message);
+    process.exit(1);
+  });
+
   process.on('SIGINT', function() {
     process.exit();
   });

src/utils.js

@@ -110,9 +110,7 @@ module.exports.getFontsPbf = function(allowedFonts, fontPath, names, range, fall
     );
   });
 
-  return new Promise(function(resolve, reject) {
-    Promise.all(queue).then(function(values) {
-      return resolve(glyphCompose.combine(values));
-    }, reject);
+  return Promise.all(queue).then(function(values) {
+    return glyphCompose.combine(values);
   });
 };