Update package version to 2.3.0

Addresses issue #197, send SIGTERM through to node

Dockerfile

@@ -1,6 +1,12 @@
-FROM debian:stretch
+FROM node:6-stretch
 MAINTAINER Petr Sloup <petr.sloup@klokantech.com>
 
+ENV NODE_ENV="production"
+VOLUME /data
+WORKDIR /data
+EXPOSE 80
+ENTRYPOINT ["/bin/bash", "/usr/src/app/run.sh"]
+
 RUN apt-get -qq update \
 && DEBIAN_FRONTEND=noninteractive apt-get -y install \
     apt-transport-https \
@@ -15,22 +21,8 @@ RUN apt-get -qq update \
     libprotobuf-dev \
     libxxf86vm-dev \
     xvfb \
-&& echo "deb https://deb.nodesource.com/node_6.x stretch main" >> /etc/apt/sources.list.d/nodejs.list \
-&& echo "deb-src https://deb.nodesource.com/node_6.x stretch main" >> /etc/apt/sources.list.d/nodejs.list \
-&& apt-get -qq update \
-&& DEBIAN_FRONTEND=noninteractive apt-get -y --allow-unauthenticated install \
-    nodejs \
-&& rm /etc/apt/sources.list.d/nodejs.list \
 && apt-get clean
 
 RUN mkdir -p /usr/src/app
 COPY / /usr/src/app
 RUN cd /usr/src/app && npm install --production
-
-VOLUME /data
-WORKDIR /data
-
-ENV NODE_ENV="production"
-
-EXPOSE 80
-ENTRYPOINT ["/usr/src/app/run.sh"]

Dockerfile_light

@@ -1,14 +1,12 @@
 FROM node:6
 MAINTAINER Petr Sloup <petr.sloup@klokantech.com>
 
+ENV NODE_ENV="production"
+EXPOSE 80
+VOLUME /data
+WORKDIR /data
+ENTRYPOINT ["node", "/usr/src/app/", "-p", "80"]
+
 RUN mkdir -p /usr/src/app
 COPY / /usr/src/app
 RUN cd /usr/src/app && npm install --production
-
-VOLUME /data
-WORKDIR /data
-
-ENV NODE_ENV="production"
-
-EXPOSE 80
-ENTRYPOINT ["node", "/usr/src/app/", "-p", "80"]

ISSUE_TEMPLATE.md

@@ -0,0 +1,19 @@
+It is great you want to help us making TileServer GL project better!
+
+This is the right place only for a software bug report or a new software feature request.
+
+The usage and installation questions belongs to https://stackoverflow.com/questions/tagged/openmaptiles
+A guaranteed support and consulting from the core developers via https://openmaptiles.com/support/
+
+Please search this GitHub repo for similar requests before posting (check also closed tickets).
+
+When reporting a problem you have with the TileServer GL software please provide:
+
+- Clear description of the problem: What steps will lead to reproducing the error on our computer? What is exactly wrong?
+- Version of the TileServer GL software you have used
+- Version and name of the operating system you use or other details of your setup
+- Information about your used config / styles / vector tiles
+- URL / link to the specific location in a live map demo where a bug is visible is always great
+- Screenshot of the problem are cool! Drag&drop an image to the report here...
+
+We love pull requests! If you are able to code, please send us your fix or code modification via GitHub... Thanks!

README.md

@@ -9,6 +9,8 @@ Vector and raster maps with GL styles. Server side rendering by Mapbox GL Native
 
 ## Get Started
 
+Make sure you have Node.js version **6** installed (running `node -v` it should output something like `v6.11.3`).
+
 Install `tileserver-gl` with server-side raster rendering of vector tiles with npm
 
 ```bash

docs/config.rst

@@ -21,9 +21,7 @@ Example::
       ],
       "formatQuality": {
         "jpeg": 80,
-        "webp": 90,
+        "webp": 90
-        "pngQuantization": false,
-        "png": 90
       },
       "maxScaleFactor": 3,
       "maxSize": 2048,
@@ -83,8 +81,6 @@ Use ``false`` to disable the front page altogether (404).
 
 Quality of the compression of individual image formats. [0-100]
 
-The value for ``png`` is only used when ``pngQuantization`` is ``true``.
-
 ``maxScaleFactor``
 -----------
 
@@ -99,6 +95,27 @@ Maximum image side length to be allowed to be rendered (including scale factor).
 Be careful when changing this value since there are hardware limits that need to be considered.
 Default is ``2048``.
 
+``minRendererPoolSizes``
+------------------------
+
+Minimum amount of raster tile renderers per scale factor.
+The value is an array: the first element is the minimum amount of renderers for scale factor one, the second for scale factor two and so on.
+If the array has less elements than ``maxScaleFactor``, then the last element is used for all remaining scale factors as well.
+Selecting renderer pool sizes is a trade-off between memory use and speed.
+A reasonable value will depend on your hardware and your amount of styles and scale factors.
+If you have plenty of memory, you'll want to set this equal to ``maxRendererPoolSizes`` to avoid increased latency due to renderer destruction and recreation.
+If you need to conserve memory, you'll want something lower than ``maxRendererPoolSizes``, possibly allocating more renderers to scale factors that are more common.
+Default is ``[8, 4, 2]``.
+
+``maxRendererPoolSizes``
+------------------------
+
+Maximum amount of raster tile renderers per scale factor.
+The value and considerations are similar to ``minRendererPoolSizes`` above.
+If you have plenty of memory, try setting these equal to or slightly above your processor count, e.g. if you have four processors, try a value of ``[6]``.
+If you need to conserve memory, try lower values for scale factors that are less common.
+Default is ``[16, 8, 4]``.
+
 ``watermark``
 -----------
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "tileserver-gl",
-  "version": "2.2.0",
+  "version": "2.3.0",
   "description": "Map tile server for JSON GL styles - vector and server side generated raster tiles",
   "main": "src/main.js",
   "bin": "src/main.js",
@@ -16,7 +16,7 @@
     "node": ">=6 <7"
   },
   "scripts": {
-    "test": "mocha test/**.js"
+    "test": "mocha test/**.js --timeout 10000"
   },
   "dependencies": {
     "@mapbox/mapbox-gl-native": "3.5.4",
@@ -34,7 +34,6 @@
     "handlebars": "4.0.10",
     "http-shutdown": "^1.2.0",
     "morgan": "1.8.2",
-    "node-pngquant-native": "1.0.5",
     "nomnom": "1.8.1",
     "pbf": "3.0.5",
     "proj4": "2.4.4",

publish.js

@@ -23,7 +23,6 @@ packageJson.name += '-light';
 packageJson.description = 'Map tile server for JSON GL styles - serving vector tiles';
 delete packageJson.dependencies['canvas'];
 delete packageJson.dependencies['@mapbox/mapbox-gl-native'];
-delete packageJson.dependencies['node-pngquant-native'];
 delete packageJson.dependencies['sharp'];
 
 delete packageJson.optionalDependencies;

run.sh

@@ -1,8 +1,23 @@
 #!/bin/bash
+
+_term() {
+	echo "Caught signal, stopping gracefully"
+	kill -TERM "$child" 2>/dev/null
+}
+
+trap _term TERM
+
+
 start-stop-daemon --start --pidfile ~/xvfb.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -screen 0 1024x768x24 -ac +extension GLX +render -noreset
-sleep 1
+echo "Waiting 3 seconds for xvfb to start..."
+sleep 3
 
 export DISPLAY=:99.0
 
 cd /data
-node /usr/src/app/ -p 80 "$@"
+node /usr/src/app/ -p 80 "$@" &
+child=$!
+wait "$child"
+
+start-stop-daemon --stop --pidfile ~/xvfb.pid # stop xvfb when exiting
+rm ~/xvfb.pid

src/serve_data.js

@@ -37,7 +37,15 @@ module.exports = function(options, repo, params, id, styles) {
   var source;
   var sourceInfoPromise = new Promise(function(resolve, reject) {
     source = new mbtiles(mbtilesFile, function(err) {
+      if (err) {
+        reject(err);
+        return;
+      }
       source.getInfo(function(err, info) {
+        if (err) {
+          reject(err);
+          return;
+        }
         tileJSON['name'] = id;
         tileJSON['format'] = 'pbf';
 
@@ -176,9 +184,7 @@ module.exports = function(options, repo, params, id, styles) {
     return res.send(info);
   });
 
-  return new Promise(function(resolve, reject) {
+  return sourceInfoPromise.then(function() {
-    sourceInfoPromise.then(function() {
+    return app;
-      resolve(app);
-    });
   });
 };

src/serve_font.js

@@ -17,14 +17,20 @@ module.exports = function(options, allowedFonts) {
   var existingFonts = {};
   var fontListingPromise = new Promise(function(resolve, reject) {
     fs.readdir(options.paths.fonts, function(err, files) {
+      if (err) {
+        reject(err);
+        return;
+      }
       files.forEach(function(file) {
         fs.stat(path.join(fontPath, file), function(err, stats) {
-          if (!err) {
+          if (err) {
+            reject(err);
+            return;
+          }
           if (stats.isDirectory() &&
               fs.existsSync(path.join(fontPath, file, '0-255.pbf'))) {
             existingFonts[path.basename(file)] = true;
           }
-          }
         });
       });
       resolve();
@@ -54,9 +60,7 @@ module.exports = function(options, allowedFonts) {
     );
   });
 
-  return new Promise(function(resolve, reject) {
+  return fontListingPromise.then(function() {
-    fontListingPromise.then(function() {
+    return app;
-      resolve(app);
-    });
   });
 };

src/serve_rendered.js

@@ -18,7 +18,6 @@ var Canvas = require('canvas'),
     mercator = new (require('@mapbox/sphericalmercator'))(),
     mbgl = require('@mapbox/mapbox-gl-native'),
     mbtiles = require('@mapbox/mbtiles'),
-    pngquant = require('node-pngquant-native'),
     proj4 = require('proj4'),
     request = require('request');
 
@@ -123,13 +122,19 @@ module.exports = function(options, repo, params, id, dataResolver) {
   var existingFonts = {};
   var fontListingPromise = new Promise(function(resolve, reject) {
     fs.readdir(options.paths.fonts, function(err, files) {
+      if (err) {
+        reject(err);
+        return;
+      }
       files.forEach(function(file) {
         fs.stat(path.join(options.paths.fonts, file), function(err, stats) {
-          if (!err) {
+          if (err) {
+            reject(err);
+            return;
+          }
           if (stats.isDirectory()) {
             existingFonts[path.basename(file)] = true;
           }
-          }
         });
       });
       resolve();
@@ -165,6 +170,7 @@ module.exports = function(options, repo, params, id, dataResolver) {
             var parts = req.url.split('/');
             var sourceId = parts[2];
             var source = map.sources[sourceId];
+            var sourceInfo = styleJSON.sources[sourceId];
             var z = parts[3] | 0,
                 x = parts[4] | 0,
                 y = parts[5].split('.')[0] | 0,
@@ -172,7 +178,7 @@ module.exports = function(options, repo, params, id, dataResolver) {
             source.getTile(z, x, y, function(err, data, headers) {
               if (err) {
                 //console.log('MBTiles error, serving empty', err);
-                createEmptyResponse(source.format, source.color, callback);
+                createEmptyResponse(sourceInfo.format, sourceInfo.color, callback);
                 return;
               }
 
@@ -182,7 +188,12 @@ module.exports = function(options, repo, params, id, dataResolver) {
               }
 
               if (format == 'pbf') {
+                try {
                   response.data = zlib.unzipSync(data);
+                }
+                catch (err) {
+                  console.log("Skipping incorrect header for tile mbtiles://%s/%s/%s/%s.pbf", id, z, x, y);
+                }
                 if (options.dataDecoratorFunc) {
                   response.data = options.dataDecoratorFunc(
                     sourceId, 'data', response.data, z, x, y);
@@ -345,18 +356,15 @@ module.exports = function(options, repo, params, id, dataResolver) {
   });
 
   var renderersReadyPromise = Promise.all(queue).then(function() {
-    // TODO: make pool sizes configurable
+    // standard and @2x tiles are much more usual -> default to larger pools
+    var minPoolSizes = options.minRendererPoolSizes || [8, 4, 2];
+    var maxPoolSizes = options.maxRendererPoolSizes || [16, 8, 4];
     for (var s = 1; s <= maxScaleFactor; s++) {
-      var minPoolSize = 2;
+      var i = Math.min(minPoolSizes.length - 1, s - 1);
-
+      var j = Math.min(maxPoolSizes.length - 1, s - 1);
-      // standard and @2x tiles are much more usual -> create larger pools
+      var minPoolSize = minPoolSizes[i];
-      if (s <= 2) {
+      var maxPoolSize = Math.max(minPoolSize, maxPoolSizes[j]);
-        minPoolSize *= 2;
+      map.renderers[s] = createPool(s, minPoolSize, maxPoolSize);
-        if (s <= 1) {
-          minPoolSize *= 2;
-        }
-      }
-      map.renderers[s] = createPool(s, minPoolSize, 2 * minPoolSize);
     }
   });
 
@@ -451,16 +459,6 @@ module.exports = function(options, repo, params, id, dataResolver) {
             return res.status(404).send('Not found');
           }
 
-          if (format == 'png') {
-            var usePngQuant =
-                (options.formatQuality || {}).pngQuantization === true;
-            if (usePngQuant) {
-              buffer = pngquant.compress(buffer, {
-                quality: [0, formatQuality || 90]
-              });
-            }
-          }
-
           res.set({
             'Last-Modified': lastModified,
             'Content-Type': 'image/' + format
@@ -753,9 +751,8 @@ module.exports = function(options, repo, params, id, dataResolver) {
     return res.send(info);
   });
 
-  return new Promise(function(resolve, reject) {
+  return Promise.all([fontListingPromise, renderersReadyPromise]).then(function() {
-    Promise.all([fontListingPromise, renderersReadyPromise]).then(function() {
+    return app;
-      resolve(app);
-    });
   });
+
 };

src/server.js

@@ -246,8 +246,9 @@ function start(opts) {
     startupPromises.push(new Promise(function(resolve, reject) {
       fs.readFile(templateFile, function(err, content) {
         if (err) {
-          console.error('Template not found:', err);
+          err = new Error('Template not found: ' + err.message);
           reject(err);
+          return;
         }
         var compiled = handlebars.compile(content.toString());
 
@@ -415,6 +416,11 @@ function start(opts) {
 module.exports = function(opts) {
   var running = start(opts);
 
+  running.startupPromise.catch(function(err) {
+    console.error(err.message);
+    process.exit(1);
+  });
+
   process.on('SIGINT', function() {
     process.exit();
   });

src/utils.js

@@ -110,9 +110,7 @@ module.exports.getFontsPbf = function(allowedFonts, fontPath, names, range, fall
     );
   });
 
-  return new Promise(function(resolve, reject) {
+  return Promise.all(queue).then(function(values) {
-    Promise.all(queue).then(function(values) {
+    return glyphCompose.combine(values);
-      return resolve(glyphCompose.combine(values));
-    }, reject);
   });
 };