Update tileserver-gl

src/server.js

@@ -1,54 +1,68 @@
 #!/usr/bin/env node
 'use strict';
 
-process.env.UV_THREADPOOL_SIZE =
-    Math.ceil(Math.max(4, require('os').cpus().length * 1.5));
+import os from 'os';
+process.env.UV_THREADPOOL_SIZE = Math.ceil(Math.max(4, os.cpus().length * 1.5));
 
-const fs = require('fs');
-const path = require('path');
+import fs from 'node:fs';
+import path from 'path';
 
-const chokidar = require('chokidar');
-const clone = require('clone');
-const cors = require('cors');
-const enableShutdown = require('http-shutdown');
-const express = require('express');
-const handlebars = require('handlebars');
-const mercator = new (require('@mapbox/sphericalmercator'))();
-const morgan = require('morgan');
+import chokidar from 'chokidar';
+import clone from 'clone';
+import cors from 'cors';
+import enableShutdown from 'http-shutdown';
+import express from 'express';
+import handlebars from 'handlebars';
+import SphericalMercator from '@mapbox/sphericalmercator';
+const mercator = new SphericalMercator();
+import morgan from 'morgan';
+import { serve_data } from './serve_data.js';
+import { serve_style } from './serve_style.js';
+import { serve_font } from './serve_font.js';
+import { getTileUrls, getPublicUrl } from './utils.js';
 
-const packageJson = require('../package');
-const serve_font = require('./serve_font');
-const serve_style = require('./serve_style');
-const serve_data = require('./serve_data');
-const utils = require('./utils');
+import { fileURLToPath } from 'url';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const packageJson = JSON.parse(
+  fs.readFileSync(__dirname + '/../package.json', 'utf8'),
+);
 
-let serve_rendered = null;
 const isLight = packageJson.name.slice(-6) === '-light';
-if (!isLight) {
-  // do not require `serve_rendered` in the light package
-  serve_rendered = require('./serve_rendered');
-}
+const serve_rendered = (
+  await import(`${!isLight ? `./serve_rendered.js` : `./serve_light.js`}`)
+).serve_rendered;
 
+/**
+ *
+ * @param opts
+ */
 function start(opts) {
   console.log('Starting server');
 
-  const app = express().disable('x-powered-by'),
-    serving = {
-      styles: {},
-      rendered: {},
-      data: {},
-      fonts: {}
-    };
+  const app = express().disable('x-powered-by');
+  const serving = {
+    styles: {},
+    rendered: {},
+    data: {},
+    fonts: {},
+  };
 
   app.enable('trust proxy');
 
   if (process.env.NODE_ENV !== 'test') {
-    const defaultLogFormat = process.env.NODE_ENV === 'production' ? 'tiny' : 'dev';
+    const defaultLogFormat =
+      process.env.NODE_ENV === 'production' ? 'tiny' : 'dev';
     const logFormat = opts.logFormat || defaultLogFormat;
-    app.use(morgan(logFormat, {
-      stream: opts.logFile ? fs.createWriteStream(opts.logFile, { flags: 'a' }) : process.stdout,
-      skip: (req, res) => opts.silent && (res.statusCode === 200 || res.statusCode === 304)
-    }));
+    app.use(
+      morgan(logFormat, {
+        stream: opts.logFile
+          ? fs.createWriteStream(opts.logFile, { flags: 'a' })
+          : process.stdout,
+        skip: (req, res) =>
+          opts.silent && (res.statusCode === 200 || res.statusCode === 304),
+      }),
+    );
   }
 
   let config = opts.config || null;
@@ -56,7 +70,7 @@ function start(opts) {
   if (opts.configPath) {
     configPath = path.resolve(opts.configPath);
     try {
-      config = clone(require(configPath));
+      config = JSON.parse(fs.readFileSync(configPath, 'utf8'));
     } catch (e) {
       console.log('ERROR: Config file not found or invalid!');
       console.log('       See README.md for instructions and sample data.');
@@ -73,17 +87,21 @@ function start(opts) {
   options.paths = paths;
   paths.root = path.resolve(
     configPath ? path.dirname(configPath) : process.cwd(),
-    paths.root || '');
+    paths.root || '',
+  );
   paths.styles = path.resolve(paths.root, paths.styles || '');
   paths.fonts = path.resolve(paths.root, paths.fonts || '');
   paths.sprites = path.resolve(paths.root, paths.sprites || '');
   paths.mbtiles = path.resolve(paths.root, paths.mbtiles || '');
+  paths.icons = path.resolve(paths.root, paths.icons || '');
 
   const startupPromises = [];
 
-  const checkPath = type => {
+  const checkPath = (type) => {
     if (!fs.existsSync(paths[type])) {
-      console.error(`The specified path for "${type}" does not exist (${paths[type]}).`);
+      console.error(
+        `The specified path for "${type}" does not exist (${paths[type]}).`,
+      );
       process.exit(1);
     }
   };
@@ -91,10 +109,50 @@ function start(opts) {
   checkPath('fonts');
   checkPath('sprites');
   checkPath('mbtiles');
+  checkPath('icons');
+
+  /**
+   * Recursively get all files within a directory.
+   * Inspired by https://stackoverflow.com/a/45130990/10133863
+   * @param {string} directory Absolute path to a directory to get files from.
+   */
+  const getFiles = async (directory) => {
+    // Fetch all entries of the directory and attach type information
+    const dirEntries = await fs.promises.readdir(directory, {
+      withFileTypes: true,
+    });
+
+    // Iterate through entries and return the relative file-path to the icon directory if it is not a directory
+    // otherwise initiate a recursive call
+    const files = await Promise.all(
+      dirEntries.map((dirEntry) => {
+        const entryPath = path.resolve(directory, dirEntry.name);
+        return dirEntry.isDirectory()
+          ? getFiles(entryPath)
+          : entryPath.replace(paths.icons + path.sep, '');
+      }),
+    );
+
+    // Flatten the list of files to a single array
+    return files.flat();
+  };
+
+  // Load all available icons into a settings object
+  startupPromises.push(
+    new Promise((resolve) => {
+      getFiles(paths.icons).then((files) => {
+        paths.availableIcons = files;
+        resolve();
+      });
+    }),
+  );
 
   if (options.dataDecorator) {
     try {
-      options.dataDecoratorFunc = require(path.resolve(paths.root, options.dataDecorator));
+      options.dataDecoratorFunc = require(path.resolve(
+        paths.root,
+        options.dataDecorator,
+      ));
     } catch (e) {}
   }
 
@@ -106,19 +164,23 @@ function start(opts) {
 
   app.use('/data/', serve_data.init(options, serving.data));
   app.use('/styles/', serve_style.init(options, serving.styles));
-  if (serve_rendered) {
+  if (!isLight) {
     startupPromises.push(
-      serve_rendered.init(options, serving.rendered)
-        .then(sub => {
-          app.use('/styles/', sub);
-        })
+      serve_rendered.init(options, serving.rendered).then((sub) => {
+        app.use('/styles/', sub);
+      }),
     );
   }
 
-  let addStyle = (id, item, allowMoreData, reportFonts) => {
+  const addStyle = (id, item, allowMoreData, reportFonts) => {
     let success = true;
     if (item.serve_data !== false) {
-        success = serve_style.add(options, serving.styles, item, id, opts.publicUrl,
+      success = serve_style.add(
+        options,
+        serving.styles,
+        item,
+        id,
+        opts.publicUrl,
         (mbtiles, fromData) => {
           let dataItemId;
           for (const id of Object.keys(serving.data)) {
@@ -132,40 +194,52 @@ function start(opts) {
               }
             }
           }
-          if (dataItemId) { // mbtiles exist in the data config
+          if (dataItemId) {
+            // mbtiles exist in the data config
             return dataItemId;
           } else {
             if (fromData || !allowMoreData) {
-              console.log(`ERROR: style "${item.style}" using unknown mbtiles "${mbtiles}"! Skipping...`);
+              console.log(
+                `ERROR: style "${item.style}" using unknown mbtiles "${mbtiles}"! Skipping...`,
+              );
               return undefined;
             } else {
               let id = mbtiles.substr(0, mbtiles.lastIndexOf('.')) || mbtiles;
               //while (data[id]) id += '_';
               data[id] = {
-                'mbtiles': mbtiles
+                mbtiles: mbtiles,
               };
               return id;
             }
           }
-        }, font => {
+        },
+        (font) => {
           if (reportFonts) {
             serving.fonts[font] = true;
           }
-        });
+        },
+      );
     }
     if (success && item.serve_rendered !== false) {
-      if (serve_rendered) {
-        startupPromises.push(serve_rendered.add(options, serving.rendered, item, id, opts.publicUrl,
-          mbtiles => {
-            let mbtilesFile;
-            for (const id of Object.keys(data)) {
-              if (id === mbtiles) {
-                mbtilesFile = data[id].mbtiles;
+      if (!isLight) {
+        startupPromises.push(
+          serve_rendered.add(
+            options,
+            serving.rendered,
+            item,
+            id,
+            opts.publicUrl,
+            (mbtiles) => {
+              let mbtilesFile;
+              for (const id of Object.keys(data)) {
+                if (id === mbtiles) {
+                  mbtilesFile = data[id].mbtiles;
+                }
               }
-            }
-            return mbtilesFile;
-          }
-        ));
+              return mbtilesFile;
+            },
+          ),
+        );
       } else {
         item.serve_rendered = false;
       }
@@ -188,9 +262,9 @@ function start(opts) {
   }
 
   startupPromises.push(
-    serve_font(options, serving.fonts).then(sub => {
+    serve_font(options, serving.fonts).then((sub) => {
       app.use('/', sub);
-    })
+    }),
   );
 
   for (const id of Object.keys(data)) {
@@ -201,7 +275,7 @@ function start(opts) {
     }
 
     startupPromises.push(
-      serve_data.add(options, serving.data, item, id, opts.publicUrl)
+      serve_data.add(options, serving.data, item, id, opts.publicUrl),
     );
   }
 
@@ -221,16 +295,15 @@ function start(opts) {
       }
     });
 
-    fs.readdir(options.paths.styles, {withFileTypes: true}, (err, files) => {
+    fs.readdir(options.paths.styles, { withFileTypes: true }, (err, files) => {
       if (err) {
         return;
       }
       for (const file of files) {
-        if (file.isFile() &&
-            path.extname(file.name).toLowerCase() == '.json') {
-          let id = path.basename(file.name, '.json');
-          let item = {
-            style: file.name
+        if (file.isFile() && path.extname(file.name).toLowerCase() == '.json') {
+          const id = path.basename(file.name, '.json');
+          const item = {
+            style: file.name,
           };
           addStyle(id, item, true, true);
         }
@@ -254,40 +327,45 @@ function start(opts) {
       }
     });
 
-    const watcher = chokidar.watch(path.join(options.paths.styles, '*.json'),
-      {
-      });
-    watcher.on('all',
-      (eventType, filename) => {
-        if (filename) {
-          let id = path.basename(filename, '.json');
-          console.log(`Style "${id}" changed, updating...`);
+    const watcher = chokidar.watch(
+      path.join(options.paths.styles, '*.json'),
+      {},
+    );
+    watcher.on('all', (eventType, filename) => {
+      if (filename) {
+        const id = path.basename(filename, '.json');
+        console.log(`Style "${id}" changed, updating...`);
 
-          serve_style.remove(serving.styles, id);
-          if (serve_rendered) {
-            serve_rendered.remove(serving.rendered, id);
-          }
-
-          if (eventType == "add" || eventType == "change") {
-            let item = {
-              style: filename
-            };
-            addStyle(id, item, true, true);
-          }
+        serve_style.remove(serving.styles, id);
+        if (!isLight) {
+          serve_rendered.remove(serving.rendered, id);
         }
-      });
+
+        if (eventType == 'add' || eventType == 'change') {
+          const item = {
+            style: filename,
+          };
+          addStyle(id, item, true, true);
+        }
+      }
+    });
   }
 
   app.get('/styles.json', (req, res, next) => {
     const result = [];
-    const query = req.query.key ? (`?key=${encodeURIComponent(req.query.key)}`) : '';
+    const query = req.query.key
+      ? `?key=${encodeURIComponent(req.query.key)}`
+      : '';
     for (const id of Object.keys(serving.styles)) {
       const styleJSON = serving.styles[id].styleJSON;
       result.push({
         version: styleJSON.version,
         name: styleJSON.name,
         id: id,
-        url: `${utils.getPublicUrl(opts.publicUrl, req)}styles/${id}/style.json${query}`
+        url: `${getPublicUrl(
+          opts.publicUrl,
+          req,
+        )}styles/${id}/style.json${query}`,
       });
     }
     res.send(result);
@@ -302,9 +380,16 @@ function start(opts) {
       } else {
         path = `${type}/${id}`;
       }
-      info.tiles = utils.getTileUrls(req, info.tiles, path, info.format, opts.publicUrl, {
-        'pbf': options.pbfAlias
-      });
+      info.tiles = getTileUrls(
+        req,
+        info.tiles,
+        path,
+        info.format,
+        opts.publicUrl,
+        {
+          pbf: options.pbfAlias,
+        },
+      );
       arr.push(info);
     }
     return arr;
@@ -320,7 +405,7 @@ function start(opts) {
     res.send(addTileJSONs(addTileJSONs([], req, 'rendered'), req, 'data'));
   });
 
-  //------------------------------------
+  // ------------------------------------
   // serve web presentations
   app.use('/', express.static(path.join(__dirname, '../public/resources')));
 
@@ -330,43 +415,52 @@ function start(opts) {
     if (template === 'index') {
       if (options.frontPage === false) {
         return;
-      } else if (options.frontPage &&
-        options.frontPage.constructor === String) {
+      } else if (
+        options.frontPage &&
+        options.frontPage.constructor === String
+      ) {
         templateFile = path.resolve(paths.root, options.frontPage);
       }
     }
-    startupPromises.push(new Promise((resolve, reject) => {
-      fs.readFile(templateFile, (err, content) => {
-        if (err) {
-          err = new Error(`Template not found: ${err.message}`);
-          reject(err);
-          return;
-        }
-        const compiled = handlebars.compile(content.toString());
-
-        app.use(urlPath, (req, res, next) => {
-          let data = {};
-          if (dataGetter) {
-            data = dataGetter(req);
-            if (!data) {
-              return res.status(404).send('Not found');
-            }
+    startupPromises.push(
+      new Promise((resolve, reject) => {
+        fs.readFile(templateFile, (err, content) => {
+          if (err) {
+            err = new Error(`Template not found: ${err.message}`);
+            reject(err);
+            return;
           }
-          data['server_version'] = `${packageJson.name} v${packageJson.version}`;
-          data['public_url'] = opts.publicUrl || '/';
-          data['is_light'] = isLight;
-          data['key_query_part'] =
-            req.query.key ? `key=${encodeURIComponent(req.query.key)}&` : '';
-          data['key_query'] = req.query.key ? `?key=${encodeURIComponent(req.query.key)}` : '';
-          if (template === 'wmts') res.set('Content-Type', 'text/xml');
-          return res.status(200).send(compiled(data));
+          const compiled = handlebars.compile(content.toString());
+
+          app.use(urlPath, (req, res, next) => {
+            let data = {};
+            if (dataGetter) {
+              data = dataGetter(req);
+              if (!data) {
+                return res.status(404).send('Not found');
+              }
+            }
+            data[
+              'server_version'
+            ] = `${packageJson.name} v${packageJson.version}`;
+            data['public_url'] = opts.publicUrl || '/';
+            data['is_light'] = isLight;
+            data['key_query_part'] = req.query.key
+              ? `key=${encodeURIComponent(req.query.key)}&`
+              : '';
+            data['key_query'] = req.query.key
+              ? `?key=${encodeURIComponent(req.query.key)}`
+              : '';
+            if (template === 'wmts') res.set('Content-Type', 'text/xml');
+            return res.status(200).send(compiled(data));
+          });
+          resolve();
         });
-        resolve();
-      });
-    }));
+      }),
+    );
   };
 
-  serveTemplate('/$', 'index', req => {
+  serveTemplate('/$', 'index', (req) => {
     const styles = clone(serving.styles || {});
     for (const id of Object.keys(styles)) {
       const style = styles[id];
@@ -376,15 +470,23 @@ function start(opts) {
       if (style.serving_rendered) {
         const center = style.serving_rendered.tileJSON.center;
         if (center) {
-          style.viewer_hash = `#${center[2]}/${center[1].toFixed(5)}/${center[0].toFixed(5)}`;
+          style.viewer_hash = `#${center[2]}/${center[1].toFixed(
+            5,
+          )}/${center[0].toFixed(5)}`;
 
           const centerPx = mercator.px([center[0], center[1]], center[2]);
-          style.thumbnail = `${center[2]}/${Math.floor(centerPx[0] / 256)}/${Math.floor(centerPx[1] / 256)}.png`;
+          style.thumbnail = `${center[2]}/${Math.floor(
+            centerPx[0] / 256,
+          )}/${Math.floor(centerPx[1] / 256)}.png`;
         }
 
-        style.xyz_link = utils.getTileUrls(
-          req, style.serving_rendered.tileJSON.tiles,
-          `styles/${id}`, style.serving_rendered.tileJSON.format, opts.publicUrl)[0];
+        style.xyz_link = getTileUrls(
+          req,
+          style.serving_rendered.tileJSON.tiles,
+          `styles/${id}`,
+          style.serving_rendered.tileJSON.format,
+          opts.publicUrl,
+        )[0];
       }
     }
     const data = clone(serving.data || {});
@@ -393,19 +495,29 @@ function start(opts) {
       const tilejson = data[id].tileJSON;
       const center = tilejson.center;
       if (center) {
-        data_.viewer_hash = `#${center[2]}/${center[1].toFixed(5)}/${center[0].toFixed(5)}`;
+        data_.viewer_hash = `#${center[2]}/${center[1].toFixed(
+          5,
+        )}/${center[0].toFixed(5)}`;
       }
       data_.is_vector = tilejson.format === 'pbf';
       if (!data_.is_vector) {
         if (center) {
           const centerPx = mercator.px([center[0], center[1]], center[2]);
-          data_.thumbnail = `${center[2]}/${Math.floor(centerPx[0] / 256)}/${Math.floor(centerPx[1] / 256)}.${data_.tileJSON.format}`;
+          data_.thumbnail = `${center[2]}/${Math.floor(
+            centerPx[0] / 256,
+          )}/${Math.floor(centerPx[1] / 256)}.${data_.tileJSON.format}`;
         }
 
-        data_.xyz_link = utils.getTileUrls(
-          req, tilejson.tiles, `data/${id}`, tilejson.format, opts.publicUrl, {
-            'pbf': options.pbfAlias
-          })[0];
+        data_.xyz_link = getTileUrls(
+          req,
+          tilejson.tiles,
+          `data/${id}`,
+          tilejson.format,
+          opts.publicUrl,
+          {
+            pbf: options.pbfAlias,
+          },
+        )[0];
       }
       if (data_.filesize) {
         let suffix = 'kB';
@@ -423,11 +535,11 @@ function start(opts) {
     }
     return {
       styles: Object.keys(styles).length ? styles : null,
-      data: Object.keys(data).length ? data : null
+      data: Object.keys(data).length ? data : null,
     };
   });
 
-  serveTemplate('/styles/:id/$', 'viewer', req => {
+  serveTemplate('/styles/:id/$', 'viewer', (req) => {
     const id = req.params.id;
     const style = clone(((serving.styles || {})[id] || {}).styleJSON);
     if (!style) {
@@ -445,22 +557,30 @@ function start(opts) {
     return res.redirect(301, '/styles/' + req.params.id + '/');
   });
   */
-  serveTemplate('/styles/:id/wmts.xml', 'wmts', req => {
+  serveTemplate('/styles/:id/wmts.xml', 'wmts', (req) => {
     const id = req.params.id;
     const wmts = clone((serving.styles || {})[id]);
     if (!wmts) {
       return null;
     }
-    if (wmts.hasOwnProperty("serve_rendered") && !wmts.serve_rendered) {
+    if (wmts.hasOwnProperty('serve_rendered') && !wmts.serve_rendered) {
       return null;
     }
     wmts.id = id;
     wmts.name = (serving.styles[id] || serving.rendered[id]).name;
-    wmts.baseUrl = `${req.get('X-Forwarded-Protocol') ? req.get('X-Forwarded-Protocol') : req.protocol}://${req.get('host')}`;
+    if (opts.publicUrl) {
+      wmts.baseUrl = opts.publicUrl;
+    } else {
+      wmts.baseUrl = `${
+        req.get('X-Forwarded-Protocol')
+          ? req.get('X-Forwarded-Protocol')
+          : req.protocol
+      }://${req.get('host')}/`;
+    }
     return wmts;
   });
 
-  serveTemplate('/data/:id/$', 'data', req => {
+  serveTemplate('/data/:id/$', 'data', (req) => {
     const id = req.params.id;
     const data = clone(serving.data[id]);
     if (!data) {
@@ -484,13 +604,17 @@ function start(opts) {
     }
   });
 
-  const server = app.listen(process.env.PORT || opts.port, process.env.BIND || opts.bind, function () {
-    let address = this.address().address;
-    if (address.indexOf('::') === 0) {
-      address = `[${address}]`; // literal IPv6 address
-    }
-    console.log(`Listening at http://${address}:${this.address().port}/`);
-  });
+  const server = app.listen(
+    process.env.PORT || opts.port,
+    process.env.BIND || opts.bind,
+    function () {
+      let address = this.address().address;
+      if (address.indexOf('::') === 0) {
+        address = `[${address}]`; // literal IPv6 address
+      }
+      console.log(`Listening at http://${address}:${this.address().port}/`);
+    },
+  );
 
   // add server.shutdown() to gracefully stop serving
   enableShutdown(server);
@@ -498,30 +622,39 @@ function start(opts) {
   return {
     app: app,
     server: server,
-    startupPromise: startupPromise
+    startupPromise: startupPromise,
   };
 }
 
-module.exports = opts => {
+/**
+ * Stop the server gracefully
+ * @param {string} signal Name of the received signal
+ */
+function stopGracefully(signal) {
+  console.log(`Caught signal ${signal}, stopping gracefully`);
+  process.exit();
+}
+
+/**
+ *
+ * @param opts
+ */
+export function server(opts) {
   const running = start(opts);
 
-  running.startupPromise.catch(err => {
+  running.startupPromise.catch((err) => {
     console.error(err.message);
     process.exit(1);
   });
 
-  process.on('SIGINT', () => {
-    process.exit();
-  });
+  process.on('SIGINT', stopGracefully);
+  process.on('SIGTERM', stopGracefully);
 
-  process.on('SIGHUP', () => {
+  process.on('SIGHUP', (signal) => {
+    console.log(`Caught signal ${signal}, refreshing`);
     console.log('Stopping server and reloading config');
 
     running.server.shutdown(() => {
-      for (const key in require.cache) {
-        delete require.cache[key];
-      }
-
       const restarted = start(opts);
       running.server = restarted.server;
       running.app = restarted.app;
@@ -529,4 +662,4 @@ module.exports = opts => {
   });
 
   return running;
-};
+}