Merge branch 'dynamic_styles'

This commit is contained in:
Petr Sloup
2020-02-10 14:00:58 +01:00
10 changed files with 1085 additions and 920 deletions
+2
View File
@@ -2,6 +2,8 @@
'use strict';
require = require('esm')(module);
const fs = require('fs');
const path = require('path');
const request = require('request');
+140 -124
View File
@@ -12,144 +12,160 @@ const VectorTile = require('@mapbox/vector-tile').VectorTile;
const utils = require('./utils');
module.exports = (options, repo, params, id, styles, publicUrl) => {
const app = express().disable('x-powered-by');
module.exports = {
init: (options, repo) => {
const app = express().disable('x-powered-by');
const mbtilesFile = path.resolve(options.paths.mbtiles, params.mbtiles);
let tileJSON = {
'tiles': params.domains || options.domains
};
repo[id] = tileJSON;
const mbtilesFileStats = fs.statSync(mbtilesFile);
if (!mbtilesFileStats.isFile() || mbtilesFileStats.size === 0) {
throw Error(`Not valid MBTiles file: ${mbtilesFile}`);
}
let source;
const sourceInfoPromise = new Promise((resolve, reject) => {
source = new MBTiles(mbtilesFile, err => {
if (err) {
reject(err);
return;
app.get('/:id/:z(\\d+)/:x(\\d+)/:y(\\d+).:format([\\w.]+)', (req, res, next) => {
const item = repo[req.params.id];
if (!item) {
return res.sendStatus(404);
}
source.getInfo((err, info) => {
let tileJSONFormat = item.tileJSON.format;
const z = req.params.z | 0;
const x = req.params.x | 0;
const y = req.params.y | 0;
let format = req.params.format;
if (format === options.pbfAlias) {
format = 'pbf';
}
if (format !== tileJSONFormat &&
!(format === 'geojson' && tileJSONFormat === 'pbf')) {
return res.status(404).send('Invalid format');
}
if (z < item.tileJSON.minzoom || 0 || x < 0 || y < 0 ||
z > item.tileJSON.maxzoom ||
x >= Math.pow(2, z) || y >= Math.pow(2, z)) {
return res.status(404).send('Out of bounds');
}
item.source.getTile(z, x, y, (err, data, headers) => {
let isGzipped;
if (err) {
reject(err);
return;
}
tileJSON['name'] = id;
tileJSON['format'] = 'pbf';
Object.assign(tileJSON, info);
tileJSON['tilejson'] = '2.0.0';
delete tileJSON['filesize'];
delete tileJSON['mtime'];
delete tileJSON['scheme'];
Object.assign(tileJSON, params.tilejson || {});
utils.fixTileJSONCenter(tileJSON);
if (options.dataDecoratorFunc) {
tileJSON = options.dataDecoratorFunc(id, 'tilejson', tileJSON);
}
resolve();
});
});
});
const tilePattern = `/${id}/:z(\\d+)/:x(\\d+)/:y(\\d+).:format([\\w.]+)`;
app.get(tilePattern, (req, res, next) => {
const z = req.params.z | 0;
const x = req.params.x | 0;
const y = req.params.y | 0;
let format = req.params.format;
if (format === options.pbfAlias) {
format = 'pbf';
}
if (format !== tileJSON.format &&
!(format === 'geojson' && tileJSON.format === 'pbf')) {
return res.status(404).send('Invalid format');
}
if (z < tileJSON.minzoom || 0 || x < 0 || y < 0 ||
z > tileJSON.maxzoom ||
x >= Math.pow(2, z) || y >= Math.pow(2, z)) {
return res.status(404).send('Out of bounds');
}
source.getTile(z, x, y, (err, data, headers) => {
let isGzipped;
if (err) {
if (/does not exist/.test(err.message)) {
return res.status(204).send();
if (/does not exist/.test(err.message)) {
return res.status(204).send();
} else {
return res.status(500).send(err.message);
}
} else {
return res.status(500).send(err.message);
}
} else {
if (data == null) {
return res.status(404).send('Not found');
} else {
if (tileJSON['format'] === 'pbf') {
isGzipped = data.slice(0, 2).indexOf(
Buffer.from([0x1f, 0x8b])) === 0;
if (options.dataDecoratorFunc) {
if (data == null) {
return res.status(404).send('Not found');
} else {
if (tileJSONFormat === 'pbf') {
isGzipped = data.slice(0, 2).indexOf(
Buffer.from([0x1f, 0x8b])) === 0;
if (options.dataDecoratorFunc) {
if (isGzipped) {
data = zlib.unzipSync(data);
isGzipped = false;
}
data = options.dataDecoratorFunc(id, 'data', data, z, x, y);
}
}
if (format === 'pbf') {
headers['Content-Type'] = 'application/x-protobuf';
} else if (format === 'geojson') {
headers['Content-Type'] = 'application/json';
if (isGzipped) {
data = zlib.unzipSync(data);
isGzipped = false;
}
data = options.dataDecoratorFunc(id, 'data', data, z, x, y);
}
}
if (format === 'pbf') {
headers['Content-Type'] = 'application/x-protobuf';
} else if (format === 'geojson') {
headers['Content-Type'] = 'application/json';
if (isGzipped) {
data = zlib.unzipSync(data);
isGzipped = false;
}
const tile = new VectorTile(new Pbf(data));
const geojson = {
"type": "FeatureCollection",
"features": []
};
for (let layerName in tile.layers) {
const layer = tile.layers[layerName];
for (let i = 0; i < layer.length; i++) {
const feature = layer.feature(i);
const featureGeoJSON = feature.toGeoJSON(x, y, z);
featureGeoJSON.properties.layer = layerName;
geojson.features.push(featureGeoJSON);
const tile = new VectorTile(new Pbf(data));
const geojson = {
"type": "FeatureCollection",
"features": []
};
for (let layerName in tile.layers) {
const layer = tile.layers[layerName];
for (let i = 0; i < layer.length; i++) {
const feature = layer.feature(i);
const featureGeoJSON = feature.toGeoJSON(x, y, z);
featureGeoJSON.properties.layer = layerName;
geojson.features.push(featureGeoJSON);
}
}
data = JSON.stringify(geojson);
}
data = JSON.stringify(geojson);
}
delete headers['ETag']; // do not trust the tile ETag -- regenerate
headers['Content-Encoding'] = 'gzip';
res.set(headers);
delete headers['ETag']; // do not trust the tile ETag -- regenerate
headers['Content-Encoding'] = 'gzip';
res.set(headers);
if (!isGzipped) {
data = zlib.gzipSync(data);
isGzipped = true;
}
if (!isGzipped) {
data = zlib.gzipSync(data);
isGzipped = true;
}
return res.status(200).send(data);
return res.status(200).send(data);
}
}
});
});
app.get('/:id.json', (req, res, next) => {
const item = repo[req.params.id];
if (!item) {
return res.sendStatus(404);
}
const info = clone(item.tileJSON);
info.tiles = utils.getTileUrls(req, info.tiles,
`data/${req.params.id}`, info.format, item.publicUrl, {
'pbf': options.pbfAlias
});
return res.send(info);
});
return app;
},
add: (options, repo, params, id, publicUrl) => {
const mbtilesFile = path.resolve(options.paths.mbtiles, params.mbtiles);
let tileJSON = {
'tiles': params.domains || options.domains
};
const mbtilesFileStats = fs.statSync(mbtilesFile);
if (!mbtilesFileStats.isFile() || mbtilesFileStats.size === 0) {
throw Error(`Not valid MBTiles file: ${mbtilesFile}`);
}
let source;
const sourceInfoPromise = new Promise((resolve, reject) => {
source = new MBTiles(mbtilesFile, err => {
if (err) {
reject(err);
return;
}
source.getInfo((err, info) => {
if (err) {
reject(err);
return;
}
tileJSON['name'] = id;
tileJSON['format'] = 'pbf';
Object.assign(tileJSON, info);
tileJSON['tilejson'] = '2.0.0';
delete tileJSON['filesize'];
delete tileJSON['mtime'];
delete tileJSON['scheme'];
Object.assign(tileJSON, params.tilejson || {});
utils.fixTileJSONCenter(tileJSON);
if (options.dataDecoratorFunc) {
tileJSON = options.dataDecoratorFunc(id, 'tilejson', tileJSON);
}
resolve();
});
});
});
return sourceInfoPromise.then(() => {
repo[id] = {
tileJSON,
publicUrl,
source
}
});
});
app.get(`/${id}.json`, (req, res, next) => {
const info = clone(tileJSON);
info.tiles = utils.getTileUrls(req, info.tiles,
`data/${id}`, info.format, publicUrl, {
'pbf': options.pbfAlias
});
return res.send(info);
});
return sourceInfoPromise.then(() => app);
}
};
+692 -653
View File
File diff suppressed because it is too large Load Diff
+135 -96
View File
@@ -5,115 +5,154 @@ const fs = require('fs');
const clone = require('clone');
const express = require('express');
import {validate} from '@mapbox/mapbox-gl-style-spec';
const utils = require('./utils');
module.exports = (options, repo, params, id, publicUrl, reportTiles, reportFont) => {
const app = express().disable('x-powered-by');
const httpTester = /^(http(s)?:)?\/\//;
const styleFile = path.resolve(options.paths.styles, params.style);
const fixUrl = (req, url, publicUrl, opt_nokey) => {
if (!url || (typeof url !== 'string') || url.indexOf('local://') !== 0) {
return url;
}
const queryParams = [];
if (!opt_nokey && req.query.key) {
queryParams.unshift(`key=${req.query.key}`);
}
let query = '';
if (queryParams.length) {
query = `?${queryParams.join('&')}`;
}
return url.replace(
'local://', utils.getPublicUrl(publicUrl, req)) + query;
};
const styleJSON = clone(require(styleFile));
for (const name of Object.keys(styleJSON.sources)) {
const source = styleJSON.sources[name];
const url = source.url;
if (url && url.lastIndexOf('mbtiles:', 0) === 0) {
let mbtilesFile = url.substring('mbtiles://'.length);
const fromData = mbtilesFile[0] === '{' &&
mbtilesFile[mbtilesFile.length - 1] === '}';
module.exports = {
init: (options, repo) => {
const app = express().disable('x-powered-by');
if (fromData) {
mbtilesFile = mbtilesFile.substr(1, mbtilesFile.length - 2);
const mapsTo = (params.mapping || {})[mbtilesFile];
if (mapsTo) {
mbtilesFile = mapsTo;
app.get('/:id/style.json', (req, res, next) => {
const item = repo[req.params.id];
if (!item) {
return res.sendStatus(404);
}
const styleJSON_ = clone(item.styleJSON);
for (const name of Object.keys(styleJSON_.sources)) {
const source = styleJSON_.sources[name];
source.url = fixUrl(req, source.url, item.publicUrl);
}
// mapbox-gl-js viewer cannot handle sprite urls with query
if (styleJSON_.sprite) {
styleJSON_.sprite = fixUrl(req, styleJSON_.sprite, item.publicUrl, true);
}
if (styleJSON_.glyphs) {
styleJSON_.glyphs = fixUrl(req, styleJSON_.glyphs, item.publicUrl, false);
}
return res.send(styleJSON_);
});
app.get('/:id/sprite:scale(@[23]x)?.:format([\\w]+)', (req, res, next) => {
const item = repo[req.params.id];
if (!item || !item.spritePath) {
return res.sendStatus(404);
}
const scale = req.params.scale,
format = req.params.format;
const filename = `${item.spritePath + (scale || '')}.${format}`;
return fs.readFile(filename, (err, data) => {
if (err) {
console.log('Sprite load error:', filename);
return res.sendStatus(404);
} else {
if (format === 'json') res.header('Content-type', 'application/json');
if (format === 'png') res.header('Content-type', 'image/png');
return res.send(data);
}
});
});
return app;
},
remove: (repo, id) => {
delete repo[id];
},
add: (options, repo, params, id, publicUrl, reportTiles, reportFont) => {
const styleFile = path.resolve(options.paths.styles, params.style);
let styleFileData;
try {
styleFileData = fs.readFileSync(styleFile);
} catch (e) {
console.log('Error reading style file');
return false;
}
let validationErrors = validate(styleFileData);
if (validationErrors.length > 0) {
console.log(`The file "${params.style}" is not valid a valid style file:`);
for (const err of validationErrors) {
console.log(`${err.line}: ${err.message}`);
}
return false;
}
let styleJSON = JSON.parse(styleFileData);
for (const name of Object.keys(styleJSON.sources)) {
const source = styleJSON.sources[name];
const url = source.url;
if (url && url.lastIndexOf('mbtiles:', 0) === 0) {
let mbtilesFile = url.substring('mbtiles://'.length);
const fromData = mbtilesFile[0] === '{' &&
mbtilesFile[mbtilesFile.length - 1] === '}';
if (fromData) {
mbtilesFile = mbtilesFile.substr(1, mbtilesFile.length - 2);
const mapsTo = (params.mapping || {})[mbtilesFile];
if (mapsTo) {
mbtilesFile = mapsTo;
}
}
const identifier = reportTiles(mbtilesFile, fromData);
if (!identifier) {
return false;
}
source.url = `local://data/${identifier}.json`;
}
}
for (let obj of styleJSON.layers) {
if (obj['type'] === 'symbol') {
const fonts = (obj['layout'] || {})['text-font'];
if (fonts && fonts.length) {
fonts.forEach(reportFont);
} else {
reportFont('Open Sans Regular');
reportFont('Arial Unicode MS Regular');
}
}
const identifier = reportTiles(mbtilesFile, fromData);
source.url = `local://data/${identifier}.json`;
}
}
for(let obj of styleJSON.layers) {
if (obj['type'] === 'symbol') {
const fonts = (obj['layout'] || {})['text-font'];
if (fonts && fonts.length) {
fonts.forEach(reportFont);
} else {
reportFont('Open Sans Regular');
reportFont('Arial Unicode MS Regular');
}
}
}
let spritePath;
let spritePath;
const httpTester = /^(http(s)?:)?\/\//;
if (styleJSON.sprite && !httpTester.test(styleJSON.sprite)) {
spritePath = path.join(options.paths.sprites,
if (styleJSON.sprite && !httpTester.test(styleJSON.sprite)) {
spritePath = path.join(options.paths.sprites,
styleJSON.sprite
.replace('{style}', path.basename(styleFile, '.json'))
.replace('{styleJsonFolder}', path.relative(options.paths.sprites, path.dirname(styleFile)))
);
styleJSON.sprite = `local://styles/${id}/sprite`;
}
if (styleJSON.glyphs && !httpTester.test(styleJSON.glyphs)) {
styleJSON.glyphs = 'local://fonts/{fontstack}/{range}.pbf';
}
.replace('{style}', path.basename(styleFile, '.json'))
.replace('{styleJsonFolder}', path.relative(options.paths.sprites, path.dirname(styleFile)))
);
styleJSON.sprite = `local://styles/${id}/sprite`;
}
if (styleJSON.glyphs && !httpTester.test(styleJSON.glyphs)) {
styleJSON.glyphs = 'local://fonts/{fontstack}/{range}.pbf';
}
repo[id] = styleJSON;
app.get(`/${id}/style.json`, (req, res, next) => {
const fixUrl = (url, opt_nokey) => {
if (!url || (typeof url !== 'string') || url.indexOf('local://') !== 0) {
return url;
}
const queryParams = [];
if (!opt_nokey && req.query.key) {
queryParams.unshift(`key=${req.query.key}`);
}
let query = '';
if (queryParams.length) {
query = `?${queryParams.join('&')}`;
}
return url.replace(
'local://', utils.getPublicUrl(publicUrl, req)) + query;
repo[id] = {
styleJSON,
spritePath,
publicUrl,
name: styleJSON.name
};
const styleJSON_ = clone(styleJSON);
for (const name of Object.keys(styleJSON_.sources)) {
const source = styleJSON_.sources[name];
source.url = fixUrl(source.url);
}
// mapbox-gl-js viewer cannot handle sprite urls with query
if (styleJSON_.sprite) {
styleJSON_.sprite = fixUrl(styleJSON_.sprite, true);
}
if (styleJSON_.glyphs) {
styleJSON_.glyphs = fixUrl(styleJSON_.glyphs, false);
}
return res.send(styleJSON_);
});
app.get(`/${id}/sprite:scale(@[23]x)?.:format([\\w]+)`,
(req, res, next) => {
if (!spritePath) {
return res.status(404).send('File not found');
}
const scale = req.params.scale,
format = req.params.format;
const filename = `${spritePath + (scale || '')}.${format}`;
return fs.readFile(filename, (err, data) => {
if (err) {
console.log('Sprite load error:', filename);
return res.status(404).send('File not found');
} else {
if (format === 'json') res.header('Content-type', 'application/json');
if (format === 'png') res.header('Content-type', 'image/png');
return res.send(data);
}
});
});
return Promise.resolve(app);
return true;
}
};
+97 -45
View File
@@ -7,6 +7,7 @@ process.env.UV_THREADPOOL_SIZE =
const fs = require('fs');
const path = require('path');
const chokidar = require('chokidar');
const clone = require('clone');
const cors = require('cors');
const enableShutdown = require('http-shutdown');
@@ -103,15 +104,21 @@ function start(opts) {
app.use(cors());
}
for (const id of Object.keys(config.styles || {})) {
const item = config.styles[id];
if (!item.style || item.style.length === 0) {
console.log(`Missing "style" property for ${id}`);
continue;
}
app.use('/data/', serve_data.init(options, serving.data));
app.use('/styles/', serve_style.init(options, serving.styles));
if (serve_rendered) {
startupPromises.push(
serve_rendered.init(options, serving.rendered)
.then(sub => {
app.use('/styles/', sub);
})
);
}
let addStyle = (id, item, allowMoreData, reportFonts) => {
let success = true;
if (item.serve_data !== false) {
startupPromises.push(serve_style(options, serving.styles, item, id, opts.publicUrl,
success = serve_style.add(options, serving.styles, item, id, opts.publicUrl,
(mbtiles, fromData) => {
let dataItemId;
for (const id of Object.keys(data)) {
@@ -127,44 +134,52 @@ function start(opts) {
}
if (dataItemId) { // mbtiles exist in the data config
return dataItemId;
} else if (fromData) {
console.log(`ERROR: data "${mbtiles}" not found!`);
process.exit(1);
} else {
let id = mbtiles.substr(0, mbtiles.lastIndexOf('.')) || mbtiles;
while (data[id]) id += '_';
data[id] = {
'mbtiles': mbtiles
};
return id;
if (fromData || !allowMoreData) {
console.log(`ERROR: style "${file.name}" using unknown mbtiles "${mbtiles}"! Skipping...`);
return undefined;
} else {
let id = mbtiles.substr(0, mbtiles.lastIndexOf('.')) || mbtiles;
while (data[id]) id += '_';
data[id] = {
'mbtiles': mbtiles
};
return id;
}
}
}, font => {
serving.fonts[font] = true;
}).then(sub => {
app.use('/styles/', sub);
}));
if (reportFonts) {
serving.fonts[font] = true;
}
});
}
if (item.serve_rendered !== false) {
if (success && item.serve_rendered !== false) {
if (serve_rendered) {
startupPromises.push(
serve_rendered(options, serving.rendered, item, id, opts.publicUrl,
mbtiles => {
let mbtilesFile;
for (const id of Object.keys(data)) {
if (id === mbtiles) {
mbtilesFile = data[id].mbtiles;
}
startupPromises.push(serve_rendered.add(options, serving.rendered, item, id, opts.publicUrl,
mbtiles => {
let mbtilesFile;
for (const id of Object.keys(data)) {
if (id === mbtiles) {
mbtilesFile = data[id].mbtiles;
}
return mbtilesFile;
}
).then(sub => {
app.use('/styles/', sub);
})
);
return mbtilesFile;
}
));
} else {
item.serve_rendered = false;
}
}
};
for (const id of Object.keys(config.styles || {})) {
const item = config.styles[id];
if (!item.style || item.style.length === 0) {
console.log(`Missing "style" property for ${id}`);
continue;
}
addStyle(id, item, true, true);
}
startupPromises.push(
@@ -181,17 +196,54 @@ function start(opts) {
}
startupPromises.push(
serve_data(options, serving.data, item, id, serving.styles, opts.publicUrl).then(sub => {
app.use('/data/', sub);
})
serve_data.add(options, serving.data, item, id, opts.publicUrl)
);
}
if (options.serveAllStyles) {
fs.readdir(options.paths.styles, {withFileTypes: true}, (err, files) => {
if (err) {
return;
}
for (const file of files) {
if (file.isFile() &&
path.extname(file.name).toLowerCase() == '.json') {
let id = path.basename(file.name, '.json');
let item = {
style: file.name
};
addStyle(id, item, false, false);
}
}
});
const watcher = chokidar.watch(path.join(options.paths.styles, '*.json'),
{
});
watcher.on('all',
(eventType, filename) => {
if (filename) {
let id = path.basename(filename, '.json');
console.log(`Style "${id}" changed, updating...`);
serve_style.remove(serving.styles, id);
serve_rendered.remove(serving.rendered, id);
if (eventType == "add" || eventType == "change") {
let item = {
style: filename
};
addStyle(id, item, false, false);
}
}
});
}
app.get('/styles.json', (req, res, next) => {
const result = [];
const query = req.query.key ? (`?key=${req.query.key}`) : '';
for (const id of Object.keys(serving.styles)) {
const styleJSON = serving.styles[id];
const styleJSON = serving.styles[id].styleJSON;
result.push({
version: styleJSON.version,
name: styleJSON.name,
@@ -204,7 +256,7 @@ function start(opts) {
const addTileJSONs = (arr, req, type) => {
for (const id of Object.keys(serving[type])) {
const info = clone(serving[type][id]);
const info = clone(serving[type][id].tileJSON);
let path = '';
if (type === 'rendered') {
path = `styles/${id}`;
@@ -276,14 +328,14 @@ function start(opts) {
};
serveTemplate('/$', 'index', req => {
const styles = clone(config.styles || {});
const styles = clone(serving.styles || {});
for (const id of Object.keys(styles)) {
const style = styles[id];
style.name = (serving.styles[id] || serving.rendered[id] || {}).name;
style.serving_data = serving.styles[id];
style.serving_rendered = serving.rendered[id];
if (style.serving_rendered) {
const center = style.serving_rendered.center;
const center = style.serving_rendered.tileJSON.center;
if (center) {
style.viewer_hash = `#${center[2]}/${center[1].toFixed(5)}/${center[0].toFixed(5)}`;
@@ -292,8 +344,8 @@ function start(opts) {
}
style.xyz_link = utils.getTileUrls(
req, style.serving_rendered.tiles,
`styles/${id}`, style.serving_rendered.format, opts.publicUrl)[0];
req, style.serving_rendered.tileJSON.tiles,
`styles/${id}`, style.serving_rendered.tileJSON.format, opts.publicUrl)[0];
}
}
const data = clone(serving.data || {});
@@ -337,7 +389,7 @@ function start(opts) {
serveTemplate('/styles/:id/$', 'viewer', req => {
const id = req.params.id;
const style = clone((config.styles || {})[id]);
const style = clone(((serving.styles || {})[id] || {}).styleJSON);
if (!style) {
return null;
}
@@ -355,7 +407,7 @@ function start(opts) {
*/
serveTemplate('/styles/:id/wmts.xml', 'wmts', req => {
const id = req.params.id;
const wmts = clone((config.styles || {})[id]);
const wmts = clone((serving.styles || {})[id]);
if (!wmts) {
return null;
}