diff --git a/lib/mbtiles.js b/lib/mbtiles.js index 596ddb4..1919b1e 100644 --- a/lib/mbtiles.js +++ b/lib/mbtiles.js @@ -1,16 +1,12 @@ -var _ = require('underscore'), - fs = require('fs'), - Step = require('step'), - crypto = require('crypto'), - zlib = require('zlib'), - path = require('path'), - url = require('url'), - qs = require('querystring'), - Buffer = require('buffer').Buffer, - sm = new (require('sphericalmercator')), - sqlite3 = require('sqlite3'); - -if (process.env.NODE_ENV === 'test') sqlite3.verbose(); +var fs = require('fs'); +var crypto = require('crypto'); +var zlib = require('zlib'); +var path = require('path'); +var url = require('url'); +var qs = require('querystring'); +var Buffer = require('buffer').Buffer; +var sm = new (require('sphericalmercator')); +var sqlite3 = require('sqlite3'); function noop(err) { if (err) throw err; @@ -53,16 +49,14 @@ function MBTiles(uri, callback) { this.setMaxListeners(0); this.filename = uri.pathname; this._batchSize = +uri.query.batch; - Step(function() { - mbtiles._db = new sqlite3.Database(mbtiles.filename, this); - }, function(err) { - if (err) throw err; - fs.stat(mbtiles.filename, this); - }, function(err, stat) { + mbtiles._db = new sqlite3.Database(mbtiles.filename, function(err) { if (err) return callback(err); - mbtiles._stat = stat; - mbtiles.open = true; - callback(null, mbtiles); + fs.stat(mbtiles.filename, function(err, stat) { + if (err) return callback(err); + mbtiles._stat = stat; + mbtiles.open = true; + callback(null, mbtiles); + }); }); return undefined; @@ -103,7 +97,7 @@ MBTiles.prototype._exists = function(table, callback) { if (typeof callback !== 'function') callback = noop; if (this._schema) { - return callback(null, _(this._schema).include(table)); + return callback(null, this._schema.indexOf(table) !== -1); } else { this._db.all( 'SELECT name FROM sqlite_master WHERE type IN (?, ?)', @@ -111,7 +105,7 @@ MBTiles.prototype._exists = function(table, callback) { 'view', function(err, rows) { if (err) return callback(err); - this._schema = _(rows).pluck('name'); + this._schema = rows.map(function(r) { return r.name }); this._exists(table, callback); }.bind(this) ); @@ -159,24 +153,22 @@ MBTiles.prototype.getTile = function(z, x, y, callback) { // Flip Y coordinate because MBTiles files are TMS. y = (1 << z) - 1 - y; + var sql = 'SELECT tile_data FROM tiles WHERE zoom_level = ? AND tile_column = ? AND tile_row = ?'; var mbtiles = this; - this._db.get('SELECT tile_data FROM tiles WHERE ' + - 'zoom_level = ? AND tile_column = ? AND tile_row = ?', - z, x, y, - function(err, row) { - if ((!err && !row) || (err && err.errno == 1)) { - return callback(new Error('Tile does not exist')); - } else if (err) { - return callback(err); - } else { - var options = { - 'Content-Type': MBTiles.utils.getMimeType(row.tile_data), - 'Last-Modified': new Date(mbtiles._stat.mtime).toUTCString(), - 'ETag': mbtiles._stat.size + '-' + Number(mbtiles._stat.mtime) - }; - return callback(null, row.tile_data, options); - } - }); + this._db.get(sql, z, x, y, function(err, row) { + if ((!err && !row) || (err && err.errno == 1)) { + return callback(new Error('Tile does not exist')); + } else if (err) { + return callback(err); + } else { + var options = { + 'Content-Type': MBTiles.utils.getMimeType(row.tile_data), + 'Last-Modified': new Date(mbtiles._stat.mtime).toUTCString(), + 'ETag': mbtiles._stat.size + '-' + Number(mbtiles._stat.mtime) + }; + return callback(null, row.tile_data, options); + } + }); }; // Select a grid and its data from an mbtiles database. Scheme is XYZ. @@ -192,190 +184,38 @@ MBTiles.prototype.getGrid = function(z, x, y, callback) { // Flip Y coordinate because MBTiles files are TMS. y = (1 << z) - 1 - y; - var that = this; - Step( - function() { - that._db.get('SELECT grid FROM grids WHERE ' + - 'zoom_level = ? AND tile_column = ? AND tile_row = ?', - z, x, y, - this.parallel() - ); - that._db.all('SELECT key_name, key_json FROM grid_data WHERE ' + - 'zoom_level = ? AND tile_column = ? AND tile_row = ?', - z, x, y, - this.parallel() - ); - }, - function(err, row, rows) { - if ((!row || !row.grid) || (err && err.errno == 1)) { - return callback(new Error('Grid does not exist')); - } - if (err) return callback(err); + var sqlgrid = 'SELECT grid FROM grids WHERE zoom_level = ? AND tile_column = ? AND tile_row = ?'; + var sqljson = 'SELECT key_name, key_json FROM grid_data WHERE zoom_level = ? AND tile_column = ? AND tile_row = ?'; - zlib.inflate(!Buffer.isBuffer(row.grid) - ? new Buffer(row.grid, 'binary') - : row.grid, - function(err,buffer) { - if (err) return callback(new Error('Grid is invalid:' + err.message)); - try { - var data = rows.reduce(function(memo, r) { - memo[r.key_name] = JSON.parse(r.key_json); - return memo; - }, {}); - var result = JSON.parse(buffer.toString()); - } catch(err) { - return callback(new Error('Grid is invalid:' + err.message)); - } - var options = { - 'Content-Type': 'text/javascript', - 'Last-Modified': new Date(that._stat.mtime).toUTCString(), - 'ETag': that._stat.size + '-' + Number(that._stat.mtime) - }; - callback(null, _(result).extend({data:data}), options); + var mbtiles = this; + mbtiles._db.get(sqlgrid, z, x, y, function(err, row) { + if (err && err.errno !== 1) return callback(err); + if (!row || !row.grid || err) return callback(new Error('Grid does not exist')); + zlib.inflate(!Buffer.isBuffer(row.grid) ? new Buffer(row.grid, 'binary') : row.grid, function(err, buffer) { + if (err) return callback(new Error('Grid is invalid:' + err.message)); + try { var grid = JSON.parse(buffer); } + catch(err) { return callback(new Error('Grid is invalid:' + err.message)) }; + mbtiles._db.all(sqljson, z, x, y, function(err, rows) { + if (err) return callback(err); + grid.data = grid.data || {}; + for (var i = 0; i < rows.length; i++) { + try { grid.data[rows[i].key_name] = JSON.parse(rows[i].key_json); } + catch(err) { return callback(new Error('Grid is invalid:' + err.message)) }; + } + callback(null, grid, { + 'Content-Type': 'text/javascript', + 'Last-Modified': new Date(mbtiles._stat.mtime).toUTCString(), + 'ETag': mbtiles._stat.size + '-' + Number(mbtiles._stat.mtime) }); - } - ); -}; - -// Select a metadata value from the database. -// -// - @param {Function} callback -MBTiles.prototype._metadata = function(key, callback) { - if (typeof callback !== 'function') callback = noop; - - this._db.get('SELECT value FROM metadata WHERE name = ?', - key, - function(err, row) { - // If the metadata table or the requested field is missing return - // null and allow the caller handle it. - if (!row || (err && err.errno == 1)) return callback(null, null); - else if (err) return callback(err); - else return callback(null, row.value); + }); }); -}; - - -MBTiles.prototype._close = function(callback) { - this._db.close(callback); + }); }; MBTiles.prototype.close = function(callback) { - this._close(callback); + this._db.close(callback); }; -MBTiles.prototype._getInfo = function(callback) { - var mbtiles = this; - var info = {}; - info.scheme = 'tms'; - info.basename = path.basename(mbtiles.filename); - info.id = path.basename(mbtiles.filename, path.extname(mbtiles.filename)); - info.filesize = mbtiles._stat.size; - Step(function() { - var end = this; - mbtiles._db.all('SELECT name, value FROM metadata', function(err, rows) { - if (rows) for (var i = 0; i < rows.length; i++) { - info[rows[i].name] = rows[i].value; - } - end(err); - }); - }, - // Determine min/max zoom if needed - function(err) { - if (err && err.errno !== 1) return callback(err); - if (info.maxzoom !== undefined - && info.minzoom !== undefined) return this(); - - var step = this; - var zoomquery = mbtiles._db.prepare('SELECT zoom_level FROM tiles ' + - 'WHERE zoom_level = ? LIMIT 1', function(err) { - if (err) { - if (err.errno === 1) step(); - else throw new Error(err); - } else { - var group = step.group(); - for (var i = 0; i < 30; i++) { - zoomquery.get(i, group()); - } - zoomquery.finalize(); - } - }); - }, - function(err, rows) { - if (err) return callback(err); - if (rows) { - var zooms = _(rows).chain() - .reject(_.isUndefined) - .pluck('zoom_level') - .value(); - info.minzoom = zooms.shift(); - info.maxzoom = zooms.length ? zooms.pop() : info.minzoom; - } - this(); - }, - // Determine bounds if needed - function(err) { - if (err) return callback(err); - if (info.bounds) return this(); - if (typeof info.minzoom === 'undefined') return this(); - - var next = this; - Step( - function() { - mbtiles._db.get( - 'SELECT MAX(tile_column) AS maxx, ' + - 'MIN(tile_column) AS minx, MAX(tile_row) AS maxy, ' + - 'MIN(tile_row) AS miny FROM tiles ' + - 'WHERE zoom_level = ?', - info.minzoom, - this - ); - }, - function(err, row) { - if (!err && row) { - // @TODO this breaks a little at zoom level zero - var urTile = sm.bbox(row.maxx, row.maxy, info.minzoom, true); - var llTile = sm.bbox(row.minx, row.miny, info.minzoom, true); - // @TODO bounds are limited to "sensible" values here - // as sometimes tilesets are rendered with "negative" - // and/or other extremity tiles. Revisit this if there - // are actual use cases for out-of-bounds bounds. - info.bounds = [ - llTile[0] > -180 ? llTile[0] : -180, - llTile[1] > -90 ? llTile[1] : -90, - urTile[2] < 180 ? urTile[2] : 180, - urTile[3] < 90 ? urTile[3] : 90 - ].join(','); - } - next(); - } - ); - }, - // Return info - function(err) { - if (err) return callback(err); - var range = parseInt(info.maxzoom, 10) - parseInt(info.minzoom, 10); - info.minzoom = parseInt(info.minzoom, 10); - if (isNaN(info.minzoom) || typeof info.minzoom !== 'number') delete info.minzoom; - info.maxzoom = parseInt(info.maxzoom, 10); - if (isNaN(info.maxzoom) || typeof info.maxzoom !== 'number') delete info.maxzoom; - - info.bounds = _((info.bounds || '').split(',')).map(parseFloat); - if (info.bounds.length !== 4 || info.bounds[0] === null) delete info.bounds; - - if (info.center) info.center = _((info.center).split(',')).map(parseFloat); - if ((!info.center || info.center.length !== 3) && info.bounds) info.center = [ - (info.bounds[2] - info.bounds[0]) / 2 + info.bounds[0], - (info.bounds[3] - info.bounds[1]) / 2 + info.bounds[1], - (range <= 1) ? info.maxzoom : Math.floor(range * 0.5) + info.minzoom - ]; - if (info.center && (info.center.length !== 3 || info.center[0] === null || isNaN(info.center[2]))) { - delete info.center; - } - - return callback(null, info); - }); -} - // Obtain metadata from the database. Performing fallback queries if certain // keys(like `bounds`, `minzoom`, `maxzoom`) have not been provided. // @@ -384,11 +224,101 @@ MBTiles.prototype.getInfo = function(callback) { if (typeof callback !== 'function') throw new Error('Callback needed'); if (!this.open) return callback(new Error('MBTiles not yet loaded')); if (this._info) return callback(null, this._info); + var mbtiles = this; - mbtiles._getInfo(function(err, info) { - mbtiles._info = info; - return callback(err, info); + var info = {}; + info.scheme = 'tms'; + info.basename = path.basename(mbtiles.filename); + info.id = path.basename(mbtiles.filename, path.extname(mbtiles.filename)); + info.filesize = mbtiles._stat.size; + mbtiles._db.all('SELECT name, value FROM metadata', function(err, rows) { + if (err && err.errno !== 1) return callback(err); + if (rows) rows.forEach(function(row) { + switch (row.name) { + case 'minzoom': + case 'maxzoom': + info[row.name] = parseInt(row.value, 10); + break; + case 'center': + case 'bounds': + info[row.name] = row.value.split(',').map(parseFloat); + break; + default: + info[row.name] = row.value; + break; + } + }); + ensureZooms(info, function(err, info) { + if (err) return callback(err); + ensureBounds(info, function(err, info) { + if (err) return callback(err); + ensureCenter(info, function(err, info) { + if (err) return callback(err); + mbtiles._info = info; + return callback(null, info); + }); + }); + }); }); + function ensureZooms(info, callback) { + if ('minzoom' in info && 'maxzoom' in info) return callback(null, info); + var remaining = 30; + var zooms = []; + var query = mbtiles._db.prepare('SELECT zoom_level FROM tiles WHERE zoom_level = ? LIMIT 1', function(err) { + if (err) return callback(err.errno === 1 ? null : err, info); + for (var i = 0; i < remaining; i++) query.get(i, function(err, row) { + if (err) return (remaining = 0) && callback(err); + if (row) zooms.push(row.zoom_level); + if (--remaining === 0) { + if (!zooms.length) return callback(null, info); + zooms.sort(function(a,b) { return a < b ? -1 : 1 }); + info.minzoom = zooms[0]; + info.maxzoom = zooms.pop(); + return callback(null, info); + } + }); + query.finalize(); + }); + }; + function ensureBounds(info, callback) { + if ('bounds' in info) return callback(null, info); + if (!('minzoom' in info)) return callback(null, info); + mbtiles._db.get( + 'SELECT MAX(tile_column) AS maxx, ' + + 'MIN(tile_column) AS minx, MAX(tile_row) AS maxy, ' + + 'MIN(tile_row) AS miny FROM tiles ' + + 'WHERE zoom_level = ?', + info.minzoom, + function(err, row) { + if (err) return callback(err); + if (!row) return callback(null, info); + + // @TODO this breaks a little at zoom level zero + var urTile = sm.bbox(row.maxx, row.maxy, info.minzoom, true); + var llTile = sm.bbox(row.minx, row.miny, info.minzoom, true); + // @TODO bounds are limited to "sensible" values here + // as sometimes tilesets are rendered with "negative" + // and/or other extremity tiles. Revisit this if there + // are actual use cases for out-of-bounds bounds. + info.bounds = [ + llTile[0] > -180 ? llTile[0] : -180, + llTile[1] > -90 ? llTile[1] : -90, + urTile[2] < 180 ? urTile[2] : 180, + urTile[3] < 90 ? urTile[3] : 90 + ].join(','); + return callback(null, info); + }); + }; + function ensureCenter(info, callback) { + if ('center' in info) return callback(null, info); + if (!('bounds' in info) || !('minzoom' in info) || !('maxzoom' in info)) return callback(null, info); + info.center = [ + (info.bounds[2] - info.bounds[0]) / 2 + info.bounds[0], + (info.bounds[3] - info.bounds[1]) / 2 + info.bounds[1], + Math.floor((info.maxzoom-info.minzoom)*0.5) + info.minzoom + ]; + return callback(null, info); + }; }; // Puts the MBTiles tilestore into write mode. @@ -400,21 +330,14 @@ MBTiles.prototype.startWriting = function(callback) { var mbtiles = this; mbtiles._clearCaches(); - Step(function() { - mbtiles._setup(this); - }, function(err) { - if (err) throw err; + mbtiles._setup(function(err) { + if (err) return callback(err); + if (mbtiles._isWritable) return callback(); + // Sets the synchronous flag to OFF for (much) faster inserts. // See http://www.sqlite3.org/pragma.html#pragma_synchronous - if (!mbtiles._isWritable) { - mbtiles._isWritable = 1; - mbtiles._db.run('PRAGMA synchronous=OFF', this); - } else { - mbtiles._isWritable++; - this(); - } - }, function(err) { - return callback(err); + mbtiles._isWritable = 1; + mbtiles._db.run('PRAGMA synchronous=OFF', callback); }); }; @@ -444,7 +367,6 @@ MBTiles.prototype._commit = function(callback) { images.finalize(); } - if (Object.keys(mbtiles._gridCache)) { // Insert grid_utfgrid table. var grids = mbtiles._db.prepare('REPLACE INTO grid_utfgrid (grid_id, grid_utfgrid) VALUES (?, ?)'); @@ -454,7 +376,6 @@ MBTiles.prototype._commit = function(callback) { grids.finalize(); } - if (Object.keys(mbtiles._keyCache)) { // Insert grid_key. var keys = mbtiles._db.prepare('INSERT OR IGNORE INTO grid_key (grid_id, key_name) VALUES (?, ?)'); @@ -466,7 +387,6 @@ MBTiles.prototype._commit = function(callback) { keys.finalize(); } - if (Object.keys(mbtiles._dataCache)) { // Insert keymap table. var keymap = mbtiles._db.prepare('REPLACE INTO keymap (key_name, key_json) VALUES (?, ?)'); @@ -520,14 +440,13 @@ MBTiles.prototype.stopWriting = function(callback) { if (!this.open) return callback(new Error('MBTiles not yet loaded')); var mbtiles = this; - if (this._isWritable) this._isWritable--; - this._commit(function(err) { + mbtiles._commit(function(err) { if (err) return callback(err); - if (!mbtiles._isWritable) { - mbtiles._db.run('PRAGMA synchronous=NORMAL', callback); - } else { - return callback(null); - } + mbtiles._db.run('PRAGMA synchronous=NORMAL', function(err) { + if (err) return callback(err); + mbtiles._isWritable = false; + return callback(); + }); }); }; @@ -542,7 +461,6 @@ MBTiles.prototype.putTile = function(z, x, y, data, callback) { if (typeof callback !== 'function') throw new Error('Callback needed'); if (!this.open) return callback(new Error('MBTiles not yet loaded')); if (!this._isWritable) return callback(new Error('MBTiles not in write mode')); - if (!Buffer.isBuffer(data)) return callback(new Error('Image needs to be a Buffer')); // Flip Y coordinate because MBTiles files are TMS. @@ -597,24 +515,26 @@ MBTiles.prototype.putGrid = function(z, x, y, data, callback) { this._mapCache[coords].grid_id = id; var mbtiles = this; - Step(function() { - if (mbtiles._gridCache[id]) return this(); - else zlib.deflate(new Buffer(json, 'utf8'), this); - }, function(err, buffer) { - if (err) throw err; - if (mbtiles._gridCache[id]) return this(); + + if (mbtiles._gridCache[id]) return callback(null); + + zlib.deflate(new Buffer(json, 'utf8'), function(err, buffer) { + if (err) return callback(err); // grid_utfgrid table. mbtiles._gridCache[id] = buffer; // grid_key table. mbtiles._keyCache[id] = Object.keys(data.data || {}); // keymap table. - _(mbtiles._dataCache).extend(data.data || {}); - this(); - }, function(err) { - if (err) return callback(err); + if (data.data) Object.keys(data.data).reduce(function(memo, key) { + memo[key] = data.data[key]; + return memo; + }, mbtiles._dataCache); // Only commit when we can insert at least batchSize rows. - if (++this._pending >= this._batchSize) return this._commit(callback); - else return callback(null); + if (++mbtiles._pending >= mbtiles._batchSize) { + mbtiles._commit(callback); + } else { + callback(null); + } }); }; @@ -623,23 +543,18 @@ MBTiles.prototype.putInfo = function(data, callback) { if (!this.open) return callback(new Error('MBTiles not yet loaded')); if (!this._isWritable) return callback(new Error('MBTiles not in write mode')); - // Valid keys. - var keys = ['name', 'type', 'description', 'version', 'formatter', 'template', - 'bounds', 'center', 'minzoom', 'maxzoom', 'legend', 'attribution']; - var stmt = this._db.prepare('REPLACE INTO metadata (name, value) VALUES (?, ?)'); stmt.on('error', callback); for (var key in data) { - if (keys.indexOf(key) !== -1) stmt.run(key, String(data[key])); + stmt.run(key, String(data[key])); } var mbtiles = this; stmt.finalize(function(err) { if (err) return callback(err); - mbtiles._getInfo(function(err, info) { - if (err) return callback(err); - mbtiles._info = info; - if (callback) callback(null); + delete mbtiles._info; + mbtiles.getInfo(function(err, info) { + return callback(err, null); }); }); }; diff --git a/package.json b/package.json index 6bc4138..ba2f369 100644 --- a/package.json +++ b/package.json @@ -29,8 +29,6 @@ "Konstantin Käfer " ], "dependencies": { - "underscore": "~1.3.3", - "step": "~0.0.5", "optimist": "~0.3.1", "sqlite3": "~2.1.1", "sphericalmercator": "~1.0.1" diff --git a/test/info.test.js b/test/info.test.js index 8e8f061..c88c7fa 100644 --- a/test/info.test.js +++ b/test/info.test.js @@ -1,4 +1,4 @@ -process.env.NODE_ENV = 'test'; +require('sqlite3').verbose(); var fs = require('fs'); var MBTiles = require('..'); diff --git a/test/list.test.js b/test/list.test.js index 0b2382b..92fa1c8 100644 --- a/test/list.test.js +++ b/test/list.test.js @@ -1,4 +1,4 @@ -process.env.NODE_ENV = 'test'; +require('sqlite3').verbose(); var fs = require('fs'); var MBTiles = require('..'); diff --git a/test/read.test.js b/test/read.test.js index 73920f7..ce0bf55 100644 --- a/test/read.test.js +++ b/test/read.test.js @@ -1,4 +1,4 @@ -process.env.NODE_ENV = 'test'; +require('sqlite3').verbose(); var fs = require('fs'); var MBTiles = require('..'); diff --git a/test/write.test.js b/test/write.test.js index 22634b8..bfb1239 100644 --- a/test/write.test.js +++ b/test/write.test.js @@ -1,4 +1,4 @@ -process.env.NODE_ENV = 'test'; +require('sqlite3').verbose(); var fs = require('fs'); var assert = require('assert'); diff --git a/test/write_grids.test.js b/test/write_grids.test.js index 7ebded4..5584ce7 100644 --- a/test/write_grids.test.js +++ b/test/write_grids.test.js @@ -1,4 +1,4 @@ -process.env.NODE_ENV = 'test'; +require('sqlite3').verbose(); var fs = require('fs'); var assert = require('assert');