diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..9d51d4f --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,9 @@ +# 0.1.0 + +- Modified interface to conform to Tilesource interface: + - `tile(x, y, z, callback)` is now `getTile(z, x, y, callback)`. Note the changed order of arguments. + - `grid(x, y, z, callback)` is now `getGrid(z, x, y, callback)`. Note the changed order of arguments. + - Added `getInfo(callback)` method. +- Removed `index.js` with `pool`, `serve` and `store` functions. +- MBTiles objects now create their databases as a singleton. There's no need to add additional pooling around MBTiles objects; you can create as many as you want. +- The constructor now takes Tilesource URIs (e.g. mbtiles:///path/to/file.mbtiles) as strings and parsed URIs as a hash. diff --git a/bin/mbcheck b/bin/mbcheck index 75e9cec..6cb1163 100755 --- a/bin/mbcheck +++ b/bin/mbcheck @@ -4,7 +4,7 @@ var options = argv = require('optimist').argv, _ = require('underscore'), Step = require('step'), utils = require('..').utils, - MBTiles = require('..').MBTiles, + MBTiles = require('..'), mbtiles, zooms = []; @@ -53,7 +53,7 @@ Step( ['name', name], ['type', type], ['description', description], - ['vesion', version], + ['version', version], ['format', format], ['bounds', bounds] ]); diff --git a/bin/mbcompact b/bin/mbcompact index 08d8278..9101421 100755 --- a/bin/mbcompact +++ b/bin/mbcompact @@ -7,7 +7,7 @@ var options = argv = require('optimist').argv, fs = require('fs'), sys = require('sys'), spawn = require('child_process').spawn, - MBTiles = require('..').MBTiles, + MBTiles = require('..'), utils = require('..').utils, mbtiles, hits = 0, diff --git a/bin/mbpipe b/bin/mbpipe index 2ae0497..5efbd3f 100755 --- a/bin/mbpipe +++ b/bin/mbpipe @@ -5,7 +5,7 @@ var options = argv = require('optimist').argv, Step = require('step'), sys = require('sys'), spawn = require('child_process').spawn, - MBTiles = require('..').MBTiles, + MBTiles = require('..'), utils = require('..').utils, mbtiles, table, diff --git a/index.js b/index.js deleted file mode 100644 index 8a2e278..0000000 --- a/index.js +++ /dev/null @@ -1,69 +0,0 @@ -var MBTiles = require('./lib/mbtiles'), - utils = require('./lib/utils'), - Step = require('step'); - -module.exports = { - MBTiles: MBTiles, - utils: utils, - pool: function(datasource) { - return { - create: function(callback) { - var resource = new MBTiles( - datasource, - function(err) { callback(err, resource); } - ); - }, - destroy: function(resource) { - resource.db.close(function() {}); - } - } - }, - serve: function(resource, options, callback) { - switch (options.format) { - case 'layer.json': - Step( - function() { - resource.metadata('formatter', this.parallel()); - resource.metadata('legend', this.parallel()); - }, - function(err, f, l) { - var layer = {}; - f && (layer.formatter = f); - l && (layer.legend = l); - callback(null, [layer, { 'Content-Type': 'text/javascript' }]); - } - ); - break; - case 'grid.json': - resource.grid(options.x, options.y, options.z, function(err, grid) { - callback(err, [grid, { 'Content-Type': 'text/javascript' }]); - }); - break; - default: - resource.tile(options.x, options.y, options.z, function(err, image) { - callback(err, [image, { 'Content-Type': 'image/png' }]); - }); - break; - } - }, - store: function(step, resource, data, callback) { - switch (step) { - case 'setup': - resource.setup(callback); - break; - case 'metadata': - resource.insertMetadata(data, callback); - break; - case 'tiles': - resource.insertTiles(data, callback); - break; - case 'grids': - resource.insertGrids(data, callback); - break; - case 'finish': - callback(); - break; - } - } -}; - diff --git a/lib/mbtiles.js b/lib/mbtiles.js index 0f7a13e..f0fc03b 100644 --- a/lib/mbtiles.js +++ b/lib/mbtiles.js @@ -3,244 +3,242 @@ var _ = require('underscore'), Step = require('step'), crypto = require('crypto'), zlib = require('zlib'), + path = require('path'), + url = require('url'), + Buffer = require('buffer').Buffer, + sm = new (require('sphericalmercator')), sqlite3 = require('sqlite3'); +if (process.env.NODE_ENV === 'test') sqlite3.verbose(); + +function noop(err) { + if (err) throw err; +} + +function hash(z, x, y) { + return (1 << z) * ((1 << z) + x) + y; +} + // MBTiles // ------- // MBTiles class for doing common operations (schema setup, tile reading, // insertion, etc.) -function MBTiles(filename, callback) { - this.filename = filename; - this.db = new sqlite3.Database(filename, callback); +module.exports = MBTiles; +MBTiles.utils = require('./utils'); + +var cache = {}; + +// Provides access to an mbtiles database file. +// - uri: A parsed URL hash, the only relevant part is `pathname`. +// - callback: Will be called when the resources have been acquired +// or acquisition failed. +require('util').inherits(MBTiles, require('events').EventEmitter) +function MBTiles(uri, callback) { + if (typeof uri === 'string') uri = url.parse(uri); + + if (!uri.pathname) { + callback(new Error('Invalid URI ' + url.format(uri))); + return; + } + + if (uri.hostname === '.' || uri.hostname == '..') { + uri.pathname = uri.hostname + uri.pathname; + delete uri.hostname; + delete uri.host; + } + + if (!cache[uri.pathname]) { + cache[uri.pathname] = this; + this._open(uri); + } + + var mbtiles = cache[uri.pathname]; + if (!mbtiles.open) { + mbtiles.once('open', callback); + } else { + callback(null, mbtiles); + } + return undefined; +} + +MBTiles.prototype._open = function(uri) { + var mbtiles = this; + function error(err) { + process.nextTick(function() { + mbtiles.emit('open', err); + }); + } + + this.filename = uri.pathname; + Step(function() { + mbtiles._db = new sqlite3.Database(mbtiles.filename, this); + }, function(err) { + if (err) return error(err); + mbtiles._setup(this); + }, function(err) { + if (err) return error(err); + fs.stat(mbtiles.filename, this); + }, function(err, stat) { + if (err) return error(err); + mbtiles._stat = stat; + fs.watchFile(mbtiles.filename, { interval: 1000 }, function(cur, prev) { + if (cur.mtime != prev.mtime) { + delete cache[uri.pathname]; + } + }); + mbtiles.open = true; + mbtiles.emit('open', null, mbtiles); + }); + + return undefined; +}; + +MBTiles.registerProtocols = function(tilelive) { + tilelive.protocols['mbtiles:'] = MBTiles; +}; + +// Finds all mbtiles file in the filepath and returns their tilesource URI. +MBTiles.list = function(filepath, callback) { + filepath = path.resolve(filepath); + fs.readdir(filepath, function(err, files) { + if (err) return callback(err); + for (var result = {}, i = 0; i < files.length; i++) { + var name = files[i].match(/^([\w-]+)\.mbtiles$/); + if (name) result[name[1]] = 'mbtiles://' + path.join(filepath, name[0]); + } + return callback(null, result); + }); +}; + +// Finds an mbtiles file with the given ID in the filepath and returns a +// tilesource URI. +MBTiles.findID = function(filepath, id, callback) { + filepath = path.resolve(filepath); + var file = path.join(filepath, id + '.mbtiles'); + fs.stat(file, function(err, stats) { + if (err) return callback(err); + else return callback(null, 'mbtiles://' + file); + }); }; // Retrieve the schema of the current mbtiles database and inform the caller of // whether the specified table exists. -MBTiles.prototype.exists = function(table, callback) { - if (this.schema) { - return callback(null, _(this.schema).include(table)); +MBTiles.prototype._exists = function(table, callback) { + if (typeof callback !== 'function') callback = noop; + + if (this._schema) { + return callback(null, _(this._schema).include(table)); } else { - this.schema = []; - this.db.all( + this._schema = []; + this._db.all( 'SELECT name FROM sqlite_master WHERE type IN (?, ?)', 'table', 'view', function(err, rows) { if (err) return callback(err); - this.schema = _(rows).pluck('name'); - this.exists(table, callback); + this._schema = _(rows).pluck('name'); + this._exists(table, callback); }.bind(this) ); } }; +MBTiles.prototype._close = function() { + fs.unwatchFile(this.filename); +}; + // DB integrity check. -MBTiles.prototype.integrity = function(callback) { - this.db.get('PRAGMA quick_check(1)', function(err, row) { +// +// - @param {Function(err)} callback +MBTiles.prototype._integrity = function(callback) { + if (typeof callback !== 'function') callback = noop; + + this._db.get('PRAGMA quick_check(1)', function(err, row) { if (!(row && row.integrity_check && row.integrity_check === 'ok')) { - callback(new Error('Corrupted database.')); + return callback(new Error('Corrupted database.')); } else { - callback(null, true); + return callback(null); } }); }; // Setup schema, indices, views for a new mbtiles database. -// Sets the synchronous flag to OFF for (much) faster inserts. -// See http://www.sqlite3.org/pragma.html#pragma_synchronous -MBTiles.prototype.setup = function(callback) { - fs.readFile(__dirname + '/schema.sql', 'utf8', function(err, sql) { - if (err) return callback(err); - this.db.serialize(function() { - this.db.run('PRAGMA synchronous = 0'); - this.db.exec(sql, callback); - }.bind(this)); - }.bind(this)); -}; - -// Generic object insert. -// -// - `table` String. The table to which objects should be inserted. -// - `objects` Array. Objects to be inserted, where each object attribute -// has key/value pairs as a hash corresponding to column name and row value. -// - `callback` Function. -MBTiles.prototype.insert = function(table, objects, callback) { - if (!objects.length) return callback(); - var keys = _(objects[0]).keys(); - var placeholders = []; - _(keys).each(function(k) { placeholders.push('?'); }); - var stmt = this.db.prepare( - 'INSERT OR IGNORE INTO ' + table + ' ' + - '(' + keys.join(',') + ') ' + - 'VALUES (' + placeholders.join(',') + ')' - ); - for (var i = 0; i < objects.length; i++) { - stmt.run.apply(stmt, _(objects[i]).values()); - } - stmt.finalize(callback); -}; - -// Insert metadata into the mbtiles database. -// -// - @param {Object} metadata key, value hash of metadata to be inserted. -// - @param {Function} callback -MBTiles.prototype.insertMetadata = function(data, callback) { - var metadata = _(data).map(function(value, key) { - return { name: key, value: value}; +// - @param {Function(err)} callback +MBTiles.prototype._setup = function(callback) { + var mbtiles = this; + mbtiles._exists('tiles', function(err, exists) { + if (exists) return callback(null); + fs.readFile(__dirname + '/schema.sql', 'utf8', function(err, sql) { + if (err) return callback(err); + mbtiles._db.exec(sql, callback); + }); }); - this.insert('metadata', metadata, callback); }; -// Insert a set of tiles into an mbtiles database. -// -// - @param {Array} renders array of images to be inserted. Each item should -// be an object of the form { z: z, x: x, y: y, data: [Image buffer] }. -// - @param {Function} callback -MBTiles.prototype.insertTiles = function(data, callback) { - var that = this, - map = [], - images = [], - ids = []; - for (var i = 0; i < data.length; i++) { - var tile_id = crypto - .createHash('md5') - .update(data[i].data) - .digest('hex'); - !_(ids).include(tile_id) && ids.push(tile_id) && images.push({ - tile_id: tile_id, - tile_data: data[i].data - }); - map.push({ - tile_id: tile_id, - zoom_level: data[i].z, - tile_column: data[i].x, - tile_row: data[i].y - }); - } - Step( - function() { - var group = this.group(); - that.insert('images', images, group()); - that.insert('map', map, group()); - }, - callback - ); -}; - -// Insert a set of grids into an mbtiles database. -// -// - @param {Array} renders array of grids to be inserted. Each item should -// be an object of the form { z: z, x: x, y: y, data: [Image buffer], keys: [] }. -// - @param {Function} callback -MBTiles.prototype.insertGrids = function(data, callback) { - var that = this, - map = [], - grids = [], - grid_keys = [], - features = {}, - ids = []; - for (var i = 0; i < data.length; i++) { - var json = JSON.stringify({ - grid: data[i].grid, - keys: data[i].keys - }); - var grid_id = crypto - .createHash('md5') - .update(json) - .digest('hex'); - !_(ids).include(grid_id) && ids.push(grid_id) && grids.push({ - grid_id: grid_id, - grid_utfgrid: zlib.deflate(new Buffer(json, 'utf8')) - }); - data[i].keys.forEach(function(k) { - grid_keys.push({ - grid_id: grid_id, - key_name: k - }); - }); - map.push({ - grid_id: grid_id, - zoom_level: data[i].z, - tile_column: data[i].x, - tile_row: data[i].y - }); - _(features).extend(data[i].data); - } - features = _(features).map(function(value, key) { - return { key_name: key, key_json: JSON.stringify(value) }; - }); - Step( - function() { - var group = this.group(); - that.insert('grid_utfgrid', grids, group()); - that.insert('grid_key', grid_keys, group()); - that.insert('keymap', features, group()); - that.insertGridTiles(map, group()); - }, - callback - ); -}; - -// Insert grids into the mbtiles database. -// -// - @param {Object} tile tile object to be inserted. -// - @param {Function} callback -MBTiles.prototype.insertGridTiles = function(map, callback) { - var stmt = this.db.prepare('UPDATE OR REPLACE map SET grid_id = ? WHERE ' + - ' zoom_level = ? AND tile_column = ? AND tile_row = ?'); - - for (var i = 0; i < map.length; i++) { - stmt.run( - map[i].grid_id, - map[i].zoom_level, - map[i].tile_column, - map[i].tile_row - ); - } - - stmt.finalize(callback); -}; - -// Select a tile from an mbtiles database. +// Select a tile from an mbtiles database. Scheme is XYZ. // +// - @param {Number} z tile z coordinate. // - @param {Number} x tile x coordinate. // - @param {Number} y tile y coordinate. -// - @param {Number} z tile z coordinate. -// - @param {Function} callback -MBTiles.prototype.tile = function(x, y, z, callback) { - this.db.get('SELECT tile_data FROM tiles WHERE ' + +// - @param {Function(err, grid, headers)} callback +MBTiles.prototype.getTile = function(z, x, y, callback) { + if (typeof callback !== 'function') throw new Error('Callback needed'); + if (!this.open) return callback(new Error('MBTiles not yet loaded')); + + // Flip Y coordinate because MBTiles files are TMS. + y = (1 << z) - 1 - y; + + var mbtiles = this; + this._db.get('SELECT tile_data FROM tiles WHERE ' + 'zoom_level = ? AND tile_column = ? AND tile_row = ?', z, x, y, function(err, row) { - if (err) callback(err); - else if (!row || !row.tile_data) callback('Tile does not exist'); - else callback(null, row.tile_data); + if (!row || (err && err.errno == 1)) { + return callback(new Error('Tile does not exist')); + } else if (err) { + return callback(err); + } else { + var options = { + 'Content-Type': MBTiles.utils.getMimeType(row.tile_data), + 'Last-Modified': mbtiles._stat.mtime, + 'ETag': mbtiles._stat.size + '-' + Number(mbtiles._stat.mtime) + }; + return callback(null, row.tile_data, options); + } }); }; -// Select a grid and its data from an mbtiles database. +// Select a grid and its data from an mbtiles database. Scheme is XYZ. // +// - @param {Number} z tile z coordinate // - @param {Number} x tile x coordinate // - @param {Number} y tile y coordinate -// - @param {Number} z tile z coordinate -// - @param {Function} callback -MBTiles.prototype.grid = function(x, y, z, callback) { +// - @param {Function(err, grid)} callback +MBTiles.prototype.getGrid = function(z, x, y, callback) { + if (typeof callback !== 'function') throw new Error('Callback needed'); + if (!this.open) return callback(new Error('MBTiles not yet loaded')); + + // Flip Y coordinate because MBTiles files are TMS. + y = (1 << z) - 1 - y; + var that = this; Step( function() { - that.db.get('SELECT grid FROM grids WHERE ' + + that._db.get('SELECT grid FROM grids WHERE ' + 'zoom_level = ? AND tile_column = ? AND tile_row = ?', z, x, y, this.parallel() ); - that.db.all('SELECT key_name, key_json FROM grid_data WHERE ' + + that._db.all('SELECT key_name, key_json FROM grid_data WHERE ' + 'zoom_level = ? AND tile_column = ? AND tile_row = ?', z, x, y, this.parallel() ); }, function(err, row, rows) { - if ((!row || !row.grid) || (err && err.errno == 1)) return callback('Grid does not exist'); + if ((!row || !row.grid) || (err && err.errno == 1)) { + return callback(new Error('Grid does not exist')); + } if (err) return callback(err); try { @@ -255,7 +253,7 @@ MBTiles.prototype.grid = function(x, y, z, callback) { }, {}); var result = _(JSON.parse(grid)).extend({ data: data }); } catch (err) { - return callback('Grid is invalid'); + return callback(new Error('Grid is invalid')); } callback(null, result); @@ -266,14 +264,330 @@ MBTiles.prototype.grid = function(x, y, z, callback) { // Select a metadata value from the database. // // - @param {Function} callback -MBTiles.prototype.metadata = function(key, callback) { - this.db.get('SELECT value FROM metadata WHERE name = ?', +MBTiles.prototype._metadata = function(key, callback) { + if (typeof callback !== 'function') callback = noop; + + this._db.get('SELECT value FROM metadata WHERE name = ?', key, function(err, row) { - if (err) callback(err); - else if (!row) callback('Key does not exist'); - else callback(null, row.value); + if (!row || (err && err.errno == 1)) return callback(new Error('Key does not exist')); + else if (err) return callback(err); + else return callback(null, row.value); }); }; -module.exports = MBTiles; +// Obtain metadata from the database. Performing fallback queries if certain +// keys(like `bounds`, `minzoom`, `maxzoom`) have not been provided. +// +// - @param {Function(err, data)} callback +MBTiles.prototype.getInfo = function(callback) { + if (typeof callback !== 'function') throw new Error('Callback needed'); + if (!this.open) return callback(new Error('MBTiles not yet loaded')); + + var that = this; + var info = {}; + info.filesize = this._stat.size; + info.scheme = 'tms'; + info.basename = path.basename(that.filename); + info.id = path.basename(that.filename, path.extname(that.filename)); + Step(function() { + var end = this; + that._db.all('SELECT name, value FROM metadata', function(err, rows) { + if (rows) for (var i = 0; i < rows.length; i++) { + info[rows[i].name] = rows[i].value; + } + end(err); + }); + }, + // Determine min/max zoom if needed + function(err) { + if (err && err.errno !== 1) return callback(err); + if (info.maxzoom !== undefined + && info.minzoom !== undefined) return this(); + + var step = this; + var zoomquery = that._db.prepare('SELECT zoom_level FROM tiles ' + + 'WHERE zoom_level = ? LIMIT 1', function(err) { + if (err) { + if (err.errno === 1) step(); + else throw new Error(err); + } else { + var group = step.group(); + for (var i = 0; i < 30; i++) { + zoomquery.get(i, group()); + } + zoomquery.finalize(); + } + }); + }, + function(err, rows) { + if (err) return callback(err); + if (rows) { + var zooms = _(rows).chain() + .reject(_.isUndefined) + .pluck('zoom_level') + .value(); + info.minzoom = zooms.shift(); + info.maxzoom = zooms.length ? zooms.pop() : info.minzoom; + } + this(); + }, + // Determine bounds if needed + function(err) { + if (err) return callback(err); + if (info.bounds) return this(); + if (typeof info.minzoom === 'undefined') return this(); + + var next = this; + Step( + function() { + that._db.get( + 'SELECT MAX(tile_column) AS maxx, ' + + 'MIN(tile_column) AS minx, MAX(tile_row) AS maxy, ' + + 'MIN(tile_row) AS miny FROM tiles ' + + 'WHERE zoom_level = ?', + info.minzoom, + this + ); + }, + function(err, row) { + if (!err && row) { + // @TODO this breaks a little at zoom level zero + var urTile = sm.bbox(row.maxx, row.maxy, info.minzoom, true); + var llTile = sm.bbox(row.minx, row.miny, info.minzoom, true); + // @TODO bounds are limited to "sensible" values here + // as sometimes tilesets are rendered with "negative" + // and/or other extremity tiles. Revisit this if there + // are actual use cases for out-of-bounds bounds. + info.bounds = [ + llTile[0] > -180 ? llTile[0] : -180, + llTile[1] > -90 ? llTile[1] : -90, + urTile[2] < 180 ? urTile[2] : 180, + urTile[3] < 90 ? urTile[3] : 90 + ].join(','); + } + next(); + } + ); + }, + // Return info + function(err) { + if (err) return callback(err); + var range = parseInt(info.maxzoom, 10) - parseInt(info.minzoom, 10); + info.minzoom = parseInt(info.minzoom, 10); + info.maxzoom = parseInt(info.maxzoom, 10); + info.bounds = _((info.bounds || '').split(',')).map(parseFloat); + if (info.center) info.center = _((info.center).split(',')).map(parseFloat); + if (!info.center || info.center.length !== 3) info.center = [ + (info.bounds[2] - info.bounds[0]) / 2 + info.bounds[0], + (info.bounds[3] - info.bounds[1]) / 2 + info.bounds[1], + (range <= 1) ? info.maxzoom : Math.floor(range * 0.5) + info.minzoom + ]; + return callback(null, info); + }); +}; + +// Puts the MBTiles tilestore into write mode. +// +// - @param {Function(err)} callback +MBTiles.prototype.startWriting = function(callback) { + if (typeof callback !== 'function') throw new Error('Callback needed'); + if (!this.open) return callback(new Error('MBTiles not yet loaded')); + + // Sets the synchronous flag to OFF for (much) faster inserts. + // See http://www.sqlite3.org/pragma.html#pragma_synchronous + + var mbtiles = this; + if (!this._isWritable) { + this._isWritable = 1; + this._pending = 0; + this._tileCache = {}; + this._gridCache = {}; + this._keyCache = {}; + this._dataCache = {}; + this._mapCache = {}; + this._db.run('PRAGMA synchronous=OFF', callback); + } else { + this._isWritable++; + return callback(null); + } +}; + +// (private) Commits the cached changes to the database. +// +// - @param {Function(err)} callback +MBTiles.prototype._commit = function(callback) { + var mbtiles = this; + mbtiles._db.serialize(function() { + mbtiles._db.run('BEGIN'); + + if (Object.keys(mbtiles._tileCache)) { + // Insert images table. + var images = mbtiles._db.prepare('REPLACE INTO images (tile_id, tile_data) VALUES (?, ?)'); + for (var id in mbtiles._tileCache) { + images.run(id, mbtiles._tileCache[id]); + } + mbtiles._tileCache = {}; + images.finalize(); + } + + + if (Object.keys(mbtiles._gridCache)) { + // Insert grid_utfgrid table. + var grids = mbtiles._db.prepare('REPLACE INTO grid_utfgrid (grid_id, grid_utfgrid) VALUES (?, ?)'); + for (var id in mbtiles._gridCache) { + grids.run(id, mbtiles._gridCache[id]); + } + mbtiles._gridCache = {}; + grids.finalize(); + } + + + if (Object.keys(mbtiles._keyCache)) { + // Insert grid_key. + var keys = mbtiles._db.prepare('INSERT OR IGNORE INTO grid_key (grid_id, key_name) VALUES (?, ?)'); + for (var id in mbtiles._keyCache) { + mbtiles._keyCache[id].forEach(function(key) { + keys.run(id, key); + }); + } + mbtiles._keyCache = {}; + keys.finalize(); + } + + + if (Object.keys(mbtiles._dataCache)) { + // Insert keymap table. + var keymap = mbtiles._db.prepare('REPLACE INTO keymap (key_name, key_json) VALUES (?, ?)'); + for (var key in mbtiles._dataCache) { + keymap.run(key, JSON.stringify(mbtiles._dataCache[key])); + } + mbtiles._dataCache = {}; + keymap.finalize(); + } + + // Insert map table. This has to be so complicate due to a design flaw + // in the tables. + // TODO: This should be remedied when we upgrade the MBTiles schema. + var mapBoth = mbtiles._db.prepare('REPLACE INTO map (zoom_level, ' + + 'tile_column, tile_row, tile_id, grid_id) VALUES (?, ?, ?, ?, ?)'); + var mapTile = mbtiles._db.prepare('REPLACE INTO map (zoom_level, ' + + 'tile_column, tile_row, tile_id, grid_id) VALUES (?, ?, ?, ?, ' + + '(SELECT grid_id FROM map WHERE zoom_level = ? ' + + 'AND tile_column = ? AND tile_row = ?))'); + var mapGrid = mbtiles._db.prepare('REPLACE INTO map (zoom_level, ' + + 'tile_column, tile_row, tile_id, grid_id) VALUES (?, ?, ?, ' + + '(SELECT tile_id FROM map WHERE zoom_level = ? ' + + 'AND tile_column = ? AND tile_row = ?), ?)'); + for (var coords in mbtiles._mapCache) { + var map = mbtiles._mapCache[coords]; + + if (typeof map.grid_id === 'undefined') { + // Only the tile_id is defined. + mapTile.run(map.z, map.x, map.y, map.tile_id, map.z, map.x, map.y); + } else if (typeof map.tile_id === 'undefined') { + // Only the grid_id is defined. + mapGrid.run(map.z, map.x, map.y, map.z, map.x, map.y, map.grid_id); + } else { + // Both tile_id and grid_id are defined. + mapBoth.run(map.z, map.x, map.y, map.tile_id, map.grid_id); + } + } + mbtiles._mapCache = {}; + mapBoth.finalize(); + mapTile.finalize(); + mapGrid.finalize(); + + mbtiles._db.run('COMMIT', callback); + }); +}; + +// Leaves write mode. +// +// - @param {Function(err)} callback +MBTiles.prototype.stopWriting = function(callback) { + if (typeof callback !== 'function') throw new Error('Callback needed'); + if (!this.open) return callback(new Error('MBTiles not yet loaded')); + + var mbtiles = this; + if (this._isWritable) this._isWritable--; + this._commit(function(err) { + if (err) return callback(err); + if (!mbtiles._isWritable) { + mbtiles._db.run('PRAGMA synchronous=NORMAL', callback); + } else { + return callback(null); + } + }); +}; + +// Inserts a tile into the MBTiles store. Scheme is XYZ. +// +// - @param {Number} z tile z coordinate +// - @param {Number} x tile x coordinate +// - @param {Number} y tile y coordinate +// - @param {Buffer} buffer tile image data +// - @param {Function(err)} callback +MBTiles.prototype.putTile = function(z, x, y, data, callback) { + if (typeof callback !== 'function') throw new Error('Callback needed'); + if (!this.open) return callback(new Error('MBTiles not yet loaded')); + if (!this._isWritable) return callback(new Error('MBTiles not in write mode')); + + if (!Buffer.isBuffer(data)) return callback(new Error('Image needs to be a Buffer')); + + // Flip Y coordinate because MBTiles files are TMS. + y = (1 << z) - 1 - y; + + var id = crypto.createHash('md5').update(data).digest('hex'); + if (!this._tileCache[id]) { + // This corresponds to the images table. + this._tileCache[id] = data; + } + + // This corresponds to the map table. + var coords = hash(z, x, y); + if (!this._mapCache[coords]) this._mapCache[coords] = { z: z, x: x, y: y }; + this._mapCache[coords].tile_id = id; + + // Only commit when we can insert at least 100 rows. + if (++this._pending < 100) return this._commit(callback); + else return callback(null); +}; + +// Inserts a grid into the MBTiles store. Scheme is XYZ. +// +// - @param {Number} z grid z coordinate +// - @param {Number} x grid x coordinate +// - @param {Number} y grid y coordinate +// - @param {Object} data grid object +// - @param {Function(err)} callback +MBTiles.prototype.putGrid = function(z, x, y, data, callback) { + if (typeof callback !== 'function') throw new Error('Callback needed'); + if (!this.open) return callback(new Error('MBTiles not yet loaded')); + + // Flip Y coordinate because MBTiles files are TMS. + y = (1 << z) - 1 - y; + + // Preprocess grid data. + var json = JSON.stringify({ grid: data.grid, keys: data.keys }); + var id = crypto.createHash('md5').update(json).digest('hex'); + if (!this._gridCache[id]) { + // This corresponds to the grid_utfgrid table. + this._gridCache[id] = zlib.deflate(new Buffer(json, 'utf8')); + + // This corresponds to the grid_key table. + this._keyCache[id] = Object.keys(data.data || {}); + + // This corresponds to the keymap table. + _(this._dataCache).extend(data.data || {}); + } + + // This corresponds to the map table. + var coords = hash(z, x, y); + if (!this._mapCache[coords]) this._mapCache[coords] = { z: z, x: x, y: y }; + this._mapCache[coords].grid_id = id; + + // Only commit when we can insert at least 100 rows. + if (++this._pending < 100) return this._commit(callback); + else return callback(null); +}; diff --git a/lib/utils.js b/lib/utils.js index 1f0387c..37146ae 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -20,6 +20,21 @@ utils.table = function(fields) { }); }; +utils.getMimeType = function(data) { + if (data[0] === 0x89 && data[1] === 0x50 && data[2] === 0x4E && + data[3] === 0x47 && data[4] === 0x0D && data[5] === 0x0A && + data[6] === 0x1A && data[7] === 0x0A) { + return 'image/png'; + } else if (data[0] === 0xFF && data[1] === 0xD8 && + data[data.length - 2] === 0xFF && data[data.length - 1] === 0xD9) { + return 'image/jpeg'; + } else if (data[0] === 0x47 && data[1] === 0x49 && data[2] === 0x46 && + data[3] === 0x38 && (data[4] === 0x39 || data[4] === 0x37) && + data[5] === 0x61) { + return 'image/gif'; + } +}; + function Queue(callback, concurrency) { this.callback = callback; this.concurrency = concurrency || 10; diff --git a/package.json b/package.json index d8accd2..5b98776 100644 --- a/package.json +++ b/package.json @@ -1,11 +1,16 @@ { "name": "mbtiles", - "version": "0.0.5", + "version": "0.1.0", "description": "Utilities and tilelive integration for the MBTiles format.", "url": "http://github.com/mapbox/node-mbtiles", + "author": { + "name": "MapBox", + "url": "http://mapbox.com/", + "email": "info@mapbox.com" + }, "keywords": ["map", "mbtiles"], "licenses": [{ "type": "BSD" }], - "main": "./index", + "main": "./lib/mbtiles", "bin": { "mbcheck": "./bin/mbcheck", "mbcompact": "./bin/mbcompact", @@ -20,12 +25,14 @@ "Tom MacWright ", "Will White ", "Dane Springmeyer ", - "Young Hahn " + "Young Hahn ", + "Konstantin Käfer " ], "dependencies": { "optimist": "0.2.x", "sqlite3": "2.0.x", "step": "0.0.x", + "sphericalmercator": "1.0.x", "underscore": "1.1.x", "zlib": "1.0.x" }, diff --git a/test/read.test.js b/test/read.test.js index e825288..efecaef 100644 --- a/test/read.test.js +++ b/test/read.test.js @@ -1,42 +1,57 @@ +process.env.NODE_ENV = 'test'; + var fs = require('fs'); var assert = require('assert'); -var MBTiles = require('..').MBTiles; +var MBTiles = require('..'); + var fixtures = { plain_1: __dirname + '/fixtures/plain_1.mbtiles', plain_2: __dirname + '/fixtures/plain_2.mbtiles', plain_3: __dirname + '/fixtures/plain_3.mbtiles', - plain_4: __dirname + '/fixtures/plain_4.mbtiles' + plain_4: __dirname + '/fixtures/plain_4.mbtiles', + non_existent: __dirname + '/fixtures/non_existent.mbtiles' }; -exports['get metadata'] = function(beforeExit) { - var completion = {}; +try { fs.unlink(fixtures.non_existent); } catch (err) {} - var mbtiles = new MBTiles(fixtures.plain_1); - mbtiles.metadata('name', function(err, value) { if (err) throw err; completion.name = value; }); - mbtiles.metadata('type', function(err, value) { if (err) throw err; completion.type = value; }); - mbtiles.metadata('description', function(err, value) { if (err) throw err; completion.description = value; }); - mbtiles.metadata('version', function(err, value) { if (err) throw err; completion.version = value; }); - mbtiles.metadata('formatter', function(err, value) { if (err) throw err; completion.formatter = value; }); - mbtiles.metadata('bounds', function(err, value) { if (err) throw err; completion.bounds = value; }); - mbtiles.metadata('invalid', function(err, value) { completion.invalid = err; }); +exports['get metadata'] = function(beforeExit) { + var completed = false; + + new MBTiles(fixtures.plain_1, function(err, mbtiles) { + mbtiles.getInfo(function(err, data) { + completed = true; + if (err) throw err; + + assert.deepEqual({ + name: 'plain_1', + description: 'demo description', + version: '1.0.3', + scheme: 'tms', + minzoom: 0, + maxzoom: 4, + formatter: null, + center: [ 0, 7.500000001278025, 2 ], + bounds: [ -179.9999999749438, -69.99999999526695, 179.9999999749438, 84.99999999782301 ], + + // These aren't part of TileJSON, but exist in an MBTiles file. + filesize: 561152, + type: 'baselayer', + id: 'plain_1', + basename: 'plain_1.mbtiles' + }, data); + }) + }); beforeExit(function() { - assert.deepEqual(completion, { - name: 'plain_1', - type: 'baselayer', - description: 'demo description', - version: '1.0.3', - formatter: null, - bounds: '-179.9999999749438,-69.99999999526695,179.9999999749438,84.99999999782301', - invalid: 'Key does not exist' - }); + assert.ok(completed); }); }; function yieldsError(status, error, msg) { return function(err) { - assert.equal(err, msg); + assert.ok(err); + assert.equal(err.message, msg); status[error]++; }; } @@ -47,25 +62,30 @@ exports['get tiles'] = function(beforeExit) { error: 0 }; - var mbtiles = new MBTiles(fixtures.plain_1); - fs.readdirSync(__dirname + '/fixtures/images/').forEach(function(file) { - var coords = file.match(/^plain_1_(\d+)_(\d+)_(\d+).png$/); - if (coords) { - mbtiles.tile(coords[1] | 0, coords[2] | 0, coords[3] | 0, function(err, tile) { - if (err) throw err; - assert.deepEqual(tile, fs.readFileSync(__dirname + '/fixtures/images/' + file)); - status.success++; - }); - } + new MBTiles(fixtures.plain_1, function(err, mbtiles) { + if (err) throw err; + fs.readdirSync(__dirname + '/fixtures/images/').forEach(function(file) { + var coords = file.match(/^plain_1_(\d+)_(\d+)_(\d+).png$/); + if (coords) { + // Flip Y coordinate because file names are TMS, but .getTile() expects XYZ. + coords[2] = Math.pow(2, coords[3]) - 1 - coords[2]; + mbtiles.getTile(coords[3] | 0, coords[1] | 0, coords[2] | 0, function(err, tile) { + if (err) throw err; + assert.deepEqual(tile, fs.readFileSync(__dirname + '/fixtures/images/' + file)); + status.success++; + }); + } + }); + + mbtiles.getTile(0, 1, 0, yieldsError(status, 'error', 'Tile does not exist')); + mbtiles.getTile(-1, 0, 0, yieldsError(status, 'error', 'Tile does not exist')); + mbtiles.getTile(0, 0, 1, yieldsError(status, 'error', 'Tile does not exist')); + mbtiles.getTile(3, 1, -1, yieldsError(status, 'error', 'Tile does not exist')); + mbtiles.getTile(2, -3, 3, yieldsError(status, 'error', 'Tile does not exist')); + mbtiles.getTile(18, 2, 262140, yieldsError(status, 'error', 'Tile does not exist')); + mbtiles.getTile(4, 0, 15, yieldsError(status, 'error', 'Tile does not exist')); }); - mbtiles.tile(1, 0, 0, yieldsError(status, 'error', 'Tile does not exist')); - mbtiles.tile(0, 0, -1, yieldsError(status, 'error', 'Tile does not exist')); - mbtiles.tile(0, -1, 0, yieldsError(status, 'error', 'Tile does not exist')); - mbtiles.tile(1, 8, 3, yieldsError(status, 'error', 'Tile does not exist')); - mbtiles.tile(-3, 0, 2, yieldsError(status, 'error', 'Tile does not exist')); - mbtiles.tile(2, 3, 18, yieldsError(status, 'error', 'Tile does not exist')); - mbtiles.tile(0, 0, 4, yieldsError(status, 'error', 'Tile does not exist')); beforeExit(function() { assert.equal(status.success, 285); @@ -79,37 +99,34 @@ exports['get grids'] = function(beforeExit) { error: 0 }; - var mbtiles = new MBTiles(fixtures.plain_2); - fs.readdirSync(__dirname + '/fixtures/grids/').forEach(function(file) { - var coords = file.match(/^plain_2_(\d+)_(\d+)_(\d+).json$/); - if (coords) { - mbtiles.grid(coords[1] | 0, coords[2] | 0, coords[3] | 0, function(err, grid) { - if (err) throw err; - assert.deepEqual(JSON.stringify(grid), fs.readFileSync(__dirname + '/fixtures/grids/' + file, 'utf8')); - status.success++; - }); - } + new MBTiles(fixtures.plain_2, function(err, mbtiles) { + if (err) throw err; + fs.readdirSync(__dirname + '/fixtures/grids/').forEach(function(file) { + var coords = file.match(/^plain_2_(\d+)_(\d+)_(\d+).json$/); + if (coords) { + // Flip Y coordinate because file names are TMS, but .getTile() expects XYZ. + coords[2] = Math.pow(2, coords[3]) - 1 - coords[2]; + mbtiles.getGrid(coords[3] | 0, coords[1] | 0, coords[2] | 0, function(err, grid) { + if (err) throw err; + assert.deepEqual(JSON.stringify(grid), fs.readFileSync(__dirname + '/fixtures/grids/' + file, 'utf8')); + status.success++; + }); + } + }); + + mbtiles.getGrid(0, 1, 0, yieldsError(status, 'error', 'Grid does not exist')); + mbtiles.getGrid(-1, 0, 0, yieldsError(status, 'error', 'Grid does not exist')); + mbtiles.getGrid(0, 0, 1, yieldsError(status, 'error', 'Grid does not exist')); + mbtiles.getGrid(3, 1, -1, yieldsError(status, 'error', 'Grid does not exist')); + mbtiles.getGrid(2, -3, 3, yieldsError(status, 'error', 'Grid does not exist')); + mbtiles.getGrid(18, 2, 262140, yieldsError(status, 'error', 'Grid does not exist')); + mbtiles.getGrid(4, 0, 15, yieldsError(status, 'error', 'Grid does not exist')); }); - mbtiles.grid(1, 0, 0, yieldsError(status, 'error', 'Grid does not exist')); - mbtiles.grid(0, 0, -1, yieldsError(status, 'error', 'Grid does not exist')); - mbtiles.grid(0, -1, 0, yieldsError(status, 'error', 'Grid does not exist')); - mbtiles.grid(1, 8, 3, yieldsError(status, 'error', 'Grid does not exist')); - mbtiles.grid(-3, 0, 2, yieldsError(status, 'error', 'Grid does not exist')); - mbtiles.grid(2, 3, 18, yieldsError(status, 'error', 'Grid does not exist')); - mbtiles.grid(0, 0, 4, yieldsError(status, 'error', 'Grid does not exist')); - - mbtiles.grid(3, 8, 4, yieldsError(status, 'error', 'Grid is invalid')); - mbtiles.grid(4, 8, 4, yieldsError(status, 'error', 'Grid is invalid')); - mbtiles.grid(5, 8, 4, yieldsError(status, 'error', 'Grid is invalid')); - mbtiles.grid(13, 4, 4, yieldsError(status, 'error', 'Grid is invalid')); - mbtiles.grid(0, 14, 4, yieldsError(status, 'error', 'Grid is invalid')); - mbtiles.grid(0, 7, 3, yieldsError(status, 'error', 'Grid is invalid')); - mbtiles.grid(6, 2, 3, yieldsError(status, 'error', 'Grid is invalid')); beforeExit(function() { assert.equal(status.success, 241); - assert.equal(status.error, 14); + assert.equal(status.error, 7); }); }; @@ -120,21 +137,23 @@ exports['get grids from file without interaction'] = function(beforeExit) { error: 0 }; - var mbtiles = new MBTiles(fixtures.plain_1); - mbtiles.grid(1, 0, 0, yieldsError(status, 'error', 'Grid does not exist')); - mbtiles.grid(0, 0, -1, yieldsError(status, 'error', 'Grid does not exist')); - mbtiles.grid(0, -1, 0, yieldsError(status, 'error', 'Grid does not exist')); - mbtiles.grid(1, 8, 3, yieldsError(status, 'error', 'Grid does not exist')); - mbtiles.grid(-3, 0, 2, yieldsError(status, 'error', 'Grid does not exist')); - mbtiles.grid(2, 3, 18, yieldsError(status, 'error', 'Grid does not exist')); - mbtiles.grid(0, 0, 4, yieldsError(status, 'error', 'Grid does not exist')); - mbtiles.grid(3, 8, 4, yieldsError(status, 'error', 'Grid does not exist')); - mbtiles.grid(4, 8, 4, yieldsError(status, 'error', 'Grid does not exist')); - mbtiles.grid(5, 8, 4, yieldsError(status, 'error', 'Grid does not exist')); - mbtiles.grid(13, 4, 4, yieldsError(status, 'error', 'Grid does not exist')); - mbtiles.grid(0, 14, 4, yieldsError(status, 'error', 'Grid does not exist')); - mbtiles.grid(0, 7, 3, yieldsError(status, 'error', 'Grid does not exist')); - mbtiles.grid(6, 2, 3, yieldsError(status, 'error', 'Grid does not exist')); + new MBTiles(fixtures.plain_1, function(err, mbtiles) { + if (err) throw err; + mbtiles.getGrid(0, 1, 0, yieldsError(status, 'error', 'Grid does not exist')); + mbtiles.getGrid(-1, 0, 0, yieldsError(status, 'error', 'Grid does not exist')); + mbtiles.getGrid(0, 0, -1, yieldsError(status, 'error', 'Grid does not exist')); + mbtiles.getGrid(3, 1, 8, yieldsError(status, 'error', 'Grid does not exist')); + mbtiles.getGrid(2, -3, 0, yieldsError(status, 'error', 'Grid does not exist')); + mbtiles.getGrid(18, 2, 3, yieldsError(status, 'error', 'Grid does not exist')); + mbtiles.getGrid(4, 0, 0, yieldsError(status, 'error', 'Grid does not exist')); + mbtiles.getGrid(4, 3, 8, yieldsError(status, 'error', 'Grid does not exist')); + mbtiles.getGrid(4, 4, 8, yieldsError(status, 'error', 'Grid does not exist')); + mbtiles.getGrid(4, 5, 8, yieldsError(status, 'error', 'Grid does not exist')); + mbtiles.getGrid(4, 13, 4, yieldsError(status, 'error', 'Grid does not exist')); + mbtiles.getGrid(4, 0, 14, yieldsError(status, 'error', 'Grid does not exist')); + mbtiles.getGrid(3, 0, 7, yieldsError(status, 'error', 'Grid does not exist')); + mbtiles.getGrid(3, 6, 2, yieldsError(status, 'error', 'Grid does not exist')); + }); beforeExit(function() { assert.equal(status.success, 0); @@ -148,37 +167,33 @@ exports['get grids with different schema'] = function(beforeExit) { error: 0 }; - var mbtiles = new MBTiles(fixtures.plain_4); - fs.readdirSync(__dirname + '/fixtures/grids/').forEach(function(file) { - var coords = file.match(/^plain_2_(\d+)_(\d+)_(\d+).json$/); - if (coords) { - mbtiles.grid(coords[1] | 0, coords[2] | 0, coords[3] | 0, function(err, grid) { - if (err) throw err; - assert.deepEqual(JSON.stringify(grid), fs.readFileSync(__dirname + '/fixtures/grids/' + file, 'utf8')); - status.success++; - }); - } + new MBTiles(fixtures.plain_4, function(err, mbtiles) { + if (err) throw err; + fs.readdirSync(__dirname + '/fixtures/grids/').forEach(function(file) { + var coords = file.match(/^plain_2_(\d+)_(\d+)_(\d+).json$/); + if (coords) { + // Flip Y coordinate because file names are TMS, but .getTile() expects XYZ. + coords[2] = Math.pow(2, coords[3]) - 1 - coords[2]; + mbtiles.getGrid(coords[3] | 0, coords[1] | 0, coords[2] | 0, function(err, grid) { + if (err) throw err; + assert.deepEqual(JSON.stringify(grid), fs.readFileSync(__dirname + '/fixtures/grids/' + file, 'utf8')); + status.success++; + }); + } + }); + + mbtiles.getGrid(0, 1, 0, yieldsError(status, 'error', 'Grid does not exist')); + mbtiles.getGrid(-1, 0, 0, yieldsError(status, 'error', 'Grid does not exist')); + mbtiles.getGrid(0, 0, 1, yieldsError(status, 'error', 'Grid does not exist')); + mbtiles.getGrid(3, 1, -1, yieldsError(status, 'error', 'Grid does not exist')); + mbtiles.getGrid(2, -3, 3, yieldsError(status, 'error', 'Grid does not exist')); + mbtiles.getGrid(18, 2, 262140, yieldsError(status, 'error', 'Grid does not exist')); + mbtiles.getGrid(4, 0, 15, yieldsError(status, 'error', 'Grid does not exist')); }); - mbtiles.grid(1, 0, 0, yieldsError(status, 'error', 'Grid does not exist')); - mbtiles.grid(0, 0, -1, yieldsError(status, 'error', 'Grid does not exist')); - mbtiles.grid(0, -1, 0, yieldsError(status, 'error', 'Grid does not exist')); - mbtiles.grid(1, 8, 3, yieldsError(status, 'error', 'Grid does not exist')); - mbtiles.grid(-3, 0, 2, yieldsError(status, 'error', 'Grid does not exist')); - mbtiles.grid(2, 3, 18, yieldsError(status, 'error', 'Grid does not exist')); - mbtiles.grid(0, 0, 4, yieldsError(status, 'error', 'Grid does not exist')); - - mbtiles.grid(3, 8, 4, yieldsError(status, 'error', 'Grid is invalid')); - mbtiles.grid(4, 8, 4, yieldsError(status, 'error', 'Grid is invalid')); - mbtiles.grid(5, 8, 4, yieldsError(status, 'error', 'Grid is invalid')); - mbtiles.grid(13, 4, 4, yieldsError(status, 'error', 'Grid is invalid')); - mbtiles.grid(0, 14, 4, yieldsError(status, 'error', 'Grid is invalid')); - mbtiles.grid(0, 7, 3, yieldsError(status, 'error', 'Grid is invalid')); - mbtiles.grid(6, 2, 3, yieldsError(status, 'error', 'Grid is invalid')); - beforeExit(function() { assert.equal(status.success, 241); - assert.equal(status.error, 14); + assert.equal(status.error, 7); }); }; @@ -189,21 +204,53 @@ exports['get grids from file without interaction'] = function(beforeExit) { error: 0 }; - var mbtiles = new MBTiles(fixtures.plain_1); - mbtiles.grid(1, 0, 0, yieldsError(status, 'error', 'Grid does not exist')); - mbtiles.grid(0, 0, -1, yieldsError(status, 'error', 'Grid does not exist')); - mbtiles.grid(0, -1, 0, yieldsError(status, 'error', 'Grid does not exist')); - mbtiles.grid(1, 8, 3, yieldsError(status, 'error', 'Grid does not exist')); - mbtiles.grid(-3, 0, 2, yieldsError(status, 'error', 'Grid does not exist')); - mbtiles.grid(2, 3, 18, yieldsError(status, 'error', 'Grid does not exist')); - mbtiles.grid(0, 0, 4, yieldsError(status, 'error', 'Grid does not exist')); - mbtiles.grid(3, 8, 4, yieldsError(status, 'error', 'Grid does not exist')); - mbtiles.grid(4, 8, 4, yieldsError(status, 'error', 'Grid does not exist')); - mbtiles.grid(5, 8, 4, yieldsError(status, 'error', 'Grid does not exist')); - mbtiles.grid(13, 4, 4, yieldsError(status, 'error', 'Grid does not exist')); - mbtiles.grid(0, 14, 4, yieldsError(status, 'error', 'Grid does not exist')); - mbtiles.grid(0, 7, 3, yieldsError(status, 'error', 'Grid does not exist')); - mbtiles.grid(6, 2, 3, yieldsError(status, 'error', 'Grid does not exist')); + new MBTiles(fixtures.plain_1, function(err, mbtiles) { + if (err) throw err; + mbtiles.getGrid(0, 1, 0, yieldsError(status, 'error', 'Grid does not exist')); + mbtiles.getGrid(-1, 0, 0, yieldsError(status, 'error', 'Grid does not exist')); + mbtiles.getGrid(0, 0, -1, yieldsError(status, 'error', 'Grid does not exist')); + mbtiles.getGrid(3, 1, 8, yieldsError(status, 'error', 'Grid does not exist')); + mbtiles.getGrid(2, -3, 0, yieldsError(status, 'error', 'Grid does not exist')); + mbtiles.getGrid(18, 2, 3, yieldsError(status, 'error', 'Grid does not exist')); + mbtiles.getGrid(4, 0, 0, yieldsError(status, 'error', 'Grid does not exist')); + mbtiles.getGrid(4, 3, 8, yieldsError(status, 'error', 'Grid does not exist')); + mbtiles.getGrid(4, 4, 8, yieldsError(status, 'error', 'Grid does not exist')); + mbtiles.getGrid(4, 5, 8, yieldsError(status, 'error', 'Grid does not exist')); + mbtiles.getGrid(4, 13, 4, yieldsError(status, 'error', 'Grid does not exist')); + mbtiles.getGrid(4, 0, 14, yieldsError(status, 'error', 'Grid does not exist')); + mbtiles.getGrid(3, 0, 7, yieldsError(status, 'error', 'Grid does not exist')); + mbtiles.getGrid(3, 6, 2, yieldsError(status, 'error', 'Grid does not exist')); + }); + + beforeExit(function() { + assert.equal(status.success, 0); + assert.equal(status.error, 14); + }); +}; + +exports['get tiles from non-existent file'] = function(beforeExit) { + var status = { + success: 0, + error: 0 + }; + + new MBTiles(fixtures.non_existent, function(err, mbtiles) { + if (err) throw err; + mbtiles.getTile(0, 1, 0, yieldsError(status, 'error', 'Tile does not exist')); + mbtiles.getTile(-1, 0, 0, yieldsError(status, 'error', 'Tile does not exist')); + mbtiles.getTile(0, 0, -1, yieldsError(status, 'error', 'Tile does not exist')); + mbtiles.getTile(3, 1, 8, yieldsError(status, 'error', 'Tile does not exist')); + mbtiles.getTile(2, -3, 0, yieldsError(status, 'error', 'Tile does not exist')); + mbtiles.getTile(18, 2, 3, yieldsError(status, 'error', 'Tile does not exist')); + mbtiles.getTile(4, 0, 0, yieldsError(status, 'error', 'Tile does not exist')); + mbtiles.getTile(4, 3, 8, yieldsError(status, 'error', 'Tile does not exist')); + mbtiles.getTile(4, 4, 8, yieldsError(status, 'error', 'Tile does not exist')); + mbtiles.getTile(4, 5, 8, yieldsError(status, 'error', 'Tile does not exist')); + mbtiles.getTile(4, 13, 4, yieldsError(status, 'error', 'Tile does not exist')); + mbtiles.getTile(4, 0, 14, yieldsError(status, 'error', 'Tile does not exist')); + mbtiles.getTile(3, 0, 7, yieldsError(status, 'error', 'Tile does not exist')); + mbtiles.getTile(3, 6, 2, yieldsError(status, 'error', 'Tile does not exist')); + }); beforeExit(function() { assert.equal(status.success, 0); diff --git a/test/reloading.test.js b/test/reloading.test.js new file mode 100644 index 0000000..9cda09e --- /dev/null +++ b/test/reloading.test.js @@ -0,0 +1,109 @@ +process.env.NODE_ENV = 'test'; + +var fs = require('fs'); +var assert = require('assert'); +var MBTiles = require('..'); + +var fixtureDir = __dirname + '/fixtures/output'; +var fixtures = { + source: __dirname + '/fixtures/plain_1.mbtiles', + destination: fixtureDir + '/write_3.mbtiles' +}; + +// Load entire database as buffer. +var file = fs.readFileSync(fixtures.source); + +// Recreate output directory to remove previous tests. +try { fs.unlinkSync(fixtures.destination); } catch(err) {} +try { fs.mkdirSync(fixtureDir, 0755); } catch(err) {} + +exports['test file reloading during copying'] = function(beforeExit) { + var completed = false; + var status = { + success: 0, + error: 0 + }; + + var tiles = [ + [ 0, 0, 0 ], + [ 1, 0, 1 ], + [ 4, 0, 5 ], + [ 4, 0, 4 ], + [ 1, 0, 0 ], + [ 3, 6, 3 ], + [ 4, 8, 6 ], + [ 4, 9, 1 ], + [ 4, 9, 10 ], + [ 4, 9, 7 ], + [ 4, 9, 6 ] + ]; + + var fd = fs.openSync(fixtures.destination, 'w'); + // Start copying the file. Write first 100 KB and last 100 KB, then wait. + fs.writeSync(fd, file, 0, 100000, 0); + fs.writeSync(fd, file, 461152, 100000, 461152); + + function writeRest() { + setTimeout(function() { + fs.writeSync(fd, file, 100000, 461152, 100000); + fs.closeSync(fd); + + setTimeout(function() { + new MBTiles(fixtures.destination, function(err, mbtiles) { + var returned = 0; + tiles.forEach(function(c) { + mbtiles.getTile(c[0], c[1], c[2], function(err, tile) { + if (++returned === tiles.length) mbtiles._close(); + if (err) assert.ok(false, "Couldn't load tile " + c[0] + '/' + c[1] + '/' + c[2]); + else status.success++; + }); + }); + }); + + }, 2000); + }, 1000); + } + + // Try reading. + new MBTiles(fixtures.destination, function(err, mbtiles) { + if (err) throw err; + mbtiles.getInfo(function(err, data) { + completed = true; + if (err) throw err; + + assert.deepEqual({ + name: 'plain_1', + description: 'demo description', + version: '1.0.3', + scheme: 'tms', + minzoom: 0, + maxzoom: 4, + formatter: null, + center: [ 0, 7.500000001278025, 2 ], + bounds: [ -179.9999999749438, -69.99999999526695, 179.9999999749438, 84.99999999782301 ], + + // These aren't part of TileJSON, but exist in an MBTiles file. + filesize: 561152, + type: 'baselayer', + id: 'write_3', + basename: 'write_3.mbtiles' + }, data); + }); + + var returned = 0; + tiles.forEach(function(c) { + mbtiles.getTile(c[0], c[1], c[2], function(err, tile) { + if (++returned === tiles.length) writeRest(); + if (err) status.error++; + else assert.ok(false, "Could unexpectedly load tile " + c[0] + '/' + c[1] + '/' + c[2]); + }); + }); + }); + + + beforeExit(function() { + assert.ok(completed); + assert.equal(status.error, 11); + assert.equal(status.success, 11); + }); +}; \ No newline at end of file diff --git a/test/write.test.js b/test/write.test.js index c8d3cf7..11b6b3b 100644 --- a/test/write.test.js +++ b/test/write.test.js @@ -1,16 +1,75 @@ +process.env.NODE_ENV = 'test'; + var fs = require('fs'); var assert = require('assert'); -var MBTiles = require('..').MBTiles; +var MBTiles = require('..'); var fixtureDir = __dirname + '/fixtures/output'; // Recreate output directory to remove previous tests. -try { fs.rmdirSync(fixtureDir); } catch(err) {} -fs.mkdirSync(fixtureDir, 0755); +try { fs.unlinkSync(fixtureDir + '/write_1.mbtiles'); } catch(err) {} +try { fs.mkdirSync(fixtureDir, 0755); } catch(err) {} exports['test mbtiles file creation'] = function(beforeExit) { - var mbtiles = new MBTiles(fixtureDir + '/test_1.mbtiles'); - mbtiles.setup(function(err) { - fs.unlinkSync(fixtureDir + '/test_1.mbtiles'); + var completed = { written: 0, read: 0 }; + new MBTiles(fixtureDir + '/write_1.mbtiles', function(err, mbtiles) { + completed.open = true; + if (err) throw err; + + mbtiles.startWriting(function(err) { + completed.started = true; + if (err) throw err; + + fs.readdirSync(__dirname + '/fixtures/images/').forEach(insertTile); + }); + + function insertTile(file) { + var coords = file.match(/^plain_1_(\d+)_(\d+)_(\d+).png$/); + if (!coords) return; + + // Flip Y coordinate because file names are TMS, but .putTile() expects XYZ. + coords[2] = Math.pow(2, coords[3]) - 1 - coords[2]; + + fs.readFile(__dirname + '/fixtures/images/' + file, function(err, tile) { + if (err) throw err; + + mbtiles.putTile(coords[3] | 0, coords[1] | 0, coords[2] | 0, tile, function(err) { + if (err) throw err; + completed.written++; + if (completed.written === 285) { + mbtiles.stopWriting(function(err) { + completed.stopped = true; + if (err) throw err; + verifyWritten(); + }); + } + }); + }); + } + + function verifyWritten() { + fs.readdirSync(__dirname + '/fixtures/images/').forEach(function(file) { + var coords = file.match(/^plain_1_(\d+)_(\d+)_(\d+).png$/); + if (coords) { + // Flip Y coordinate because file names are TMS, but .getTile() expects XYZ. + coords[2] = Math.pow(2, coords[3]) - 1 - coords[2]; + mbtiles.getTile(coords[3] | 0, coords[1] | 0, coords[2] | 0, function(err, tile) { + if (err) throw err; + assert.deepEqual(tile, fs.readFileSync(__dirname + '/fixtures/images/' + file)); + completed.read++; + }); + } + }); + } }); + + beforeExit(function() { + assert.deepEqual({ + open: true, + started: true, + written: 285, + read: 285, + stopped: true + }, completed); + }) }; diff --git a/test/write_grids.test.js b/test/write_grids.test.js new file mode 100644 index 0000000..4d30c1e --- /dev/null +++ b/test/write_grids.test.js @@ -0,0 +1,75 @@ +process.env.NODE_ENV = 'test'; + +var fs = require('fs'); +var assert = require('assert'); +var MBTiles = require('..'); + +var fixtureDir = __dirname + '/fixtures/output'; + +// Recreate output directory to remove previous tests. +try { fs.unlinkSync(fixtureDir + '/write_2.mbtiles'); } catch(err) {} +try { fs.mkdirSync(fixtureDir, 0755); } catch(err) {} + +exports['test mbtiles file creation'] = function(beforeExit) { + var completed = { written: 0, read: 0 }; + new MBTiles(fixtureDir + '/write_2.mbtiles', function(err, mbtiles) { + completed.open = true; + if (err) throw err; + + mbtiles.startWriting(function(err) { + completed.started = true; + if (err) throw err; + + fs.readdirSync(__dirname + '/fixtures/grids/').forEach(insertGrid); + }); + + function insertGrid(file) { + var coords = file.match(/^plain_2_(\d+)_(\d+)_(\d+).json$/); + if (!coords) return; + + // Flip Y coordinate because file names are TMS, but .putGrid() expects XYZ. + coords[2] = Math.pow(2, coords[3]) - 1 - coords[2]; + + fs.readFile(__dirname + '/fixtures/grids/' + file, 'utf8', function(err, grid) { + if (err) throw err; + + mbtiles.putGrid(coords[3] | 0, coords[1] | 0, coords[2] | 0, JSON.parse(grid), function(err) { + if (err) throw err; + completed.written++; + if (completed.written === 241) { + mbtiles.stopWriting(function(err) { + completed.stopped = true; + if (err) throw err; + verifyWritten(); + }); + } + }); + }); + } + + function verifyWritten() { + fs.readdirSync(__dirname + '/fixtures/grids/').forEach(function(file) { + var coords = file.match(/^plain_2_(\d+)_(\d+)_(\d+).json$/); + if (coords) { + // Flip Y coordinate because file names are TMS, but .getTile() expects XYZ. + coords[2] = Math.pow(2, coords[3]) - 1 - coords[2]; + mbtiles.getGrid(coords[3] | 0, coords[1] | 0, coords[2] | 0, function(err, grid) { + if (err) throw err; + assert.deepEqual(JSON.stringify(grid), fs.readFileSync(__dirname + '/fixtures/grids/' + file, 'utf8')); + completed.read++; + }); + } + }); + } + }); + + beforeExit(function() { + assert.deepEqual({ + open: true, + started: true, + written: 241, + read: 241, + stopped: true + }, completed); + }) +};