diff --git a/lib/mbtiles.js b/lib/mbtiles.js index b5b70da..cf7a15d 100644 --- a/lib/mbtiles.js +++ b/lib/mbtiles.js @@ -5,6 +5,7 @@ var _ = require('underscore'), zlib = require('zlib'), path = require('path'), url = require('url'), + Buffer = require('buffer').Buffer, sm = new (require('sphericalmercator')), sqlite3 = require('sqlite3'); @@ -14,6 +15,10 @@ function noop(err) { if (err) throw err; } +function hash(z, x, y) { + return (1 << z) * ((1 << z) + x) + y; +} + // MBTiles // ------- // MBTiles class for doing common operations (schema setup, tile reading, @@ -32,7 +37,7 @@ function MBTiles(uri, callback) { this.filename = uri.pathname; Step(function() { - mbtiles._db = new sqlite3.cached.Database(mbtiles.filename, this) + mbtiles._db = new sqlite3.cached.Database(mbtiles.filename, this); }, function(err) { if (err) return callback(err); mbtiles._setup(this); @@ -198,56 +203,56 @@ MBTiles.prototype._setup = function(callback) { // // - @param {Array} renders array of grids to be inserted. Each item should // // be an object of the form { z: z, x: x, y: y, data: [Image buffer], keys: [] }. // // - @param {Function} callback -// MBTiles.prototype._insertGrids = function(data, callback) { -// if (typeof callback !== 'function') callback = noop; -// -// var that = this, -// map = [], -// grids = [], -// grid_keys = [], -// features = {}, -// ids = []; -// for (var i = 0; i < data.length; i++) { -// var json = JSON.stringify({ -// grid: data[i].grid, -// keys: data[i].keys -// }); -// var grid_id = crypto -// .createHash('md5') -// .update(json) -// .digest('hex'); -// !_(ids).include(grid_id) && ids.push(grid_id) && grids.push({ -// grid_id: grid_id, -// grid_utfgrid: zlib.deflate(new Buffer(json, 'utf8')) -// }); -// data[i].keys.forEach(function(k) { -// grid_keys.push({ -// grid_id: grid_id, -// key_name: k -// }); -// }); -// map.push({ -// grid_id: grid_id, -// zoom_level: data[i].z, -// tile_column: data[i].x, -// tile_row: data[i].y -// }); -// _(features).extend(data[i].data); -// } -// features = _(features).map(function(value, key) { -// return { key_name: key, key_json: JSON.stringify(value) }; -// }); -// Step( -// function() { -// var group = this.group(); -// that._insert('grid_utfgrid', grids, group()); -// that._insert('grid_key', grid_keys, group()); -// that._insert('keymap', features, group()); -// that._insertGridTiles(map, group()); -// }, -// callback -// ); -// }; +MBTiles.prototype._insertGrids = function(data, callback) { + if (typeof callback !== 'function') callback = noop; + + var that = this, + map = [], + grids = [], + grid_keys = [], + features = {}, + ids = []; + for (var i = 0; i < data.length; i++) { + var json = JSON.stringify({ + grid: data[i].grid, + keys: data[i].keys + }); + var grid_id = crypto + .createHash('md5') + .update(json) + .digest('hex'); + !_(ids).include(grid_id) && ids.push(grid_id) && grids.push({ + grid_id: grid_id, + grid_utfgrid: zlib.deflate(new Buffer(json, 'utf8')) + }); + data[i].keys.forEach(function(k) { + grid_keys.push({ + grid_id: grid_id, + key_name: k + }); + }); + map.push({ + grid_id: grid_id, + zoom_level: data[i].z, + tile_column: data[i].x, + tile_row: data[i].y + }); + _(features).extend(data[i].data); + } + features = _(features).map(function(value, key) { + return { key_name: key, key_json: JSON.stringify(value) }; + }); + Step( + function() { + var group = this.group(); + that._insert('grid_utfgrid', grids, group()); + that._insert('grid_key', grid_keys, group()); + that._insert('keymap', features, group()); + that._insertGridTiles(map, group()); + }, + callback + ); +}; // // // Insert grids into the mbtiles database. // // @@ -282,7 +287,7 @@ MBTiles.prototype.getTile = function(z, x, y, callback) { if (!this.open) return callback(new Error('MBTiles not yet loaded')); // Flip Y coordinate because MBTiles files are TMS. - y = Math.pow(2, z) - 1 - y; + y = (1 << z) - 1 - y; var mbtiles = this; this._db.get('SELECT tile_data FROM tiles WHERE ' + @@ -315,7 +320,7 @@ MBTiles.prototype.getGrid = function(z, x, y, callback) { if (!this.open) return callback(new Error('MBTiles not yet loaded')); // Flip Y coordinate because MBTiles files are TMS. - y = Math.pow(2, z) - 1 - y; + y = (1 << z) - 1 - y; var that = this; Step( @@ -492,8 +497,12 @@ MBTiles.prototype.startWriting = function(callback) { var mbtiles = this; if (!this._isWritable) { this._isWritable = 1; - this._tileCache = []; - this._gridCache = []; + this._pending = 0; + this._tileCache = {}; + this._gridCache = {}; + this._keyCache = {}; + this._dataCache = {}; + this._mapCache = {}; this._db.run('PRAGMA synchronous=OFF', callback); } else { this._isWritable++; @@ -502,22 +511,91 @@ MBTiles.prototype.startWriting = function(callback) { }; MBTiles.prototype._commit = function(callback) { + // Only commit when we can insert at least 100 rows. + if (++this._pending < 100) { + return callback(null); + } + var mbtiles = this; - this._db.serialize(function() { + mbtiles._db.serialize(function() { mbtiles._db.run('BEGIN'); - var tile_data = mbtiles._db.prepare('REPLACE INTO images (tile_id, tile_data) VALUES (?, ?)'); - var tile_map = mbtiles._db.prepare('REPLACE INTO map (zoom_level, ' + + if (Object.keys(mbtiles._tileCache)) { + // Insert images table. + var images = mbtiles._db.prepare('REPLACE INTO images (tile_id, tile_data) VALUES (?, ?)'); + for (var id in mbtiles._tileCache) { + images.run(id, mbtiles._tileCache[id]); + } + mbtiles._tileCache = {}; + images.finalize(); + } + + + if (Object.keys(mbtiles._gridCache)) { + // Insert grid_utfgrid table. + var grids = mbtiles._db.prepare('REPLACE INTO grid_utfgrid (grid_id, grid_utfgrid) VALUES (?, ?)'); + for (var id in mbtiles._gridCache) { + grids.run(id, mbtiles._gridCache[id]); + } + mbtiles._gridCache = {}; + grids.finalize(); + } + + + if (Object.keys(mbtiles._keyCache)) { + // Insert grid_key. + var keys = mbtiles._db.prepare('INSERT OR IGNORE INTO grid_key (grid_id, key_name) VALUES (?, ?)'); + for (var id in mbtiles._keyCache) { + mbtiles._keyCache.forEach(function(key) { + keys.run(id, key); + }); + } + mbtiles._keyCache = {}; + keys.finalize(); + } + + + if (Object.keys(mbtiles._dataCache)) { + // Insert keymap table. + var keymap = mbtiles._db.prepare('REPLACE INTO keymap (key_name, key_json) VALUES (?, ?)'); + for (var key in mbtiles._dataCache) { + keymap.run(key, JSON.stringify(mbtiles._dataCache[key])); + } + mbtiles._dataCache = {}; + keymap.finalize(); + } + + // Insert map table. This has to be so complicate due to a design flaw + // in the tables. + // TODO: This should be remedied when we upgrade the MBTiles schema. + var mapBoth = mbtiles._db.prepare('REPLACE INTO map (zoom_level, ' + + 'tile_column, tile_row, tile_id, grid_id) VALUES (?, ?, ?, ?, ?)'); + var mapTile = mbtiles._db.prepare('REPLACE INTO map (zoom_level, ' + 'tile_column, tile_row, tile_id, grid_id) VALUES (?, ?, ?, ?, ' + '(SELECT grid_id FROM map WHERE zoom_level = ? ' + 'AND tile_column = ? AND tile_row = ?))'); - mbtiles._tileCache.forEach(function(row) { - var id = crypto.createHash('md5').update(row[3]).digest('hex'); - tile_data.run(id, row[3]); - tile_map.run(row[0], row[1], row[2], id, row[0], row[1], row[2]); - }); - tile_data.finalize(); - tile_map.finalize(); + var mapGrid = mbtiles._db.prepare('REPLACE INTO map (zoom_level, ' + + 'tile_column, tile_row, tile_id, grid_id) VALUES (?, ?, ?, ' + + '(SELECT tile_id FROM map WHERE zoom_level = ? ' + + 'AND tile_column = ? AND tile_row = ?), ?)'); + for (var coords in mbtiles._mapCache) { + var map = mbtiles._mapCache[coords]; + + if (typeof map.grid_id === 'undefined') { + // Only the tile_id is defined. + mapTile.run(map.z, map.x, map.y, map.tile_id, map.z, map.x, map.y); + } else if (typeof map.tile_id === 'undefined') { + // Only the grid_id is defined. + mapGrid.run(map.z, map.x, map.y, map.z, map.x, map.y, map.grid_id); + } else { + // Both tile_id and grid_id are defined. + mapBoth.run(map.z, map.x, map.y, map.tile_id, map.grid_id); + } + } + mbtiles._mapCache = {}; + mapBoth.finalize(); + mapTile.finalize(); + mapGrid.finalize(); mbtiles._db.run('COMMIT', callback); }); @@ -545,16 +623,23 @@ MBTiles.prototype.putTile = function(z, x, y, data, callback) { if (!this.open) return callback(new Error('MBTiles not yet loaded')); if (!this._isWritable) return callback(new Error('MBTiles not in write mode')); + if (!Buffer.isBuffer(data)) return callback(new Error('Image needs to be a Buffer')); + // Flip Y coordinate because MBTiles files are TMS. - y = Math.pow(2, z) - 1 - y; + y = (1 << z) - 1 - y; - this._tileCache.push([ z, x, y, data ]); - - if (this._tileCache.length + this._gridCache.length >= 100) { - this._commit(callback); - } else { - return callback(null); + var id = crypto.createHash('md5').update(data).digest('hex'); + if (!this._tileCache[id]) { + // This corresponds to the images table. + this._tileCache[id] = data; } + + // This corresponds to the map table. + var coords = hash(z, x, y); + if (!this._mapCache[coords]) this._mapCache[coords] = { z: z, x: x, y: y }; + this._mapCache[coords].tile_id = id; + + this._commit(callback); }; // Insert a tile. Scheme is XYZ. @@ -563,13 +648,26 @@ MBTiles.prototype.putGrid = function(z, x, y, data, callback) { if (!this.open) return callback(new Error('MBTiles not yet loaded')); // Flip Y coordinate because MBTiles files are TMS. - y = Math.pow(2, z) - 1 - y; + y = (1 << z) - 1 - y; - this._tileCache.push([ z, x, y, data ]); + // Preprocess grid data. + var json = JSON.stringify({ grid: data.grid, keys: data.keys }); + var id = crypto.createHash('md5').update(json).digest('hex'); + if (!this._gridCache[id]) { + // This corresponds to the grid_utfgrid table. + this._gridCache[id] = zlib.deflate(new Buffer(json, 'utf8')); - if (this._tileCache.length + this._gridCache.length >= 100) { - this._commit(callback); - } else { - return callback(null); + // This corresponds to the grid_key table. + this._keyCache[id] = Object.keys(data.data || {}); + + // This corresponds to the keymap table. + this._dataCache.extend(data.data || {}); } + + // This corresponds to the map table. + var coords = hash(z, x, y); + if (!this._mapCache[coords]) this._mapCache[coords] = { z: z, x: x, y: y }; + this._mapCache[coords].grid_id = id; + + this._commit(callback); };