diff --git a/index.js b/index.js index 94abdf8..8fa814c 100644 --- a/index.js +++ b/index.js @@ -1,16 +1,13 @@ var MBTiles = require('./lib/mbtiles'), - zlib = require('zlib'), - Buffer = require('buffer').Buffer, Step = require('step'); module.exports = { MBTiles: MBTiles, - pool: function(datasource, options) { + pool: function(datasource) { return { create: function(callback) { var resource = new MBTiles( datasource, - options, function() { callback(resource); } ); }, @@ -37,42 +34,11 @@ module.exports = { ); break; case 'grid.json': - var grid; - Step( - function() { - resource.grid(options.x, options.y, options.z, this); - }, - function(err, buf) { - if (err) throw err; - if (!Buffer.isBuffer(buf)) - buf = new Buffer(buf, 'binary'); - var inflated = zlib.inflate(buf); - this(null,inflated); - }, - function(err, buf) { - if (err) throw err; - grid = buf.toString(); - resource.grid_data(options.x, options.y, options.z, this); - }, - function(err, gd) { - if (err) return callback(err); - // Manually append grid data as a string to the grid buffer. - // Ideally we would - // - // JSON.stringify(_.extend(JSON.parse(grid), { data: gd })) - // - // But calling JSON stringify will escape UTF8 characters of a - // high enough ordinal making the grid data unusable. Instead, - // manipulate the JSON string directly, popping the trailing } - // off and splicing the grid data in at the "data" key. - grid = grid.substr(0, grid.length - 1) - + ', "data":' - + JSON.stringify(gd) - + '}'; - options.jsonp && (grid = options.jsonp + '(' + grid + ');'); - callback(err, [grid, { 'Content-Type': 'text/javascript' }]); - } - ); + resource.grid(options.x, options.y, options.z, function(err, grid) { + grid = JSON.stringify(grid); + options.jsonp && (grid = options.jsonp + '(' + grid + ');'); + callback(err, [grid, { 'Content-Type': 'text/javascript' }]); + }); break; default: resource.tile(options.x, options.y, options.z, function(err, image) { @@ -86,6 +52,9 @@ module.exports = { case 'setup': resource.setup(callback); break; + case 'metadata': + resource.insertMetadata(data, callback); + break; case 'tiles': resource.insertTiles(data, callback); break; diff --git a/lib/mbtiles.js b/lib/mbtiles.js index 1f68853..c29365c 100644 --- a/lib/mbtiles.js +++ b/lib/mbtiles.js @@ -1,67 +1,89 @@ -var fs = require('fs'); -var Step = require('step'); -var crypto = require('crypto'); -var sys = require('sys'); -var Buffer = require('buffer').Buffer; -var zlib = require('zlib'); -var sqlite3 = require('sqlite3'); +var _ = require('underscore'), + fs = require('fs'), + Step = require('step'), + crypto = require('crypto'), + zlib = require('zlib'), + sqlite3 = require('sqlite3'); // MBTiles // ------- // MBTiles class for doing common operations (schema setup, tile reading, // insertion, etc.) -function MBTiles(filename, options, callback) { - this.options = options || {}; +function MBTiles(filename, callback) { this.filename = filename; - this.compress = true; // @TODO this.db = new sqlite3.Database(filename, callback); -} +}; // Retrieve the schema of the current mbtiles database and inform the caller of // whether the specified table exists. MBTiles.prototype.exists = function(table, callback) { if (this.schema) { - if (this.schema.indexOf(table) !== -1) { - return callback(null, true); - } else { - return callback(null, false); - } + return callback(null, _(this.schema).include(table)); + } else { + this.schema = []; + this.db.all( + 'SELECT name FROM sqlite_master WHERE type IN (?, ?)', + 'table', + 'view', + function(err, rows) { + if (err) return callback(err); + this.schema = _(rows).pluck('name'); + this.exists(table, callback); + }.bind(this) + ); } - - var that = this; - that.schema = []; - - this.db.all('SELECT name FROM sqlite_master WHERE type IN (?, ?)', - 'table', - 'view', - function(err, rows) { - if (err) return callback(err); - for (var i = 0; i < rows.length; i++) { - that.schema.push(rows[i].name); - } - that.exists(table, callback); - }); -} +}; // Setup schema, indices, views for a new mbtiles database. +// Sets the synchronous flag to OFF for (much) faster inserts. +// See http://www.sqlite3.org/pragma.html#pragma_synchronous MBTiles.prototype.setup = function(callback) { - var db = this.db; fs.readFile(__dirname + '/schema.sql', 'utf8', function(err, sql) { if (err) return callback(err); + this.db.serialize(function() { + this.db.run('PRAGMA synchronous = 0'); + this.db.exec(sql, callback); + }.bind(this)); + }.bind(this)); +}; - db.serialize(function() { - // Set the synchronous flag to OFF for (much) faster inserts. - // See http://www.sqlite3.org/pragma.html#pragma_synchronous - db.run('PRAGMA synchronous = 0'); - db.exec(sql, callback); - }); +// Generic object insert. +// +// - `table` String. The table to which objects should be inserted. +// - `objects` Array. Objects to be inserted, where each object attribute +// has key/value pairs as a hash corresponding to column name and row value. +// - `callback` Function. +MBTiles.prototype.insert = function(table, objects, callback) { + if (!objects.length) return callback(); + var keys = _(objects[0]).keys(); + var placeholders = []; + _(keys).each(function(k) { placeholders.push('?'); }); + var stmt = this.db.prepare( + 'INSERT OR IGNORE INTO ' + table + ' ' + + '(' + keys.join(',') + ') ' + + 'VALUES (' + placeholders.join(',') + ')' + ); + for (var i = 0; i < objects.length; i++) { + stmt.run.apply(stmt, _(objects[i]).values()); + } + stmt.finalize(callback); +}; + +// Insert metadata into the mbtiles database. +// +// - @param {Object} metadata key, value hash of metadata to be inserted. +// - @param {Function} callback +MBTiles.prototype.insertMetadata = function(data, callback) { + var metadata = _(data).map(function(value, key) { + return { name: key, value: value}; }); + this.insert('metadata', metadata, callback); }; // Insert a set of tiles into an mbtiles database. // // - @param {Array} renders array of images to be inserted. Each item should -// be an object of the form { tile: [z, x, y], data: [Image buffer] }. +// be an object of the form { z: z, x: x, y: y, data: [Image buffer] }. // - @param {Function} callback MBTiles.prototype.insertTiles = function(data, callback) { var that = this, @@ -69,31 +91,26 @@ MBTiles.prototype.insertTiles = function(data, callback) { images = [], ids = []; for (var i = 0; i < data.length; i++) { - // Generate ID from MD5 hash of actual image data. - // Generate ID from tile coordinates (unique). - var tile_id = this.compress - ? crypto.createHash('md5').update(data[i].data).digest('hex') - : data[i].tile.join('.'); - if (ids.indexOf(tile_id) === -1) { - ids.push(tile_id); - images.push({ tile_id: tile_id, tile_data: data[i].data }); - } + var tile_id = crypto + .createHash('md5') + .update(data[i].data) + .digest('hex'); + !_(ids).include(tile_id) && ids.push(tile_id) && images.push({ + tile_id: tile_id, + tile_data: data[i].data + }); map.push({ tile_id: tile_id, - zoom_level: data[i].tile[0], - tile_column: data[i].tile[1], - tile_row: data[i].tile[2] + zoom_level: data[i].z, + tile_column: data[i].x, + tile_row: data[i].y }); } Step( function() { var group = this.group(); - for (var i = 0; i < images.length; i++) { - that.insertImage(images[i], group()); - } - for (var i = 0; i < map.length; i++) { - that.insertTile(map[i], group()); - } + that.insert('images', images, group()); + that.insert('map', map, group()); }, callback ); @@ -102,88 +119,58 @@ MBTiles.prototype.insertTiles = function(data, callback) { // Insert a set of grids into an mbtiles database. // // - @param {Array} renders array of grids to be inserted. Each item should -// be an object of the form { tile: [z, x, y], data: [Grid data] }. +// be an object of the form { z: z, x: x, y: y, data: [Image buffer], keys: [] }. // - @param {Function} callback MBTiles.prototype.insertGrids = function(data, callback) { var that = this, map = [], grids = [], grid_keys = [], + features = {}, ids = []; - for (var i = 0; i < tiles.length; i++) { - var grid_id; - // Generate ID from MD5 hash of grid data. - // Generate ID from tile coordinates (unique). - var grid_id = this.compress - ? crypto.createHash('md5').update(data[i].data[0]).digest('hex') - : data[i].tile.join('.'); - if (ids.indexOf(grid_id) === -1) { - ids.push(grid_id); - grids.push({ grid_id: grid_id, grid_utfgrid: renders[i][0] }); - } - data[i].data[2].keys.forEach(function(k) { - grid_keys.push({ grid_id: grid_id, key_name: k }); + for (var i = 0; i < data.length; i++) { + var json = JSON.stringify({ + grid: data[i].grid, + keys: data[i].keys + }); + var grid_id = crypto + .createHash('md5') + .update(json) + .digest('hex'); + !_(ids).include(grid_id) && ids.push(grid_id) && grids.push({ + grid_id: grid_id, + grid_utfgrid: zlib.deflate(new Buffer(json, 'utf8')) + }); + data[i].keys.forEach(function(k) { + grid_keys.push({ + grid_id: grid_id, + key_name: k + }); }); map.push({ grid_id: grid_id, - zoom_level: data[i].tile[0], - tile_column: data[i].tile[1], - tile_row: data[i].tile[2] + zoom_level: data[i].z, + tile_column: data[i].x, + tile_row: data[i].y }); + _(features).extend(data[i].data); } - + features = _(features).map(function(value, key) { + return { key_name: key, key_json: JSON.stringify(value) }; + }); Step( function() { var group = this.group(); - that.insertUTFGrids(grids, group()); - that.insertGridKeys(grid_keys, group()); + that.insert('grid_utfgrid', grids, group()); + that.insert('grid_key', grid_keys, group()); + that.insert('keymap', features, group()); that.insertGridTiles(map, group()); }, callback ); }; -// Given an array of mappings of grid_id to key_name, insert -// them all into the `grid_key` table. -MBTiles.prototype.insertGridKeys = function(grid_keys, callback) { - var stmt = this.db.prepare('INSERT OR IGNORE INTO grid_key' + - ' (grid_id, key_name) VALUES (?, ?)'); - for (var i = 0; i < grid_keys.length; i++) { - stmt.run( - grid_keys[i].grid_id, - grid_keys[i].key_name - ); - } - stmt.finalize(callback); -}; - -// Insert a single feature into the grid_data table, -// with the key/axis of `key` -MBTiles.prototype.insertGridData = function(data, key_name, callback) { - this.db.run( - 'INSERT OR IGNORE INTO keymap (key_name, key_json) VALUES (?, ?)', - data[key_name], - JSON.stringify(data), - callback - ); -}; - -// Insert a single tile into the mbtiles database. -// -// - @param {Object} tile tile object to be inserted. -// - @param {Function} callback -MBTiles.prototype.insertTile = function(tile, callback) { - this.db.run( - 'INSERT INTO map (tile_id, zoom_level, tile_column, tile_row) VALUES (?, ?, ?, ?)', - tile.tile_id, - tile.zoom_level, - tile.tile_column, - tile.tile_row, - callback - ); -}; - -// Insert a single tile into the mbtiles database. +// Insert grids into the mbtiles database. // // - @param {Object} tile tile object to be inserted. // - @param {Function} callback @@ -203,47 +190,6 @@ MBTiles.prototype.insertGridTiles = function(map, callback) { stmt.finalize(callback); }; -// Insert a single grid into the mbtiles database. -// -// - @param {Object} image object to be inserted. -// - @param {Function} callback -MBTiles.prototype.insertUTFGrids = function(grids, callback) { - var stmt = this.db.prepare('INSERT OR IGNORE INTO grid_utfgrid' - + ' (grid_id, grid_utfgrid) VALUES (?, ?)'); - - var total = grids.length, ran = 0; - grids.forEach(function(grid) { - var buf = zlib.deflate(grid.grid_utfgrid); - stmt.run(grid.grid_id, buf); - if (++ran === total) stmt.finalize(callback); - }); -}; - -// Insert a single image into the mbtiles database. -// -// - @param {Object} image object to be inserted. -// - @param {Function} callback -MBTiles.prototype.insertImage = function(image, callback) { - this.db.run( - 'INSERT OR IGNORE INTO images (tile_id, tile_data) VALUES (?, ?)', - image.tile_id, - image.tile_data, - callback - ); -}; - -// Insert metadata into the mbtiles database. -// -// - @param {Object} metadata key, value hash of metadata to be inserted. -// - @param {Function} callback -MBTiles.prototype.insertMetadata = function(metadata, callback) { - var stmt = this.db.prepare('INSERT INTO metadata (name, value) VALUES (?, ?)'); - for (var name in metadata) { - stmt.run(name, metadata[name]); - } - stmt.finalize(callback); -}; - // Select a tile from an mbtiles database. // // - @param {Number} x tile x coordinate. @@ -261,37 +207,42 @@ MBTiles.prototype.tile = function(x, y, z, callback) { }); }; -// Get grid data at a certain `x, y, z` coordinate, calling -// back with an error argument if the grid is not found. -MBTiles.prototype.grid_data = function(x, y, z, callback) { - this.db.all('SELECT key_name, key_json FROM grid_data WHERE ' + - 'zoom_level = ? AND tile_column = ? AND tile_row = ?', - z, x, y, - function(err, rows) { - if (err) callback(err); - else if (!rows.length) callback('Grid data does not exist'); - else callback(null, rows.reduce(function(memo, r) { - memo[r.key_name] = JSON.parse(r.key_json); - return memo; - }, {})); - }); -}; - -// Select a tile from an mbtiles database. +// Select a grid and its data from an mbtiles database. // // - @param {Number} x tile x coordinate // - @param {Number} y tile y coordinate // - @param {Number} z tile z coordinate // - @param {Function} callback MBTiles.prototype.grid = function(x, y, z, callback) { - this.db.get('SELECT grid FROM grids WHERE ' + - 'zoom_level = ? AND tile_column = ? AND tile_row = ?', - z, x, y, - function(err, row) { - if (err) callback(err); - else if (!row || !row.grid) callback('Grid does not exist'); - else callback(null, row.grid); - }); + var that = this; + Step( + function() { + that.db.get('SELECT grid FROM grids WHERE ' + + 'zoom_level = ? AND tile_column = ? AND tile_row = ?', + z, x, y, + this.parallel() + ); + that.db.all('SELECT key_name, key_json FROM grid_data WHERE ' + + 'zoom_level = ? AND tile_column = ? AND tile_row = ?', + z, x, y, + this.parallel() + ); + }, + function(err, row, rows) { + if (err) return callback(err); + if (!row || !row.grid) return callback('Grid does not exist'); + var grid = zlib.inflate( + !Buffer.isBuffer(row.grid) + ? new Buffer(grid, 'binary') + : row.grid + ).toString(); + var data = rows.reduce(function(memo, r) { + memo[r.key_name] = JSON.parse(r.key_json); + return memo; + }, {}); + callback(null, _(JSON.parse(grid)).extend({ data: data })); + } + ); }; // Select a metadata value from the database.