Simplify, update for tilelive batch rendering API.

index.js

@@ -1,16 +1,13 @@
 var MBTiles = require('./lib/mbtiles'),
-    zlib = require('zlib'),
-    Buffer = require('buffer').Buffer,
     Step = require('step');
 
 module.exports = {
     MBTiles: MBTiles,
-    pool: function(datasource, options) {
+    pool: function(datasource) {
         return {
             create: function(callback) {
                 var resource = new MBTiles(
                     datasource,
-                    options,
                     function() { callback(resource); }
                 );
             },
@@ -37,42 +34,11 @@ module.exports = {
             );
             break;
         case 'grid.json':
-            var grid;
+            resource.grid(options.x, options.y, options.z, function(err, grid) {
-            Step(
+                grid = JSON.stringify(grid);
-                function() {
+                options.jsonp && (grid = options.jsonp + '(' + grid + ');');
-                    resource.grid(options.x, options.y, options.z, this);
+                callback(err, [grid, { 'Content-Type': 'text/javascript' }]);
-                },
+            });
-                function(err, buf) {
-                    if (err) throw err;
-                    if (!Buffer.isBuffer(buf))
-                        buf = new Buffer(buf, 'binary');
-                    var inflated = zlib.inflate(buf);
-                    this(null,inflated);
-                },
-                function(err, buf) {
-                    if (err) throw err;
-                    grid = buf.toString();
-                    resource.grid_data(options.x, options.y, options.z, this);
-                },
-                function(err, gd) {
-                    if (err) return callback(err);
-                    // Manually append grid data as a string to the grid buffer.
-                    // Ideally we would
-                    //
-                    //     JSON.stringify(_.extend(JSON.parse(grid), { data: gd }))
-                    //
-                    // But calling JSON stringify will escape UTF8 characters of a
-                    // high enough ordinal making the grid data unusable. Instead,
-                    // manipulate the JSON string directly, popping the trailing }
-                    // off and splicing the grid data in at the "data" key.
-                    grid = grid.substr(0, grid.length - 1)
-                        + ', "data":'
-                        + JSON.stringify(gd)
-                        + '}';
-                    options.jsonp && (grid = options.jsonp + '(' + grid + ');');
-                    callback(err, [grid, { 'Content-Type': 'text/javascript' }]);
-                }
-            );
             break;
         default:
             resource.tile(options.x, options.y, options.z, function(err, image) {
@@ -86,6 +52,9 @@ module.exports = {
         case 'setup':
             resource.setup(callback);
             break;
+        case 'metadata':
+            resource.insertMetadata(data, callback);
+            break;
         case 'tiles':
             resource.insertTiles(data, callback);
             break;

lib/mbtiles.js

@@ -1,67 +1,89 @@
-var fs = require('fs');
+var _ = require('underscore'),
-var Step = require('step');
+    fs = require('fs'),
-var crypto = require('crypto');
+    Step = require('step'),
-var sys = require('sys');
+    crypto = require('crypto'),
-var Buffer = require('buffer').Buffer;
+    zlib = require('zlib'),
-var zlib = require('zlib');
+    sqlite3 = require('sqlite3');
-var sqlite3 = require('sqlite3');
 
 // MBTiles
 // -------
 // MBTiles class for doing common operations (schema setup, tile reading,
 // insertion, etc.)
-function MBTiles(filename, options, callback) {
+function MBTiles(filename, callback) {
-    this.options = options || {};
     this.filename = filename;
-    this.compress = true; // @TODO
     this.db = new sqlite3.Database(filename, callback);
-}
+};
 
 // Retrieve the schema of the current mbtiles database and inform the caller of
 // whether the specified table exists.
 MBTiles.prototype.exists = function(table, callback) {
     if (this.schema) {
-        if (this.schema.indexOf(table) !== -1) {
+        return callback(null, _(this.schema).include(table));
-            return callback(null, true);
+    } else {
-        } else {
+        this.schema = [];
-            return callback(null, false);
+        this.db.all(
-        }
+            'SELECT name FROM sqlite_master WHERE type IN (?, ?)',
+            'table',
+            'view',
+            function(err, rows) {
+                if (err) return callback(err);
+                this.schema = _(rows).pluck('name');
+                this.exists(table, callback);
+            }.bind(this)
+        );
     }
-
+};
-    var that = this;
-    that.schema = [];
-
-    this.db.all('SELECT name FROM sqlite_master WHERE type IN (?, ?)',
-        'table',
-        'view',
-        function(err, rows) {
-            if (err) return callback(err);
-            for (var i = 0; i < rows.length; i++) {
-                that.schema.push(rows[i].name);
-            }
-            that.exists(table, callback);
-        });
-}
 
 // Setup schema, indices, views for a new mbtiles database.
+// Sets the synchronous flag to OFF for (much) faster inserts.
+// See http://www.sqlite3.org/pragma.html#pragma_synchronous
 MBTiles.prototype.setup = function(callback) {
-    var db = this.db;
     fs.readFile(__dirname + '/schema.sql', 'utf8', function(err, sql) {
         if (err) return callback(err);
+        this.db.serialize(function() {
+            this.db.run('PRAGMA synchronous = 0');
+            this.db.exec(sql, callback);
+        }.bind(this));
+    }.bind(this));
+};
 
-        db.serialize(function() {
+// Generic object insert.
-            // Set the synchronous flag to OFF for (much) faster inserts.
+//
-            // See http://www.sqlite3.org/pragma.html#pragma_synchronous
+// - `table` String. The table to which objects should be inserted.
-            db.run('PRAGMA synchronous = 0');
+// - `objects` Array. Objects to be inserted, where each object attribute
-            db.exec(sql, callback);
+//   has key/value pairs as a hash corresponding to column name and row value.
-        });
+// - `callback` Function.
+MBTiles.prototype.insert = function(table, objects, callback) {
+    if (!objects.length) return callback();
+    var keys = _(objects[0]).keys();
+    var placeholders = [];
+    _(keys).each(function(k) { placeholders.push('?'); });
+    var stmt = this.db.prepare(
+        'INSERT OR IGNORE INTO ' + table + ' ' +
+        '(' + keys.join(',') + ') ' +
+        'VALUES (' + placeholders.join(',') + ')'
+    );
+    for (var i = 0; i < objects.length; i++) {
+        stmt.run.apply(stmt, _(objects[i]).values());
+    }
+    stmt.finalize(callback);
+};
+
+// Insert metadata into the mbtiles database.
+//
+// - @param {Object} metadata key, value hash of metadata to be inserted.
+// - @param {Function} callback
+MBTiles.prototype.insertMetadata = function(data, callback) {
+    var metadata = _(data).map(function(value, key) {
+        return { name: key, value: value};
     });
+    this.insert('metadata', metadata, callback);
 };
 
 // Insert a set of tiles into an mbtiles database.
 //
 // - @param {Array} renders array of images to be inserted. Each item should
-//   be an object of the form { tile: [z, x, y], data: [Image buffer] }.
+//   be an object of the form { z: z, x: x, y: y, data: [Image buffer] }.
 // - @param {Function} callback
 MBTiles.prototype.insertTiles = function(data, callback) {
     var that = this,
@@ -69,31 +91,26 @@ MBTiles.prototype.insertTiles = function(data, callback) {
         images = [],
         ids = [];
     for (var i = 0; i < data.length; i++) {
-        // Generate ID from MD5 hash of actual image data.
+        var tile_id = crypto
-        // Generate ID from tile coordinates (unique).
+            .createHash('md5')
-        var tile_id = this.compress
+            .update(data[i].data)
-            ? crypto.createHash('md5').update(data[i].data).digest('hex')
+            .digest('hex');
-            : data[i].tile.join('.');
+        !_(ids).include(tile_id) && ids.push(tile_id) && images.push({
-        if (ids.indexOf(tile_id) === -1) {
+            tile_id: tile_id,
-            ids.push(tile_id);
+            tile_data: data[i].data
-            images.push({ tile_id: tile_id, tile_data: data[i].data });
+        });
-        }
         map.push({
             tile_id: tile_id,
-            zoom_level: data[i].tile[0],
+            zoom_level: data[i].z,
-            tile_column: data[i].tile[1],
+            tile_column: data[i].x,
-            tile_row: data[i].tile[2]
+            tile_row: data[i].y
         });
     }
     Step(
         function() {
             var group = this.group();
-            for (var i = 0; i < images.length; i++) {
+            that.insert('images', images, group());
-                that.insertImage(images[i], group());
+            that.insert('map', map, group());
-            }
-            for (var i = 0; i < map.length; i++) {
-                that.insertTile(map[i], group());
-            }
         },
         callback
     );
@@ -102,88 +119,58 @@ MBTiles.prototype.insertTiles = function(data, callback) {
 // Insert a set of grids into an mbtiles database.
 //
 // - @param {Array} renders array of grids to be inserted. Each item should
-//   be an object of the form { tile: [z, x, y], data: [Grid data] }.
+//   be an object of the form { z: z, x: x, y: y, data: [Image buffer], keys: [] }.
 // - @param {Function} callback
 MBTiles.prototype.insertGrids = function(data, callback) {
     var that = this,
         map = [],
         grids = [],
         grid_keys = [],
+        features = {},
         ids = [];
-    for (var i = 0; i < tiles.length; i++) {
+    for (var i = 0; i < data.length; i++) {
-        var grid_id;
+        var json = JSON.stringify({
-        // Generate ID from MD5 hash of grid data.
+            grid: data[i].grid,
-        // Generate ID from tile coordinates (unique).
+            keys: data[i].keys
-        var grid_id = this.compress
+        });
-            ? crypto.createHash('md5').update(data[i].data[0]).digest('hex')
+        var grid_id = crypto
-            : data[i].tile.join('.');
+            .createHash('md5')
-        if (ids.indexOf(grid_id) === -1) {
+            .update(json)
-            ids.push(grid_id);
+            .digest('hex');
-            grids.push({ grid_id: grid_id, grid_utfgrid: renders[i][0] });
+        !_(ids).include(grid_id) && ids.push(grid_id) && grids.push({
-        }
+            grid_id: grid_id,
-        data[i].data[2].keys.forEach(function(k) {
+            grid_utfgrid: zlib.deflate(new Buffer(json, 'utf8'))
-            grid_keys.push({ grid_id: grid_id, key_name: k });
+        });
+        data[i].keys.forEach(function(k) {
+            grid_keys.push({
+                grid_id: grid_id,
+                key_name: k
+            });
         });
         map.push({
             grid_id: grid_id,
-            zoom_level: data[i].tile[0],
+            zoom_level: data[i].z,
-            tile_column: data[i].tile[1],
+            tile_column: data[i].x,
-            tile_row: data[i].tile[2]
+            tile_row: data[i].y
         });
+        _(features).extend(data[i].data);
     }
-
+    features = _(features).map(function(value, key) {
+        return { key_name: key, key_json: JSON.stringify(value) };
+    });
     Step(
         function() {
             var group = this.group();
-            that.insertUTFGrids(grids, group());
+            that.insert('grid_utfgrid', grids, group());
-            that.insertGridKeys(grid_keys, group());
+            that.insert('grid_key', grid_keys, group());
+            that.insert('keymap', features, group());
             that.insertGridTiles(map, group());
         },
         callback
     );
 };
 
-// Given an array of mappings of grid_id to key_name, insert
+// Insert grids into the mbtiles database.
-// them all into the `grid_key` table.
-MBTiles.prototype.insertGridKeys = function(grid_keys, callback) {
-    var stmt = this.db.prepare('INSERT OR IGNORE INTO grid_key' +
-                          '    (grid_id, key_name) VALUES (?, ?)');
-    for (var i = 0; i < grid_keys.length; i++) {
-        stmt.run(
-            grid_keys[i].grid_id,
-            grid_keys[i].key_name
-        );
-    }
-    stmt.finalize(callback);
-};
-
-// Insert a single feature into the grid_data table,
-// with the key/axis of `key`
-MBTiles.prototype.insertGridData = function(data, key_name, callback) {
-    this.db.run(
-        'INSERT OR IGNORE INTO keymap (key_name, key_json) VALUES (?, ?)',
-        data[key_name],
-        JSON.stringify(data),
-        callback
-    );
-};
-
-// Insert a single tile into the mbtiles database.
-//
-// - @param {Object} tile tile object to be inserted.
-// - @param {Function} callback
-MBTiles.prototype.insertTile = function(tile, callback) {
-    this.db.run(
-        'INSERT INTO map (tile_id, zoom_level, tile_column, tile_row) VALUES (?, ?, ?, ?)',
-        tile.tile_id,
-        tile.zoom_level,
-        tile.tile_column,
-        tile.tile_row,
-        callback
-    );
-};
-
-// Insert a single tile into the mbtiles database.
 //
 // - @param {Object} tile tile object to be inserted.
 // - @param {Function} callback
@@ -203,47 +190,6 @@ MBTiles.prototype.insertGridTiles = function(map, callback) {
     stmt.finalize(callback);
 };
 
-// Insert a single grid into the mbtiles database.
-//
-// - @param {Object} image object to be inserted.
-// - @param {Function} callback
-MBTiles.prototype.insertUTFGrids = function(grids, callback) {
-    var stmt = this.db.prepare('INSERT OR IGNORE INTO grid_utfgrid'
-                             + '    (grid_id, grid_utfgrid) VALUES (?, ?)');
-
-    var total = grids.length, ran = 0;
-    grids.forEach(function(grid) {
-        var buf = zlib.deflate(grid.grid_utfgrid);
-        stmt.run(grid.grid_id, buf);
-        if (++ran === total) stmt.finalize(callback);
-    });
-};
-
-// Insert a single image into the mbtiles database.
-//
-// - @param {Object} image object to be inserted.
-// - @param {Function} callback
-MBTiles.prototype.insertImage = function(image, callback) {
-    this.db.run(
-        'INSERT OR IGNORE INTO images (tile_id, tile_data) VALUES (?, ?)',
-        image.tile_id,
-        image.tile_data,
-        callback
-    );
-};
-
-// Insert metadata into the mbtiles database.
-//
-// - @param {Object} metadata key, value hash of metadata to be inserted.
-// - @param {Function} callback
-MBTiles.prototype.insertMetadata = function(metadata, callback) {
-    var stmt = this.db.prepare('INSERT INTO metadata (name, value) VALUES (?, ?)');
-    for (var name in metadata) {
-        stmt.run(name, metadata[name]);
-    }
-    stmt.finalize(callback);
-};
-
 // Select a tile from an mbtiles database.
 //
 // - @param {Number} x tile x coordinate.
@@ -261,37 +207,42 @@ MBTiles.prototype.tile = function(x, y, z, callback) {
         });
 };
 
-// Get grid data at a certain `x, y, z` coordinate, calling
+// Select a grid and its data from an mbtiles database.
-// back with an error argument if the grid is not found.
-MBTiles.prototype.grid_data = function(x, y, z, callback) {
-    this.db.all('SELECT key_name, key_json FROM grid_data WHERE ' +
-        'zoom_level = ? AND tile_column = ? AND tile_row = ?',
-        z, x, y,
-        function(err, rows) {
-            if (err) callback(err);
-            else if (!rows.length) callback('Grid data does not exist');
-            else callback(null, rows.reduce(function(memo, r) {
-                memo[r.key_name] = JSON.parse(r.key_json);
-                return memo;
-            }, {}));
-        });
-};
-
-// Select a tile from an mbtiles database.
 //
 // - @param {Number} x tile x coordinate
 // - @param {Number} y tile y coordinate
 // - @param {Number} z tile z coordinate
 // - @param {Function} callback
 MBTiles.prototype.grid = function(x, y, z, callback) {
-    this.db.get('SELECT grid FROM grids WHERE ' +
+    var that = this;
-        'zoom_level = ? AND tile_column = ? AND tile_row = ?',
+    Step(
-        z, x, y,
+        function() {
-        function(err, row) {
+            that.db.get('SELECT grid FROM grids WHERE ' +
-            if (err) callback(err);
+                'zoom_level = ? AND tile_column = ? AND tile_row = ?',
-            else if (!row || !row.grid) callback('Grid does not exist');
+                z, x, y,
-            else callback(null, row.grid);
+                this.parallel()
-        });
+            );
+            that.db.all('SELECT key_name, key_json FROM grid_data WHERE ' +
+                'zoom_level = ? AND tile_column = ? AND tile_row = ?',
+                z, x, y,
+                this.parallel()
+            );
+        },
+        function(err, row, rows) {
+            if (err) return callback(err);
+            if (!row || !row.grid) return callback('Grid does not exist');
+            var grid = zlib.inflate(
+                    !Buffer.isBuffer(row.grid)
+                    ? new Buffer(grid, 'binary')
+                    : row.grid
+                ).toString();
+            var data = rows.reduce(function(memo, r) {
+                    memo[r.key_name] = JSON.parse(r.key_json);
+                    return memo;
+                }, {});
+            callback(null, _(JSON.parse(grid)).extend({ data: data }));
+        }
+    );
 };
 
 // Select a metadata value from the database.