first shot at putTile

lib/mbtiles.js

@@ -8,6 +8,8 @@ var _ = require('underscore'),
     sm = new (require('sphericalmercator')),
     sqlite3 = require('sqlite3');
 
+if (process.env.NODE_ENV === 'test') sqlite3.verbose();
+
 function noop(err) {
     if (err) throw err;
 }
@@ -31,27 +33,24 @@ function MBTiles(uri, callback) {
     this.filename = uri.pathname;
     Step(function() {
         mbtiles._db = new sqlite3.cached.Database(mbtiles.filename, this)
+    }, function(err) {
+        if (err) return callback(err);
+        mbtiles._setup(this);
     }, function(err) {
         if (err) return callback(err);
         fs.stat(mbtiles.filename, this);
     }, function(err, stat) {
         if (err) return callback(err);
         mbtiles._stat = stat;
-        mbtiles._exists('map', this);
-    }, function(err, exists) {
-        if (err) return callback(err);
-        else if (!exists) mbtiles._setup(this);
-        else this(null);
-    }, function(err) {
-        if (err) callback(err);
-        callback(null, mbtiles);
+        mbtiles.open = true;
+        return callback(null, mbtiles);
     });
+
+    return undefined;
 };
 
 // Finds all mbtiles file in the filepath and returns their tilesource URI.
 MBTiles.list = function(filepath, callback) {
-    if (typeof callback !== 'function') callback = noop;
-
     filepath = path.resolve(filepath);
     fs.readdir(filepath, function(err, files) {
         if (err) return callback(err);
@@ -59,20 +58,18 @@ MBTiles.list = function(filepath, callback) {
             var name = files[i].match(/^([\w-]+)\.mbtiles$/);
             if (name) result[name[1]] = 'mbtiles://' + path.join(filepath, name[0]);
         }
-        callback(null, result);
+        return callback(null, result);
     });
 };
 
 // Finds an mbtiles file with the given ID in the filepath and returns a
 // tilesource URI.
 MBTiles.findID = function(filepath, id, callback) {
-    if (typeof callback !== 'function') callback = noop;
-
     filepath = path.resolve(filepath);
     var file = path.join(filepath, id + '.mbtiles');
     fs.stat(file, function(err, stats) {
-        if (err) callback(err);
-        else callback(null, 'mbtiles://' + file);
+        if (err) return callback(err);
+        else return callback(null, 'mbtiles://' + file);
     });
 };
 
@@ -104,178 +101,175 @@ MBTiles.prototype._integrity = function(callback) {
 
     this._db.get('PRAGMA quick_check(1)', function(err, row) {
         if (!(row && row.integrity_check && row.integrity_check === 'ok')) {
-            callback(new Error('Corrupted database.'));
+            return callback(new Error('Corrupted database.'));
         } else {
-            callback(null, true);
+            return callback(null, true);
         }
     });
 };
 
 // Setup schema, indices, views for a new mbtiles database.
-// Sets the synchronous flag to OFF for (much) faster inserts.
-// See http://www.sqlite3.org/pragma.html#pragma_synchronous
 MBTiles.prototype._setup = function(callback) {
-    if (typeof callback !== 'function') callback = noop;
-
-    fs.readFile(__dirname + '/schema.sql', 'utf8', function(err, sql) {
-        if (err) return callback(err);
-        this._db.serialize(function() {
-            this._db.run('PRAGMA synchronous = 0');
-            this._db.exec(sql, callback);
-        }.bind(this));
-    }.bind(this));
-};
-
-// Generic object insert.
-//
-// - `table` String. The table to which objects should be inserted.
-// - `objects` Array. Objects to be inserted, where each object attribute
-//   has key/value pairs as a hash corresponding to column name and row value.
-// - `callback` Function.
-MBTiles.prototype._insert = function(table, objects, callback) {
-    if (typeof callback !== 'function') callback = noop;
-
-    if (!objects.length) return callback(null);
-    var keys = _(objects[0]).keys();
-    var placeholders = [];
-    _(keys).each(function(k) { placeholders.push('?'); });
-    var stmt = this._db.prepare(
-        'INSERT OR IGNORE INTO ' + table + ' ' +
-        '(' + keys.join(',') + ') ' +
-        'VALUES (' + placeholders.join(',') + ')'
-    );
-    for (var i = 0; i < objects.length; i++) {
-        stmt.run.apply(stmt, _(objects[i]).values());
-    }
-    stmt.finalize(callback);
-};
-
-// Insert metadata into the mbtiles database.
-//
-// - @param {Object} metadata key, value hash of metadata to be inserted.
-// - @param {Function} callback
-MBTiles.prototype._insertMetadata = function(data, callback) {
-    if (typeof callback !== 'function') callback = noop;
-
-    var metadata = _(data).map(function(value, key) {
-        return { name: key, value: value};
+    var mbtiles = this;
+    mbtiles._exists('tiles', function(err, exists) {
+        if (exists) return callback(null);
+        fs.readFile(__dirname + '/schema.sql', 'utf8', function(err, sql) {
+            if (err) return callback(err);
+            mbtiles._db.exec(sql, callback);
+        });
     });
-    this._insert('metadata', metadata, callback);
 };
 
-// Insert a set of tiles into an mbtiles database. Scheme is TMS.
+// // Generic object insert.
+// //
+// // - `table` String. The table to which objects should be inserted.
+// // - `objects` Array. Objects to be inserted, where each object attribute
+// //   has key/value pairs as a hash corresponding to column name and row value.
+// // - `callback` Function.
+// MBTiles.prototype._insert = function(table, objects, callback) {
+//     if (typeof callback !== 'function') callback = noop;
 //
-// - @param {Array} renders array of images to be inserted. Each item should
-//   be an object of the form { z: z, x: x, y: y, data: [Image buffer] }.
-// - @param {Function} callback
-MBTiles.prototype._insertTiles = function(data, callback) {
-    var that = this,
-        map = [],
-        images = [],
-        ids = [];
-    for (var i = 0; i < data.length; i++) {
-        var tile_id = crypto
-            .createHash('md5')
-            .update(data[i].data)
-            .digest('hex');
-        !_(ids).include(tile_id) && ids.push(tile_id) && images.push({
-            tile_id: tile_id,
-            tile_data: data[i].data
-        });
-        map.push({
-            tile_id: tile_id,
-            zoom_level: data[i].z,
-            tile_column: data[i].x,
-            tile_row: data[i].y
-        });
-    }
-    Step(
-        function() {
-            var group = this.group();
-            that._insert('images', images, group());
-            that._insert('map', map, group());
-        },
-        callback
-    );
-};
-
-// Insert a set of grids into an mbtiles database.
+//     if (!objects.length) return callback(null);
+//     var keys = _(objects[0]).keys();
+//     var placeholders = [];
+//     _(keys).each(function(k) { placeholders.push('?'); });
+//     var stmt = this._db.prepare(
+//         'INSERT OR IGNORE INTO ' + table + ' ' +
+//         '(' + keys.join(',') + ') ' +
+//         'VALUES (' + placeholders.join(',') + ')'
+//     );
+//     for (var i = 0; i < objects.length; i++) {
+//         stmt.run.apply(stmt, _(objects[i]).values());
+//     }
+//     stmt.finalize(callback);
+// };
 //
-// - @param {Array} renders array of grids to be inserted. Each item should
-//   be an object of the form { z: z, x: x, y: y, data: [Image buffer], keys: [] }.
-// - @param {Function} callback
-MBTiles.prototype._insertGrids = function(data, callback) {
-    if (typeof callback !== 'function') callback = noop;
-
-    var that = this,
-        map = [],
-        grids = [],
-        grid_keys = [],
-        features = {},
-        ids = [];
-    for (var i = 0; i < data.length; i++) {
-        var json = JSON.stringify({
-            grid: data[i].grid,
-            keys: data[i].keys
-        });
-        var grid_id = crypto
-            .createHash('md5')
-            .update(json)
-            .digest('hex');
-        !_(ids).include(grid_id) && ids.push(grid_id) && grids.push({
-            grid_id: grid_id,
-            grid_utfgrid: zlib.deflate(new Buffer(json, 'utf8'))
-        });
-        data[i].keys.forEach(function(k) {
-            grid_keys.push({
-                grid_id: grid_id,
-                key_name: k
-            });
-        });
-        map.push({
-            grid_id: grid_id,
-            zoom_level: data[i].z,
-            tile_column: data[i].x,
-            tile_row: data[i].y
-        });
-        _(features).extend(data[i].data);
-    }
-    features = _(features).map(function(value, key) {
-        return { key_name: key, key_json: JSON.stringify(value) };
-    });
-    Step(
-        function() {
-            var group = this.group();
-            that._insert('grid_utfgrid', grids, group());
-            that._insert('grid_key', grid_keys, group());
-            that._insert('keymap', features, group());
-            that._insertGridTiles(map, group());
-        },
-        callback
-    );
-};
-
-// Insert grids into the mbtiles database.
+// // Insert metadata into the mbtiles database.
+// //
+// // - @param {Object} metadata key, value hash of metadata to be inserted.
+// // - @param {Function} callback
+// MBTiles.prototype._insertMetadata = function(data, callback) {
+//     if (typeof callback !== 'function') callback = noop;
 //
-// - @param {Object} tile tile object to be inserted.
-// - @param {Function} callback
-MBTiles.prototype._insertGridTiles = function(map, callback) {
-    if (typeof callback !== 'function') callback = noop;
-
-    var stmt = this._db.prepare('UPDATE OR REPLACE map SET grid_id = ? WHERE ' +
-        ' zoom_level = ? AND tile_column = ? AND tile_row = ?');
-
-    for (var i = 0; i < map.length; i++) {
-        stmt.run(
-            map[i].grid_id,
-            map[i].zoom_level,
-            map[i].tile_column,
-            map[i].tile_row
-        );
-    }
-
-    stmt.finalize(callback);
-};
+//     var metadata = _(data).map(function(value, key) {
+//         return { name: key, value: value};
+//     });
+//     this._insert('metadata', metadata, callback);
+// };
+//
+// // Insert a set of tiles into an mbtiles database. Scheme is TMS.
+// //
+// // - @param {Array} renders array of images to be inserted. Each item should
+// //   be an object of the form { z: z, x: x, y: y, data: [Image buffer] }.
+// // - @param {Function} callback
+// MBTiles.prototype._insertTiles = function(data, callback) {
+//     var that = this,
+//         map = [],
+//         images = [],
+//         ids = [];
+//     for (var i = 0; i < data.length; i++) {
+//         var tile_id = crypto
+//             .createHash('md5')
+//             .update(data[i].data)
+//             .digest('hex');
+//         !_(ids).include(tile_id) && ids.push(tile_id) && images.push({
+//             tile_id: tile_id,
+//             tile_data: data[i].data
+//         });
+//         map.push({
+//             tile_id: tile_id,
+//             zoom_level: data[i].z,
+//             tile_column: data[i].x,
+//             tile_row: data[i].y
+//         });
+//     }
+//     Step(
+//         function() {
+//             var group = this.group();
+//             that._insert('images', images, group());
+//             that._insert('map', map, group());
+//         },
+//         callback
+//     );
+// };
+//
+// // Insert a set of grids into an mbtiles database.
+// //
+// // - @param {Array} renders array of grids to be inserted. Each item should
+// //   be an object of the form { z: z, x: x, y: y, data: [Image buffer], keys: [] }.
+// // - @param {Function} callback
+// MBTiles.prototype._insertGrids = function(data, callback) {
+//     if (typeof callback !== 'function') callback = noop;
+//
+//     var that = this,
+//         map = [],
+//         grids = [],
+//         grid_keys = [],
+//         features = {},
+//         ids = [];
+//     for (var i = 0; i < data.length; i++) {
+//         var json = JSON.stringify({
+//             grid: data[i].grid,
+//             keys: data[i].keys
+//         });
+//         var grid_id = crypto
+//             .createHash('md5')
+//             .update(json)
+//             .digest('hex');
+//         !_(ids).include(grid_id) && ids.push(grid_id) && grids.push({
+//             grid_id: grid_id,
+//             grid_utfgrid: zlib.deflate(new Buffer(json, 'utf8'))
+//         });
+//         data[i].keys.forEach(function(k) {
+//             grid_keys.push({
+//                 grid_id: grid_id,
+//                 key_name: k
+//             });
+//         });
+//         map.push({
+//             grid_id: grid_id,
+//             zoom_level: data[i].z,
+//             tile_column: data[i].x,
+//             tile_row: data[i].y
+//         });
+//         _(features).extend(data[i].data);
+//     }
+//     features = _(features).map(function(value, key) {
+//         return { key_name: key, key_json: JSON.stringify(value) };
+//     });
+//     Step(
+//         function() {
+//             var group = this.group();
+//             that._insert('grid_utfgrid', grids, group());
+//             that._insert('grid_key', grid_keys, group());
+//             that._insert('keymap', features, group());
+//             that._insertGridTiles(map, group());
+//         },
+//         callback
+//     );
+// };
+//
+// // Insert grids into the mbtiles database.
+// //
+// // - @param {Object} tile tile object to be inserted.
+// // - @param {Function} callback
+// MBTiles.prototype._insertGridTiles = function(map, callback) {
+//     if (typeof callback !== 'function') callback = noop;
+//
+//     var stmt = this._db.prepare('UPDATE OR REPLACE map SET grid_id = ? WHERE ' +
+//         ' zoom_level = ? AND tile_column = ? AND tile_row = ?');
+//
+//     for (var i = 0; i < map.length; i++) {
+//         stmt.run(
+//             map[i].grid_id,
+//             map[i].zoom_level,
+//             map[i].tile_column,
+//             map[i].tile_row
+//         );
+//     }
+//
+//     stmt.finalize(callback);
+// };
 
 // Select a tile from an mbtiles database. Scheme is XYZ.
 //
@@ -284,7 +278,8 @@ MBTiles.prototype._insertGridTiles = function(map, callback) {
 // - @param {Number} z tile z coordinate.
 // - @param {Function} callback
 MBTiles.prototype.getTile = function(z, x, y, callback) {
-    if (typeof callback !== 'function') callback = noop;
+    if (typeof callback !== 'function') throw new Error('Callback needed');
+    if (!this.open) return callback(new Error('MBTiles not yet loaded'));
 
     // Flip Y coordinate because MBTiles files are TMS.
     y = Math.pow(2, z) - 1 - y;
@@ -316,7 +311,8 @@ MBTiles.prototype.getTile = function(z, x, y, callback) {
 // - @param {Number} z tile z coordinate
 // - @param {Function} callback
 MBTiles.prototype.getGrid = function(z, x, y, callback) {
-    if (typeof callback !== 'function') callback = noop;
+    if (typeof callback !== 'function') throw new Error('Callback needed');
+    if (!this.open) return callback(new Error('MBTiles not yet loaded'));
 
     // Flip Y coordinate because MBTiles files are TMS.
     y = Math.pow(2, z) - 1 - y;
@@ -353,7 +349,7 @@ MBTiles.prototype.getGrid = function(z, x, y, callback) {
                     }, {});
                 var result = _(JSON.parse(grid)).extend({ data: data });
             } catch (err) {
-                callback(new Error('Grid is invalid'));
+                return callback(new Error('Grid is invalid'));
             }
 
             callback(null, result);
@@ -370,9 +366,9 @@ MBTiles.prototype._metadata = function(key, callback) {
     this._db.get('SELECT value FROM metadata WHERE name = ?',
         key,
         function(err, row) {
-            if (!row || (err && err.errno == 1)) callback(new Error('Key does not exist'));
-            else if (err) callback(err);
-            else callback(null, row.value);
+            if (!row || (err && err.errno == 1)) return callback(new Error('Key does not exist'));
+            else if (err) return callback(err);
+            else return callback(null, row.value);
         });
 };
 
@@ -380,7 +376,8 @@ MBTiles.prototype._metadata = function(key, callback) {
 // performing fallback queries if certain keys (like `bounds`, `minzoom`,
 // `maxzoom`) have not been provided.
 MBTiles.prototype.getInfo = function(callback) {
-    if (typeof callback !== 'function') callback = noop;
+    if (typeof callback !== 'function') throw new Error('Callback needed');
+    if (!this.open) return callback(new Error('MBTiles not yet loaded'));
 
     var that = this;
     var info = {};
@@ -481,18 +478,98 @@ MBTiles.prototype.getInfo = function(callback) {
             (info.bounds[3] - info.bounds[1]) / 2 + info.bounds[1],
             (range <= 1) ? info.maxzoom : Math.floor(range * 0.5) + info.minzoom
         ];
-        callback(null, info);
+        return callback(null, info);
+    });
+};
+
+MBTiles.prototype.startWriting = function(callback) {
+    if (typeof callback !== 'function') throw new Error('Callback needed');
+    if (!this.open) return callback(new Error('MBTiles not yet loaded'));
+
+    // Sets the synchronous flag to OFF for (much) faster inserts.
+    // See http://www.sqlite3.org/pragma.html#pragma_synchronous
+
+    var mbtiles = this;
+    if (!this._isWritable) {
+        this._isWritable = 1;
+        this._tileCache = [];
+        this._gridCache = [];
+        this._db.run('PRAGMA synchronous=OFF', callback);
+    } else {
+        this._isWritable++;
+        return callback(null);
+    }
+};
+
+MBTiles.prototype._commit = function(callback) {
+    var mbtiles = this;
+    this._db.serialize(function() {
+        mbtiles._db.run('BEGIN');
+
+        var tile_data = mbtiles._db.prepare('REPLACE INTO images (tile_id, tile_data) VALUES (?, ?)');
+        var tile_map = mbtiles._db.prepare('REPLACE INTO map (zoom_level, ' +
+            'tile_column, tile_row, tile_id, grid_id) VALUES (?, ?, ?, ?, ' +
+            '(SELECT grid_id FROM map WHERE zoom_level = ? ' +
+            'AND tile_column = ? AND tile_row = ?))');
+        mbtiles._tileCache.forEach(function(row) {
+            var id = crypto.createHash('md5').update(row[3]).digest('hex');
+            tile_data.run(id, row[3]);
+            tile_map.run(row[0], row[1], row[2], id, row[0], row[1], row[2]);
+        });
+        tile_data.finalize();
+        tile_map.finalize();
+
+        mbtiles._db.run('COMMIT', callback);
+    });
+};
+
+MBTiles.prototype.stopWriting = function(callback) {
+    if (typeof callback !== 'function') throw new Error('Callback needed');
+    if (!this.open) return callback(new Error('MBTiles not yet loaded'));
+
+    var mbtiles = this;
+    if (this._isWritable) this._isWritable--;
+    this._commit(function(err) {
+        if (err) return callback(err);
+        if (!mbtiles._isWritable) {
+            mbtiles._db.run('PRAGMA synchronous=NORMAL', callback);
+        } else {
+            return callback(null);
+        }
     });
 };
 
 // Insert a tile. Scheme is XYZ.
 MBTiles.prototype.putTile = function(z, x, y, data, callback) {
+    if (typeof callback !== 'function') throw new Error('Callback needed');
+    if (!this.open) return callback(new Error('MBTiles not yet loaded'));
+    if (!this._isWritable) return callback(new Error('MBTiles not in write mode'));
 
     // Flip Y coordinate because MBTiles files are TMS.
     y = Math.pow(2, z) - 1 - y;
 
-    this.run('INSERT INGORE INTO ')
+    this._tileCache.push([ z, x, y, data ]);
 
-    this._insertTiles([ { z: z, x: x, y: y, data: data }], callback);
+    if (this._tileCache.length + this._gridCache.length >= 100) {
+        this._commit(callback);
+    } else {
+        return callback(null);
+    }
 };
 
+// Insert a tile. Scheme is XYZ.
+MBTiles.prototype.putGrid = function(z, x, y, data, callback) {
+    if (typeof callback !== 'function') throw new Error('Callback needed');
+    if (!this.open) return callback(new Error('MBTiles not yet loaded'));
+
+    // Flip Y coordinate because MBTiles files are TMS.
+    y = Math.pow(2, z) - 1 - y;
+
+    this._tileCache.push([ z, x, y, data ]);
+
+    if (this._tileCache.length + this._gridCache.length >= 100) {
+        this._commit(callback);
+    } else {
+        return callback(null);
+    }
+};