node-mbtiles/lib/mbtiles.js

var _ = require('underscore'),
    fs = require('fs'),
    Step = require('step'),
    crypto = require('crypto'),
    zlib = require('zlib'),
    path = require('path'),
    url = require('url'),
    sm = new (require('sphericalmercator')),
    sqlite3 = require('sqlite3');

if (process.env.NODE_ENV === 'test') sqlite3.verbose();

function noop(err) {
    if (err) throw err;
}

// MBTiles
// -------
// MBTiles class for doing common operations (schema setup, tile reading,
// insertion, etc.)
module.exports = MBTiles;
MBTiles.utils = require('./utils');

// Provides access to an mbtiles database file.
// - uri: A parsed URL hash, the only relevant part is `pathname`.
// - callback: Will be called when the resources have been acquired
//       or acquisition failed.
function MBTiles(uri, callback) {
    var mbtiles = this;
    if (typeof callback !== 'function') callback = noop;
    if (typeof uri === 'string') uri = url.parse(uri);

    this.filename = uri.pathname;
    Step(function() {
        mbtiles._db = new sqlite3.cached.Database(mbtiles.filename, this)
    }, function(err) {
        if (err) return callback(err);
        mbtiles._setup(this);
    }, function(err) {
        if (err) return callback(err);
        fs.stat(mbtiles.filename, this);
    }, function(err, stat) {
        if (err) return callback(err);
        mbtiles._stat = stat;
        mbtiles.open = true;
        return callback(null, mbtiles);
    });

    return undefined;
};

// Finds all mbtiles file in the filepath and returns their tilesource URI.
MBTiles.list = function(filepath, callback) {
    filepath = path.resolve(filepath);
    fs.readdir(filepath, function(err, files) {
        if (err) return callback(err);
        for (var result = {}, i = 0; i < files.length; i++) {
            var name = files[i].match(/^([\w-]+)\.mbtiles$/);
            if (name) result[name[1]] = 'mbtiles://' + path.join(filepath, name[0]);
        }
        return callback(null, result);
    });
};

// Finds an mbtiles file with the given ID in the filepath and returns a
// tilesource URI.
MBTiles.findID = function(filepath, id, callback) {
    filepath = path.resolve(filepath);
    var file = path.join(filepath, id + '.mbtiles');
    fs.stat(file, function(err, stats) {
        if (err) return callback(err);
        else return callback(null, 'mbtiles://' + file);
    });
};

// Retrieve the schema of the current mbtiles database and inform the caller of
// whether the specified table exists.
MBTiles.prototype._exists = function(table, callback) {
    if (typeof callback !== 'function') callback = noop;

    if (this._schema) {
        return callback(null, _(this._schema).include(table));
    } else {
        this._schema = [];
        this._db.all(
            'SELECT name FROM sqlite_master WHERE type IN (?, ?)',
            'table',
            'view',
            function(err, rows) {
                if (err) return callback(err);
                this._schema = _(rows).pluck('name');
                this._exists(table, callback);
            }.bind(this)
        );
    }
};

// DB integrity check.
MBTiles.prototype._integrity = function(callback) {
    if (typeof callback !== 'function') callback = noop;

    this._db.get('PRAGMA quick_check(1)', function(err, row) {
        if (!(row && row.integrity_check && row.integrity_check === 'ok')) {
            return callback(new Error('Corrupted database.'));
        } else {
            return callback(null, true);
        }
    });
};

// Setup schema, indices, views for a new mbtiles database.
MBTiles.prototype._setup = function(callback) {
    var mbtiles = this;
    mbtiles._exists('tiles', function(err, exists) {
        if (exists) return callback(null);
        fs.readFile(__dirname + '/schema.sql', 'utf8', function(err, sql) {
            if (err) return callback(err);
            mbtiles._db.exec(sql, callback);
        });
    });
};

// // Generic object insert.
// //
// // - `table` String. The table to which objects should be inserted.
// // - `objects` Array. Objects to be inserted, where each object attribute
// //   has key/value pairs as a hash corresponding to column name and row value.
// // - `callback` Function.
// MBTiles.prototype._insert = function(table, objects, callback) {
//     if (typeof callback !== 'function') callback = noop;
//
//     if (!objects.length) return callback(null);
//     var keys = _(objects[0]).keys();
//     var placeholders = [];
//     _(keys).each(function(k) { placeholders.push('?'); });
//     var stmt = this._db.prepare(
//         'INSERT OR IGNORE INTO ' + table + ' ' +
//         '(' + keys.join(',') + ') ' +
//         'VALUES (' + placeholders.join(',') + ')'
//     );
//     for (var i = 0; i < objects.length; i++) {
//         stmt.run.apply(stmt, _(objects[i]).values());
//     }
//     stmt.finalize(callback);
// };
//
// // Insert metadata into the mbtiles database.
// //
// // - @param {Object} metadata key, value hash of metadata to be inserted.
// // - @param {Function} callback
// MBTiles.prototype._insertMetadata = function(data, callback) {
//     if (typeof callback !== 'function') callback = noop;
//
//     var metadata = _(data).map(function(value, key) {
//         return { name: key, value: value};
//     });
//     this._insert('metadata', metadata, callback);
// };
//
// // Insert a set of tiles into an mbtiles database. Scheme is TMS.
// //
// // - @param {Array} renders array of images to be inserted. Each item should
// //   be an object of the form { z: z, x: x, y: y, data: [Image buffer] }.
// // - @param {Function} callback
// MBTiles.prototype._insertTiles = function(data, callback) {
//     var that = this,
//         map = [],
//         images = [],
//         ids = [];
//     for (var i = 0; i < data.length; i++) {
//         var tile_id = crypto
//             .createHash('md5')
//             .update(data[i].data)
//             .digest('hex');
//         !_(ids).include(tile_id) && ids.push(tile_id) && images.push({
//             tile_id: tile_id,
//             tile_data: data[i].data
//         });
//         map.push({
//             tile_id: tile_id,
//             zoom_level: data[i].z,
//             tile_column: data[i].x,
//             tile_row: data[i].y
//         });
//     }
//     Step(
//         function() {
//             var group = this.group();
//             that._insert('images', images, group());
//             that._insert('map', map, group());
//         },
//         callback
//     );
// };
//
// // Insert a set of grids into an mbtiles database.
// //
// // - @param {Array} renders array of grids to be inserted. Each item should
// //   be an object of the form { z: z, x: x, y: y, data: [Image buffer], keys: [] }.
// // - @param {Function} callback
// MBTiles.prototype._insertGrids = function(data, callback) {
//     if (typeof callback !== 'function') callback = noop;
//
//     var that = this,
//         map = [],
//         grids = [],
//         grid_keys = [],
//         features = {},
//         ids = [];
//     for (var i = 0; i < data.length; i++) {
//         var json = JSON.stringify({
//             grid: data[i].grid,
//             keys: data[i].keys
//         });
//         var grid_id = crypto
//             .createHash('md5')
//             .update(json)
//             .digest('hex');
//         !_(ids).include(grid_id) && ids.push(grid_id) && grids.push({
//             grid_id: grid_id,
//             grid_utfgrid: zlib.deflate(new Buffer(json, 'utf8'))
//         });
//         data[i].keys.forEach(function(k) {
//             grid_keys.push({
//                 grid_id: grid_id,
//                 key_name: k
//             });
//         });
//         map.push({
//             grid_id: grid_id,
//             zoom_level: data[i].z,
//             tile_column: data[i].x,
//             tile_row: data[i].y
//         });
//         _(features).extend(data[i].data);
//     }
//     features = _(features).map(function(value, key) {
//         return { key_name: key, key_json: JSON.stringify(value) };
//     });
//     Step(
//         function() {
//             var group = this.group();
//             that._insert('grid_utfgrid', grids, group());
//             that._insert('grid_key', grid_keys, group());
//             that._insert('keymap', features, group());
//             that._insertGridTiles(map, group());
//         },
//         callback
//     );
// };
//
// // Insert grids into the mbtiles database.
// //
// // - @param {Object} tile tile object to be inserted.
// // - @param {Function} callback
// MBTiles.prototype._insertGridTiles = function(map, callback) {
//     if (typeof callback !== 'function') callback = noop;
//
//     var stmt = this._db.prepare('UPDATE OR REPLACE map SET grid_id = ? WHERE ' +
//         ' zoom_level = ? AND tile_column = ? AND tile_row = ?');
//
//     for (var i = 0; i < map.length; i++) {
//         stmt.run(
//             map[i].grid_id,
//             map[i].zoom_level,
//             map[i].tile_column,
//             map[i].tile_row
//         );
//     }
//
//     stmt.finalize(callback);
// };

// Select a tile from an mbtiles database. Scheme is XYZ.
//
// - @param {Number} x tile x coordinate.
// - @param {Number} y tile y coordinate.
// - @param {Number} z tile z coordinate.
// - @param {Function} callback
MBTiles.prototype.getTile = function(z, x, y, callback) {
    if (typeof callback !== 'function') throw new Error('Callback needed');
    if (!this.open) return callback(new Error('MBTiles not yet loaded'));

    // Flip Y coordinate because MBTiles files are TMS.
    y = Math.pow(2, z) - 1 - y;

    var mbtiles = this;
    this._db.get('SELECT tile_data FROM tiles WHERE ' +
        'zoom_level = ? AND tile_column = ? AND tile_row = ?',
        z, x, y,
        function(err, row) {
            if (!row || (err && err.errno == 1)) {
                return callback(new Error('Tile does not exist'));
            } else if (err) {
                return callback(err);
            } else {
                var options = {
                    'Content-Type': MBTiles.utils.getMimeType(row.tile_data),
                    'Last-Modified': mbtiles._stat.mtime,
                    'ETag': mbtiles._stat.size + '-' + Number(mbtiles._stat.mtime)
                };
                return callback(null, row.tile_data, options);
            }
        });
};

// Select a grid and its data from an mbtiles database. Scheme is XYZ.
//
// - @param {Number} x tile x coordinate
// - @param {Number} y tile y coordinate
// - @param {Number} z tile z coordinate
// - @param {Function} callback
MBTiles.prototype.getGrid = function(z, x, y, callback) {
    if (typeof callback !== 'function') throw new Error('Callback needed');
    if (!this.open) return callback(new Error('MBTiles not yet loaded'));

    // Flip Y coordinate because MBTiles files are TMS.
    y = Math.pow(2, z) - 1 - y;

    var that = this;
    Step(
        function() {
            that._db.get('SELECT grid FROM grids WHERE ' +
                'zoom_level = ? AND tile_column = ? AND tile_row = ?',
                z, x, y,
                this.parallel()
            );
            that._db.all('SELECT key_name, key_json FROM grid_data WHERE ' +
                'zoom_level = ? AND tile_column = ? AND tile_row = ?',
                z, x, y,
                this.parallel()
            );
        },
        function(err, row, rows) {
            if ((!row || !row.grid) || (err && err.errno == 1)) {
                return callback(new Error('Grid does not exist'));
            }
            if (err) return callback(err);

            try {
                var grid = zlib.inflate(
                           !Buffer.isBuffer(row.grid)
                           ? new Buffer(row.grid, 'binary')
                           : row.grid
                       ).toString();
                var data = rows.reduce(function(memo, r) {
                        memo[r.key_name] = JSON.parse(r.key_json);
                        return memo;
                    }, {});
                var result = _(JSON.parse(grid)).extend({ data: data });
            } catch (err) {
                return callback(new Error('Grid is invalid'));
            }

            callback(null, result);
        }
    );
};

// Select a metadata value from the database.
//
// - @param {Function} callback
MBTiles.prototype._metadata = function(key, callback) {
    if (typeof callback !== 'function') callback = noop;

    this._db.get('SELECT value FROM metadata WHERE name = ?',
        key,
        function(err, row) {
            if (!row || (err && err.errno == 1)) return callback(new Error('Key does not exist'));
            else if (err) return callback(err);
            else return callback(null, row.value);
        });
};

// Extend `MBTiles` class with an `info` method for retrieving metadata and
// performing fallback queries if certain keys (like `bounds`, `minzoom`,
// `maxzoom`) have not been provided.
MBTiles.prototype.getInfo = function(callback) {
    if (typeof callback !== 'function') throw new Error('Callback needed');
    if (!this.open) return callback(new Error('MBTiles not yet loaded'));

    var that = this;
    var info = {};
    info.filesize = this._stat.size;
    info.scheme = 'tms';
    info.basename = path.basename(that.filename);
    info.id = path.basename(that.filename, path.extname(that.filename));
    Step(function() {
        var end = this;
        that._db.all('SELECT name, value FROM metadata', function(err, rows) {
            if (rows) for (var i = 0; i < rows.length; i++) {
                info[rows[i].name] = rows[i].value;
            }
            end(err);
        });
    },
    // Determine min/max zoom if needed
    function(err) {
        if (err && err.errno !== 1) return callback(err);
        if (info.maxzoom !== undefined
            && info.minzoom !== undefined) return this();

        var step = this;
        var zoomquery = that._db.prepare('SELECT zoom_level FROM tiles ' +
                                        'WHERE zoom_level = ? LIMIT 1', function(err) {
            if (err) {
                if (err.errno === 1) step();
                else throw new Error(err);
            } else {
                var group = step.group();
                for (var i = 0; i < 30; i++) {
                    zoomquery.get(i, group());
                }
                zoomquery.finalize();
            }
        });
    },
    function(err, rows) {
        if (err) return callback(err);
        if (rows) {
            var zooms = _(rows).chain()
                .reject(_.isUndefined)
                .pluck('zoom_level')
                .value();
            info.minzoom = zooms.shift();
            info.maxzoom = zooms.length ? zooms.pop() : info.minzoom;
        }
        this();
    },
    // Determine bounds if needed
    function(err) {
        if (err) return callback(err);
        if (info.bounds) return this();
        if (typeof info.minzoom === 'undefined') return this();

        var next = this;
        Step(
            function() {
                that._db.get(
                    'SELECT MAX(tile_column) AS maxx, ' +
                    'MIN(tile_column) AS minx, MAX(tile_row) AS maxy, ' +
                    'MIN(tile_row) AS miny FROM tiles ' +
                    'WHERE zoom_level = ?',
                    info.minzoom,
                    this
                );
            },
            function(err, row) {
                if (!err && row) {
                    // @TODO this breaks a little at zoom level zero
                    var urTile = sm.bbox(row.maxx, row.maxy, info.minzoom, true);
                    var llTile = sm.bbox(row.minx, row.miny, info.minzoom, true);
                    // @TODO bounds are limited to "sensible" values here
                    // as sometimes tilesets are rendered with "negative"
                    // and/or other extremity tiles. Revisit this if there
                    // are actual use cases for out-of-bounds bounds.
                    info.bounds = [
                        llTile[0] > -180 ? llTile[0] : -180,
                        llTile[1] > -90 ? llTile[1] : -90,
                        urTile[2] < 180 ? urTile[2] : 180,
                        urTile[3] < 90 ? urTile[3] : 90
                    ].join(',');
                }
                next();
            }
        );
    },
    // Return info
    function(err) {
        if (err) return callback(err);
        var range = parseInt(info.maxzoom, 10) - parseInt(info.minzoom, 10);
        info.minzoom = parseInt(info.minzoom, 10);
        info.maxzoom = parseInt(info.maxzoom, 10);
        info.bounds = _((info.bounds || '').split(',')).map(parseFloat);
        if (info.center) info.center = _((info.center).split(',')).map(parseFloat);
        if (!info.center || info.center.length !== 3) info.center = [
            (info.bounds[2] - info.bounds[0]) / 2 + info.bounds[0],
            (info.bounds[3] - info.bounds[1]) / 2 + info.bounds[1],
            (range <= 1) ? info.maxzoom : Math.floor(range * 0.5) + info.minzoom
        ];
        return callback(null, info);
    });
};

MBTiles.prototype.startWriting = function(callback) {
    if (typeof callback !== 'function') throw new Error('Callback needed');
    if (!this.open) return callback(new Error('MBTiles not yet loaded'));

    // Sets the synchronous flag to OFF for (much) faster inserts.
    // See http://www.sqlite3.org/pragma.html#pragma_synchronous

    var mbtiles = this;
    if (!this._isWritable) {
        this._isWritable = 1;
        this._tileCache = [];
        this._gridCache = [];
        this._db.run('PRAGMA synchronous=OFF', callback);
    } else {
        this._isWritable++;
        return callback(null);
    }
};

MBTiles.prototype._commit = function(callback) {
    var mbtiles = this;
    this._db.serialize(function() {
        mbtiles._db.run('BEGIN');

        var tile_data = mbtiles._db.prepare('REPLACE INTO images (tile_id, tile_data) VALUES (?, ?)');
        var tile_map = mbtiles._db.prepare('REPLACE INTO map (zoom_level, ' +
            'tile_column, tile_row, tile_id, grid_id) VALUES (?, ?, ?, ?, ' +
            '(SELECT grid_id FROM map WHERE zoom_level = ? ' +
            'AND tile_column = ? AND tile_row = ?))');
        mbtiles._tileCache.forEach(function(row) {
            var id = crypto.createHash('md5').update(row[3]).digest('hex');
            tile_data.run(id, row[3]);
            tile_map.run(row[0], row[1], row[2], id, row[0], row[1], row[2]);
        });
        tile_data.finalize();
        tile_map.finalize();

        mbtiles._db.run('COMMIT', callback);
    });
};

MBTiles.prototype.stopWriting = function(callback) {
    if (typeof callback !== 'function') throw new Error('Callback needed');
    if (!this.open) return callback(new Error('MBTiles not yet loaded'));

    var mbtiles = this;
    if (this._isWritable) this._isWritable--;
    this._commit(function(err) {
        if (err) return callback(err);
        if (!mbtiles._isWritable) {
            mbtiles._db.run('PRAGMA synchronous=NORMAL', callback);
        } else {
            return callback(null);
        }
    });
};

// Insert a tile. Scheme is XYZ.
MBTiles.prototype.putTile = function(z, x, y, data, callback) {
    if (typeof callback !== 'function') throw new Error('Callback needed');
    if (!this.open) return callback(new Error('MBTiles not yet loaded'));
    if (!this._isWritable) return callback(new Error('MBTiles not in write mode'));

    // Flip Y coordinate because MBTiles files are TMS.
    y = Math.pow(2, z) - 1 - y;

    this._tileCache.push([ z, x, y, data ]);

    if (this._tileCache.length + this._gridCache.length >= 100) {
        this._commit(callback);
    } else {
        return callback(null);
    }
};

// Insert a tile. Scheme is XYZ.
MBTiles.prototype.putGrid = function(z, x, y, data, callback) {
    if (typeof callback !== 'function') throw new Error('Callback needed');
    if (!this.open) return callback(new Error('MBTiles not yet loaded'));

    // Flip Y coordinate because MBTiles files are TMS.
    y = Math.pow(2, z) - 1 - y;

    this._tileCache.push([ z, x, y, data ]);

    if (this._tileCache.length + this._gridCache.length >= 100) {
        this._commit(callback);
    } else {
        return callback(null);
    }
};