ashatora/node-mbtiles
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove Step, underscore dependencies from main module.

Browse Source
This commit is contained in:
Young Hahn
2013-07-12 05:25:03 -04:00
parent f0714fe167
commit 906d6a444d
7 changed files with 185 additions and 272 deletions
Download Patch File Download Diff File Expand all files Collapse all files

445
lib/mbtiles.js
View File

@@ -1,16 +1,12 @@
var _ = require('underscore'),
fs = require('fs'),
Step = require('step'),
crypto = require('crypto'),
zlib = require('zlib'),
path = require('path'),
url = require('url'),
qs = require('querystring'),
Buffer = require('buffer').Buffer,
sm = new (require('sphericalmercator')),
sqlite3 = require('sqlite3');
if (process.env.NODE_ENV === 'test') sqlite3.verbose();
var fs = require('fs');
var crypto = require('crypto');
var zlib = require('zlib');
var path = require('path');
var url = require('url');
var qs = require('querystring');
var Buffer = require('buffer').Buffer;
var sm = new (require('sphericalmercator'));
var sqlite3 = require('sqlite3');
function noop(err) {
if (err) throw err;
@@ -53,16 +49,14 @@ function MBTiles(uri, callback) {
this.setMaxListeners(0);
this.filename = uri.pathname;
this._batchSize = +uri.query.batch;
Step(function() {
mbtiles._db = new sqlite3.Database(mbtiles.filename, this);
}, function(err) {
if (err) throw err;
fs.stat(mbtiles.filename, this);
}, function(err, stat) {
mbtiles._db = new sqlite3.Database(mbtiles.filename, function(err) {
if (err) return callback(err);
mbtiles._stat = stat;
mbtiles.open = true;
callback(null, mbtiles);
fs.stat(mbtiles.filename, function(err, stat) {
if (err) return callback(err);
mbtiles._stat = stat;
mbtiles.open = true;
callback(null, mbtiles);
});
});
return undefined;
@@ -103,7 +97,7 @@ MBTiles.prototype._exists = function(table, callback) {
if (typeof callback !== 'function') callback = noop;
if (this._schema) {
return callback(null, _(this._schema).include(table));
return callback(null, this._schema.indexOf(table) !== -1);
} else {
this._db.all(
'SELECT name FROM sqlite_master WHERE type IN (?, ?)',
@@ -111,7 +105,7 @@ MBTiles.prototype._exists = function(table, callback) {
'view',
function(err, rows) {
if (err) return callback(err);
this._schema = _(rows).pluck('name');
this._schema = rows.map(function(r) { return r.name });
this._exists(table, callback);
}.bind(this)
);
@@ -159,24 +153,22 @@ MBTiles.prototype.getTile = function(z, x, y, callback) {
// Flip Y coordinate because MBTiles files are TMS.
y = (1 << z) - 1 - y;
var sql = 'SELECT tile_data FROM tiles WHERE zoom_level = ? AND tile_column = ? AND tile_row = ?';
var mbtiles = this;
this._db.get('SELECT tile_data FROM tiles WHERE ' +
'zoom_level = ? AND tile_column = ? AND tile_row = ?',
z, x, y,
function(err, row) {
if ((!err && !row) || (err && err.errno == 1)) {
return callback(new Error('Tile does not exist'));
} else if (err) {
return callback(err);
} else {
var options = {
'Content-Type': MBTiles.utils.getMimeType(row.tile_data),
'Last-Modified': new Date(mbtiles._stat.mtime).toUTCString(),
'ETag': mbtiles._stat.size + '-' + Number(mbtiles._stat.mtime)
};
return callback(null, row.tile_data, options);
}
});
this._db.get(sql, z, x, y, function(err, row) {
if ((!err && !row) || (err && err.errno == 1)) {
return callback(new Error('Tile does not exist'));
} else if (err) {
return callback(err);
} else {
var options = {
'Content-Type': MBTiles.utils.getMimeType(row.tile_data),
'Last-Modified': new Date(mbtiles._stat.mtime).toUTCString(),
'ETag': mbtiles._stat.size + '-' + Number(mbtiles._stat.mtime)
};
return callback(null, row.tile_data, options);
}
});
};
// Select a grid and its data from an mbtiles database. Scheme is XYZ.
@@ -192,190 +184,38 @@ MBTiles.prototype.getGrid = function(z, x, y, callback) {
// Flip Y coordinate because MBTiles files are TMS.
y = (1 << z) - 1 - y;
var that = this;
Step(
function() {
that._db.get('SELECT grid FROM grids WHERE ' +
'zoom_level = ? AND tile_column = ? AND tile_row = ?',
z, x, y,
this.parallel()
);
that._db.all('SELECT key_name, key_json FROM grid_data WHERE ' +
'zoom_level = ? AND tile_column = ? AND tile_row = ?',
z, x, y,
this.parallel()
);
},
function(err, row, rows) {
if ((!row || !row.grid) || (err && err.errno == 1)) {
return callback(new Error('Grid does not exist'));
}
if (err) return callback(err);
var sqlgrid = 'SELECT grid FROM grids WHERE zoom_level = ? AND tile_column = ? AND tile_row = ?';
var sqljson = 'SELECT key_name, key_json FROM grid_data WHERE zoom_level = ? AND tile_column = ? AND tile_row = ?';
zlib.inflate(!Buffer.isBuffer(row.grid)
? new Buffer(row.grid, 'binary')
: row.grid,
function(err,buffer) {
if (err) return callback(new Error('Grid is invalid:' + err.message));
try {
var data = rows.reduce(function(memo, r) {
memo[r.key_name] = JSON.parse(r.key_json);
return memo;
}, {});
var result = JSON.parse(buffer.toString());
} catch(err) {
return callback(new Error('Grid is invalid:' + err.message));
}
var options = {
'Content-Type': 'text/javascript',
'Last-Modified': new Date(that._stat.mtime).toUTCString(),
'ETag': that._stat.size + '-' + Number(that._stat.mtime)
};
callback(null, _(result).extend({data:data}), options);
var mbtiles = this;
mbtiles._db.get(sqlgrid, z, x, y, function(err, row) {
if (err && err.errno !== 1) return callback(err);
if (!row || !row.grid || err) return callback(new Error('Grid does not exist'));
zlib.inflate(!Buffer.isBuffer(row.grid) ? new Buffer(row.grid, 'binary') : row.grid, function(err, buffer) {
if (err) return callback(new Error('Grid is invalid:' + err.message));
try { var grid = JSON.parse(buffer); }
catch(err) { return callback(new Error('Grid is invalid:' + err.message)) };
mbtiles._db.all(sqljson, z, x, y, function(err, rows) {
if (err) return callback(err);
grid.data = grid.data || {};
for (var i = 0; i < rows.length; i++) {
try { grid.data[rows[i].key_name] = JSON.parse(rows[i].key_json); }
catch(err) { return callback(new Error('Grid is invalid:' + err.message)) };
}
callback(null, grid, {
'Content-Type': 'text/javascript',
'Last-Modified': new Date(mbtiles._stat.mtime).toUTCString(),
'ETag': mbtiles._stat.size + '-' + Number(mbtiles._stat.mtime)
});
}
);
};
// Select a metadata value from the database.
//
// - @param {Function} callback
MBTiles.prototype._metadata = function(key, callback) {
if (typeof callback !== 'function') callback = noop;
this._db.get('SELECT value FROM metadata WHERE name = ?',
key,
function(err, row) {
// If the metadata table or the requested field is missing return
// null and allow the caller handle it.
if (!row || (err && err.errno == 1)) return callback(null, null);
else if (err) return callback(err);
else return callback(null, row.value);
});
});
};
MBTiles.prototype._close = function(callback) {
this._db.close(callback);
});
};
MBTiles.prototype.close = function(callback) {
this._close(callback);
this._db.close(callback);
};
MBTiles.prototype._getInfo = function(callback) {
var mbtiles = this;
var info = {};
info.scheme = 'tms';
info.basename = path.basename(mbtiles.filename);
info.id = path.basename(mbtiles.filename, path.extname(mbtiles.filename));
info.filesize = mbtiles._stat.size;
Step(function() {
var end = this;
mbtiles._db.all('SELECT name, value FROM metadata', function(err, rows) {
if (rows) for (var i = 0; i < rows.length; i++) {
info[rows[i].name] = rows[i].value;
}
end(err);
});
},
// Determine min/max zoom if needed
function(err) {
if (err && err.errno !== 1) return callback(err);
if (info.maxzoom !== undefined
&& info.minzoom !== undefined) return this();
var step = this;
var zoomquery = mbtiles._db.prepare('SELECT zoom_level FROM tiles ' +
'WHERE zoom_level = ? LIMIT 1', function(err) {
if (err) {
if (err.errno === 1) step();
else throw new Error(err);
} else {
var group = step.group();
for (var i = 0; i < 30; i++) {
zoomquery.get(i, group());
}
zoomquery.finalize();
}
});
},
function(err, rows) {
if (err) return callback(err);
if (rows) {
var zooms = _(rows).chain()
.reject(_.isUndefined)
.pluck('zoom_level')
.value();
info.minzoom = zooms.shift();
info.maxzoom = zooms.length ? zooms.pop() : info.minzoom;
}
this();
},
// Determine bounds if needed
function(err) {
if (err) return callback(err);
if (info.bounds) return this();
if (typeof info.minzoom === 'undefined') return this();
var next = this;
Step(
function() {
mbtiles._db.get(
'SELECT MAX(tile_column) AS maxx, ' +
'MIN(tile_column) AS minx, MAX(tile_row) AS maxy, ' +
'MIN(tile_row) AS miny FROM tiles ' +
'WHERE zoom_level = ?',
info.minzoom,
this
);
},
function(err, row) {
if (!err && row) {
// @TODO this breaks a little at zoom level zero
var urTile = sm.bbox(row.maxx, row.maxy, info.minzoom, true);
var llTile = sm.bbox(row.minx, row.miny, info.minzoom, true);
// @TODO bounds are limited to "sensible" values here
// as sometimes tilesets are rendered with "negative"
// and/or other extremity tiles. Revisit this if there
// are actual use cases for out-of-bounds bounds.
info.bounds = [
llTile[0] > -180 ? llTile[0] : -180,
llTile[1] > -90 ? llTile[1] : -90,
urTile[2] < 180 ? urTile[2] : 180,
urTile[3] < 90 ? urTile[3] : 90
].join(',');
}
next();
}
);
},
// Return info
function(err) {
if (err) return callback(err);
var range = parseInt(info.maxzoom, 10) - parseInt(info.minzoom, 10);
info.minzoom = parseInt(info.minzoom, 10);
if (isNaN(info.minzoom) || typeof info.minzoom !== 'number') delete info.minzoom;
info.maxzoom = parseInt(info.maxzoom, 10);
if (isNaN(info.maxzoom) || typeof info.maxzoom !== 'number') delete info.maxzoom;
info.bounds = _((info.bounds || '').split(',')).map(parseFloat);
if (info.bounds.length !== 4 || info.bounds[0] === null) delete info.bounds;
if (info.center) info.center = _((info.center).split(',')).map(parseFloat);
if ((!info.center || info.center.length !== 3) && info.bounds) info.center = [
(info.bounds[2] - info.bounds[0]) / 2 + info.bounds[0],
(info.bounds[3] - info.bounds[1]) / 2 + info.bounds[1],
(range <= 1) ? info.maxzoom : Math.floor(range * 0.5) + info.minzoom
];
if (info.center && (info.center.length !== 3 || info.center[0] === null || isNaN(info.center[2]))) {
delete info.center;
}
return callback(null, info);
});
}
// Obtain metadata from the database. Performing fallback queries if certain
// keys(like `bounds`, `minzoom`, `maxzoom`) have not been provided.
//
@@ -384,11 +224,101 @@ MBTiles.prototype.getInfo = function(callback) {
if (typeof callback !== 'function') throw new Error('Callback needed');
if (!this.open) return callback(new Error('MBTiles not yet loaded'));
if (this._info) return callback(null, this._info);
var mbtiles = this;
mbtiles._getInfo(function(err, info) {
mbtiles._info = info;
return callback(err, info);
var info = {};
info.scheme = 'tms';
info.basename = path.basename(mbtiles.filename);
info.id = path.basename(mbtiles.filename, path.extname(mbtiles.filename));
info.filesize = mbtiles._stat.size;
mbtiles._db.all('SELECT name, value FROM metadata', function(err, rows) {
if (err && err.errno !== 1) return callback(err);
if (rows) rows.forEach(function(row) {
switch (row.name) {
case 'minzoom':
case 'maxzoom':
info[row.name] = parseInt(row.value, 10);
break;
case 'center':
case 'bounds':
info[row.name] = row.value.split(',').map(parseFloat);
break;
default:
info[row.name] = row.value;
break;
}
});
ensureZooms(info, function(err, info) {
if (err) return callback(err);
ensureBounds(info, function(err, info) {
if (err) return callback(err);
ensureCenter(info, function(err, info) {
if (err) return callback(err);
mbtiles._info = info;
return callback(null, info);
});
});
});
});
function ensureZooms(info, callback) {
if ('minzoom' in info && 'maxzoom' in info) return callback(null, info);
var remaining = 30;
var zooms = [];
var query = mbtiles._db.prepare('SELECT zoom_level FROM tiles WHERE zoom_level = ? LIMIT 1', function(err) {
if (err) return callback(err.errno === 1 ? null : err, info);
for (var i = 0; i < remaining; i++) query.get(i, function(err, row) {
if (err) return (remaining = 0) && callback(err);
if (row) zooms.push(row.zoom_level);
if (--remaining === 0) {
if (!zooms.length) return callback(null, info);
zooms.sort(function(a,b) { return a < b ? -1 : 1 });
info.minzoom = zooms[0];
info.maxzoom = zooms.pop();
return callback(null, info);
}
});
query.finalize();
});
};
function ensureBounds(info, callback) {
if ('bounds' in info) return callback(null, info);
if (!('minzoom' in info)) return callback(null, info);
mbtiles._db.get(
'SELECT MAX(tile_column) AS maxx, ' +
'MIN(tile_column) AS minx, MAX(tile_row) AS maxy, ' +
'MIN(tile_row) AS miny FROM tiles ' +
'WHERE zoom_level = ?',
info.minzoom,
function(err, row) {
if (err) return callback(err);
if (!row) return callback(null, info);
// @TODO this breaks a little at zoom level zero
var urTile = sm.bbox(row.maxx, row.maxy, info.minzoom, true);
var llTile = sm.bbox(row.minx, row.miny, info.minzoom, true);
// @TODO bounds are limited to "sensible" values here
// as sometimes tilesets are rendered with "negative"
// and/or other extremity tiles. Revisit this if there
// are actual use cases for out-of-bounds bounds.
info.bounds = [
llTile[0] > -180 ? llTile[0] : -180,
llTile[1] > -90 ? llTile[1] : -90,
urTile[2] < 180 ? urTile[2] : 180,
urTile[3] < 90 ? urTile[3] : 90
].join(',');
return callback(null, info);
});
};
function ensureCenter(info, callback) {
if ('center' in info) return callback(null, info);
if (!('bounds' in info) || !('minzoom' in info) || !('maxzoom' in info)) return callback(null, info);
info.center = [
(info.bounds[2] - info.bounds[0]) / 2 + info.bounds[0],
(info.bounds[3] - info.bounds[1]) / 2 + info.bounds[1],
Math.floor((info.maxzoom-info.minzoom)*0.5) + info.minzoom
];
return callback(null, info);
};
};
// Puts the MBTiles tilestore into write mode.
@@ -400,21 +330,14 @@ MBTiles.prototype.startWriting = function(callback) {
var mbtiles = this;
mbtiles._clearCaches();
Step(function() {
mbtiles._setup(this);
}, function(err) {
if (err) throw err;
mbtiles._setup(function(err) {
if (err) return callback(err);
if (mbtiles._isWritable) return callback();
// Sets the synchronous flag to OFF for (much) faster inserts.
// See http://www.sqlite3.org/pragma.html#pragma_synchronous
if (!mbtiles._isWritable) {
mbtiles._isWritable = 1;
mbtiles._db.run('PRAGMA synchronous=OFF', this);
} else {
mbtiles._isWritable++;
this();
}
}, function(err) {
return callback(err);
mbtiles._isWritable = 1;
mbtiles._db.run('PRAGMA synchronous=OFF', callback);
});
};
@@ -444,7 +367,6 @@ MBTiles.prototype._commit = function(callback) {
images.finalize();
}
if (Object.keys(mbtiles._gridCache)) {
// Insert grid_utfgrid table.
var grids = mbtiles._db.prepare('REPLACE INTO grid_utfgrid (grid_id, grid_utfgrid) VALUES (?, ?)');
@@ -454,7 +376,6 @@ MBTiles.prototype._commit = function(callback) {
grids.finalize();
}
if (Object.keys(mbtiles._keyCache)) {
// Insert grid_key.
var keys = mbtiles._db.prepare('INSERT OR IGNORE INTO grid_key (grid_id, key_name) VALUES (?, ?)');
@@ -466,7 +387,6 @@ MBTiles.prototype._commit = function(callback) {
keys.finalize();
}
if (Object.keys(mbtiles._dataCache)) {
// Insert keymap table.
var keymap = mbtiles._db.prepare('REPLACE INTO keymap (key_name, key_json) VALUES (?, ?)');
@@ -520,14 +440,13 @@ MBTiles.prototype.stopWriting = function(callback) {
if (!this.open) return callback(new Error('MBTiles not yet loaded'));
var mbtiles = this;
if (this._isWritable) this._isWritable--;
this._commit(function(err) {
mbtiles._commit(function(err) {
if (err) return callback(err);
if (!mbtiles._isWritable) {
mbtiles._db.run('PRAGMA synchronous=NORMAL', callback);
} else {
return callback(null);
}
mbtiles._db.run('PRAGMA synchronous=NORMAL', function(err) {
if (err) return callback(err);
mbtiles._isWritable = false;
return callback();
});
});
};
@@ -542,7 +461,6 @@ MBTiles.prototype.putTile = function(z, x, y, data, callback) {
if (typeof callback !== 'function') throw new Error('Callback needed');
if (!this.open) return callback(new Error('MBTiles not yet loaded'));
if (!this._isWritable) return callback(new Error('MBTiles not in write mode'));
if (!Buffer.isBuffer(data)) return callback(new Error('Image needs to be a Buffer'));
// Flip Y coordinate because MBTiles files are TMS.
@@ -597,24 +515,26 @@ MBTiles.prototype.putGrid = function(z, x, y, data, callback) {
this._mapCache[coords].grid_id = id;
var mbtiles = this;
Step(function() {
if (mbtiles._gridCache[id]) return this();
else zlib.deflate(new Buffer(json, 'utf8'), this);
}, function(err, buffer) {
if (err) throw err;
if (mbtiles._gridCache[id]) return this();
if (mbtiles._gridCache[id]) return callback(null);
zlib.deflate(new Buffer(json, 'utf8'), function(err, buffer) {
if (err) return callback(err);
// grid_utfgrid table.
mbtiles._gridCache[id] = buffer;
// grid_key table.
mbtiles._keyCache[id] = Object.keys(data.data || {});
// keymap table.
_(mbtiles._dataCache).extend(data.data || {});
this();
}, function(err) {
if (err) return callback(err);
if (data.data) Object.keys(data.data).reduce(function(memo, key) {
memo[key] = data.data[key];
return memo;
}, mbtiles._dataCache);
// Only commit when we can insert at least batchSize rows.
if (++this._pending >= this._batchSize) return this._commit(callback);
else return callback(null);
if (++mbtiles._pending >= mbtiles._batchSize) {
mbtiles._commit(callback);
} else {
callback(null);
}
});
};
@@ -623,23 +543,18 @@ MBTiles.prototype.putInfo = function(data, callback) {
if (!this.open) return callback(new Error('MBTiles not yet loaded'));
if (!this._isWritable) return callback(new Error('MBTiles not in write mode'));
// Valid keys.
var keys = ['name', 'type', 'description', 'version', 'formatter', 'template',
'bounds', 'center', 'minzoom', 'maxzoom', 'legend', 'attribution'];
var stmt = this._db.prepare('REPLACE INTO metadata (name, value) VALUES (?, ?)');
stmt.on('error', callback);
for (var key in data) {
if (keys.indexOf(key) !== -1) stmt.run(key, String(data[key]));
stmt.run(key, String(data[key]));
}
var mbtiles = this;
stmt.finalize(function(err) {
if (err) return callback(err);
mbtiles._getInfo(function(err, info) {
if (err) return callback(err);
mbtiles._info = info;
if (callback) callback(null);
delete mbtiles._info;
mbtiles.getInfo(function(err, info) {
return callback(err, null);
});
});
};