ashatora/node-mbtiles
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove Step, underscore dependencies from main module.

Browse Source
This commit is contained in:
Young Hahn
2013-07-12 05:25:03 -04:00
parent f0714fe167
commit 906d6a444d
7 changed files with 185 additions and 272 deletions
Download Patch File Download Diff File Expand all files Collapse all files

357
lib/mbtiles.js
View File

@@ -1,16 +1,12 @@
var _ = require('underscore'), var fs = require('fs');
fs = require('fs'), var crypto = require('crypto');
Step = require('step'), var zlib = require('zlib');
crypto = require('crypto'), var path = require('path');
zlib = require('zlib'), var url = require('url');
path = require('path'), var qs = require('querystring');
url = require('url'), var Buffer = require('buffer').Buffer;
qs = require('querystring'), var sm = new (require('sphericalmercator'));
Buffer = require('buffer').Buffer, var sqlite3 = require('sqlite3');
sm = new (require('sphericalmercator')),
sqlite3 = require('sqlite3');
if (process.env.NODE_ENV === 'test') sqlite3.verbose();
function noop(err) { function noop(err) {
if (err) throw err; if (err) throw err;
@@ -53,17 +49,15 @@ function MBTiles(uri, callback) {
this.setMaxListeners(0); this.setMaxListeners(0);
this.filename = uri.pathname; this.filename = uri.pathname;
this._batchSize = +uri.query.batch; this._batchSize = +uri.query.batch;
Step(function() { mbtiles._db = new sqlite3.Database(mbtiles.filename, function(err) {
mbtiles._db = new sqlite3.Database(mbtiles.filename, this); if (err) return callback(err);
}, function(err) { fs.stat(mbtiles.filename, function(err, stat) {
if (err) throw err;
fs.stat(mbtiles.filename, this);
}, function(err, stat) {
if (err) return callback(err); if (err) return callback(err);
mbtiles._stat = stat; mbtiles._stat = stat;
mbtiles.open = true; mbtiles.open = true;
callback(null, mbtiles); callback(null, mbtiles);
}); });
});
return undefined; return undefined;
}; };
@@ -103,7 +97,7 @@ MBTiles.prototype._exists = function(table, callback) {
if (typeof callback !== 'function') callback = noop; if (typeof callback !== 'function') callback = noop;
if (this._schema) { if (this._schema) {
return callback(null, _(this._schema).include(table)); return callback(null, this._schema.indexOf(table) !== -1);
} else { } else {
this._db.all( this._db.all(
'SELECT name FROM sqlite_master WHERE type IN (?, ?)', 'SELECT name FROM sqlite_master WHERE type IN (?, ?)',
@@ -111,7 +105,7 @@ MBTiles.prototype._exists = function(table, callback) {
'view', 'view',
function(err, rows) { function(err, rows) {
if (err) return callback(err); if (err) return callback(err);
this._schema = _(rows).pluck('name'); this._schema = rows.map(function(r) { return r.name });
this._exists(table, callback); this._exists(table, callback);
}.bind(this) }.bind(this)
); );
@@ -159,11 +153,9 @@ MBTiles.prototype.getTile = function(z, x, y, callback) {
// Flip Y coordinate because MBTiles files are TMS. // Flip Y coordinate because MBTiles files are TMS.
y = (1 << z) - 1 - y; y = (1 << z) - 1 - y;
var sql = 'SELECT tile_data FROM tiles WHERE zoom_level = ? AND tile_column = ? AND tile_row = ?';
var mbtiles = this; var mbtiles = this;
this._db.get('SELECT tile_data FROM tiles WHERE ' + this._db.get(sql, z, x, y, function(err, row) {
'zoom_level = ? AND tile_column = ? AND tile_row = ?',
z, x, y,
function(err, row) {
if ((!err && !row) || (err && err.errno == 1)) { if ((!err && !row) || (err && err.errno == 1)) {
return callback(new Error('Tile does not exist')); return callback(new Error('Tile does not exist'));
} else if (err) { } else if (err) {
@@ -192,146 +184,115 @@ MBTiles.prototype.getGrid = function(z, x, y, callback) {
// Flip Y coordinate because MBTiles files are TMS. // Flip Y coordinate because MBTiles files are TMS.
y = (1 << z) - 1 - y; y = (1 << z) - 1 - y;
var that = this; var sqlgrid = 'SELECT grid FROM grids WHERE zoom_level = ? AND tile_column = ? AND tile_row = ?';
Step( var sqljson = 'SELECT key_name, key_json FROM grid_data WHERE zoom_level = ? AND tile_column = ? AND tile_row = ?';
function() {
that._db.get('SELECT grid FROM grids WHERE ' +
'zoom_level = ? AND tile_column = ? AND tile_row = ?',
z, x, y,
this.parallel()
);
that._db.all('SELECT key_name, key_json FROM grid_data WHERE ' +
'zoom_level = ? AND tile_column = ? AND tile_row = ?',
z, x, y,
this.parallel()
);
},
function(err, row, rows) {
if ((!row || !row.grid) || (err && err.errno == 1)) {
return callback(new Error('Grid does not exist'));
}
if (err) return callback(err);
zlib.inflate(!Buffer.isBuffer(row.grid) var mbtiles = this;
? new Buffer(row.grid, 'binary') mbtiles._db.get(sqlgrid, z, x, y, function(err, row) {
: row.grid, if (err && err.errno !== 1) return callback(err);
function(err,buffer) { if (!row || !row.grid || err) return callback(new Error('Grid does not exist'));
zlib.inflate(!Buffer.isBuffer(row.grid) ? new Buffer(row.grid, 'binary') : row.grid, function(err, buffer) {
if (err) return callback(new Error('Grid is invalid:' + err.message)); if (err) return callback(new Error('Grid is invalid:' + err.message));
try { try { var grid = JSON.parse(buffer); }
var data = rows.reduce(function(memo, r) { catch(err) { return callback(new Error('Grid is invalid:' + err.message)) };
memo[r.key_name] = JSON.parse(r.key_json); mbtiles._db.all(sqljson, z, x, y, function(err, rows) {
return memo; if (err) return callback(err);
}, {}); grid.data = grid.data || {};
var result = JSON.parse(buffer.toString()); for (var i = 0; i < rows.length; i++) {
} catch(err) { try { grid.data[rows[i].key_name] = JSON.parse(rows[i].key_json); }
return callback(new Error('Grid is invalid:' + err.message)); catch(err) { return callback(new Error('Grid is invalid:' + err.message)) };
} }
var options = { callback(null, grid, {
'Content-Type': 'text/javascript', 'Content-Type': 'text/javascript',
'Last-Modified': new Date(that._stat.mtime).toUTCString(), 'Last-Modified': new Date(mbtiles._stat.mtime).toUTCString(),
'ETag': that._stat.size + '-' + Number(that._stat.mtime) 'ETag': mbtiles._stat.size + '-' + Number(mbtiles._stat.mtime)
}; });
callback(null, _(result).extend({data:data}), options); });
}); });
}
);
};
// Select a metadata value from the database.
//
// - @param {Function} callback
MBTiles.prototype._metadata = function(key, callback) {
if (typeof callback !== 'function') callback = noop;
this._db.get('SELECT value FROM metadata WHERE name = ?',
key,
function(err, row) {
// If the metadata table or the requested field is missing return
// null and allow the caller handle it.
if (!row || (err && err.errno == 1)) return callback(null, null);
else if (err) return callback(err);
else return callback(null, row.value);
}); });
};
MBTiles.prototype._close = function(callback) {
this._db.close(callback);
}; };
MBTiles.prototype.close = function(callback) { MBTiles.prototype.close = function(callback) {
this._close(callback); this._db.close(callback);
}; };
MBTiles.prototype._getInfo = function(callback) { // Obtain metadata from the database. Performing fallback queries if certain
// keys(like `bounds`, `minzoom`, `maxzoom`) have not been provided.
//
// - @param {Function(err, data)} callback
MBTiles.prototype.getInfo = function(callback) {
if (typeof callback !== 'function') throw new Error('Callback needed');
if (!this.open) return callback(new Error('MBTiles not yet loaded'));
if (this._info) return callback(null, this._info);
var mbtiles = this; var mbtiles = this;
var info = {}; var info = {};
info.scheme = 'tms'; info.scheme = 'tms';
info.basename = path.basename(mbtiles.filename); info.basename = path.basename(mbtiles.filename);
info.id = path.basename(mbtiles.filename, path.extname(mbtiles.filename)); info.id = path.basename(mbtiles.filename, path.extname(mbtiles.filename));
info.filesize = mbtiles._stat.size; info.filesize = mbtiles._stat.size;
Step(function() {
var end = this;
mbtiles._db.all('SELECT name, value FROM metadata', function(err, rows) { mbtiles._db.all('SELECT name, value FROM metadata', function(err, rows) {
if (rows) for (var i = 0; i < rows.length; i++) {
info[rows[i].name] = rows[i].value;
}
end(err);
});
},
// Determine min/max zoom if needed
function(err) {
if (err && err.errno !== 1) return callback(err); if (err && err.errno !== 1) return callback(err);
if (info.maxzoom !== undefined if (rows) rows.forEach(function(row) {
&& info.minzoom !== undefined) return this(); switch (row.name) {
case 'minzoom':
var step = this; case 'maxzoom':
var zoomquery = mbtiles._db.prepare('SELECT zoom_level FROM tiles ' + info[row.name] = parseInt(row.value, 10);
'WHERE zoom_level = ? LIMIT 1', function(err) { break;
if (err) { case 'center':
if (err.errno === 1) step(); case 'bounds':
else throw new Error(err); info[row.name] = row.value.split(',').map(parseFloat);
} else { break;
var group = step.group(); default:
for (var i = 0; i < 30; i++) { info[row.name] = row.value;
zoomquery.get(i, group()); break;
}
zoomquery.finalize();
} }
}); });
}, ensureZooms(info, function(err, info) {
function(err, rows) {
if (err) return callback(err); if (err) return callback(err);
if (rows) { ensureBounds(info, function(err, info) {
var zooms = _(rows).chain() if (err) return callback(err);
.reject(_.isUndefined) ensureCenter(info, function(err, info) {
.pluck('zoom_level') if (err) return callback(err);
.value(); mbtiles._info = info;
info.minzoom = zooms.shift(); return callback(null, info);
info.maxzoom = zooms.length ? zooms.pop() : info.minzoom; });
});
});
});
function ensureZooms(info, callback) {
if ('minzoom' in info && 'maxzoom' in info) return callback(null, info);
var remaining = 30;
var zooms = [];
var query = mbtiles._db.prepare('SELECT zoom_level FROM tiles WHERE zoom_level = ? LIMIT 1', function(err) {
if (err) return callback(err.errno === 1 ? null : err, info);
for (var i = 0; i < remaining; i++) query.get(i, function(err, row) {
if (err) return (remaining = 0) && callback(err);
if (row) zooms.push(row.zoom_level);
if (--remaining === 0) {
if (!zooms.length) return callback(null, info);
zooms.sort(function(a,b) { return a < b ? -1 : 1 });
info.minzoom = zooms[0];
info.maxzoom = zooms.pop();
return callback(null, info);
} }
this(); });
}, query.finalize();
// Determine bounds if needed });
function(err) { };
if (err) return callback(err); function ensureBounds(info, callback) {
if (info.bounds) return this(); if ('bounds' in info) return callback(null, info);
if (typeof info.minzoom === 'undefined') return this(); if (!('minzoom' in info)) return callback(null, info);
var next = this;
Step(
function() {
mbtiles._db.get( mbtiles._db.get(
'SELECT MAX(tile_column) AS maxx, ' + 'SELECT MAX(tile_column) AS maxx, ' +
'MIN(tile_column) AS minx, MAX(tile_row) AS maxy, ' + 'MIN(tile_column) AS minx, MAX(tile_row) AS maxy, ' +
'MIN(tile_row) AS miny FROM tiles ' + 'MIN(tile_row) AS miny FROM tiles ' +
'WHERE zoom_level = ?', 'WHERE zoom_level = ?',
info.minzoom, info.minzoom,
this
);
},
function(err, row) { function(err, row) {
if (!err && row) { if (err) return callback(err);
if (!row) return callback(null, info);
// @TODO this breaks a little at zoom level zero // @TODO this breaks a little at zoom level zero
var urTile = sm.bbox(row.maxx, row.maxy, info.minzoom, true); var urTile = sm.bbox(row.maxx, row.maxy, info.minzoom, true);
var llTile = sm.bbox(row.minx, row.miny, info.minzoom, true); var llTile = sm.bbox(row.minx, row.miny, info.minzoom, true);
@@ -345,50 +306,19 @@ MBTiles.prototype._getInfo = function(callback) {
urTile[2] < 180 ? urTile[2] : 180, urTile[2] < 180 ? urTile[2] : 180,
urTile[3] < 90 ? urTile[3] : 90 urTile[3] < 90 ? urTile[3] : 90
].join(','); ].join(',');
}
next();
}
);
},
// Return info
function(err) {
if (err) return callback(err);
var range = parseInt(info.maxzoom, 10) - parseInt(info.minzoom, 10);
info.minzoom = parseInt(info.minzoom, 10);
if (isNaN(info.minzoom) || typeof info.minzoom !== 'number') delete info.minzoom;
info.maxzoom = parseInt(info.maxzoom, 10);
if (isNaN(info.maxzoom) || typeof info.maxzoom !== 'number') delete info.maxzoom;
info.bounds = _((info.bounds || '').split(',')).map(parseFloat);
if (info.bounds.length !== 4 || info.bounds[0] === null) delete info.bounds;
if (info.center) info.center = _((info.center).split(',')).map(parseFloat);
if ((!info.center || info.center.length !== 3) && info.bounds) info.center = [
(info.bounds[2] - info.bounds[0]) / 2 + info.bounds[0],
(info.bounds[3] - info.bounds[1]) / 2 + info.bounds[1],
(range <= 1) ? info.maxzoom : Math.floor(range * 0.5) + info.minzoom
];
if (info.center && (info.center.length !== 3 || info.center[0] === null || isNaN(info.center[2]))) {
delete info.center;
}
return callback(null, info); return callback(null, info);
}); });
} };
function ensureCenter(info, callback) {
// Obtain metadata from the database. Performing fallback queries if certain if ('center' in info) return callback(null, info);
// keys(like `bounds`, `minzoom`, `maxzoom`) have not been provided. if (!('bounds' in info) || !('minzoom' in info) || !('maxzoom' in info)) return callback(null, info);
// info.center = [
// - @param {Function(err, data)} callback (info.bounds[2] - info.bounds[0]) / 2 + info.bounds[0],
MBTiles.prototype.getInfo = function(callback) { (info.bounds[3] - info.bounds[1]) / 2 + info.bounds[1],
if (typeof callback !== 'function') throw new Error('Callback needed'); Math.floor((info.maxzoom-info.minzoom)*0.5) + info.minzoom
if (!this.open) return callback(new Error('MBTiles not yet loaded')); ];
if (this._info) return callback(null, this._info); return callback(null, info);
var mbtiles = this; };
mbtiles._getInfo(function(err, info) {
mbtiles._info = info;
return callback(err, info);
});
}; };
// Puts the MBTiles tilestore into write mode. // Puts the MBTiles tilestore into write mode.
@@ -400,21 +330,14 @@ MBTiles.prototype.startWriting = function(callback) {
var mbtiles = this; var mbtiles = this;
mbtiles._clearCaches(); mbtiles._clearCaches();
Step(function() { mbtiles._setup(function(err) {
mbtiles._setup(this); if (err) return callback(err);
}, function(err) { if (mbtiles._isWritable) return callback();
if (err) throw err;
// Sets the synchronous flag to OFF for (much) faster inserts. // Sets the synchronous flag to OFF for (much) faster inserts.
// See http://www.sqlite3.org/pragma.html#pragma_synchronous // See http://www.sqlite3.org/pragma.html#pragma_synchronous
if (!mbtiles._isWritable) {
mbtiles._isWritable = 1; mbtiles._isWritable = 1;
mbtiles._db.run('PRAGMA synchronous=OFF', this); mbtiles._db.run('PRAGMA synchronous=OFF', callback);
} else {
mbtiles._isWritable++;
this();
}
}, function(err) {
return callback(err);
}); });
}; };
@@ -444,7 +367,6 @@ MBTiles.prototype._commit = function(callback) {
images.finalize(); images.finalize();
} }
if (Object.keys(mbtiles._gridCache)) { if (Object.keys(mbtiles._gridCache)) {
// Insert grid_utfgrid table. // Insert grid_utfgrid table.
var grids = mbtiles._db.prepare('REPLACE INTO grid_utfgrid (grid_id, grid_utfgrid) VALUES (?, ?)'); var grids = mbtiles._db.prepare('REPLACE INTO grid_utfgrid (grid_id, grid_utfgrid) VALUES (?, ?)');
@@ -454,7 +376,6 @@ MBTiles.prototype._commit = function(callback) {
grids.finalize(); grids.finalize();
} }
if (Object.keys(mbtiles._keyCache)) { if (Object.keys(mbtiles._keyCache)) {
// Insert grid_key. // Insert grid_key.
var keys = mbtiles._db.prepare('INSERT OR IGNORE INTO grid_key (grid_id, key_name) VALUES (?, ?)'); var keys = mbtiles._db.prepare('INSERT OR IGNORE INTO grid_key (grid_id, key_name) VALUES (?, ?)');
@@ -466,7 +387,6 @@ MBTiles.prototype._commit = function(callback) {
keys.finalize(); keys.finalize();
} }
if (Object.keys(mbtiles._dataCache)) { if (Object.keys(mbtiles._dataCache)) {
// Insert keymap table. // Insert keymap table.
var keymap = mbtiles._db.prepare('REPLACE INTO keymap (key_name, key_json) VALUES (?, ?)'); var keymap = mbtiles._db.prepare('REPLACE INTO keymap (key_name, key_json) VALUES (?, ?)');
@@ -520,14 +440,13 @@ MBTiles.prototype.stopWriting = function(callback) {
if (!this.open) return callback(new Error('MBTiles not yet loaded')); if (!this.open) return callback(new Error('MBTiles not yet loaded'));
var mbtiles = this; var mbtiles = this;
if (this._isWritable) this._isWritable--; mbtiles._commit(function(err) {
this._commit(function(err) {
if (err) return callback(err); if (err) return callback(err);
if (!mbtiles._isWritable) { mbtiles._db.run('PRAGMA synchronous=NORMAL', function(err) {
mbtiles._db.run('PRAGMA synchronous=NORMAL', callback); if (err) return callback(err);
} else { mbtiles._isWritable = false;
return callback(null); return callback();
} });
}); });
}; };
@@ -542,7 +461,6 @@ MBTiles.prototype.putTile = function(z, x, y, data, callback) {
if (typeof callback !== 'function') throw new Error('Callback needed'); if (typeof callback !== 'function') throw new Error('Callback needed');
if (!this.open) return callback(new Error('MBTiles not yet loaded')); if (!this.open) return callback(new Error('MBTiles not yet loaded'));
if (!this._isWritable) return callback(new Error('MBTiles not in write mode')); if (!this._isWritable) return callback(new Error('MBTiles not in write mode'));
if (!Buffer.isBuffer(data)) return callback(new Error('Image needs to be a Buffer')); if (!Buffer.isBuffer(data)) return callback(new Error('Image needs to be a Buffer'));
// Flip Y coordinate because MBTiles files are TMS. // Flip Y coordinate because MBTiles files are TMS.
@@ -597,24 +515,26 @@ MBTiles.prototype.putGrid = function(z, x, y, data, callback) {
this._mapCache[coords].grid_id = id; this._mapCache[coords].grid_id = id;
var mbtiles = this; var mbtiles = this;
Step(function() {
if (mbtiles._gridCache[id]) return this(); if (mbtiles._gridCache[id]) return callback(null);
else zlib.deflate(new Buffer(json, 'utf8'), this);
}, function(err, buffer) { zlib.deflate(new Buffer(json, 'utf8'), function(err, buffer) {
if (err) throw err; if (err) return callback(err);
if (mbtiles._gridCache[id]) return this();
// grid_utfgrid table. // grid_utfgrid table.
mbtiles._gridCache[id] = buffer; mbtiles._gridCache[id] = buffer;
// grid_key table. // grid_key table.
mbtiles._keyCache[id] = Object.keys(data.data || {}); mbtiles._keyCache[id] = Object.keys(data.data || {});
// keymap table. // keymap table.
_(mbtiles._dataCache).extend(data.data || {}); if (data.data) Object.keys(data.data).reduce(function(memo, key) {
this(); memo[key] = data.data[key];
}, function(err) { return memo;
if (err) return callback(err); }, mbtiles._dataCache);
// Only commit when we can insert at least batchSize rows. // Only commit when we can insert at least batchSize rows.
if (++this._pending >= this._batchSize) return this._commit(callback); if (++mbtiles._pending >= mbtiles._batchSize) {
else return callback(null); mbtiles._commit(callback);
} else {
callback(null);
}
}); });
}; };
@@ -623,23 +543,18 @@ MBTiles.prototype.putInfo = function(data, callback) {
if (!this.open) return callback(new Error('MBTiles not yet loaded')); if (!this.open) return callback(new Error('MBTiles not yet loaded'));
if (!this._isWritable) return callback(new Error('MBTiles not in write mode')); if (!this._isWritable) return callback(new Error('MBTiles not in write mode'));
// Valid keys.
var keys = ['name', 'type', 'description', 'version', 'formatter', 'template',
'bounds', 'center', 'minzoom', 'maxzoom', 'legend', 'attribution'];
var stmt = this._db.prepare('REPLACE INTO metadata (name, value) VALUES (?, ?)'); var stmt = this._db.prepare('REPLACE INTO metadata (name, value) VALUES (?, ?)');
stmt.on('error', callback); stmt.on('error', callback);
for (var key in data) { for (var key in data) {
if (keys.indexOf(key) !== -1) stmt.run(key, String(data[key])); stmt.run(key, String(data[key]));
} }
var mbtiles = this; var mbtiles = this;
stmt.finalize(function(err) { stmt.finalize(function(err) {
if (err) return callback(err); if (err) return callback(err);
mbtiles._getInfo(function(err, info) { delete mbtiles._info;
if (err) return callback(err); mbtiles.getInfo(function(err, info) {
mbtiles._info = info; return callback(err, null);
if (callback) callback(null);
}); });
}); });
}; };

2
package.json
View File

@@ -29,8 +29,6 @@
"Konstantin Käfer <kkaefer>" "Konstantin Käfer <kkaefer>"
], ],
"dependencies": { "dependencies": {
"underscore": "~1.3.3",
"step": "~0.0.5",
"optimist": "~0.3.1", "optimist": "~0.3.1",
"sqlite3": "~2.1.1", "sqlite3": "~2.1.1",
"sphericalmercator": "~1.0.1" "sphericalmercator": "~1.0.1"

2
test/info.test.js
View File

@@ -1,4 +1,4 @@
process.env.NODE_ENV = 'test'; require('sqlite3').verbose();
var fs = require('fs'); var fs = require('fs');
var MBTiles = require('..'); var MBTiles = require('..');

2
test/list.test.js
View File

@@ -1,4 +1,4 @@
process.env.NODE_ENV = 'test'; require('sqlite3').verbose();
var fs = require('fs'); var fs = require('fs');
var MBTiles = require('..'); var MBTiles = require('..');

2
test/read.test.js
View File

@@ -1,4 +1,4 @@
process.env.NODE_ENV = 'test'; require('sqlite3').verbose();
var fs = require('fs'); var fs = require('fs');
var MBTiles = require('..'); var MBTiles = require('..');

2
test/write.test.js
View File

@@ -1,4 +1,4 @@
process.env.NODE_ENV = 'test'; require('sqlite3').verbose();
var fs = require('fs'); var fs = require('fs');
var assert = require('assert'); var assert = require('assert');

2
test/write_grids.test.js
View File

@@ -1,4 +1,4 @@
process.env.NODE_ENV = 'test'; require('sqlite3').verbose();
var fs = require('fs'); var fs = require('fs');
var assert = require('assert'); var assert = require('assert');