ashatora/node-mbtiles
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor writing into external write API.

Browse Source
This commit is contained in:
Young Hahn
2013-07-18 09:09:14 -04:00
parent c8c1a5cb72
commit a0f5c5ee6b
1 changed files with 92 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

195
lib/mbtiles.js
View File

@@ -350,11 +350,23 @@ MBTiles.prototype.startWriting = function(callback) {
MBTiles.prototype._clearCaches = function() {
this._pending = 0;
this._tileCache = {};
this._gridCache = {};
this._keyCache = {};
this._dataCache = {};
this._mapCache = {};
this._writes = {};
};
// Queue a row to be written to a table.
MBTiles.prototype.write = function(table, id, row, callback) {
callback = callback || function() {};
this._writes = this._writes || {};
this._writes[table] = this._writes[table] || {};
this._writes[table][id] = this._writes[table][id] || {};
// Merge row data.
for (var key in row) this._writes[table][id][key] = row[key];
return ++this._pending >= this._batchSize
? this._commit(callback)
: callback();
};
// (private) Commits the cached changes to the database.
@@ -365,74 +377,51 @@ MBTiles.prototype._commit = function(callback) {
mbtiles._db.serialize(function() {
mbtiles._db.run('BEGIN');
if (Object.keys(mbtiles._tileCache)) {
// Insert images table.
var images = mbtiles._db.prepare('REPLACE INTO images (tile_id, tile_data) VALUES (?, ?)');
for (var id in mbtiles._tileCache) {
images.run(id, mbtiles._tileCache[id]);
var statements = {};
Object.keys(mbtiles._writes).forEach(function(table) {
switch (table) {
case 'map':
// Insert map table. This has to be so complicate due to a design flaw
// in the tables.
// TODO: This should be remedied when we upgrade the MBTiles schema.
var sql = '\
REPLACE INTO map (zoom_level, tile_column, tile_row, tile_id, grid_id)\
VALUES (?, ?, ?,\
COALESCE(?, (SELECT tile_id FROM map WHERE zoom_level = ? AND tile_column = ? AND tile_row = ?)),\
COALESCE(?, (SELECT grid_id FROM map WHERE zoom_level = ? AND tile_column = ? AND tile_row = ?)))';
statements['map'] = mbtiles._db.prepare(sql);
for (var id in mbtiles._writes[table]) {
var row = mbtiles._writes[table][id];
statements['map'].run(
row.zoom_level, row.tile_column, row.tile_row,
row.tile_id, row.zoom_level, row.tile_column, row.tile_row,
row.grid_id, row.zoom_level, row.tile_column, row.tile_row);
}
statements['map'].finalize();
break;
default:
var rows = [];
var args = [];
var fields = [];
for (var id in mbtiles._writes[table]) {
var record = mbtiles._writes[table][id];
var row = [];
for (var field in record) {
row.push(record[field]);
if (fields.indexOf(field) === -1) {
fields.push(field);
args.push('?');
}
}
rows.push(row);
}
var sql = 'REPLACE INTO ' + table + ' ( ' + fields.join(',') + ' ) VALUES(' + args.join(',') + ')';
statements[table] = mbtiles._db.prepare(sql);
while (rows.length) statements[table].run.apply(statements[table], rows.shift());
statements[table].finalize();
break;
}
images.finalize();
}
if (Object.keys(mbtiles._gridCache)) {
// Insert grid_utfgrid table.
var grids = mbtiles._db.prepare('REPLACE INTO grid_utfgrid (grid_id, grid_utfgrid) VALUES (?, ?)');
for (var id in mbtiles._gridCache) {
grids.run(id, mbtiles._gridCache[id]);
}
grids.finalize();
}
if (Object.keys(mbtiles._keyCache)) {
// Insert grid_key.
var keys = mbtiles._db.prepare('INSERT OR IGNORE INTO grid_key (grid_id, key_name) VALUES (?, ?)');
for (var id in mbtiles._keyCache) {
mbtiles._keyCache[id].forEach(function(key) {
keys.run(id, key);
});
}
keys.finalize();
}
if (Object.keys(mbtiles._dataCache)) {
// Insert keymap table.
var keymap = mbtiles._db.prepare('REPLACE INTO keymap (key_name, key_json) VALUES (?, ?)');
for (var key in mbtiles._dataCache) {
keymap.run(key, JSON.stringify(mbtiles._dataCache[key]));
}
keymap.finalize();
}
// Insert map table. This has to be so complicate due to a design flaw
// in the tables.
// TODO: This should be remedied when we upgrade the MBTiles schema.
var mapBoth = mbtiles._db.prepare('REPLACE INTO map (zoom_level, ' +
'tile_column, tile_row, tile_id, grid_id) VALUES (?, ?, ?, ?, ?)');
var mapTile = mbtiles._db.prepare('REPLACE INTO map (zoom_level, ' +
'tile_column, tile_row, tile_id, grid_id) VALUES (?, ?, ?, ?, ' +
'(SELECT grid_id FROM map WHERE zoom_level = ? ' +
'AND tile_column = ? AND tile_row = ?))');
var mapGrid = mbtiles._db.prepare('REPLACE INTO map (zoom_level, ' +
'tile_column, tile_row, tile_id, grid_id) VALUES (?, ?, ?, ' +
'(SELECT tile_id FROM map WHERE zoom_level = ? ' +
'AND tile_column = ? AND tile_row = ?), ?)');
for (var coords in mbtiles._mapCache) {
var map = mbtiles._mapCache[coords];
if (typeof map.grid_id === 'undefined') {
// Only the tile_id is defined.
mapTile.run(map.z, map.x, map.y, map.tile_id, map.z, map.x, map.y);
} else if (typeof map.tile_id === 'undefined') {
// Only the grid_id is defined.
mapGrid.run(map.z, map.x, map.y, map.z, map.x, map.y, map.grid_id);
} else {
// Both tile_id and grid_id are defined.
mapBoth.run(map.z, map.x, map.y, map.tile_id, map.grid_id);
}
}
mapBoth.finalize();
mapTile.finalize();
mapGrid.finalize();
});
mbtiles._db.run('COMMIT', callback);
mbtiles._clearCaches();
@@ -479,17 +468,18 @@ MBTiles.prototype.putTile = function(z, x, y, data, callback) {
? String(data.key)
: crypto.createHash('md5').update(data).digest('hex');
// This corresponds to the images table.
if (!this._tileCache[id]) this._tileCache[id] = data;
// This corresponds to the map table.
// Queue writes for images, map table.
var coords = hash(z, x, y);
if (!this._mapCache[coords]) this._mapCache[coords] = { z: z, x: x, y: y };
this._mapCache[coords].tile_id = id;
// Only commit when we can insert at least batchSize rows.
if (++this._pending >= this._batchSize) return this._commit(callback);
else return callback(null);
this.write('images', id, {
tile_id: id,
tile_data: data
});
this.write('map', coords, {
zoom_level: z,
tile_column: x,
tile_row: y,
tile_id: id
}, callback);
};
// Inserts a grid into the MBTiles store. Scheme is XYZ.
@@ -516,32 +506,31 @@ MBTiles.prototype.putGrid = function(z, x, y, data, callback) {
? String(data.key)
: crypto.createHash('md5').update(json).digest('hex');
// This corresponds to the map table.
var coords = hash(z, x, y);
if (!this._mapCache[coords]) this._mapCache[coords] = { z: z, x: x, y: y };
this._mapCache[coords].grid_id = id;
var mbtiles = this;
if (mbtiles._gridCache[id]) return callback(null);
this.write('map', coords, {
zoom_level: z,
tile_column: x,
tile_row: y,
grid_id: id
});
zlib.deflate(new Buffer(json, 'utf8'), function(err, buffer) {
if (err) return callback(err);
// grid_utfgrid table.
mbtiles._gridCache[id] = buffer;
// grid_key table.
mbtiles._keyCache[id] = Object.keys(data.data || {});
// keymap table.
if (data.data) Object.keys(data.data).reduce(function(memo, key) {
memo[key] = data.data[key];
return memo;
}, mbtiles._dataCache);
// Only commit when we can insert at least batchSize rows.
if (++mbtiles._pending >= mbtiles._batchSize) {
mbtiles._commit(callback);
} else {
callback(null);
}
Object.keys(data.data || {}).forEach(function(key) {
mbtiles.write('grid_key', id + '_' + key, {
grid_id: id,
key_name: key
});
mbtiles.write('keymap', key, {
key_name: key,
key_json: JSON.stringify(data.data[key])
});
});
mbtiles.write('grid_utfgrid', id, {
grid_id: id,
grid_utfgrid: buffer
}, callback);
});
};