ashatora/node-mbtiles
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor tile writing

Browse Source
This commit is contained in:
Konstantin Käfer
2011-07-07 22:52:29 +02:00
parent c0fb0fd2a6
commit f244fe5517
1 changed files with 176 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

254
lib/mbtiles.js
View File

@@ -5,6 +5,7 @@ var _ = require('underscore'),
zlib = require('zlib'),
path = require('path'),
url = require('url'),
Buffer = require('buffer').Buffer,
sm = new (require('sphericalmercator')),
sqlite3 = require('sqlite3');
@@ -14,6 +15,10 @@ function noop(err) {
if (err) throw err;
}
function hash(z, x, y) {
return (1 << z) * ((1 << z) + x) + y;
}
// MBTiles
// -------
// MBTiles class for doing common operations (schema setup, tile reading,
@@ -32,7 +37,7 @@ function MBTiles(uri, callback) {
this.filename = uri.pathname;
Step(function() {
mbtiles._db = new sqlite3.cached.Database(mbtiles.filename, this)
mbtiles._db = new sqlite3.cached.Database(mbtiles.filename, this);
}, function(err) {
if (err) return callback(err);
mbtiles._setup(this);
@@ -198,56 +203,56 @@ MBTiles.prototype._setup = function(callback) {
// // - @param {Array} renders array of grids to be inserted. Each item should
// // be an object of the form { z: z, x: x, y: y, data: [Image buffer], keys: [] }.
// // - @param {Function} callback
// MBTiles.prototype._insertGrids = function(data, callback) {
// if (typeof callback !== 'function') callback = noop;
//
// var that = this,
// map = [],
// grids = [],
// grid_keys = [],
// features = {},
// ids = [];
// for (var i = 0; i < data.length; i++) {
// var json = JSON.stringify({
// grid: data[i].grid,
// keys: data[i].keys
// });
// var grid_id = crypto
// .createHash('md5')
// .update(json)
// .digest('hex');
// !_(ids).include(grid_id) && ids.push(grid_id) && grids.push({
// grid_id: grid_id,
// grid_utfgrid: zlib.deflate(new Buffer(json, 'utf8'))
// });
// data[i].keys.forEach(function(k) {
// grid_keys.push({
// grid_id: grid_id,
// key_name: k
// });
// });
// map.push({
// grid_id: grid_id,
// zoom_level: data[i].z,
// tile_column: data[i].x,
// tile_row: data[i].y
// });
// _(features).extend(data[i].data);
// }
// features = _(features).map(function(value, key) {
// return { key_name: key, key_json: JSON.stringify(value) };
// });
// Step(
// function() {
// var group = this.group();
// that._insert('grid_utfgrid', grids, group());
// that._insert('grid_key', grid_keys, group());
// that._insert('keymap', features, group());
// that._insertGridTiles(map, group());
// },
// callback
// );
// };
MBTiles.prototype._insertGrids = function(data, callback) {
if (typeof callback !== 'function') callback = noop;
var that = this,
map = [],
grids = [],
grid_keys = [],
features = {},
ids = [];
for (var i = 0; i < data.length; i++) {
var json = JSON.stringify({
grid: data[i].grid,
keys: data[i].keys
});
var grid_id = crypto
.createHash('md5')
.update(json)
.digest('hex');
!_(ids).include(grid_id) && ids.push(grid_id) && grids.push({
grid_id: grid_id,
grid_utfgrid: zlib.deflate(new Buffer(json, 'utf8'))
});
data[i].keys.forEach(function(k) {
grid_keys.push({
grid_id: grid_id,
key_name: k
});
});
map.push({
grid_id: grid_id,
zoom_level: data[i].z,
tile_column: data[i].x,
tile_row: data[i].y
});
_(features).extend(data[i].data);
}
features = _(features).map(function(value, key) {
return { key_name: key, key_json: JSON.stringify(value) };
});
Step(
function() {
var group = this.group();
that._insert('grid_utfgrid', grids, group());
that._insert('grid_key', grid_keys, group());
that._insert('keymap', features, group());
that._insertGridTiles(map, group());
},
callback
);
};
//
// // Insert grids into the mbtiles database.
// //
@@ -282,7 +287,7 @@ MBTiles.prototype.getTile = function(z, x, y, callback) {
if (!this.open) return callback(new Error('MBTiles not yet loaded'));
// Flip Y coordinate because MBTiles files are TMS.
y = Math.pow(2, z) - 1 - y;
y = (1 << z) - 1 - y;
var mbtiles = this;
this._db.get('SELECT tile_data FROM tiles WHERE ' +
@@ -315,7 +320,7 @@ MBTiles.prototype.getGrid = function(z, x, y, callback) {
if (!this.open) return callback(new Error('MBTiles not yet loaded'));
// Flip Y coordinate because MBTiles files are TMS.
y = Math.pow(2, z) - 1 - y;
y = (1 << z) - 1 - y;
var that = this;
Step(
@@ -492,8 +497,12 @@ MBTiles.prototype.startWriting = function(callback) {
var mbtiles = this;
if (!this._isWritable) {
this._isWritable = 1;
this._tileCache = [];
this._gridCache = [];
this._pending = 0;
this._tileCache = {};
this._gridCache = {};
this._keyCache = {};
this._dataCache = {};
this._mapCache = {};
this._db.run('PRAGMA synchronous=OFF', callback);
} else {
this._isWritable++;
@@ -502,22 +511,91 @@ MBTiles.prototype.startWriting = function(callback) {
};
MBTiles.prototype._commit = function(callback) {
// Only commit when we can insert at least 100 rows.
if (++this._pending < 100) {
return callback(null);
}
var mbtiles = this;
this._db.serialize(function() {
mbtiles._db.serialize(function() {
mbtiles._db.run('BEGIN');
var tile_data = mbtiles._db.prepare('REPLACE INTO images (tile_id, tile_data) VALUES (?, ?)');
var tile_map = mbtiles._db.prepare('REPLACE INTO map (zoom_level, ' +
if (Object.keys(mbtiles._tileCache)) {
// Insert images table.
var images = mbtiles._db.prepare('REPLACE INTO images (tile_id, tile_data) VALUES (?, ?)');
for (var id in mbtiles._tileCache) {
images.run(id, mbtiles._tileCache[id]);
}
mbtiles._tileCache = {};
images.finalize();
}
if (Object.keys(mbtiles._gridCache)) {
// Insert grid_utfgrid table.
var grids = mbtiles._db.prepare('REPLACE INTO grid_utfgrid (grid_id, grid_utfgrid) VALUES (?, ?)');
for (var id in mbtiles._gridCache) {
grids.run(id, mbtiles._gridCache[id]);
}
mbtiles._gridCache = {};
grids.finalize();
}
if (Object.keys(mbtiles._keyCache)) {
// Insert grid_key.
var keys = mbtiles._db.prepare('INSERT OR IGNORE INTO grid_key (grid_id, key_name) VALUES (?, ?)');
for (var id in mbtiles._keyCache) {
mbtiles._keyCache.forEach(function(key) {
keys.run(id, key);
});
}
mbtiles._keyCache = {};
keys.finalize();
}
if (Object.keys(mbtiles._dataCache)) {
// Insert keymap table.
var keymap = mbtiles._db.prepare('REPLACE INTO keymap (key_name, key_json) VALUES (?, ?)');
for (var key in mbtiles._dataCache) {
keymap.run(key, JSON.stringify(mbtiles._dataCache[key]));
}
mbtiles._dataCache = {};
keymap.finalize();
}
// Insert map table. This has to be so complicate due to a design flaw
// in the tables.
// TODO: This should be remedied when we upgrade the MBTiles schema.
var mapBoth = mbtiles._db.prepare('REPLACE INTO map (zoom_level, ' +
'tile_column, tile_row, tile_id, grid_id) VALUES (?, ?, ?, ?, ?)');
var mapTile = mbtiles._db.prepare('REPLACE INTO map (zoom_level, ' +
'tile_column, tile_row, tile_id, grid_id) VALUES (?, ?, ?, ?, ' +
'(SELECT grid_id FROM map WHERE zoom_level = ? ' +
'AND tile_column = ? AND tile_row = ?))');
mbtiles._tileCache.forEach(function(row) {
var id = crypto.createHash('md5').update(row[3]).digest('hex');
tile_data.run(id, row[3]);
tile_map.run(row[0], row[1], row[2], id, row[0], row[1], row[2]);
});
tile_data.finalize();
tile_map.finalize();
var mapGrid = mbtiles._db.prepare('REPLACE INTO map (zoom_level, ' +
'tile_column, tile_row, tile_id, grid_id) VALUES (?, ?, ?, ' +
'(SELECT tile_id FROM map WHERE zoom_level = ? ' +
'AND tile_column = ? AND tile_row = ?), ?)');
for (var coords in mbtiles._mapCache) {
var map = mbtiles._mapCache[coords];
if (typeof map.grid_id === 'undefined') {
// Only the tile_id is defined.
mapTile.run(map.z, map.x, map.y, map.tile_id, map.z, map.x, map.y);
} else if (typeof map.tile_id === 'undefined') {
// Only the grid_id is defined.
mapGrid.run(map.z, map.x, map.y, map.z, map.x, map.y, map.grid_id);
} else {
// Both tile_id and grid_id are defined.
mapBoth.run(map.z, map.x, map.y, map.tile_id, map.grid_id);
}
}
mbtiles._mapCache = {};
mapBoth.finalize();
mapTile.finalize();
mapGrid.finalize();
mbtiles._db.run('COMMIT', callback);
});
@@ -545,16 +623,23 @@ MBTiles.prototype.putTile = function(z, x, y, data, callback) {
if (!this.open) return callback(new Error('MBTiles not yet loaded'));
if (!this._isWritable) return callback(new Error('MBTiles not in write mode'));
if (!Buffer.isBuffer(data)) return callback(new Error('Image needs to be a Buffer'));
// Flip Y coordinate because MBTiles files are TMS.
y = Math.pow(2, z) - 1 - y;
y = (1 << z) - 1 - y;
this._tileCache.push([ z, x, y, data ]);
if (this._tileCache.length + this._gridCache.length >= 100) {
this._commit(callback);
} else {
return callback(null);
var id = crypto.createHash('md5').update(data).digest('hex');
if (!this._tileCache[id]) {
// This corresponds to the images table.
this._tileCache[id] = data;
}
// This corresponds to the map table.
var coords = hash(z, x, y);
if (!this._mapCache[coords]) this._mapCache[coords] = { z: z, x: x, y: y };
this._mapCache[coords].tile_id = id;
this._commit(callback);
};
// Insert a tile. Scheme is XYZ.
@@ -563,13 +648,26 @@ MBTiles.prototype.putGrid = function(z, x, y, data, callback) {
if (!this.open) return callback(new Error('MBTiles not yet loaded'));
// Flip Y coordinate because MBTiles files are TMS.
y = Math.pow(2, z) - 1 - y;
y = (1 << z) - 1 - y;
this._tileCache.push([ z, x, y, data ]);
// Preprocess grid data.
var json = JSON.stringify({ grid: data.grid, keys: data.keys });
var id = crypto.createHash('md5').update(json).digest('hex');
if (!this._gridCache[id]) {
// This corresponds to the grid_utfgrid table.
this._gridCache[id] = zlib.deflate(new Buffer(json, 'utf8'));
if (this._tileCache.length + this._gridCache.length >= 100) {
this._commit(callback);
} else {
return callback(null);
// This corresponds to the grid_key table.
this._keyCache[id] = Object.keys(data.data || {});
// This corresponds to the keymap table.
this._dataCache.extend(data.data || {});
}
// This corresponds to the map table.
var coords = hash(z, x, y);
if (!this._mapCache[coords]) this._mapCache[coords] = { z: z, x: x, y: y };
this._mapCache[coords].grid_id = id;
this._commit(callback);
};