Initial commit of tilelive-mbtiles
This commit is contained in:
Young Hahn
99
index.js
Normal file
99
index.js
Normal file
@@ -0,0 +1,99 @@
|
||||
var MBTiles = require('./lib/mbtiles'),
|
||||
zlib = require('zlib'),
|
||||
Buffer = require('buffer').Buffer,
|
||||
Step = require('step');
|
||||
|
||||
module.exports = {
|
||||
MBTiles: MBTiles,
|
||||
pool: function(datasource, options) {
|
||||
return {
|
||||
create: function(callback) {
|
||||
var resource = new MBTiles(
|
||||
datasource,
|
||||
options,
|
||||
function() { callback(resource); }
|
||||
);
|
||||
},
|
||||
destroy: function(resource) {
|
||||
resource.db.close(function() {});
|
||||
}
|
||||
}
|
||||
},
|
||||
serve: function(resource, options, callback) {
|
||||
switch (options.format) {
|
||||
case 'layer.json':
|
||||
Step(
|
||||
function() {
|
||||
resource.metadata('formatter', this.parallel());
|
||||
resource.metadata('legend', this.parallel());
|
||||
},
|
||||
function(err, f, l) {
|
||||
var layer = {};
|
||||
f && (layer.formatter = f);
|
||||
l && (layer.legend = l);
|
||||
callback(null, layer);
|
||||
}
|
||||
);
|
||||
break;
|
||||
case 'grid.json':
|
||||
var grid;
|
||||
Step(
|
||||
function() {
|
||||
resource.grid(options.x, options.y, options.z, this);
|
||||
},
|
||||
function(err, buf) {
|
||||
if (err) throw err;
|
||||
if (!Buffer.isBuffer(buf))
|
||||
buf = new Buffer(buf, 'binary');
|
||||
var inflated = zlib.inflate(buf);
|
||||
this(null,inflated);
|
||||
},
|
||||
function(err, buf) {
|
||||
if (err) throw err;
|
||||
grid = buf.toString();
|
||||
resource.grid_data(options.x, options.y, options.z, this);
|
||||
},
|
||||
function(err, gd) {
|
||||
if (err) return callback(err);
|
||||
// Manually append grid data as a string to the grid buffer.
|
||||
// Ideally we would
|
||||
//
|
||||
// JSON.stringify(_.extend(JSON.parse(grid), { data: gd }))
|
||||
//
|
||||
// But calling JSON stringify will escape UTF8 characters of a
|
||||
// high enough ordinal making the grid data unusable. Instead,
|
||||
// manipulate the JSON string directly, popping the trailing }
|
||||
// off and splicing the grid data in at the "data" key.
|
||||
grid = grid.substr(0, grid.length - 1)
|
||||
+ ', "data":'
|
||||
+ JSON.stringify(gd)
|
||||
+ '}';
|
||||
callback(err, grid);
|
||||
}
|
||||
);
|
||||
break;
|
||||
default:
|
||||
resource.tile(options.x, options.y, options.z, function(err, image) {
|
||||
callback(err, [image, { 'Content-Type': 'image/png' }]);
|
||||
});
|
||||
break;
|
||||
}
|
||||
},
|
||||
store: function(step, resource, data, callback) {
|
||||
switch (step) {
|
||||
case 'setup':
|
||||
resource.setup(callback);
|
||||
break;
|
||||
case 'tiles':
|
||||
resource.insertTiles(data, callback);
|
||||
break;
|
||||
case 'grids':
|
||||
resource.insertGrids(data, callback);
|
||||
break;
|
||||
case 'finish':
|
||||
callback();
|
||||
break;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
310
lib/mbtiles.js
Normal file
310
lib/mbtiles.js
Normal file
@@ -0,0 +1,310 @@
|
||||
var fs = require('fs');
|
||||
var Step = require('step');
|
||||
var crypto = require('crypto');
|
||||
var sys = require('sys');
|
||||
var Buffer = require('buffer').Buffer;
|
||||
var zlib = require('zlib');
|
||||
var sqlite3 = require('sqlite3');
|
||||
|
||||
// MBTiles
|
||||
// -------
|
||||
// MBTiles class for doing common operations (schema setup, tile reading,
|
||||
// insertion, etc.)
|
||||
function MBTiles(filename, options, callback) {
|
||||
this.options = options || {};
|
||||
this.filename = filename;
|
||||
this.compress = true; // @TODO
|
||||
this.db = new sqlite3.Database(filename, callback);
|
||||
}
|
||||
|
||||
// Retrieve the schema of the current mbtiles database and inform the caller of
|
||||
// whether the specified table exists.
|
||||
MBTiles.prototype.exists = function(table, callback) {
|
||||
if (this.schema) {
|
||||
if (this.schema.indexOf(table) !== -1) {
|
||||
return callback(null, true);
|
||||
} else {
|
||||
return callback(null, false);
|
||||
}
|
||||
}
|
||||
|
||||
var that = this;
|
||||
that.schema = [];
|
||||
|
||||
this.db.all('SELECT name FROM sqlite_master WHERE type IN (?, ?)',
|
||||
'table',
|
||||
'view',
|
||||
function(err, rows) {
|
||||
if (err) return callback(err);
|
||||
for (var i = 0; i < rows.length; i++) {
|
||||
that.schema.push(rows[i].name);
|
||||
}
|
||||
that.exists(table, callback);
|
||||
});
|
||||
}
|
||||
|
||||
// Setup schema, indices, views for a new mbtiles database.
|
||||
MBTiles.prototype.setup = function(callback) {
|
||||
var db = this.db;
|
||||
fs.readFile(__dirname + '/schema.sql', 'utf8', function(err, sql) {
|
||||
if (err) return callback(err);
|
||||
|
||||
db.serialize(function() {
|
||||
// Set the synchronous flag to OFF for (much) faster inserts.
|
||||
// See http://www.sqlite3.org/pragma.html#pragma_synchronous
|
||||
db.run('PRAGMA synchronous = 0');
|
||||
db.exec(sql, callback);
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
// Insert a set of tiles into an mbtiles database.
|
||||
//
|
||||
// - @param {Array} renders array of images to be inserted. Each item should
|
||||
// be an object of the form { tile: [z, x, y], data: [Image buffer] }.
|
||||
// - @param {Function} callback
|
||||
MBTiles.prototype.insertTiles = function(data, callback) {
|
||||
var that = this,
|
||||
map = [],
|
||||
images = [],
|
||||
ids = [];
|
||||
for (var i = 0; i < data.length; i++) {
|
||||
// Generate ID from MD5 hash of actual image data.
|
||||
// Generate ID from tile coordinates (unique).
|
||||
var tile_id = this.compress
|
||||
? crypto.createHash('md5').update(data[i].data).digest('hex')
|
||||
: data[i].tile.join('.');
|
||||
if (ids.indexOf(tile_id) === -1) {
|
||||
ids.push(tile_id);
|
||||
images.push({ tile_id: tile_id, tile_data: data[i].data });
|
||||
}
|
||||
map.push({
|
||||
tile_id: tile_id,
|
||||
zoom_level: data[i].tile[0],
|
||||
tile_column: data[i].tile[1],
|
||||
tile_row: data[i].tile[2]
|
||||
});
|
||||
}
|
||||
Step(
|
||||
function() {
|
||||
var group = this.group();
|
||||
for (var i = 0; i < images.length; i++) {
|
||||
that.insertImage(images[i], group());
|
||||
}
|
||||
for (var i = 0; i < map.length; i++) {
|
||||
that.insertTile(map[i], group());
|
||||
}
|
||||
},
|
||||
callback
|
||||
);
|
||||
};
|
||||
|
||||
// Insert a set of grids into an mbtiles database.
|
||||
//
|
||||
// - @param {Array} renders array of grids to be inserted. Each item should
|
||||
// be an object of the form { tile: [z, x, y], data: [Grid data] }.
|
||||
// - @param {Function} callback
|
||||
MBTiles.prototype.insertGrids = function(data, callback) {
|
||||
var that = this,
|
||||
map = [],
|
||||
grids = [],
|
||||
grid_keys = [],
|
||||
ids = [];
|
||||
for (var i = 0; i < tiles.length; i++) {
|
||||
var grid_id;
|
||||
// Generate ID from MD5 hash of grid data.
|
||||
// Generate ID from tile coordinates (unique).
|
||||
var grid_id = this.compress
|
||||
? crypto.createHash('md5').update(data[i].data[0]).digest('hex')
|
||||
: data[i].tile.join('.');
|
||||
if (ids.indexOf(grid_id) === -1) {
|
||||
ids.push(grid_id);
|
||||
grids.push({ grid_id: grid_id, grid_utfgrid: renders[i][0] });
|
||||
}
|
||||
data[i].data[2].keys.forEach(function(k) {
|
||||
grid_keys.push({ grid_id: grid_id, key_name: k });
|
||||
});
|
||||
map.push({
|
||||
grid_id: grid_id,
|
||||
zoom_level: data[i].tile[0],
|
||||
tile_column: data[i].tile[1],
|
||||
tile_row: data[i].tile[2]
|
||||
});
|
||||
}
|
||||
|
||||
Step(
|
||||
function() {
|
||||
var group = this.group();
|
||||
that.insertUTFGrids(grids, group());
|
||||
that.insertGridKeys(grid_keys, group());
|
||||
that.insertGridTiles(map, group());
|
||||
},
|
||||
callback
|
||||
);
|
||||
};
|
||||
|
||||
// Given an array of mappings of grid_id to key_name, insert
|
||||
// them all into the `grid_key` table.
|
||||
MBTiles.prototype.insertGridKeys = function(grid_keys, callback) {
|
||||
var stmt = this.db.prepare('INSERT OR IGNORE INTO grid_key' +
|
||||
' (grid_id, key_name) VALUES (?, ?)');
|
||||
for (var i = 0; i < grid_keys.length; i++) {
|
||||
stmt.run(
|
||||
grid_keys[i].grid_id,
|
||||
grid_keys[i].key_name
|
||||
);
|
||||
}
|
||||
stmt.finalize(callback);
|
||||
};
|
||||
|
||||
// Insert a single feature into the grid_data table,
|
||||
// with the key/axis of `key`
|
||||
MBTiles.prototype.insertGridData = function(data, key_name, callback) {
|
||||
this.db.run(
|
||||
'INSERT OR IGNORE INTO keymap (key_name, key_json) VALUES (?, ?)',
|
||||
data[key_name],
|
||||
JSON.stringify(data),
|
||||
callback
|
||||
);
|
||||
};
|
||||
|
||||
// Insert a single tile into the mbtiles database.
|
||||
//
|
||||
// - @param {Object} tile tile object to be inserted.
|
||||
// - @param {Function} callback
|
||||
MBTiles.prototype.insertTile = function(tile, callback) {
|
||||
this.db.run(
|
||||
'INSERT INTO map (tile_id, zoom_level, tile_column, tile_row) VALUES (?, ?, ?, ?)',
|
||||
tile.tile_id,
|
||||
tile.zoom_level,
|
||||
tile.tile_column,
|
||||
tile.tile_row,
|
||||
callback
|
||||
);
|
||||
};
|
||||
|
||||
// Insert a single tile into the mbtiles database.
|
||||
//
|
||||
// - @param {Object} tile tile object to be inserted.
|
||||
// - @param {Function} callback
|
||||
MBTiles.prototype.insertGridTiles = function(map, callback) {
|
||||
var stmt = this.db.prepare('UPDATE OR REPLACE map SET grid_id = ? WHERE ' +
|
||||
' zoom_level = ? AND tile_column = ? AND tile_row = ?');
|
||||
|
||||
for (var i = 0; i < map.length; i++) {
|
||||
stmt.run(
|
||||
map[i].grid_id,
|
||||
map[i].zoom_level,
|
||||
map[i].tile_column,
|
||||
map[i].tile_row
|
||||
);
|
||||
}
|
||||
|
||||
stmt.finalize(callback);
|
||||
};
|
||||
|
||||
// Insert a single grid into the mbtiles database.
|
||||
//
|
||||
// - @param {Object} image object to be inserted.
|
||||
// - @param {Function} callback
|
||||
MBTiles.prototype.insertUTFGrids = function(grids, callback) {
|
||||
var stmt = this.db.prepare('INSERT OR IGNORE INTO grid_utfgrid'
|
||||
+ ' (grid_id, grid_utfgrid) VALUES (?, ?)');
|
||||
|
||||
var total = grids.length, ran = 0;
|
||||
grids.forEach(function(grid) {
|
||||
var buf = zlib.deflate(grid.grid_utfgrid);
|
||||
stmt.run(grid.grid_id, buf);
|
||||
if (++ran === total) stmt.finalize(callback);
|
||||
});
|
||||
};
|
||||
|
||||
// Insert a single image into the mbtiles database.
|
||||
//
|
||||
// - @param {Object} image object to be inserted.
|
||||
// - @param {Function} callback
|
||||
MBTiles.prototype.insertImage = function(image, callback) {
|
||||
this.db.run(
|
||||
'INSERT OR IGNORE INTO images (tile_id, tile_data) VALUES (?, ?)',
|
||||
image.tile_id,
|
||||
image.tile_data,
|
||||
callback
|
||||
);
|
||||
};
|
||||
|
||||
// Insert metadata into the mbtiles database.
|
||||
//
|
||||
// - @param {Object} metadata key, value hash of metadata to be inserted.
|
||||
// - @param {Function} callback
|
||||
MBTiles.prototype.insertMetadata = function(metadata, callback) {
|
||||
var stmt = this.db.prepare('INSERT INTO metadata (name, value) VALUES (?, ?)');
|
||||
for (var name in metadata) {
|
||||
stmt.run(name, metadata[name]);
|
||||
}
|
||||
stmt.finalize(callback);
|
||||
};
|
||||
|
||||
// Select a tile from an mbtiles database.
|
||||
//
|
||||
// - @param {Number} x tile x coordinate.
|
||||
// - @param {Number} y tile y coordinate.
|
||||
// - @param {Number} z tile z coordinate.
|
||||
// - @param {Function} callback
|
||||
MBTiles.prototype.tile = function(x, y, z, callback) {
|
||||
this.db.get('SELECT tile_data FROM tiles WHERE ' +
|
||||
'zoom_level = ? AND tile_column = ? AND tile_row = ?',
|
||||
z, x, y,
|
||||
function(err, row) {
|
||||
if (err) callback(err);
|
||||
else if (!row || !row.tile_data) callback('Tile does not exist');
|
||||
else callback(null, row.tile_data);
|
||||
});
|
||||
};
|
||||
|
||||
// Get grid data at a certain `x, y, z` coordinate, calling
|
||||
// back with an error argument if the grid is not found.
|
||||
MBTiles.prototype.grid_data = function(x, y, z, callback) {
|
||||
this.db.all('SELECT key_name, key_json FROM grid_data WHERE ' +
|
||||
'zoom_level = ? AND tile_column = ? AND tile_row = ?',
|
||||
z, x, y,
|
||||
function(err, rows) {
|
||||
if (err) callback(err);
|
||||
else if (!rows.length) callback('Grid data does not exist');
|
||||
else callback(null, rows.reduce(function(memo, r) {
|
||||
memo[r.key_name] = JSON.parse(r.key_json);
|
||||
return memo;
|
||||
}, {}));
|
||||
});
|
||||
};
|
||||
|
||||
// Select a tile from an mbtiles database.
|
||||
//
|
||||
// - @param {Number} x tile x coordinate
|
||||
// - @param {Number} y tile y coordinate
|
||||
// - @param {Number} z tile z coordinate
|
||||
// - @param {Function} callback
|
||||
MBTiles.prototype.grid = function(x, y, z, callback) {
|
||||
this.db.get('SELECT grid FROM grids WHERE ' +
|
||||
'zoom_level = ? AND tile_column = ? AND tile_row = ?',
|
||||
z, x, y,
|
||||
function(err, row) {
|
||||
if (err) callback(err);
|
||||
else if (!row || !row.grid) callback('Grid does not exist');
|
||||
else callback(null, row.grid);
|
||||
});
|
||||
};
|
||||
|
||||
// Select a metadata value from the database.
|
||||
//
|
||||
// - @param {Function} callback
|
||||
MBTiles.prototype.metadata = function(key, callback) {
|
||||
this.db.get('SELECT value FROM metadata WHERE name = ?',
|
||||
key,
|
||||
function(err, row) {
|
||||
if (err) callback(err);
|
||||
else if (!row) callback('Key does not exist');
|
||||
else callback(null, row.value);
|
||||
});
|
||||
};
|
||||
|
||||
module.exports = MBTiles;
|
||||
72
lib/schema.sql
Normal file
72
lib/schema.sql
Normal file
@@ -0,0 +1,72 @@
|
||||
-- MBTiles schema
|
||||
|
||||
CREATE TABLE IF NOT EXISTS map (
|
||||
zoom_level INTEGER,
|
||||
tile_column INTEGER,
|
||||
tile_row INTEGER,
|
||||
tile_id TEXT,
|
||||
grid_id TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS grid_key (
|
||||
grid_id TEXT,
|
||||
key_name TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS keymap (
|
||||
key_name TEXT,
|
||||
key_json TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS grid_utfgrid (
|
||||
grid_id TEXT,
|
||||
grid_utfgrid BLOB
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS images (
|
||||
tile_data blob,
|
||||
tile_id text
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS metadata (
|
||||
name text,
|
||||
value text
|
||||
);
|
||||
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS map_index ON map (zoom_level, tile_column, tile_row);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS grid_key_lookup ON grid_key (grid_id, key_name);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS keymap_lookup ON keymap (key_name);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS grid_utfgrid_lookup ON grid_utfgrid (grid_id);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS images_id ON images (tile_id);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS name ON metadata (name);
|
||||
|
||||
|
||||
CREATE VIEW IF NOT EXISTS tiles AS
|
||||
SELECT
|
||||
map.zoom_level AS zoom_level,
|
||||
map.tile_column AS tile_column,
|
||||
map.tile_row AS tile_row,
|
||||
images.tile_data AS tile_data
|
||||
FROM map
|
||||
JOIN images ON images.tile_id = map.tile_id;
|
||||
|
||||
CREATE VIEW IF NOT EXISTS grids AS
|
||||
SELECT
|
||||
map.zoom_level AS zoom_level,
|
||||
map.tile_column AS tile_column,
|
||||
map.tile_row AS tile_row,
|
||||
grid_utfgrid.grid_utfgrid AS grid
|
||||
FROM map
|
||||
JOIN grid_utfgrid ON grid_utfgrid.grid_id = map.grid_id;
|
||||
|
||||
CREATE VIEW IF NOT EXISTS grid_data AS
|
||||
SELECT
|
||||
map.zoom_level AS zoom_level,
|
||||
map.tile_column AS tile_column,
|
||||
map.tile_row AS tile_row,
|
||||
keymap.key_name AS key_name,
|
||||
keymap.key_json AS key_json
|
||||
FROM map
|
||||
JOIN grid_key ON map.grid_id = grid_key.grid_id
|
||||
JOIN keymap ON grid_key.key_name = keymap.key_name;
|
||||
30
package.json
Normal file
30
package.json
Normal file
@@ -0,0 +1,30 @@
|
||||
{
|
||||
"name" : "tilelive-mbtiles",
|
||||
"main" : "./index",
|
||||
"description" : "mbtiles backend for tilelive",
|
||||
"url" : "http://github.com/mapbox/tilelive-mbtiles",
|
||||
"keywords" : ["map", "server", "mbtiles", "tms"],
|
||||
"licenses" : [{
|
||||
"type": "BSD"
|
||||
}],
|
||||
"repositories": [{
|
||||
"type": "git",
|
||||
"url": "http://github.com/mapbox/tilelive-mbtiles.git"
|
||||
}],
|
||||
"contributors": [
|
||||
"Tom MacWright <tmcw>",
|
||||
"Will White <willwhite>",
|
||||
"Dane Springmeyer <springmeyer>",
|
||||
"Young Hahn <yhahn>"
|
||||
],
|
||||
"dependencies": {
|
||||
"sqlite3" : ">= 2.0.8 < 2.1.0",
|
||||
"step" : ">= 0.0.4 < 0.1.0",
|
||||
"underscore" : ">= 1.1.1 < 1.2.0",
|
||||
"zlib" : ">= 1.0.3 < 1.1.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 0.2.6"
|
||||
},
|
||||
"version" : "0.0.1"
|
||||
}
|
||||
Reference in New Issue
Block a user