ashatora/node-mbtiles
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove commands.

Browse Source
This commit is contained in:
Young Hahn
2013-07-12 05:54:50 -04:00
parent 46f0b33bc8
commit a98037dbdf
5 changed files with 0 additions and 594 deletions
Download Patch File Download Diff File Expand all files Collapse all files

158
bin/mbcheck
View File

@@ -1,158 +0,0 @@
#!/usr/bin/env node
var options = argv = require('optimist').argv,
_ = require('underscore'),
Step = require('step'),
utils = require('..').utils,
MBTiles = require('..'),
mbtiles,
zooms = [];
// Show help.
if (argv.help || argv._.length === 0) {
console.warn('Usage: mbcheck [FILE]');
console.warn(' Check an mbtiles for missing metadata and tiles.');
console.warn('Examples:');
console.warn(' mbcheck world.mbtiles');
process.exit();
}
// Wishlist:
// ---------
// - check that tiles exist for specified bounds
Step(
function() { mbtiles = new MBTiles(argv._[0], this); },
function() { mbtiles._integrity(this); },
function(err) {
if (err) throw err;
mbtiles._exists('tiles', function(err, exists) {
if (!exists) throw new Error('`tiles` table does not exist.');
this();
}.bind(this));
},
function(err) {
if (err) throw err;
mbtiles._exists('metadata', function(err, exists) {
if (!exists) throw new Error('`metadata` table does not exist.');
this();
}.bind(this));
},
function(err) {
if (err) throw err;
mbtiles._metadata('name', this.parallel());
mbtiles._metadata('type', this.parallel());
mbtiles._metadata('desciption', this.parallel());
mbtiles._metadata('version', this.parallel());
mbtiles._metadata('format', this.parallel());
mbtiles._metadata('bounds', this.parallel());
},
function(err, name, type, description, version, format, bounds) {
console.warn('Metadata');
utils.table([
['name', name],
['type', type],
['description', description],
['version', version],
['format', format],
['bounds', bounds]
]);
this();
},
function(err) {
if (err) throw err;
Step(
function() {
var group = this.group(),
query = mbtiles._db.prepare(
'SELECT zoom_level AS zoom, ' +
'COUNT(zoom_level) AS count ' +
'FROM tiles ' +
'WHERE zoom_level = ?'
);
for (var i = 0; i < 30; i++) {
query.get(i, group());
}
query.finalize();
},
function(err, rows) {
if (err) throw err;
zooms = rows;
do {
var last = zooms.pop();
if (last.count) zooms.push(last);
} while(!last.count);
do {
var first = zooms.shift();
if (first.count) zooms.unshift(first);
} while(!first.count);
this();
}.bind(this)
);
},
function(err) {
if (err) throw err;
Step(
function() {
var group = this.group(),
query = mbtiles._db.prepare(
'SELECT MAX(tile_column) AS maxx, ' +
'MIN(tile_column) AS minx, ' +
'MAX(tile_row) AS maxy, ' +
'MIN(tile_row) AS miny, ' +
'zoom_level AS zoom ' +
'FROM tiles ' +
'WHERE zoom_level = ?'
);
for (var i = 0; i < zooms.length; i++) {
query.get(zooms[i].zoom, group());
}
query.finalize();
},
function(err, rows) {
if (err) throw err;
var group = this.group(),
query = mbtiles._db.prepare(
'SELECT zoom_level AS zoom, ' +
'COUNT(zoom_level) AS count, ' +
'? AS expected ' +
'FROM tiles ' +
'WHERE zoom_level = ? AND ' +
'tile_column >= ? AND ' +
'tile_column <= ? AND ' +
'tile_row >= ? AND ' +
'tile_row <= ?'
);
for (var i = 0; i < rows.length; i++) {
var expected = (rows[i].maxx - rows[i].minx + 1) * (rows[i].maxy - rows[i].miny + 1);
query.get(
expected,
rows[i].zoom,
rows[i].minx,
rows[i].maxx,
rows[i].miny,
rows[i].maxy,
group()
);
}
query.finalize();
},
function(err, rows) {
if (err) throw err;
console.warn('');
var output = [];
output.push(['Z', 'Tiles', 'Missing']);
_(rows).each(function(row, index) {
output.push([
row.zoom,
zooms[index].count,
row.expected - row.count
]);
});
utils.table(output);
this();
}.bind(this)
);
}
);

145
bin/mbcompact
View File

@@ -1,145 +0,0 @@
#!/usr/bin/env node
var options = argv = require('optimist').argv,
_ = require('underscore'),
Step = require('step'),
crypto = require('crypto'),
fs = require('fs'),
sys = require('sys'),
spawn = require('child_process').spawn,
MBTiles = require('..'),
utils = require('..').utils,
mbtiles,
hits = 0,
ids = [];
// Show help.
if (argv.help || argv._.length < 1) {
console.warn('Usage: mbcompact [FILE]');
console.warn(' Eliminate duplicate images to reduce mbtiles filesize.');
console.warn('Examples:');
console.warn(' mbcompact world.mbtiles');
process.exit();
}
// Grab args.
var filename = argv._[0];
Step(
function() { mbtiles = new MBTiles(filename, this); },
function(err) {
if (err) throw err;
mbtiles._db.all('SELECT name, type '
+ 'FROM sqlite_master '
+ 'WHERE type IN (?, ?)',
'table',
'view',
function(err, rows) {
if (err) throw err;
if (_(rows).any(function(row) { return row.name === 'tiles' && row.type === 'view' })) {
throw new Error('Table is already compacted.');
}
if (!_(rows).any(function(row) { return row.name === 'tiles' && row.type === 'table' })) {
throw new Error('Tiles table does not exist.');
}
this();
}.bind(this)
);
},
function(err) {
if (err) throw err;
mbtiles._setup(this);
},
function(err) {
if (err) throw err;
mbtiles._db.get('SELECT COUNT(*) AS total FROM tiles', this.parallel());
mbtiles._db.run('PRAGMA locking_mode=EXCLUSIVE', this.parallel());
mbtiles._db.run('PRAGMA journal_mode=TRUNCATE', this.parallel());
},
function(err, row) {
if (err) throw err;
if (!row.total) throw new Error('No tiles found');
var total = row.total;
var printed = 0;
var done = this;
var doit = function(limit, offset) {
process.nextTick(function() {
mbtiles._db
.prepare('SELECT tile_data AS tile_data, zoom_level AS z, tile_column AS x, tile_row AS y FROM tiles LIMIT ? OFFSET ?')
.all(limit, offset, function(err, rows) {
var images = [];
var map = [];
for (var i = 0; i < rows.length; i++) {
var tile_id = crypto
.createHash('md5')
.update(rows[i].tile_data)
.digest('hex');
if (!_(ids).include(tile_id)) {
ids.unshift(tile_id);
images.push({
tile_id: tile_id,
tile_data: rows[i].tile_data
});
} else {
hits++;
}
map.push({
tile_id: tile_id,
zoom_level: rows[i].z,
tile_column: rows[i].x,
tile_row: rows[i].y
});
}
Step(
function() {
mbtiles._insert('images', images, this.parallel());
mbtiles._insert('map', map, this.parallel());
},
function(err) {
if (err) throw err;
// If IDs has grown over threshold, trim back down.
(ids.length > 1000) && (ids = ids.slice(0, 1000 - 900));
var progress = Math.floor(offset / total * 40);
if (progress > printed) {
sys.print((new Array(progress - printed + 1)).join('#'));
printed = progress;
}
if (rows.length === limit) {
doit(limit, offset + limit);
} else {
done();
}
}
);
})
.finalize();
});
};
console.warn('00 -------------- 50 -------------- 100');
sys.print('');
doit(1000, 0);
},
function(err) {
if (err) throw err;
mbtiles._db.run('DROP TABLE tiles', this);
},
function(err) {
if (err) throw err;
mbtiles._db.run('CREATE VIEW IF NOT EXISTS tiles AS '
+ 'SELECT map.zoom_level AS zoom_level, '
+ 'map.tile_column AS tile_column, '
+ 'map.tile_row AS tile_row, '
+ 'images.tile_data AS tile_data '
+ 'FROM map JOIN images ON images.tile_id = map.tile_id;', this.parallel());
mbtiles._db.run('VACUUM', this.parallel());
},
function(err) {
if (err) throw err;
sys.print('\n');
console.warn('Compact hits %s.', hits);
console.warn('Compaction complete.');
}
);

145
bin/mbpipe
View File

@@ -1,145 +0,0 @@
#!/usr/bin/env node
var options = argv = require('optimist').argv,
_ = require('underscore'),
Step = require('step'),
sys = require('sys'),
spawn = require('child_process').spawn,
MBTiles = require('..'),
utils = require('..').utils,
mbtiles,
table,
total;
// @TODO: support specifying a table to be processed, e.g. for processing
// grids or grid data.
// Show help.
if (argv.help || argv._.length < 2) {
console.warn('Usage: mbpipe [COMMAND] [FILE]');
console.warn(' Pipe each tile to stdin of the specified command and');
console.warn(' write stdout into the mbtiles file.');
console.warn('Examples:');
console.warn(' mbpipe "pngquant 32" world.mbtiles');
process.exit();
}
// Grab args.
var filename = argv._[1],
command = {
command: argv._[0].split(' ')[0],
args: argv._[0].split(' ').slice(1)
};
Step(
function() { mbtiles = new MBTiles(filename, this); },
function(err) {
if (err) throw err;
Step(
function() { mbtiles._db.run('PRAGMA synchronous=0', this); },
function() { mbtiles._db.run('PRAGMA locking_mode=EXCLUSIVE', this); },
function() { mbtiles._db.run('PRAGMA journal_mode=TRUNCATE', this); },
this
);
},
function(err) {
if (err) throw err;
mbtiles._exists('tiles', function(err, exists) {
if (exists) table = 'tiles';
this();
}.bind(this));
},
function(err) {
if (err) throw err;
mbtiles._exists('images', function(err, exists) {
if (exists) table = 'images';
this();
}.bind(this));
},
function(err) {
if (err) throw err;
if (!table) throw new Error('No usable image table found.');
mbtiles._db.get('SELECT COUNT(tile_data) AS total FROM ' + table, function(err, row) {
total = row.total;
this();
}.bind(this));
},
function(err) {
if (err) throw err;
if (!total) throw new Error('No tiles found');
var printed = 0;
var done = this;
var doit = function(table, limit, offset) {
process.nextTick(function() {
var query = table === 'images'
? mbtiles._db.prepare('SELECT tile_data AS tile_data, tile_id AS id FROM images ORDER BY tile_id ASC LIMIT ? OFFSET ?')
: mbtiles._db.prepare('SELECT tile_data AS tile_data, zoom_level AS z, tile_column AS x, tile_row AS y FROM tiles ORDER BY z, x, y LIMIT ? OFFSET ?');
query.all(limit, offset, function(err, rows) {
Step(
function() {
var group = this.group();
var exec = function(row, callback) {
row.piped = new Buffer(0);
var child = spawn(command.command, command.args);
var stream = function(chunk) {
var joined = (new Buffer(row.piped.length + chunk.length));
row.piped.copy(joined, 0, 0);
chunk.copy(joined, row.piped.length, 0);
row.piped = joined;
};
child.stdin.write(row.tile_data);
child.stdin.end();
child.stdout.on('data', stream);
child.stderr.on('data', stream);
child.on('exit', function(code) {
if (code) return callback(null, null);
if (table === 'images') {
mbtiles._db.run(
'INSERT OR REPLACE INTO images (tile_id, tile_data) VALUES(?, ?)',
row.id,
row.piped,
callback
);
} else {
mbtiles._db.run(
'INSERT OR REPLACE INTO tiles (zoom_level, tile_column, tile_row, tile_data) VALUES(?, ?, ?, ?)',
row.z,
row.x,
row.y,
row.piped,
callback
);
}
});
};
for (var i = 0; i < rows.length; i++) {
exec(rows[i], group());
}
},
function(err) {
var progress = Math.floor(offset / total * 40);
if (progress > printed) {
sys.print((new Array(progress - printed + 1)).join('#'));
printed = progress;
}
if (rows.length === limit) {
doit(table, limit, offset + limit);
} else {
mbtiles._db.run('VACUUM', function() {
sys.print('\n');
console.warn('Pipe complete.');
done();
});
}
}
);
});
query.finalize();
});
};
console.warn('00 -------------- 50 -------------- 100');
doit(table, 255, 0);
}
);

139
bin/mbrekey
View File

@@ -1,139 +0,0 @@
#!/usr/bin/env node
var options = argv = require('optimist').argv;
var _ = require('underscore');
var sqlite3 = require('sqlite3');
var utils = require('..').utils;
// Show help.
if (argv.help || argv._.length < 1) {
console.warn('Usage: mbrekey [FILE]');
console.warn(' Rekey a compacted mbtiles to save space.');
console.warn('Examples:');
console.warn(' mbrekey world.mbtiles');
process.exit();
}
// Grab args.
var filename = argv._[0];
var db = new sqlite3.Database(filename);
db.exec("PRAGMA journal_mode = OFF; PRAGMA locking_mode = EXCLUSIVE; PRAGMA cache_size = 400000; PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;");
var tables = {};
db.all("SELECT tbl_name FROM sqlite_master WHERE TYPE = 'table'", function(err, map) {
if (err) throw err;
map.forEach(function(row) {
tables[row.tbl_name] = true;
});
});
var tileQueue = new utils.Queue(changeTileID, 1);
var gridQueue = new utils.Queue(changeGridID, 1);
db.run('CREATE INDEX IF NOT EXISTS "temp_tile_id_idx" ON "map" ("tile_id")', function(err) {
if (err) throw err;
console.warn('Created temporary index.');
db.run('CREATE TEMP TABLE "tile_hash_id" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, "hash" TEXT UNIQUE NOT NULL)', function(err) {
if (err) throw err;
db.run('REPLACE INTO tile_hash_id (hash) SELECT tile_id FROM images', function(err) {
if (err) throw err;
db.get('SELECT MAX(id) AS max FROM tile_hash_id', function(err, max) {
if (err) throw err;
if (!max.max) return tileQueue.emit('empty');
max = max.max;
console.warn('Starting tile update... (%d total)', max);
for (var i = 1; i < max; i += 10000) {
tileQueue.add(i);
}
});
});
});
})
tileQueue.on('empty', function() {
if (!tables.grid_utfgrid) {
deleteTempKey();
console.warn("MBtiles file doesn't have interactivity");
return;
}
db.run('CREATE TEMP TABLE "grid_hash_id" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, "hash" TEXT UNIQUE NOT NULL)', function(err) {
if (err) throw err;
db.run('REPLACE INTO grid_hash_id (hash) SELECT grid_id FROM grid_utfgrid', function(err) {
if (err) throw err;
db.get('SELECT MAX(id) AS max FROM grid_hash_id', function(err, max) {
if (err) throw err;
if (!max.max) return gridQueue.emit('empty');
max = max.max;
console.warn('Starting grid update... (%d total)', max);
for (var i = 1; i < max; i += 10000) {
gridQueue.add(i);
}
});
});
});
});
gridQueue.on('empty', deleteTempKey);
var changedTiles = 0;
function changeTileID(start, done) {
db.all('SELECT id, hash FROM tile_hash_id WHERE id >= ? AND id < ?', start, start + 10000, function(err, rows) {
if (err) throw err;
db.serialize(function() {
db.run('BEGIN');
var st1 = db.prepare('UPDATE images SET tile_id = ? WHERE tile_id = ?');
var st2 = db.prepare('UPDATE map SET tile_id = ? WHERE tile_id = ?');
for (var i = 0; i < rows.length; i++) {
st1.run(rows[i].id, rows[i].hash);
st2.run(rows[i].id, rows[i].hash);
}
db.run('COMMIT', function(err) {
if (err) throw err;
st1.finalize(function(err) {
st2.finalize(function(err) {
changedTiles += rows.length;
console.warn('Updated %d tiles.', changedTiles);
done();
})});
});
});
});
};
var changedGrids = 0;
function changeGridID(start, done) {
db.all('SELECT id, hash FROM grid_hash_id WHERE id >= ? AND id < ?', start, start + 10000, function(err, rows) {
if (err) throw err;
db.serialize(function() {
db.run('BEGIN');
rows.forEach(function(row) {
db.run('UPDATE grid_key SET grid_id = ? WHERE grid_id = ?', row.id, row.hash);
db.run('UPDATE grid_utfgrid SET grid_id = ? WHERE grid_id = ?', row.id, row.hash);
db.run('UPDATE map SET grid_id = ? WHERE grid_id = ?', row.id, row.hash);
});
db.run('COMMIT', function(err) {
if (err) throw err;
changedGrids += rows.length;
console.warn('Updated %d grids.', changedGrids);
done();
});
});
});
};
function deleteTempKey() {
db.run('DROP INDEX "temp_tile_id_idx"', function(err) {
if (err) throw err;
console.warn('Vacuuming...');
db.run('VACUUM;', function(err) {
if (err) throw err;
db.close();
});
});
}

7
package.json
View File

@@ -11,12 +11,6 @@
"keywords": ["map", "mbtiles"],
"licenses": [{ "type": "BSD" }],
"main": "./lib/mbtiles",
"bin": {
"mbcheck": "./bin/mbcheck",
"mbcompact": "./bin/mbcompact",
"mbpipe": "./bin/mbpipe",
"mbrekey": "./bin/mbrekey"
},
"repository": {
"type": "git",
"url": "http://github.com/mapbox/node-mbtiles.git"
@@ -29,7 +23,6 @@
"Konstantin Käfer <kkaefer>"
],
"dependencies": {
"optimist": "~0.3.1",
"sqlite3": "~2.1.1",
"sphericalmercator": "~1.0.1"
},