From a98037dbdfd19f6cf5a72737cf338654686f734e Mon Sep 17 00:00:00 2001 From: Young Hahn Date: Fri, 12 Jul 2013 05:54:50 -0400 Subject: [PATCH] Remove commands. --- bin/mbcheck | 158 -------------------------------------------------- bin/mbcompact | 145 --------------------------------------------- bin/mbpipe | 145 --------------------------------------------- bin/mbrekey | 139 -------------------------------------------- package.json | 7 --- 5 files changed, 594 deletions(-) delete mode 100755 bin/mbcheck delete mode 100755 bin/mbcompact delete mode 100755 bin/mbpipe delete mode 100755 bin/mbrekey diff --git a/bin/mbcheck b/bin/mbcheck deleted file mode 100755 index f8d5966..0000000 --- a/bin/mbcheck +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/env node - -var options = argv = require('optimist').argv, - _ = require('underscore'), - Step = require('step'), - utils = require('..').utils, - MBTiles = require('..'), - mbtiles, - zooms = []; - -// Show help. -if (argv.help || argv._.length === 0) { - console.warn('Usage: mbcheck [FILE]'); - console.warn(' Check an mbtiles for missing metadata and tiles.'); - console.warn('Examples:'); - console.warn(' mbcheck world.mbtiles'); - process.exit(); -} - -// Wishlist: -// --------- -// - check that tiles exist for specified bounds - -Step( - function() { mbtiles = new MBTiles(argv._[0], this); }, - function() { mbtiles._integrity(this); }, - function(err) { - if (err) throw err; - mbtiles._exists('tiles', function(err, exists) { - if (!exists) throw new Error('`tiles` table does not exist.'); - this(); - }.bind(this)); - }, - function(err) { - if (err) throw err; - mbtiles._exists('metadata', function(err, exists) { - if (!exists) throw new Error('`metadata` table does not exist.'); - this(); - }.bind(this)); - }, - function(err) { - if (err) throw err; - mbtiles._metadata('name', this.parallel()); - mbtiles._metadata('type', this.parallel()); - mbtiles._metadata('desciption', this.parallel()); - mbtiles._metadata('version', this.parallel()); - mbtiles._metadata('format', this.parallel()); - mbtiles._metadata('bounds', this.parallel()); - }, - function(err, name, type, description, version, format, bounds) { - console.warn('Metadata'); - utils.table([ - ['name', name], - ['type', type], - ['description', description], - ['version', version], - ['format', format], - ['bounds', bounds] - ]); - this(); - }, - function(err) { - if (err) throw err; - Step( - function() { - var group = this.group(), - query = mbtiles._db.prepare( - 'SELECT zoom_level AS zoom, ' + - 'COUNT(zoom_level) AS count ' + - 'FROM tiles ' + - 'WHERE zoom_level = ?' - ); - for (var i = 0; i < 30; i++) { - query.get(i, group()); - } - query.finalize(); - }, - function(err, rows) { - if (err) throw err; - zooms = rows; - do { - var last = zooms.pop(); - if (last.count) zooms.push(last); - } while(!last.count); - do { - var first = zooms.shift(); - if (first.count) zooms.unshift(first); - } while(!first.count); - this(); - }.bind(this) - ); - }, - function(err) { - if (err) throw err; - Step( - function() { - var group = this.group(), - query = mbtiles._db.prepare( - 'SELECT MAX(tile_column) AS maxx, ' + - 'MIN(tile_column) AS minx, ' + - 'MAX(tile_row) AS maxy, ' + - 'MIN(tile_row) AS miny, ' + - 'zoom_level AS zoom ' + - 'FROM tiles ' + - 'WHERE zoom_level = ?' - ); - for (var i = 0; i < zooms.length; i++) { - query.get(zooms[i].zoom, group()); - } - query.finalize(); - }, - function(err, rows) { - if (err) throw err; - - var group = this.group(), - query = mbtiles._db.prepare( - 'SELECT zoom_level AS zoom, ' + - 'COUNT(zoom_level) AS count, ' + - '? AS expected ' + - 'FROM tiles ' + - 'WHERE zoom_level = ? AND ' + - 'tile_column >= ? AND ' + - 'tile_column <= ? AND ' + - 'tile_row >= ? AND ' + - 'tile_row <= ?' - ); - for (var i = 0; i < rows.length; i++) { - var expected = (rows[i].maxx - rows[i].minx + 1) * (rows[i].maxy - rows[i].miny + 1); - query.get( - expected, - rows[i].zoom, - rows[i].minx, - rows[i].maxx, - rows[i].miny, - rows[i].maxy, - group() - ); - } - query.finalize(); - }, - function(err, rows) { - if (err) throw err; - console.warn(''); - var output = []; - output.push(['Z', 'Tiles', 'Missing']); - _(rows).each(function(row, index) { - output.push([ - row.zoom, - zooms[index].count, - row.expected - row.count - ]); - }); - utils.table(output); - this(); - }.bind(this) - ); - } -); diff --git a/bin/mbcompact b/bin/mbcompact deleted file mode 100755 index 0bd8f9f..0000000 --- a/bin/mbcompact +++ /dev/null @@ -1,145 +0,0 @@ -#!/usr/bin/env node - -var options = argv = require('optimist').argv, - _ = require('underscore'), - Step = require('step'), - crypto = require('crypto'), - fs = require('fs'), - sys = require('sys'), - spawn = require('child_process').spawn, - MBTiles = require('..'), - utils = require('..').utils, - mbtiles, - hits = 0, - ids = []; - -// Show help. -if (argv.help || argv._.length < 1) { - console.warn('Usage: mbcompact [FILE]'); - console.warn(' Eliminate duplicate images to reduce mbtiles filesize.'); - console.warn('Examples:'); - console.warn(' mbcompact world.mbtiles'); - process.exit(); -} - -// Grab args. -var filename = argv._[0]; -Step( - function() { mbtiles = new MBTiles(filename, this); }, - function(err) { - if (err) throw err; - mbtiles._db.all('SELECT name, type ' - + 'FROM sqlite_master ' - + 'WHERE type IN (?, ?)', - 'table', - 'view', - function(err, rows) { - if (err) throw err; - if (_(rows).any(function(row) { return row.name === 'tiles' && row.type === 'view' })) { - throw new Error('Table is already compacted.'); - } - if (!_(rows).any(function(row) { return row.name === 'tiles' && row.type === 'table' })) { - throw new Error('Tiles table does not exist.'); - } - this(); - }.bind(this) - ); - }, - function(err) { - if (err) throw err; - mbtiles._setup(this); - }, - function(err) { - if (err) throw err; - mbtiles._db.get('SELECT COUNT(*) AS total FROM tiles', this.parallel()); - mbtiles._db.run('PRAGMA locking_mode=EXCLUSIVE', this.parallel()); - mbtiles._db.run('PRAGMA journal_mode=TRUNCATE', this.parallel()); - }, - function(err, row) { - if (err) throw err; - if (!row.total) throw new Error('No tiles found'); - - var total = row.total; - var printed = 0; - var done = this; - var doit = function(limit, offset) { - process.nextTick(function() { - mbtiles._db - .prepare('SELECT tile_data AS tile_data, zoom_level AS z, tile_column AS x, tile_row AS y FROM tiles LIMIT ? OFFSET ?') - .all(limit, offset, function(err, rows) { - var images = []; - var map = []; - for (var i = 0; i < rows.length; i++) { - var tile_id = crypto - .createHash('md5') - .update(rows[i].tile_data) - .digest('hex'); - if (!_(ids).include(tile_id)) { - ids.unshift(tile_id); - images.push({ - tile_id: tile_id, - tile_data: rows[i].tile_data - }); - } else { - hits++; - } - map.push({ - tile_id: tile_id, - zoom_level: rows[i].z, - tile_column: rows[i].x, - tile_row: rows[i].y - }); - } - Step( - function() { - mbtiles._insert('images', images, this.parallel()); - mbtiles._insert('map', map, this.parallel()); - }, - function(err) { - if (err) throw err; - - // If IDs has grown over threshold, trim back down. - (ids.length > 1000) && (ids = ids.slice(0, 1000 - 900)); - - var progress = Math.floor(offset / total * 40); - if (progress > printed) { - sys.print((new Array(progress - printed + 1)).join('#')); - printed = progress; - } - if (rows.length === limit) { - doit(limit, offset + limit); - } else { - done(); - } - } - ); - }) - .finalize(); - }); - }; - console.warn('00 -------------- 50 -------------- 100'); - sys.print(''); - doit(1000, 0); - }, - function(err) { - if (err) throw err; - mbtiles._db.run('DROP TABLE tiles', this); - }, - function(err) { - if (err) throw err; - mbtiles._db.run('CREATE VIEW IF NOT EXISTS tiles AS ' - + 'SELECT map.zoom_level AS zoom_level, ' - + 'map.tile_column AS tile_column, ' - + 'map.tile_row AS tile_row, ' - + 'images.tile_data AS tile_data ' - + 'FROM map JOIN images ON images.tile_id = map.tile_id;', this.parallel()); - mbtiles._db.run('VACUUM', this.parallel()); - }, - function(err) { - if (err) throw err; - sys.print('\n'); - console.warn('Compact hits %s.', hits); - console.warn('Compaction complete.'); - } -); - diff --git a/bin/mbpipe b/bin/mbpipe deleted file mode 100755 index f06d8b2..0000000 --- a/bin/mbpipe +++ /dev/null @@ -1,145 +0,0 @@ -#!/usr/bin/env node - -var options = argv = require('optimist').argv, - _ = require('underscore'), - Step = require('step'), - sys = require('sys'), - spawn = require('child_process').spawn, - MBTiles = require('..'), - utils = require('..').utils, - mbtiles, - table, - total; - -// @TODO: support specifying a table to be processed, e.g. for processing -// grids or grid data. - -// Show help. -if (argv.help || argv._.length < 2) { - console.warn('Usage: mbpipe [COMMAND] [FILE]'); - console.warn(' Pipe each tile to stdin of the specified command and'); - console.warn(' write stdout into the mbtiles file.'); - console.warn('Examples:'); - console.warn(' mbpipe "pngquant 32" world.mbtiles'); - process.exit(); -} - -// Grab args. -var filename = argv._[1], - command = { - command: argv._[0].split(' ')[0], - args: argv._[0].split(' ').slice(1) - }; - -Step( - function() { mbtiles = new MBTiles(filename, this); }, - function(err) { - if (err) throw err; - Step( - function() { mbtiles._db.run('PRAGMA synchronous=0', this); }, - function() { mbtiles._db.run('PRAGMA locking_mode=EXCLUSIVE', this); }, - function() { mbtiles._db.run('PRAGMA journal_mode=TRUNCATE', this); }, - this - ); - }, - function(err) { - if (err) throw err; - mbtiles._exists('tiles', function(err, exists) { - if (exists) table = 'tiles'; - this(); - }.bind(this)); - }, - function(err) { - if (err) throw err; - mbtiles._exists('images', function(err, exists) { - if (exists) table = 'images'; - this(); - }.bind(this)); - }, - function(err) { - if (err) throw err; - if (!table) throw new Error('No usable image table found.'); - mbtiles._db.get('SELECT COUNT(tile_data) AS total FROM ' + table, function(err, row) { - total = row.total; - this(); - }.bind(this)); - }, - function(err) { - if (err) throw err; - if (!total) throw new Error('No tiles found'); - - var printed = 0; - var done = this; - var doit = function(table, limit, offset) { - process.nextTick(function() { - var query = table === 'images' - ? mbtiles._db.prepare('SELECT tile_data AS tile_data, tile_id AS id FROM images ORDER BY tile_id ASC LIMIT ? OFFSET ?') - : mbtiles._db.prepare('SELECT tile_data AS tile_data, zoom_level AS z, tile_column AS x, tile_row AS y FROM tiles ORDER BY z, x, y LIMIT ? OFFSET ?'); - query.all(limit, offset, function(err, rows) { - Step( - function() { - var group = this.group(); - var exec = function(row, callback) { - row.piped = new Buffer(0); - var child = spawn(command.command, command.args); - var stream = function(chunk) { - var joined = (new Buffer(row.piped.length + chunk.length)); - row.piped.copy(joined, 0, 0); - chunk.copy(joined, row.piped.length, 0); - row.piped = joined; - }; - child.stdin.write(row.tile_data); - child.stdin.end(); - child.stdout.on('data', stream); - child.stderr.on('data', stream); - child.on('exit', function(code) { - if (code) return callback(null, null); - if (table === 'images') { - mbtiles._db.run( - 'INSERT OR REPLACE INTO images (tile_id, tile_data) VALUES(?, ?)', - row.id, - row.piped, - callback - ); - } else { - mbtiles._db.run( - 'INSERT OR REPLACE INTO tiles (zoom_level, tile_column, tile_row, tile_data) VALUES(?, ?, ?, ?)', - row.z, - row.x, - row.y, - row.piped, - callback - ); - } - }); - }; - for (var i = 0; i < rows.length; i++) { - exec(rows[i], group()); - } - }, - function(err) { - var progress = Math.floor(offset / total * 40); - if (progress > printed) { - sys.print((new Array(progress - printed + 1)).join('#')); - printed = progress; - } - if (rows.length === limit) { - doit(table, limit, offset + limit); - } else { - mbtiles._db.run('VACUUM', function() { - sys.print('\n'); - console.warn('Pipe complete.'); - done(); - }); - } - } - ); - }); - query.finalize(); - }); - }; - console.warn('00 -------------- 50 -------------- 100'); - doit(table, 255, 0); - } -); - diff --git a/bin/mbrekey b/bin/mbrekey deleted file mode 100755 index 6e5ee3f..0000000 --- a/bin/mbrekey +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env node - -var options = argv = require('optimist').argv; -var _ = require('underscore'); -var sqlite3 = require('sqlite3'); -var utils = require('..').utils; - -// Show help. -if (argv.help || argv._.length < 1) { - console.warn('Usage: mbrekey [FILE]'); - console.warn(' Rekey a compacted mbtiles to save space.'); - console.warn('Examples:'); - console.warn(' mbrekey world.mbtiles'); - process.exit(); -} - -// Grab args. -var filename = argv._[0]; - -var db = new sqlite3.Database(filename); - -db.exec("PRAGMA journal_mode = OFF; PRAGMA locking_mode = EXCLUSIVE; PRAGMA cache_size = 400000; PRAGMA synchronous = OFF; PRAGMA temp_store = MEMORY;"); - -var tables = {}; -db.all("SELECT tbl_name FROM sqlite_master WHERE TYPE = 'table'", function(err, map) { - if (err) throw err; - map.forEach(function(row) { - tables[row.tbl_name] = true; - }); -}); - -var tileQueue = new utils.Queue(changeTileID, 1); -var gridQueue = new utils.Queue(changeGridID, 1); - -db.run('CREATE INDEX IF NOT EXISTS "temp_tile_id_idx" ON "map" ("tile_id")', function(err) { - if (err) throw err; - console.warn('Created temporary index.'); - db.run('CREATE TEMP TABLE "tile_hash_id" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, "hash" TEXT UNIQUE NOT NULL)', function(err) { - if (err) throw err; - db.run('REPLACE INTO tile_hash_id (hash) SELECT tile_id FROM images', function(err) { - if (err) throw err; - db.get('SELECT MAX(id) AS max FROM tile_hash_id', function(err, max) { - if (err) throw err; - if (!max.max) return tileQueue.emit('empty'); - max = max.max; - console.warn('Starting tile update... (%d total)', max); - - for (var i = 1; i < max; i += 10000) { - tileQueue.add(i); - } - }); - }); - }); -}) - -tileQueue.on('empty', function() { - if (!tables.grid_utfgrid) { - deleteTempKey(); - console.warn("MBtiles file doesn't have interactivity"); - return; - } - - db.run('CREATE TEMP TABLE "grid_hash_id" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, "hash" TEXT UNIQUE NOT NULL)', function(err) { - if (err) throw err; - db.run('REPLACE INTO grid_hash_id (hash) SELECT grid_id FROM grid_utfgrid', function(err) { - if (err) throw err; - db.get('SELECT MAX(id) AS max FROM grid_hash_id', function(err, max) { - if (err) throw err; - if (!max.max) return gridQueue.emit('empty'); - max = max.max; - console.warn('Starting grid update... (%d total)', max); - for (var i = 1; i < max; i += 10000) { - gridQueue.add(i); - } - }); - }); - }); -}); - -gridQueue.on('empty', deleteTempKey); - -var changedTiles = 0; - -function changeTileID(start, done) { - db.all('SELECT id, hash FROM tile_hash_id WHERE id >= ? AND id < ?', start, start + 10000, function(err, rows) { - if (err) throw err; - db.serialize(function() { - db.run('BEGIN'); - var st1 = db.prepare('UPDATE images SET tile_id = ? WHERE tile_id = ?'); - var st2 = db.prepare('UPDATE map SET tile_id = ? WHERE tile_id = ?'); - for (var i = 0; i < rows.length; i++) { - st1.run(rows[i].id, rows[i].hash); - st2.run(rows[i].id, rows[i].hash); - } - db.run('COMMIT', function(err) { - if (err) throw err; - st1.finalize(function(err) { - st2.finalize(function(err) { - changedTiles += rows.length; - console.warn('Updated %d tiles.', changedTiles); - done(); - })}); - }); - }); - }); -}; - - -var changedGrids = 0; -function changeGridID(start, done) { - db.all('SELECT id, hash FROM grid_hash_id WHERE id >= ? AND id < ?', start, start + 10000, function(err, rows) { - if (err) throw err; - db.serialize(function() { - db.run('BEGIN'); - rows.forEach(function(row) { - db.run('UPDATE grid_key SET grid_id = ? WHERE grid_id = ?', row.id, row.hash); - db.run('UPDATE grid_utfgrid SET grid_id = ? WHERE grid_id = ?', row.id, row.hash); - db.run('UPDATE map SET grid_id = ? WHERE grid_id = ?', row.id, row.hash); - }); - db.run('COMMIT', function(err) { - if (err) throw err; - changedGrids += rows.length; - console.warn('Updated %d grids.', changedGrids); - done(); - }); - }); - }); -}; - -function deleteTempKey() { - db.run('DROP INDEX "temp_tile_id_idx"', function(err) { - if (err) throw err; - console.warn('Vacuuming...'); - db.run('VACUUM;', function(err) { - if (err) throw err; - db.close(); - }); - }); -} diff --git a/package.json b/package.json index ba2f369..221b7ed 100644 --- a/package.json +++ b/package.json @@ -11,12 +11,6 @@ "keywords": ["map", "mbtiles"], "licenses": [{ "type": "BSD" }], "main": "./lib/mbtiles", - "bin": { - "mbcheck": "./bin/mbcheck", - "mbcompact": "./bin/mbcompact", - "mbpipe": "./bin/mbpipe", - "mbrekey": "./bin/mbrekey" - }, "repository": { "type": "git", "url": "http://github.com/mapbox/node-mbtiles.git" @@ -29,7 +23,6 @@ "Konstantin Käfer " ], "dependencies": { - "optimist": "~0.3.1", "sqlite3": "~2.1.1", "sphericalmercator": "~1.0.1" },