Add support for geocoder api (carmen dev).

lib/mbtiles.js

@@ -56,6 +56,7 @@ function MBTiles(uri, callback) {
             if (err) return callback(err);
             mbtiles._stat = stat;
             mbtiles.open = true;
+            mbtiles.emit('open', err);
             callback(null, mbtiles);
         });
     });
@@ -562,3 +563,82 @@ MBTiles.prototype.putInfo = function(data, callback) {
     });
 };
 
+// Implements carmen#getGeocoderData method.
+MBTiles.prototype.getGeocoderData = function(type, shard, callback) {
+    return this._db.get('SELECT data FROM geocoder_data WHERE type = ? AND shard = ?', type, shard, function(err, row) {
+        if (err) return callback(err);
+        if (!row) return callback();
+        zlib.inflate(row.data, callback);
+    });
+};
+
+// Implements carmen#putGeocoderData method.
+MBTiles.prototype.putGeocoderData = function(type, shard, data, callback) {
+    var source = this;
+    zlib.deflate(data, function(err, zdata) {
+        if (err) return callback(err);
+        source.write('geocoder_data', type + '.' + shard, { type:type, shard: shard, data: zdata }, callback);
+    });
+};
+
+// Implements carmen#getIndexableDocs method.
+MBTiles.prototype.getIndexableDocs = function(pointer, callback) {
+    pointer = pointer || {};
+    pointer.limit = pointer.limit || 10000;
+    pointer.offset = pointer.offset || 0;
+    pointer.nogrids = 'nogrids' in pointer ? pointer.nogrids : false;
+
+    // Converts MBTiles native TMS coords to ZXY.
+    function tms2zxy(zxys) {
+        return zxys.split(',').map(function(tms) {
+            var zxy = tms.split('/').map(function(v) { return parseInt(v, 10); });
+            zxy[2] = (1 << zxy[0]) - 1 - zxy[2];
+            return zxy.join('/');
+        });
+    }
+
+    // If 'carmen' option is passed in initial pointer, retrieve indexables from
+    // carmen table. This option can be used to access the previously indexed
+    // documents from an MBTiles database without having to know what search
+    // field was used in the past (see comment below).
+    if (pointer.table === 'carmen') {
+        return this._db.all('SELECT c.id AS id, c.text AS text, c.zxy, k.key_json FROM carmen c JOIN keymap k ON c.id = k.key_name LIMIT ? OFFSET ?', pointer.limit, pointer.offset, function(err, rows) {
+            if (err) return callback(err);
+            var docs = rows.map(function(row) {
+                var doc = JSON.parse(row.key_json);
+                doc._id = row.id;
+                doc._text = row.text;
+                if (row.zxy) doc._zxy = tms2zxy(row.zxy);
+                return doc;
+            });
+            pointer.offset += pointer.limit;
+            return callback(null, docs, pointer);
+        }.bind(this));
+    }
+
+    // By default the keymap table contains all indexable documents.
+    this.getInfo(function(err, info) {
+        if (err) return callback(err);
+        var sql, args;
+        if (pointer.nogrids) {
+            sql = "SELECT key_name, key_json FROM keymap LIMIT ? OFFSET ?;";
+            args = [pointer.limit, pointer.offset];
+        } else {
+            sql = "SELECT k.key_name AS id, k.key_json, GROUP_CONCAT(zoom_level||'/'||tile_column ||'/'||tile_row,',') AS zxy FROM keymap k JOIN grid_key g ON k.key_name = g.key_name JOIN map m ON g.grid_id = m.grid_id WHERE m.zoom_level=? GROUP BY k.key_name LIMIT ? OFFSET ?;";
+            args = [info.maxzoom, pointer.limit, pointer.offset];
+        }
+        this._db.all(sql, args, function(err, rows) {
+            if (err) return callback(err);
+            var docs = rows.map(function(row) {
+                var doc = JSON.parse(row.key_json);
+                doc._id = row.id;
+                doc._text = doc.search || doc.name || '';
+                if (row.zxy) doc._zxy = tms2zxy(row.zxy);
+                return doc;
+            });
+            pointer.offset += pointer.limit;
+            return callback(null, docs, pointer);
+        }.bind(this));
+    }.bind(this));
+};
+

lib/schema.sql

@@ -35,6 +35,11 @@ CREATE TABLE IF NOT EXISTS metadata (
     value text
 );
 
+CREATE TABLE IF NOT EXISTS geocoder_data (
+    type TEXT,
+    shard INTEGER,
+    data BLOB
+);
 
 CREATE UNIQUE INDEX IF NOT EXISTS map_index ON map (zoom_level, tile_column, tile_row);
 CREATE UNIQUE INDEX IF NOT EXISTS grid_key_lookup ON grid_key (grid_id, key_name);
@@ -42,7 +47,9 @@ CREATE UNIQUE INDEX IF NOT EXISTS keymap_lookup ON keymap (key_name);
 CREATE UNIQUE INDEX IF NOT EXISTS grid_utfgrid_lookup ON grid_utfgrid (grid_id);
 CREATE UNIQUE INDEX IF NOT EXISTS images_id ON images (tile_id);
 CREATE UNIQUE INDEX IF NOT EXISTS name ON metadata (name);
-
+CREATE INDEX IF NOT EXISTS map_grid_id ON map (grid_id);
+CREATE INDEX IF NOT EXISTS geocoder_type_index ON geocoder_data (type);
+CREATE UNIQUE INDEX IF NOT EXISTS geocoder_shard_index ON geocoder_data (type, shard);
 
 CREATE VIEW IF NOT EXISTS tiles AS
     SELECT

test/geocoder.test.js

@@ -0,0 +1,92 @@
+var fs = require('fs');
+var assert = require('assert');
+var util = require('util');
+var MBTiles = require('..');
+
+describe('geocoder (carmen) API', function() {
+
+var expected = {
+    bounds: '-141.005548666451,41.6690855919108,-52.615930948992,83.1161164353916',
+    lat: 56.8354595949484,
+    lon: -110.424643384994,
+    name: 'Canada',
+    population: 33487208,
+    search: 'Canada, CA'
+};
+
+var tmp = '/tmp/mbtiles-test-' + (+new Date).toString(16);
+var index;
+var from;
+var to;
+
+before(function() {
+    try { fs.mkdirSync(tmp); } catch(err) { throw err; }
+});
+before(function(done) {
+    index = new MBTiles(__dirname + '/fixtures/geocoder_data.mbtiles', done);
+});
+before(function(done) {
+    from = new MBTiles(__dirname + '/fixtures/plain_4.mbtiles', done);
+});
+before(function(done) {
+    to = new MBTiles(tmp + '/indexed.mbtiles', done);
+});
+
+after(function(done) {
+    this.timeout(5000);
+    index.close(function(err) {
+        if (err) throw err;
+        from.close(function(err) {
+            if (err) throw err;
+            to.close(function(err) {
+                if (err) throw err;
+                try { fs.unlinkSync(tmp + '/indexed.mbtiles'); } catch(err) { throw err; }
+                try { fs.rmdirSync(tmp); } catch(err) { throw err; }
+                done();
+            });
+        });
+    });
+});
+
+it('getGeocoderData', function(done) {
+    index.getGeocoderData('term', 0, function(err, buffer) {
+        assert.ifError(err);
+        assert.equal(3891, buffer.length);
+        done();
+    });
+});
+
+it('putGeocoderData', function(done) {
+    this.timeout(5000);
+    to.startWriting(function(err) {
+        assert.ifError(err);
+        to.putGeocoderData('term', 0, new Buffer('asdf'), function(err) {
+            assert.ifError(err);
+            to.stopWriting(function(err) {
+                assert.ifError(err);
+                to.getGeocoderData('term', 0, function(err, buffer) {
+                    assert.ifError(err);
+                    assert.deepEqual('asdf', buffer.toString());
+                    done();
+                });
+            });
+        });
+    });
+});
+
+it('getIndexableDocs', function(done) {
+    from.getIndexableDocs({ limit: 10 }, function(err, docs, pointer) {
+        assert.ifError(err);
+        assert.equal(docs.length, 10);
+        assert.deepEqual(pointer, { limit: 10, offset: 10, nogrids: false });
+        from.getIndexableDocs(pointer, function(err, docs, pointer) {
+            assert.ifError(err);
+            assert.equal(docs.length, 10);
+            assert.deepEqual(pointer, { limit: 10, offset: 20, nogrids: false });
+            done();
+        });
+    });
+});
+
+});
+