Parse PBF directly in ol.format.MVT

This change also fixes a previous issue, where MultiPolygon geometries for
ol.Feature features were never created.
This commit is contained in:
Andreas Hocevar
2017-08-28 19:05:10 +02:00
parent 4b4f383043
commit 3cb5060497
5 changed files with 364 additions and 174 deletions

View File

@@ -224,3 +224,7 @@ At least 2 conditions are required.
### 58
Duplicate item added to a unique collection. For example, it may be that you tried to add the same layer to a map twice. Check for calls to `map.addLayer()` or other places where the map's layer collection is modified.
### 59
Invalid command found in the PBF. This indicates that the loaded vector tile may be corrupt.

View File

@@ -50,7 +50,6 @@
"rollup-plugin-commonjs": "^8.0.2",
"rollup-plugin-node-resolve": "^3.0.0",
"temp": "0.8.3",
"@mapbox/vector-tile": "1.3.0",
"walk": "2.3.9"
},
"devDependencies": {
@@ -148,11 +147,6 @@
{
"module": "pixelworks",
"import": "Processor"
},
{
"module": "@mapbox/vector-tile",
"name": "vectortile",
"import": "VectorTile"
}
]
}

View File

@@ -3,8 +3,8 @@
goog.provide('ol.format.MVT');
goog.require('ol');
goog.require('ol.asserts');
goog.require('ol.ext.PBF');
goog.require('ol.ext.vectortile.VectorTile');
goog.require('ol.format.Feature');
goog.require('ol.format.FormatType');
goog.require('ol.geom.GeometryLayout');
@@ -12,8 +12,10 @@ goog.require('ol.geom.GeometryType');
goog.require('ol.geom.LineString');
goog.require('ol.geom.MultiLineString');
goog.require('ol.geom.MultiPoint');
goog.require('ol.geom.MultiPolygon');
goog.require('ol.geom.Point');
goog.require('ol.geom.Polygon');
goog.require('ol.geom.flat.orient');
goog.require('ol.proj.Projection');
goog.require('ol.proj.Units');
goog.require('ol.render.Feature');
@@ -79,6 +81,256 @@ ol.format.MVT = function(opt_options) {
ol.inherits(ol.format.MVT, ol.format.Feature);
/**
* Reader callbacks for parsing the PBF.
* @type {Object.<string, function(number, Object, ol.ext.PBF)>}
*/
ol.format.MVT.pbfReaders_ = {
layers: function(tag, layers, pbf) {
if (tag === 3) {
var layer = {
keys: [],
values: [],
features: []
};
var end = pbf.readVarint() + pbf.pos;
pbf.readFields(ol.format.MVT.pbfReaders_.layer, layer, end);
layer.length = layer.features.length;
if (layer.length) {
layers[layer.name] = layer;
}
}
},
layer: function(tag, layer, pbf) {
if (tag === 15) {
layer.version = pbf.readVarint();
} else if (tag === 1) {
layer.name = pbf.readString();
} else if (tag === 5) {
layer.extent = pbf.readVarint();
} else if (tag === 2) {
layer.features.push(pbf.pos);
} else if (tag === 3) {
layer.keys.push(pbf.readString());
} else if (tag === 4) {
var value = null;
var end = pbf.readVarint() + pbf.pos;
while (pbf.pos < end) {
tag = pbf.readVarint() >> 3;
value = tag === 1 ? pbf.readString() :
tag === 2 ? pbf.readFloat() :
tag === 3 ? pbf.readDouble() :
tag === 4 ? pbf.readVarint64() :
tag === 5 ? pbf.readVarint() :
tag === 6 ? pbf.readSVarint() :
tag === 7 ? pbf.readBoolean() : null;
}
layer.values.push(value);
}
},
feature: function(tag, feature, pbf) {
if (tag == 1) {
feature.id = pbf.readVarint();
} else if (tag == 2) {
var end = pbf.readVarint() + pbf.pos;
while (pbf.pos < end) {
var key = feature.layer.keys[pbf.readVarint()];
var value = feature.layer.values[pbf.readVarint()];
feature.properties[key] = value;
}
} else if (tag == 3) {
feature.type = pbf.readVarint();
} else if (tag == 4) {
feature.geometry = pbf.pos;
}
}
};
/**
* Read a raw feature from the pbf offset stored at index `i` in the raw layer.
* @suppress {missingProperties}
* @private
* @param {ol.ext.PBF} pbf PBF.
* @param {Object} layer Raw layer.
* @param {number} i Index of the feature in the raw layer's `features` array.
* @return {Object} Raw feature.
*/
ol.format.MVT.readRawFeature_ = function(pbf, layer, i) {
pbf.pos = layer.features[i];
var end = pbf.readVarint() + pbf.pos;
var feature = {
layer: layer,
type: 0,
properties: {}
};
pbf.readFields(ol.format.MVT.pbfReaders_.feature, feature, end);
return feature;
};
/**
* Read the raw geometry from the pbf offset stored in a raw feature's geometry
* proeprty.
* @suppress {missingProperties}
* @private
* @param {ol.ext.PBF} pbf PBF.
* @param {Object} feature Raw feature.
* @param {Array.<number>} flatCoordinates Array to store flat coordinates in.
* @param {Array.<number>} ends Array to store ends in.
*/
ol.format.MVT.readRawGeometry_ = function(pbf, feature, flatCoordinates, ends) {
pbf.pos = feature.geometry;
var end = pbf.readVarint() + pbf.pos;
var cmd = 1;
var length = 0;
var x = 0;
var y = 0;
var coordsLen = 0;
var currentEnd = 0;
while (pbf.pos < end) {
if (!length) {
var cmdLen = pbf.readVarint();
cmd = cmdLen & 0x7;
length = cmdLen >> 3;
}
length--;
if (cmd === 1 || cmd === 2) {
x += pbf.readSVarint();
y += pbf.readSVarint();
if (cmd === 1) { // moveTo
if (coordsLen > currentEnd) {
ends.push(coordsLen);
currentEnd = coordsLen;
}
}
flatCoordinates.push(x, y);
coordsLen += 2;
} else if (cmd === 7) {
if (coordsLen > currentEnd) {
// close polygon
flatCoordinates.push(
flatCoordinates[currentEnd], flatCoordinates[currentEnd + 1]);
coordsLen += 2;
}
} else {
ol.asserts.assert(false, 59); // Invalid command found in the PBF
}
}
if (coordsLen > currentEnd) {
ends.push(coordsLen);
currentEnd = coordsLen;
}
};
/**
* @suppress {missingProperties}
* @private
* @param {number} type The raw feature's geometry type
* @param {number} numEnds Number of ends of the flat coordinates of the
* geometry.
* @return {ol.geom.GeometryType} The geometry type.
*/
ol.format.MVT.getGeometryType_ = function(type, numEnds) {
/** @type {ol.geom.GeometryType} */
var geometryType;
if (type === 1) {
geometryType = numEnds === 1 ?
ol.geom.GeometryType.POINT : ol.geom.GeometryType.MULTI_POINT;
} else if (type === 2) {
geometryType = numEnds === 1 ?
ol.geom.GeometryType.LINE_STRING :
ol.geom.GeometryType.MULTI_LINE_STRING;
} else if (type === 3) {
geometryType = ol.geom.GeometryType.POLYGON;
// MultiPolygon not relevant for rendering - winding order determines
// outer rings of polygons.
}
return geometryType;
};
/**
* @private
* @param {ol.ext.PBF} pbf PBF
* @param {Object} rawFeature Raw Mapbox feature.
* @param {olx.format.ReadOptions=} opt_options Read options.
* @return {ol.Feature|ol.render.Feature} Feature.
*/
ol.format.MVT.prototype.createFeature_ = function(pbf, rawFeature, opt_options) {
var type = rawFeature.type;
if (type === 0) {
return null;
}
var feature;
var id = rawFeature.id;
var values = rawFeature.properties;
values[this.layerName_] = rawFeature.layer.name;
var flatCoordinates = [];
var ends = [];
ol.format.MVT.readRawGeometry_(pbf, rawFeature, flatCoordinates, ends);
var geometryType = ol.format.MVT.getGeometryType_(type, ends.length);
if (this.featureClass_ === ol.render.Feature) {
feature = new this.featureClass_(geometryType, flatCoordinates, ends, values, id);
} else {
var geom;
if (geometryType == ol.geom.GeometryType.POLYGON) {
var endss = [];
var offset = 0;
var prevEndIndex = 0;
for (var i = 0, ii = ends.length; i < ii; ++i) {
var end = ends[i];
if (!ol.geom.flat.orient.linearRingIsClockwise(flatCoordinates, offset, end, 2)) {
endss.push(ends.slice(prevEndIndex, i));
prevEndIndex = i;
}
offset = end;
}
if (endss.length > 1) {
ends = endss;
geom = new ol.geom.MultiPolygon(null);
} else {
geom = new ol.geom.Polygon(null);
}
} else {
geom = geometryType === ol.geom.GeometryType.POINT ? new ol.geom.Point(null) :
geometryType === ol.geom.GeometryType.LINE_STRING ? new ol.geom.LineString(null) :
geometryType === ol.geom.GeometryType.POLYGON ? new ol.geom.Polygon(null) :
geometryType === ol.geom.GeometryType.MULTI_POINT ? new ol.geom.MultiPoint (null) :
geometryType === ol.geom.GeometryType.MULTI_LINE_STRING ? new ol.geom.MultiLineString(null) :
null;
}
geom.setFlatCoordinates(ol.geom.GeometryLayout.XY, flatCoordinates, ends);
feature = new this.featureClass_();
if (this.geometryName_) {
feature.setGeometryName(this.geometryName_);
}
var geometry = ol.format.Feature.transformWithOptions(geom, false, this.adaptOptions(opt_options));
feature.setGeometry(geometry);
feature.setId(id);
feature.setProperties(values);
}
return feature;
};
/**
* @inheritDoc
* @api
@@ -96,68 +348,6 @@ ol.format.MVT.prototype.getType = function() {
};
/**
* @private
* @param {Object} rawFeature Raw Mapbox feature.
* @param {string} layer Layer.
* @param {olx.format.ReadOptions=} opt_options Read options.
* @return {ol.Feature} Feature.
*/
ol.format.MVT.prototype.readFeature_ = function(
rawFeature, layer, opt_options) {
var feature = new this.featureClass_();
var id = rawFeature.id;
var values = rawFeature.properties;
values[this.layerName_] = layer;
if (this.geometryName_) {
feature.setGeometryName(this.geometryName_);
}
var geometry = ol.format.Feature.transformWithOptions(
ol.format.MVT.readGeometry_(rawFeature), false,
this.adaptOptions(opt_options));
feature.setGeometry(geometry);
feature.setId(id);
feature.setProperties(values);
return feature;
};
/**
* @private
* @param {Object} rawFeature Raw Mapbox feature.
* @param {string} layer Layer.
* @return {ol.render.Feature} Feature.
*/
ol.format.MVT.prototype.readRenderFeature_ = function(rawFeature, layer) {
var coords = rawFeature.loadGeometry();
var ends = [];
var flatCoordinates = [];
ol.format.MVT.calculateFlatCoordinates_(coords, flatCoordinates, ends);
var type = rawFeature.type;
/** @type {ol.geom.GeometryType} */
var geometryType;
if (type === 1) {
geometryType = coords.length === 1 ?
ol.geom.GeometryType.POINT : ol.geom.GeometryType.MULTI_POINT;
} else if (type === 2) {
if (coords.length === 1) {
geometryType = ol.geom.GeometryType.LINE_STRING;
} else {
geometryType = ol.geom.GeometryType.MULTI_LINE_STRING;
}
} else if (type === 3) {
geometryType = ol.geom.GeometryType.POLYGON;
}
var values = rawFeature.properties;
values[this.layerName_] = layer;
var id = rawFeature.id;
return new this.featureClass_(geometryType, flatCoordinates, ends, values, id);
};
/**
* @inheritDoc
* @api
@@ -166,27 +356,22 @@ ol.format.MVT.prototype.readFeatures = function(source, opt_options) {
var layers = this.layers_;
var pbf = new ol.ext.PBF(/** @type {ArrayBuffer} */ (source));
var tile = new ol.ext.vectortile.VectorTile(pbf);
var pbfLayers = pbf.readFields(ol.format.MVT.pbfReaders_.layers, {});
/** @type {Array.<ol.Feature|ol.render.Feature>} */
var features = [];
var featureClass = this.featureClass_;
var layer, feature;
for (var name in tile.layers) {
var pbfLayer;
for (var name in pbfLayers) {
if (layers && layers.indexOf(name) == -1) {
continue;
}
layer = tile.layers[name];
pbfLayer = pbfLayers[name];
var rawFeature;
for (var i = 0, ii = layer.length; i < ii; ++i) {
rawFeature = layer.feature(i);
if (featureClass === ol.render.Feature) {
feature = this.readRenderFeature_(rawFeature, name);
} else {
feature = this.readFeature_(rawFeature, name, opt_options);
}
features.push(feature);
for (var i = 0, ii = pbfLayer.length; i < ii; ++i) {
rawFeature = ol.format.MVT.readRawFeature_(pbf, pbfLayer, i);
features.push(this.createFeature_(pbf, rawFeature));
}
this.extent_ = layer ? [0, 0, layer.extent, layer.extent] : null;
this.extent_ = pbfLayer ? [0, 0, pbfLayer.extent, pbfLayer.extent] : null;
}
return features;
@@ -212,68 +397,6 @@ ol.format.MVT.prototype.setLayers = function(layers) {
};
/**
* @private
* @param {Object} coords Raw feature coordinates.
* @param {Array.<number>} flatCoordinates Flat coordinates to be populated by
* this function.
* @param {Array.<number>} ends Ends to be populated by this function.
*/
ol.format.MVT.calculateFlatCoordinates_ = function(
coords, flatCoordinates, ends) {
var end = 0;
for (var i = 0, ii = coords.length; i < ii; ++i) {
var line = coords[i];
var j, jj;
for (j = 0, jj = line.length; j < jj; ++j) {
var coord = line[j];
// Non-tilespace coords can be calculated here when a TileGrid and
// TileCoord are known.
flatCoordinates.push(coord.x, coord.y);
}
end += 2 * j;
ends.push(end);
}
};
/**
* @private
* @param {Object} rawFeature Raw Mapbox feature.
* @return {ol.geom.Geometry} Geometry.
*/
ol.format.MVT.readGeometry_ = function(rawFeature) {
var type = rawFeature.type;
if (type === 0) {
return null;
}
var coords = rawFeature.loadGeometry();
var ends = [];
var flatCoordinates = [];
ol.format.MVT.calculateFlatCoordinates_(coords, flatCoordinates, ends);
var geom;
if (type === 1) {
geom = coords.length === 1 ?
new ol.geom.Point(null) : new ol.geom.MultiPoint(null);
} else if (type === 2) {
if (coords.length === 1) {
geom = new ol.geom.LineString(null);
} else {
geom = new ol.geom.MultiLineString(null);
}
} else if (type === 3) {
geom = new ol.geom.Polygon(null);
}
geom.setFlatCoordinates(ol.geom.GeometryLayout.XY, flatCoordinates,
ends);
return geom;
};
/**
* Not implemented.
* @override

View File

@@ -4,7 +4,6 @@ goog.require('ol');
goog.require('ol.TileState');
goog.require('ol.VectorImageTile');
goog.require('ol.VectorTile');
goog.require('ol.proj');
goog.require('ol.size');
goog.require('ol.tilegrid');
goog.require('ol.source.UrlTile');

View File

@@ -1,11 +1,11 @@
goog.require('ol.Feature');
goog.require('ol.ext.PBF');
goog.require('ol.ext.vectortile.VectorTile');
goog.require('ol.extent');
goog.require('ol.format.MVT');
goog.require('ol.geom.Point');
goog.require('ol.geom.Polygon');
goog.require('ol.geom.MultiPolygon');
goog.require('ol.render.Feature');
where('ArrayBuffer.isView').describe('ol.format.MVT', function() {
@@ -41,36 +41,23 @@ where('ArrayBuffer.isView').describe('ol.format.MVT', function() {
featureClass: ol.Feature,
layers: ['poi_label']
});
var pbf = new ol.ext.PBF(data);
var tile = new ol.ext.vectortile.VectorTile(pbf);
var geometry, rawGeometry;
var geometry;
rawGeometry = tile.layers['poi_label'].feature(0).loadGeometry();
geometry = format.readFeatures(data)[0]
.getGeometry();
geometry = format.readFeatures(data)[0].getGeometry();
expect(geometry.getType()).to.be('Point');
expect(geometry.getCoordinates())
.to.eql([rawGeometry[0][0].x, rawGeometry[0][0].y]);
expect(geometry.getCoordinates()).to.eql([-1210, 2681]);
rawGeometry = tile.layers['water'].feature(0).loadGeometry();
format.setLayers(['water']);
geometry = format.readFeatures(data)[0]
.getGeometry();
geometry = format.readFeatures(data)[0].getGeometry();
expect(geometry.getType()).to.be('Polygon');
expect(rawGeometry[0].length)
.to.equal(geometry.getCoordinates()[0].length);
expect(geometry.getCoordinates()[0][0])
.to.eql([rawGeometry[0][0].x, rawGeometry[0][0].y]);
expect(geometry.getCoordinates()[0].length).to.be(10);
expect(geometry.getCoordinates()[0][0]).to.eql([1007, 2302]);
rawGeometry = tile.layers['barrier_line'].feature(0).loadGeometry();
format.setLayers(['barrier_line']);
geometry = format.readFeatures(data)[0]
.getGeometry();
geometry = format.readFeatures(data)[0].getGeometry();
expect(geometry.getType()).to.be('MultiLineString');
expect(rawGeometry[1].length)
.to.equal(geometry.getCoordinates()[1].length);
expect(geometry.getCoordinates()[1][0])
.to.eql([rawGeometry[1][0].x, rawGeometry[1][0].y]);
expect(geometry.getCoordinates()[1].length).to.be(6);
expect(geometry.getCoordinates()[1][0]).to.eql([4160, 3489]);
});
it('parses id property', function() {
@@ -102,7 +89,7 @@ where('ArrayBuffer.isView').describe('ol.format.MVT', function() {
describe('ol.format.MVT', function() {
describe('#readFeature_', function() {
describe('#createFeature_', function() {
it('accepts a geometryName', function() {
var format = new ol.format.MVT({
featureClass: ol.Feature,
@@ -114,16 +101,99 @@ describe('ol.format.MVT', function() {
geometry: 'foo'
},
type: 1,
loadGeometry: function() {
return [[0, 0]];
layer: {
name: 'layer1'
}
};
var feature = format.readFeature_(rawFeature, 'mapbox');
var readRawGeometry_ = ol.format.MVT.readRawGeometry_;
ol.format.MVT.readRawGeometry_ = function({}, rawFeature, flatCoordinates, ends) {
flatCoordinates.push(0, 0);
ends.push(2);
};
var feature = format.createFeature_({}, rawFeature);
ol.format.MVT.readRawGeometry_ = readRawGeometry_;
var geometry = feature.getGeometry();
expect(geometry).to.be.a(ol.geom.Point);
expect(feature.get('myGeom')).to.equal(geometry);
expect(feature.get('geometry')).to.be('foo');
});
it('detects a Polygon', function() {
var format = new ol.format.MVT({
featureClass: ol.Feature
});
var rawFeature = {
type: 3,
properties: {},
layer: {
name: 'layer1'
}
};
var readRawGeometry_ = ol.format.MVT.readRawGeometry_;
ol.format.MVT.readRawGeometry_ = function({}, rawFeature, flatCoordinates, ends) {
flatCoordinates.push(0, 0, 3, 0, 3, 3, 3, 0, 0, 0);
flatCoordinates.push(1, 1, 1, 2, 2, 2, 2, 1, 1, 1);
ends.push(10, 20);
};
var feature = format.createFeature_({}, rawFeature);
ol.format.MVT.readRawGeometry_ = readRawGeometry_;
var geometry = feature.getGeometry();
expect(geometry).to.be.a(ol.geom.Polygon);
});
it('detects a MultiPolygon', function() {
var format = new ol.format.MVT({
featureClass: ol.Feature
});
var rawFeature = {
type: 3,
properties: {},
layer: {
name: 'layer1'
}
};
var readRawGeometry_ = ol.format.MVT.readRawGeometry_;
ol.format.MVT.readRawGeometry_ = function({}, rawFeature, flatCoordinates, ends) {
flatCoordinates.push(0, 0, 1, 0, 1, 1, 1, 0, 0, 0);
flatCoordinates.push(1, 1, 2, 1, 2, 2, 2, 1, 1, 1);
ends.push(10, 20);
};
var feature = format.createFeature_({}, rawFeature);
ol.format.MVT.readRawGeometry_ = readRawGeometry_;
var geometry = feature.getGeometry();
expect(geometry).to.be.a(ol.geom.MultiPolygon);
});
it('creates ol.render.Feature instances', function() {
var format = new ol.format.MVT();
var rawFeature = {
type: 3,
properties: {
foo: 'bar'
},
layer: {
name: 'layer1'
}
};
var readRawGeometry_ = ol.format.MVT.readRawGeometry_;
var createdFlatCoordinates;
var createdEnds;
ol.format.MVT.readRawGeometry_ = function({}, rawFeature, flatCoordinates, ends) {
flatCoordinates.push(0, 0, 1, 0, 1, 1, 1, 0, 0, 0);
flatCoordinates.push(1, 1, 2, 1, 2, 2, 2, 1, 1, 1);
createdFlatCoordinates = flatCoordinates;
ends.push(10, 20);
createdEnds = ends;
};
var feature = format.createFeature_({}, rawFeature);
ol.format.MVT.readRawGeometry_ = readRawGeometry_;
expect(feature).to.be.a(ol.render.Feature);
expect(feature.getType()).to.be('Polygon');
expect(feature.getFlatCoordinates()).to.equal(createdFlatCoordinates);
expect(feature.getEnds()).to.equal(createdEnds);
expect(feature.get('foo')).to.be('bar');
});
});
});