Parse PBF directly in ol.format.MVT

This change also fixes a previous issue, where MultiPolygon geometries for
ol.Feature features were never created.

doc/errors/index.md

@@ -224,3 +224,7 @@ At least 2 conditions are required.
 ### 58
 
 Duplicate item added to a unique collection.  For example, it may be that you tried to add the same layer to a map twice.  Check for calls to `map.addLayer()` or other places where the map's layer collection is modified.
+
+### 59
+
+Invalid command found in the PBF.  This indicates that the loaded vector tile may be corrupt.

package.json

@@ -50,7 +50,6 @@
     "rollup-plugin-commonjs": "^8.0.2",
     "rollup-plugin-node-resolve": "^3.0.0",
     "temp": "0.8.3",
-    "@mapbox/vector-tile": "1.3.0",
     "walk": "2.3.9"
   },
   "devDependencies": {
@@ -148,11 +147,6 @@
     {
       "module": "pixelworks",
       "import": "Processor"
-    },
-    {
-      "module": "@mapbox/vector-tile",
-      "name": "vectortile",
-      "import": "VectorTile"
     }
   ]
 }

src/ol/format/mvt.js

@@ -3,8 +3,8 @@
 goog.provide('ol.format.MVT');
 
 goog.require('ol');
+goog.require('ol.asserts');
 goog.require('ol.ext.PBF');
-goog.require('ol.ext.vectortile.VectorTile');
 goog.require('ol.format.Feature');
 goog.require('ol.format.FormatType');
 goog.require('ol.geom.GeometryLayout');
@@ -12,8 +12,10 @@ goog.require('ol.geom.GeometryType');
 goog.require('ol.geom.LineString');
 goog.require('ol.geom.MultiLineString');
 goog.require('ol.geom.MultiPoint');
+goog.require('ol.geom.MultiPolygon');
 goog.require('ol.geom.Point');
 goog.require('ol.geom.Polygon');
+goog.require('ol.geom.flat.orient');
 goog.require('ol.proj.Projection');
 goog.require('ol.proj.Units');
 goog.require('ol.render.Feature');
@@ -79,6 +81,256 @@ ol.format.MVT = function(opt_options) {
 ol.inherits(ol.format.MVT, ol.format.Feature);
 
 
+/**
+ * Reader callbacks for parsing the PBF.
+ * @type {Object.<string, function(number, Object, ol.ext.PBF)>}
+ */
+ol.format.MVT.pbfReaders_ = {
+  layers: function(tag, layers, pbf) {
+    if (tag === 3) {
+      var layer = {
+        keys: [],
+        values: [],
+        features: []
+      };
+      var end = pbf.readVarint() + pbf.pos;
+      pbf.readFields(ol.format.MVT.pbfReaders_.layer, layer, end);
+      layer.length = layer.features.length;
+      if (layer.length) {
+        layers[layer.name] = layer;
+      }
+    }
+  },
+  layer: function(tag, layer, pbf) {
+    if (tag === 15) {
+      layer.version = pbf.readVarint();
+    } else if (tag === 1) {
+      layer.name = pbf.readString();
+    } else if (tag === 5) {
+      layer.extent = pbf.readVarint();
+    } else if (tag === 2) {
+      layer.features.push(pbf.pos);
+    } else if (tag === 3) {
+      layer.keys.push(pbf.readString());
+    } else if (tag === 4) {
+      var value = null;
+      var end = pbf.readVarint() + pbf.pos;
+      while (pbf.pos < end) {
+        tag = pbf.readVarint() >> 3;
+        value = tag === 1 ? pbf.readString() :
+          tag === 2 ? pbf.readFloat() :
+            tag === 3 ? pbf.readDouble() :
+              tag === 4 ? pbf.readVarint64() :
+                tag === 5 ? pbf.readVarint() :
+                  tag === 6 ? pbf.readSVarint() :
+                    tag === 7 ? pbf.readBoolean() : null;
+      }
+      layer.values.push(value);
+    }
+  },
+  feature: function(tag, feature, pbf) {
+    if (tag == 1) {
+      feature.id = pbf.readVarint();
+    } else if (tag == 2) {
+      var end = pbf.readVarint() + pbf.pos;
+      while (pbf.pos < end) {
+        var key = feature.layer.keys[pbf.readVarint()];
+        var value = feature.layer.values[pbf.readVarint()];
+        feature.properties[key] = value;
+      }
+    } else if (tag == 3) {
+      feature.type = pbf.readVarint();
+    } else if (tag == 4) {
+      feature.geometry = pbf.pos;
+    }
+  }
+};
+
+
+/**
+ * Read a raw feature from the pbf offset stored at index `i` in the raw layer.
+ * @suppress {missingProperties}
+ * @private
+ * @param {ol.ext.PBF} pbf PBF.
+ * @param {Object} layer Raw layer.
+ * @param {number} i Index of the feature in the raw layer's `features` array.
+ * @return {Object} Raw feature.
+ */
+ol.format.MVT.readRawFeature_ = function(pbf, layer, i) {
+  pbf.pos = layer.features[i];
+  var end = pbf.readVarint() + pbf.pos;
+
+  var feature = {
+    layer: layer,
+    type: 0,
+    properties: {}
+  };
+  pbf.readFields(ol.format.MVT.pbfReaders_.feature, feature, end);
+  return feature;
+};
+
+
+/**
+ * Read the raw geometry from the pbf offset stored in a raw feature's geometry
+ * proeprty.
+ * @suppress {missingProperties}
+ * @private
+ * @param {ol.ext.PBF} pbf PBF.
+ * @param {Object} feature Raw feature.
+ * @param {Array.<number>} flatCoordinates Array to store flat coordinates in.
+ * @param {Array.<number>} ends Array to store ends in.
+ */
+ol.format.MVT.readRawGeometry_ = function(pbf, feature, flatCoordinates, ends) {
+  pbf.pos = feature.geometry;
+
+  var end = pbf.readVarint() + pbf.pos;
+  var cmd = 1;
+  var length = 0;
+  var x = 0;
+  var y = 0;
+  var coordsLen = 0;
+  var currentEnd = 0;
+
+  while (pbf.pos < end) {
+    if (!length) {
+      var cmdLen = pbf.readVarint();
+      cmd = cmdLen & 0x7;
+      length = cmdLen >> 3;
+    }
+
+    length--;
+
+    if (cmd === 1 || cmd === 2) {
+      x += pbf.readSVarint();
+      y += pbf.readSVarint();
+
+      if (cmd === 1) { // moveTo
+        if (coordsLen > currentEnd) {
+          ends.push(coordsLen);
+          currentEnd = coordsLen;
+        }
+      }
+
+      flatCoordinates.push(x, y);
+      coordsLen += 2;
+
+    } else if (cmd === 7) {
+
+      if (coordsLen > currentEnd) {
+        // close polygon
+        flatCoordinates.push(
+            flatCoordinates[currentEnd], flatCoordinates[currentEnd + 1]);
+        coordsLen += 2;
+      }
+
+    } else {
+      ol.asserts.assert(false, 59); // Invalid command found in the PBF
+    }
+  }
+
+  if (coordsLen > currentEnd) {
+    ends.push(coordsLen);
+    currentEnd = coordsLen;
+  }
+
+};
+
+
+/**
+ * @suppress {missingProperties}
+ * @private
+ * @param {number} type The raw feature's geometry type
+ * @param {number} numEnds Number of ends of the flat coordinates of the
+ * geometry.
+ * @return {ol.geom.GeometryType} The geometry type.
+ */
+ol.format.MVT.getGeometryType_ = function(type, numEnds) {
+  /** @type {ol.geom.GeometryType} */
+  var geometryType;
+  if (type === 1) {
+    geometryType = numEnds === 1 ?
+      ol.geom.GeometryType.POINT : ol.geom.GeometryType.MULTI_POINT;
+  } else if (type === 2) {
+    geometryType = numEnds === 1 ?
+      ol.geom.GeometryType.LINE_STRING :
+      ol.geom.GeometryType.MULTI_LINE_STRING;
+  } else if (type === 3) {
+    geometryType = ol.geom.GeometryType.POLYGON;
+    // MultiPolygon not relevant for rendering - winding order determines
+    // outer rings of polygons.
+  }
+  return geometryType;
+};
+
+/**
+ * @private
+ * @param {ol.ext.PBF} pbf PBF
+ * @param {Object} rawFeature Raw Mapbox feature.
+ * @param {olx.format.ReadOptions=} opt_options Read options.
+ * @return {ol.Feature|ol.render.Feature} Feature.
+ */
+ol.format.MVT.prototype.createFeature_ = function(pbf, rawFeature, opt_options) {
+  var type = rawFeature.type;
+  if (type === 0) {
+    return null;
+  }
+
+  var feature;
+  var id = rawFeature.id;
+  var values = rawFeature.properties;
+  values[this.layerName_] = rawFeature.layer.name;
+
+  var flatCoordinates = [];
+  var ends = [];
+  ol.format.MVT.readRawGeometry_(pbf, rawFeature, flatCoordinates, ends);
+
+  var geometryType = ol.format.MVT.getGeometryType_(type, ends.length);
+
+  if (this.featureClass_ === ol.render.Feature) {
+    feature = new this.featureClass_(geometryType, flatCoordinates, ends, values, id);
+  } else {
+    var geom;
+    if (geometryType == ol.geom.GeometryType.POLYGON) {
+      var endss = [];
+      var offset = 0;
+      var prevEndIndex = 0;
+      for (var i = 0, ii = ends.length; i < ii; ++i) {
+        var end = ends[i];
+        if (!ol.geom.flat.orient.linearRingIsClockwise(flatCoordinates, offset, end, 2)) {
+          endss.push(ends.slice(prevEndIndex, i));
+          prevEndIndex = i;
+        }
+        offset = end;
+      }
+      if (endss.length > 1) {
+        ends = endss;
+        geom = new ol.geom.MultiPolygon(null);
+      } else {
+        geom = new ol.geom.Polygon(null);
+      }
+    } else {
+      geom = geometryType === ol.geom.GeometryType.POINT ? new ol.geom.Point(null) :
+        geometryType === ol.geom.GeometryType.LINE_STRING ? new ol.geom.LineString(null) :
+          geometryType === ol.geom.GeometryType.POLYGON ? new ol.geom.Polygon(null) :
+            geometryType === ol.geom.GeometryType.MULTI_POINT ? new ol.geom.MultiPoint (null) :
+              geometryType === ol.geom.GeometryType.MULTI_LINE_STRING ? new ol.geom.MultiLineString(null) :
+                null;
+    }
+    geom.setFlatCoordinates(ol.geom.GeometryLayout.XY, flatCoordinates, ends);
+    feature = new this.featureClass_();
+    if (this.geometryName_) {
+      feature.setGeometryName(this.geometryName_);
+    }
+    var geometry = ol.format.Feature.transformWithOptions(geom, false, this.adaptOptions(opt_options));
+    feature.setGeometry(geometry);
+    feature.setId(id);
+    feature.setProperties(values);
+  }
+
+  return feature;
+};
+
+
 /**
  * @inheritDoc
  * @api
@@ -96,68 +348,6 @@ ol.format.MVT.prototype.getType = function() {
 };
 
 
-/**
- * @private
- * @param {Object} rawFeature Raw Mapbox feature.
- * @param {string} layer Layer.
- * @param {olx.format.ReadOptions=} opt_options Read options.
- * @return {ol.Feature} Feature.
- */
-ol.format.MVT.prototype.readFeature_ = function(
-    rawFeature, layer, opt_options) {
-  var feature = new this.featureClass_();
-  var id = rawFeature.id;
-  var values = rawFeature.properties;
-  values[this.layerName_] = layer;
-  if (this.geometryName_) {
-    feature.setGeometryName(this.geometryName_);
-  }
-  var geometry = ol.format.Feature.transformWithOptions(
-      ol.format.MVT.readGeometry_(rawFeature), false,
-      this.adaptOptions(opt_options));
-  feature.setGeometry(geometry);
-  feature.setId(id);
-  feature.setProperties(values);
-  return feature;
-};
-
-
-/**
- * @private
- * @param {Object} rawFeature Raw Mapbox feature.
- * @param {string} layer Layer.
- * @return {ol.render.Feature} Feature.
- */
-ol.format.MVT.prototype.readRenderFeature_ = function(rawFeature, layer) {
-  var coords = rawFeature.loadGeometry();
-  var ends = [];
-  var flatCoordinates = [];
-  ol.format.MVT.calculateFlatCoordinates_(coords, flatCoordinates, ends);
-
-  var type = rawFeature.type;
-  /** @type {ol.geom.GeometryType} */
-  var geometryType;
-  if (type === 1) {
-    geometryType = coords.length === 1 ?
-      ol.geom.GeometryType.POINT : ol.geom.GeometryType.MULTI_POINT;
-  } else if (type === 2) {
-    if (coords.length === 1) {
-      geometryType = ol.geom.GeometryType.LINE_STRING;
-    } else {
-      geometryType = ol.geom.GeometryType.MULTI_LINE_STRING;
-    }
-  } else if (type === 3) {
-    geometryType = ol.geom.GeometryType.POLYGON;
-  }
-
-  var values = rawFeature.properties;
-  values[this.layerName_] = layer;
-  var id = rawFeature.id;
-
-  return new this.featureClass_(geometryType, flatCoordinates, ends, values, id);
-};
-
-
 /**
  * @inheritDoc
  * @api
@@ -166,27 +356,22 @@ ol.format.MVT.prototype.readFeatures = function(source, opt_options) {
   var layers = this.layers_;
 
   var pbf = new ol.ext.PBF(/** @type {ArrayBuffer} */ (source));
-  var tile = new ol.ext.vectortile.VectorTile(pbf);
+  var pbfLayers = pbf.readFields(ol.format.MVT.pbfReaders_.layers, {});
+  /** @type {Array.<ol.Feature|ol.render.Feature>} */
   var features = [];
-  var featureClass = this.featureClass_;
+  var pbfLayer;
-  var layer, feature;
+  for (var name in pbfLayers) {
-  for (var name in tile.layers) {
     if (layers && layers.indexOf(name) == -1) {
       continue;
     }
-    layer = tile.layers[name];
+    pbfLayer = pbfLayers[name];
 
     var rawFeature;
-    for (var i = 0, ii = layer.length; i < ii; ++i) {
+    for (var i = 0, ii = pbfLayer.length; i < ii; ++i) {
-      rawFeature = layer.feature(i);
+      rawFeature = ol.format.MVT.readRawFeature_(pbf, pbfLayer, i);
-      if (featureClass === ol.render.Feature) {
+      features.push(this.createFeature_(pbf, rawFeature));
-        feature = this.readRenderFeature_(rawFeature, name);
-      } else {
-        feature = this.readFeature_(rawFeature, name, opt_options);
-      }
-      features.push(feature);
     }
-    this.extent_ = layer ? [0, 0, layer.extent, layer.extent] : null;
+    this.extent_ = pbfLayer ? [0, 0, pbfLayer.extent, pbfLayer.extent] : null;
   }
 
   return features;
@@ -212,68 +397,6 @@ ol.format.MVT.prototype.setLayers = function(layers) {
 };
 
 
-/**
- * @private
- * @param {Object} coords Raw feature coordinates.
- * @param {Array.<number>} flatCoordinates Flat coordinates to be populated by
- *     this function.
- * @param {Array.<number>} ends Ends to be populated by this function.
- */
-ol.format.MVT.calculateFlatCoordinates_ = function(
-    coords, flatCoordinates, ends) {
-  var end = 0;
-  for (var i = 0, ii = coords.length; i < ii; ++i) {
-    var line = coords[i];
-    var j, jj;
-    for (j = 0, jj = line.length; j < jj; ++j) {
-      var coord = line[j];
-      // Non-tilespace coords can be calculated here when a TileGrid and
-      // TileCoord are known.
-      flatCoordinates.push(coord.x, coord.y);
-    }
-    end += 2 * j;
-    ends.push(end);
-  }
-};
-
-
-/**
- * @private
- * @param {Object} rawFeature Raw Mapbox feature.
- * @return {ol.geom.Geometry} Geometry.
- */
-ol.format.MVT.readGeometry_ = function(rawFeature) {
-  var type = rawFeature.type;
-  if (type === 0) {
-    return null;
-  }
-
-  var coords = rawFeature.loadGeometry();
-  var ends = [];
-  var flatCoordinates = [];
-  ol.format.MVT.calculateFlatCoordinates_(coords, flatCoordinates, ends);
-
-  var geom;
-  if (type === 1) {
-    geom = coords.length === 1 ?
-      new ol.geom.Point(null) : new ol.geom.MultiPoint(null);
-  } else if (type === 2) {
-    if (coords.length === 1) {
-      geom = new ol.geom.LineString(null);
-    } else {
-      geom = new ol.geom.MultiLineString(null);
-    }
-  } else if (type === 3) {
-    geom = new ol.geom.Polygon(null);
-  }
-
-  geom.setFlatCoordinates(ol.geom.GeometryLayout.XY, flatCoordinates,
-      ends);
-
-  return geom;
-};
-
-
 /**
  * Not implemented.
  * @override

src/ol/source/vectortile.js

@@ -4,7 +4,6 @@ goog.require('ol');
 goog.require('ol.TileState');
 goog.require('ol.VectorImageTile');
 goog.require('ol.VectorTile');
-goog.require('ol.proj');
 goog.require('ol.size');
 goog.require('ol.tilegrid');
 goog.require('ol.source.UrlTile');

test/spec/ol/format/mvt.test.js

@@ -1,11 +1,11 @@
 
 
 goog.require('ol.Feature');
-goog.require('ol.ext.PBF');
-goog.require('ol.ext.vectortile.VectorTile');
 goog.require('ol.extent');
 goog.require('ol.format.MVT');
 goog.require('ol.geom.Point');
+goog.require('ol.geom.Polygon');
+goog.require('ol.geom.MultiPolygon');
 goog.require('ol.render.Feature');
 
 where('ArrayBuffer.isView').describe('ol.format.MVT', function() {
@@ -41,36 +41,23 @@ where('ArrayBuffer.isView').describe('ol.format.MVT', function() {
         featureClass: ol.Feature,
         layers: ['poi_label']
       });
-      var pbf = new ol.ext.PBF(data);
+      var geometry;
-      var tile = new ol.ext.vectortile.VectorTile(pbf);
-      var geometry, rawGeometry;
 
-      rawGeometry = tile.layers['poi_label'].feature(0).loadGeometry();
+      geometry = format.readFeatures(data)[0].getGeometry();
-      geometry = format.readFeatures(data)[0]
-          .getGeometry();
       expect(geometry.getType()).to.be('Point');
-      expect(geometry.getCoordinates())
+      expect(geometry.getCoordinates()).to.eql([-1210, 2681]);
-          .to.eql([rawGeometry[0][0].x, rawGeometry[0][0].y]);
 
-      rawGeometry = tile.layers['water'].feature(0).loadGeometry();
       format.setLayers(['water']);
-      geometry = format.readFeatures(data)[0]
+      geometry = format.readFeatures(data)[0].getGeometry();
-          .getGeometry();
       expect(geometry.getType()).to.be('Polygon');
-      expect(rawGeometry[0].length)
+      expect(geometry.getCoordinates()[0].length).to.be(10);
-          .to.equal(geometry.getCoordinates()[0].length);
+      expect(geometry.getCoordinates()[0][0]).to.eql([1007, 2302]);
-      expect(geometry.getCoordinates()[0][0])
-          .to.eql([rawGeometry[0][0].x, rawGeometry[0][0].y]);
 
-      rawGeometry = tile.layers['barrier_line'].feature(0).loadGeometry();
       format.setLayers(['barrier_line']);
-      geometry = format.readFeatures(data)[0]
+      geometry = format.readFeatures(data)[0].getGeometry();
-          .getGeometry();
       expect(geometry.getType()).to.be('MultiLineString');
-      expect(rawGeometry[1].length)
+      expect(geometry.getCoordinates()[1].length).to.be(6);
-          .to.equal(geometry.getCoordinates()[1].length);
+      expect(geometry.getCoordinates()[1][0]).to.eql([4160, 3489]);
-      expect(geometry.getCoordinates()[1][0])
-          .to.eql([rawGeometry[1][0].x, rawGeometry[1][0].y]);
     });
 
     it('parses id property', function() {
@@ -102,7 +89,7 @@ where('ArrayBuffer.isView').describe('ol.format.MVT', function() {
 
 describe('ol.format.MVT', function() {
 
-  describe('#readFeature_', function() {
+  describe('#createFeature_', function() {
     it('accepts a geometryName', function() {
       var format = new ol.format.MVT({
         featureClass: ol.Feature,
@@ -114,16 +101,99 @@ describe('ol.format.MVT', function() {
           geometry: 'foo'
         },
         type: 1,
-        loadGeometry: function() {
+        layer: {
-          return [[0, 0]];
+          name: 'layer1'
         }
       };
-      var feature = format.readFeature_(rawFeature, 'mapbox');
+      var readRawGeometry_ = ol.format.MVT.readRawGeometry_;
+      ol.format.MVT.readRawGeometry_ = function({}, rawFeature, flatCoordinates, ends) {
+        flatCoordinates.push(0, 0);
+        ends.push(2);
+      };
+      var feature = format.createFeature_({}, rawFeature);
+      ol.format.MVT.readRawGeometry_ = readRawGeometry_;
       var geometry = feature.getGeometry();
       expect(geometry).to.be.a(ol.geom.Point);
       expect(feature.get('myGeom')).to.equal(geometry);
       expect(feature.get('geometry')).to.be('foo');
     });
+
+    it('detects a Polygon', function() {
+      var format = new ol.format.MVT({
+        featureClass: ol.Feature
+      });
+      var rawFeature = {
+        type: 3,
+        properties: {},
+        layer: {
+          name: 'layer1'
+        }
+      };
+      var readRawGeometry_ = ol.format.MVT.readRawGeometry_;
+      ol.format.MVT.readRawGeometry_ = function({}, rawFeature, flatCoordinates, ends) {
+        flatCoordinates.push(0, 0, 3, 0, 3, 3, 3, 0, 0, 0);
+        flatCoordinates.push(1, 1, 1, 2, 2, 2, 2, 1, 1, 1);
+        ends.push(10, 20);
+      };
+      var feature = format.createFeature_({}, rawFeature);
+      ol.format.MVT.readRawGeometry_ = readRawGeometry_;
+      var geometry = feature.getGeometry();
+      expect(geometry).to.be.a(ol.geom.Polygon);
+    });
+
+    it('detects a MultiPolygon', function() {
+      var format = new ol.format.MVT({
+        featureClass: ol.Feature
+      });
+      var rawFeature = {
+        type: 3,
+        properties: {},
+        layer: {
+          name: 'layer1'
+        }
+      };
+      var readRawGeometry_ = ol.format.MVT.readRawGeometry_;
+      ol.format.MVT.readRawGeometry_ = function({}, rawFeature, flatCoordinates, ends) {
+        flatCoordinates.push(0, 0, 1, 0, 1, 1, 1, 0, 0, 0);
+        flatCoordinates.push(1, 1, 2, 1, 2, 2, 2, 1, 1, 1);
+        ends.push(10, 20);
+      };
+      var feature = format.createFeature_({}, rawFeature);
+      ol.format.MVT.readRawGeometry_ = readRawGeometry_;
+      var geometry = feature.getGeometry();
+      expect(geometry).to.be.a(ol.geom.MultiPolygon);
+    });
+
+    it('creates ol.render.Feature instances', function() {
+      var format = new ol.format.MVT();
+      var rawFeature = {
+        type: 3,
+        properties: {
+          foo: 'bar'
+        },
+        layer: {
+          name: 'layer1'
+        }
+      };
+      var readRawGeometry_ = ol.format.MVT.readRawGeometry_;
+      var createdFlatCoordinates;
+      var createdEnds;
+      ol.format.MVT.readRawGeometry_ = function({}, rawFeature, flatCoordinates, ends) {
+        flatCoordinates.push(0, 0, 1, 0, 1, 1, 1, 0, 0, 0);
+        flatCoordinates.push(1, 1, 2, 1, 2, 2, 2, 1, 1, 1);
+        createdFlatCoordinates = flatCoordinates;
+        ends.push(10, 20);
+        createdEnds = ends;
+      };
+      var feature = format.createFeature_({}, rawFeature);
+      ol.format.MVT.readRawGeometry_ = readRawGeometry_;
+      expect(feature).to.be.a(ol.render.Feature);
+      expect(feature.getType()).to.be('Polygon');
+      expect(feature.getFlatCoordinates()).to.equal(createdFlatCoordinates);
+      expect(feature.getEnds()).to.equal(createdEnds);
+      expect(feature.get('foo')).to.be('bar');
+    });
+
   });
 
 });