diff --git a/src/ol/interaction/pointerinteraction.js b/src/ol/interaction/pointerinteraction.js
index c1797be6bf..559ce021ba 100644
--- a/src/ol/interaction/pointerinteraction.js
+++ b/src/ol/interaction/pointerinteraction.js
@@ -84,7 +84,11 @@ ol.interaction.PointerInteraction.prototype.updateTrackedTouches_ =
 
     if (mapBrowserEvent.type == ol.MapBrowserEvent.EventType.POINTERUP) {
       delete this.trackedTouches_[event.pointerId];
-    } else {
+    } else if (mapBrowserEvent.type ==
+        ol.MapBrowserEvent.EventType.POINTERDOWN) {
+      this.trackedTouches_[event.pointerId] = event;
+    } else if (event.pointerId in this.trackedTouches_) {
+      // update only when there was a pointerdown event for this pointer
       this.trackedTouches_[event.pointerId] = event;
     }
     this.targetTouches = goog.object.getValues(this.trackedTouches_);
@@ -96,7 +100,8 @@ ol.interaction.PointerInteraction.prototype.updateTrackedTouches_ =
  * @param {ol.MapBrowserEvent} mapBrowserEvent Event.
  * @protected
  */
-ol.interaction.PointerInteraction.prototype.handlePointerMove = goog.nullFunction;
+ol.interaction.PointerInteraction.prototype.handlePointerMove =
+    goog.nullFunction;
 
 
 /**
diff --git a/src/ol/mapbrowserevent.js b/src/ol/mapbrowserevent.js
index cad9f0ce0e..ca0d48c774 100644
--- a/src/ol/mapbrowserevent.js
+++ b/src/ol/mapbrowserevent.js
@@ -130,6 +130,7 @@ ol.MapBrowserEventHandler = function(map) {
    * @private
    */
   this.activePointers_ = 0;
+  this.trackedTouches_ = {};
 
   /**
    * @type {ol.pointer.PointerEventHandler}
@@ -206,10 +207,11 @@ ol.MapBrowserEventHandler.prototype.updateActivePointers_ =
 
   if (event.type == ol.MapBrowserEvent.EventType.POINTERUP ||
       event.type == ol.MapBrowserEvent.EventType.POINTERCANCEL) {
-    this.activePointers_--;
+    delete this.trackedTouches_[event.pointerId];
   } else if (event.type == ol.MapBrowserEvent.EventType.POINTERDOWN) {
-    this.activePointers_++;
+    this.trackedTouches_[event.pointerId] = true;
   }
+  this.activePointers_ = goog.object.getCount(this.trackedTouches_);
 };