diff --git a/src/ol/mapbrowserevent.js b/src/ol/mapbrowserevent.js
index 2d586dd614..566169d561 100644
--- a/src/ol/mapbrowserevent.js
+++ b/src/ol/mapbrowserevent.js
@@ -312,10 +312,16 @@ ol.MapBrowserEventHandler.prototype.handleTouchStart_ = function(browserEvent) {
  * @private
  */
 ol.MapBrowserEventHandler.prototype.handleTouchMove_ = function(browserEvent) {
-  this.dragged_ = true;
-  var newEvent = new ol.MapBrowserEvent(
-      ol.MapBrowserEvent.EventType.TOUCHMOVE, this.map_, browserEvent);
-  this.dispatchEvent(newEvent);
+  if (this.down_) {
+    // 'touchmove' events are dispatched only when this.down_ is set
+    // (set after a touch start) to prevent unwanted events when the
+    // mouse hover the page. This only happens with the IE pointer
+    // event system.
+    this.dragged_ = true;
+    var newEvent = new ol.MapBrowserEvent(
+        ol.MapBrowserEvent.EventType.TOUCHMOVE, this.map_, browserEvent);
+    this.dispatchEvent(newEvent);
+  }
 };