diff --git a/src/ol/mapbrowserevent.js b/src/ol/mapbrowserevent.js
index aa695f810b..cdb40d3e66 100644
--- a/src/ol/mapbrowserevent.js
+++ b/src/ol/mapbrowserevent.js
@@ -123,6 +123,12 @@ ol.MapBrowserEventHandler = function(map) {
    */
   this.clickListenerKey_ = null;
 
+  /**
+   * @type {?number}
+   * @private
+   */
+  this.downListenerKey_ = null;
+
   /**
    * @type {Array.<number>}
    * @private
@@ -141,7 +147,7 @@ ol.MapBrowserEventHandler = function(map) {
         [goog.events.EventType.CLICK, goog.events.EventType.DBLCLICK],
         this.click_, false, this);
   }
-  goog.events.listen(element,
+  this.downListenerKey_ = goog.events.listen(element,
       ol.BrowserFeature.HAS_TOUCH ?
           goog.events.EventType.TOUCHSTART :
           goog.events.EventType.MOUSEDOWN,
@@ -281,13 +287,8 @@ ol.MapBrowserEventHandler.prototype.drag_ = function(browserEvent) {
  * FIXME empty description for jsdoc
  */
 ol.MapBrowserEventHandler.prototype.disposeInternal = function() {
-  var element = this.map_.getViewport();
-  goog.events.unlisten(element,
-      ol.BrowserFeature.HAS_TOUCH ?
-          goog.events.EventType.TOUCHSTART :
-          goog.events.EventType.MOUSEDOWN,
-      this.handleDown_, false, this);
   goog.events.unlistenByKey(this.clickListenerKey_);
+  goog.events.unlistenByKey(this.downListenerKey_);
   if (!goog.isNull(this.dragListenerKeys_)) {
     goog.array.forEach(this.dragListenerKeys_, goog.events.unlistenByKey);
     this.dragListenerKeys_ = null;