diff --git a/src/ol/interaction/selectinteraction.js b/src/ol/interaction/selectinteraction.js
index 628a6c1c80..93e604019d 100644
--- a/src/ol/interaction/selectinteraction.js
+++ b/src/ol/interaction/selectinteraction.js
@@ -322,7 +322,8 @@ ol.interaction.Select.handleEvent = function(mapBrowserEvent) {
          * @param {ol.layer.Layer} layer Layer.
          */
         function(feature, layer) {
-          if (layer !== this.featureOverlay_) {
+          goog.asserts.assertInstanceof(feature, ol.Feature);
+          if (layer !== null) {
             if (add || toggle) {
               if (this.filter_(feature, layer) &&
                   !ol.array.includes(features.getArray(), feature) &&
@@ -331,7 +332,7 @@ ol.interaction.Select.handleEvent = function(mapBrowserEvent) {
                 this.addFeatureLayerAssociation_(feature, layer);
               }
             }
-          } else {
+          } else if (this.featureOverlay_.getSource().hasFeature(feature)) {
             if (remove || toggle) {
               deselected.push(feature);
               this.removeFeatureLayerAssociation_(feature);
diff --git a/src/ol/source/vectorsource.js b/src/ol/source/vectorsource.js
index 9a3352726c..df58419883 100644
--- a/src/ol/source/vectorsource.js
+++ b/src/ol/source/vectorsource.js
@@ -739,6 +739,16 @@ ol.source.Vector.prototype.handleFeatureChange_ = function(event) {
 };
 
 
+/**
+ * @param {ol.Feature} feature Feature.
+ * @return {boolean} Feature is in source.
+ */
+ol.source.Vector.prototype.hasFeature = function(feature) {
+  var id = feature.getId();
+  return id ? id in this.idIndex_ : goog.getUid(feature) in this.undefIdIndex_;
+};
+
+
 /**
  * @return {boolean} Is empty.
  */
diff --git a/test/spec/ol/interaction/selectinteraction.test.js b/test/spec/ol/interaction/selectinteraction.test.js
index 4feac188ac..e032b3f28c 100644
--- a/test/spec/ol/interaction/selectinteraction.test.js
+++ b/test/spec/ol/interaction/selectinteraction.test.js
@@ -158,6 +158,38 @@ describe('ol.interaction.Select', function() {
     });
   });
 
+  describe('toggle selecting polygons', function() {
+    var select;
+
+    beforeEach(function() {
+      select = new ol.interaction.Select({
+        multi: true
+      });
+      map.addInteraction(select);
+    });
+
+    it('with SHIFT + single-click', function() {
+      var listenerSpy = sinon.spy();
+      select.on('select', listenerSpy);
+
+      simulateEvent(ol.MapBrowserEvent.EventType.SINGLECLICK, 10, -20, true);
+
+      expect(listenerSpy.callCount).to.be(1);
+
+      var features = select.getFeatures();
+      expect(features.getLength()).to.equal(4);
+
+      map.renderSync();
+
+      simulateEvent(ol.MapBrowserEvent.EventType.SINGLECLICK, 10, -20, true);
+
+      expect(listenerSpy.callCount).to.be(2);
+
+      features = select.getFeatures();
+      expect(features.getLength()).to.equal(0);
+    });
+  });
+
   describe('filter features using the filter option', function() {
 
     describe('with multi set to true', function() {