diff --git a/src/ol/interaction/Modify.js b/src/ol/interaction/Modify.js
index 279c12e38e..dc6d73d5c4 100644
--- a/src/ol/interaction/Modify.js
+++ b/src/ol/interaction/Modify.js
@@ -392,7 +392,13 @@ class Modify extends PointerInteraction {
         }
       });
     for (let i = nodesToRemove.length - 1; i >= 0; --i) {
-      rBush.remove(nodesToRemove[i]);
+      const nodeToRemove = nodesToRemove[i];
+      for (let j = this.dragSegments_.length - 1; j >= 0; --j) {
+        if (this.dragSegments_[j][0] === nodeToRemove) {
+          this.dragSegments_.splice(j, 1);
+        }
+      }
+      rBush.remove(nodeToRemove);
     }
   }
 
diff --git a/test/spec/ol/interaction/modify.test.js b/test/spec/ol/interaction/modify.test.js
index aa6ca05749..5e5474424e 100644
--- a/test/spec/ol/interaction/modify.test.js
+++ b/test/spec/ol/interaction/modify.test.js
@@ -696,6 +696,24 @@ describe('ol.interaction.Modify', function() {
     });
   });
 
+  describe('handle feature removal during down-up sequence', function() {
+    it('removes segment data of removed features from dragSegments_', function() {
+      const collection = new Collection(features);
+      const modify = new Modify({
+        features: collection
+      });
+      map.addInteraction(modify);
+      simulateEvent('pointermove', 0, 0, null, 0);
+      simulateEvent('pointerdown', 0, 0, null, 0);
+      simulateEvent('pointermove', -10, -10, null, 0);
+      simulateEvent('pointerdrag', -10, -10, null, 0);
+      collection.remove(features[0]);
+      expect(function() {
+        simulateEvent('pointerup', -10, -10, null, 0);
+      }).to.not.throwError();
+    });
+  });
+
   describe('#setActive', function() {
     it('removes the vertexFeature of deactivation', function() {
       const modify = new Modify({