Merge pull request #10691 from ahocevar/hitdetect-color-range

Do not exceed color range

src/ol/render/canvas/hitdetect.js

@@ -33,7 +33,7 @@ export function createHitDetectionImageData(size, transforms, features, styleFun
   const renderer = new CanvasImmediateRenderer(context, 0.5, extent, null, rotation);
   const featureCount = features.length;
   // Stretch hit detection index to use the whole available color range
-  const indexFactor = Math.ceil((256 * 256 * 256 - 1) / featureCount);
+  const indexFactor = Math.floor((256 * 256 * 256 - 1) / featureCount);
   const featuresByZIndex = {};
   for (let i = 1; i <= featureCount; ++i) {
     const feature = features[i - 1];
@@ -121,6 +121,7 @@ export function createHitDetectionImageData(size, transforms, features, styleFun
       }
     }
   }
+  document.body.appendChild(context.canvas);
   return context.getImageData(0, 0, canvas.width, canvas.height);
 }
 
@@ -141,7 +142,7 @@ export function hitDetect(pixel, features, imageData) {
     const g = imageData.data[index + 1];
     const b = imageData.data[index + 2];
     const i = b + (256 * (g + (256 * r)));
-    const indexFactor = Math.ceil((256 * 256 * 256 - 1) / features.length);
+    const indexFactor = Math.floor((256 * 256 * 256 - 1) / features.length);
     if (i && i % indexFactor === 0) {
       resultFeatures.push(features[i / indexFactor - 1]);
     }

test/spec/ol/render/canvas/hitdetect.test.js

@@ -0,0 +1,33 @@
+import {createHitDetectionImageData} from '../../../../../src/ol/render/canvas/hitdetect.js';
+import {create} from '../../../../../src/ol/transform.js';
+import Feature from '../../../../../src/ol/Feature.js';
+import Point from '../../../../../src/ol/geom/Point.js';
+import {Style} from '../../../../../src/ol/style.js';
+import Circle from '../../../../../src/ol/style/Circle.js';
+
+describe('hitdetect', function() {
+
+  let features, styleFunction;
+
+  beforeEach(function() {
+    features = [
+      new Feature(new Point([0, 75])),
+      new Feature(new Point([0, 50])),
+      new Feature(new Point([0, 25])),
+      new Feature(new Point([0, 0]))
+    ];
+    styleFunction = function() {
+      return new Style({
+        image: new Circle({
+          radius: 5
+        })
+      });
+    };
+  });
+
+  it ('does not exceed the color range', function() {
+    const imageData = createHitDetectionImageData([2, 2], [create()], features, styleFunction, [0, 0, 0, 0], 1, 0);
+    expect(Array.prototype.slice.call(imageData.data, 0, 3)).to.eql([255, 255, 252]);
+  });
+
+});