feat(web): OCR overlay interactivity during zoom (#27039)

Change-Id: Id62e1a0264df2de0f3177a59b24bc5176a6a6964

web/src/lib/utils/container-utils.spec.ts

@@ -1,4 +1,11 @@
-import { getContentMetrics, getNaturalSize, scaleToFit } from '$lib/utils/container-utils';
+import {
+  getContentMetrics,
+  getNaturalSize,
+  mapNormalizedRectToContent,
+  mapNormalizedToContent,
+  scaleToCover,
+  scaleToFit,
+} from '$lib/utils/container-utils';
 
 const mockImage = (props: {
   naturalWidth: number;
@@ -92,3 +99,81 @@ describe('getNaturalSize', () => {
     expect(getNaturalSize(video)).toEqual({ width: 1920, height: 1080 });
   });
 });
+
+describe('scaleToCover', () => {
+  it('should scale up to cover container when image is smaller', () => {
+    expect(scaleToCover({ width: 400, height: 300 }, { width: 800, height: 600 })).toEqual({
+      width: 800,
+      height: 600,
+    });
+  });
+
+  it('should use height scale when image is wider than container', () => {
+    expect(scaleToCover({ width: 2000, height: 1000 }, { width: 800, height: 600 })).toEqual({
+      width: 1200,
+      height: 600,
+    });
+  });
+
+  it('should use width scale when image is taller than container', () => {
+    expect(scaleToCover({ width: 1000, height: 2000 }, { width: 800, height: 600 })).toEqual({
+      width: 800,
+      height: 1600,
+    });
+  });
+});
+
+describe('mapNormalizedToContent', () => {
+  const metrics = { contentWidth: 800, contentHeight: 400, offsetX: 0, offsetY: 100 };
+
+  it('should map top-left corner', () => {
+    expect(mapNormalizedToContent({ x: 0, y: 0 }, metrics)).toEqual({ x: 0, y: 100 });
+  });
+
+  it('should map bottom-right corner', () => {
+    expect(mapNormalizedToContent({ x: 1, y: 1 }, metrics)).toEqual({ x: 800, y: 500 });
+  });
+
+  it('should map center point', () => {
+    expect(mapNormalizedToContent({ x: 0.5, y: 0.5 }, metrics)).toEqual({ x: 400, y: 300 });
+  });
+
+  it('should apply offsets correctly for letterboxed content', () => {
+    const letterboxed = { contentWidth: 300, contentHeight: 600, offsetX: 250, offsetY: 0 };
+    expect(mapNormalizedToContent({ x: 0, y: 0 }, letterboxed)).toEqual({ x: 250, y: 0 });
+    expect(mapNormalizedToContent({ x: 1, y: 1 }, letterboxed)).toEqual({ x: 550, y: 600 });
+  });
+
+  it('should accept Size (zero offsets)', () => {
+    const size = { width: 800, height: 400 };
+    expect(mapNormalizedToContent({ x: 0, y: 0 }, size)).toEqual({ x: 0, y: 0 });
+    expect(mapNormalizedToContent({ x: 1, y: 1 }, size)).toEqual({ x: 800, y: 400 });
+    expect(mapNormalizedToContent({ x: 0.5, y: 0.5 }, size)).toEqual({ x: 400, y: 200 });
+  });
+});
+
+describe('mapNormalizedRectToContent', () => {
+  const metrics = { contentWidth: 800, contentHeight: 400, offsetX: 0, offsetY: 100 };
+
+  it('should map a normalized rect to content pixel coordinates', () => {
+    const rect = mapNormalizedRectToContent({ x: 0.25, y: 0.25 }, { x: 0.75, y: 0.75 }, metrics);
+    expect(rect).toEqual({ left: 200, top: 200, width: 400, height: 200 });
+  });
+
+  it('should map full image rect', () => {
+    const rect = mapNormalizedRectToContent({ x: 0, y: 0 }, { x: 1, y: 1 }, metrics);
+    expect(rect).toEqual({ left: 0, top: 100, width: 800, height: 400 });
+  });
+
+  it('should handle letterboxed content with horizontal offsets', () => {
+    const letterboxed = { contentWidth: 300, contentHeight: 600, offsetX: 250, offsetY: 0 };
+    const rect = mapNormalizedRectToContent({ x: 0, y: 0 }, { x: 1, y: 1 }, letterboxed);
+    expect(rect).toEqual({ left: 250, top: 0, width: 300, height: 600 });
+  });
+
+  it('should accept Size (zero offsets)', () => {
+    const size = { width: 800, height: 400 };
+    const rect = mapNormalizedRectToContent({ x: 0.25, y: 0.25 }, { x: 0.75, y: 0.75 }, size);
+    expect(rect).toEqual({ left: 200, top: 100, width: 400, height: 200 });
+  });
+});

web/src/lib/utils/container-utils.ts

@@ -1,14 +1,35 @@
-export interface ContentMetrics {
+// Coordinate spaces used throughout the viewer:
+//
+// "Normalized": 0–1 range, (0,0) = top-left, (1,1) = bottom-right. Resolution-independent.
+//   Example: OCR coordinates, or face coords after dividing by metadata dimensions.
+//
+// "Content": pixel position within the container after scaling (scaleToFit/scaleToCover)
+//   and centering. Used for DOM overlay positioning (face boxes, OCR text).
+//
+// "Natural": pixel position in the original full-resolution image file (e.g. 4000×3000).
+//   Used when cropping or drawing on the source image.
+//
+// "Metadata pixel space": coordinates from face detection / OCR models, in pixels relative
+//   to face.imageWidth/imageHeight. Divide by those dimensions to get normalized coords.
+
+export type Point = {
+  x: number;
+  y: number;
+};
+
+export type Size = {
+  width: number;
+  height: number;
+};
+
+export type ContentMetrics = {
   contentWidth: number;
   contentHeight: number;
   offsetX: number;
   offsetY: number;
-}
+};
 
-export const scaleToCover = (
-  dimensions: { width: number; height: number },
-  container: { width: number; height: number },
-): { width: number; height: number } => {
+export const scaleToCover = (dimensions: Size, container: Size): Size => {
   const scaleX = container.width / dimensions.width;
   const scaleY = container.height / dimensions.height;
   const scale = Math.max(scaleX, scaleY);
@@ -18,10 +39,7 @@ export const scaleToCover = (
   };
 };
 
-export const scaleToFit = (
-  dimensions: { width: number; height: number },
-  container: { width: number; height: number },
-): { width: number; height: number } => {
+export const scaleToFit = (dimensions: Size, container: Size): Size => {
   const scaleX = container.width / dimensions.width;
   const scaleY = container.height / dimensions.height;
   const scale = Math.min(scaleX, scaleY);
@@ -31,14 +49,14 @@ export const scaleToFit = (
   };
 };
 
-const getElementSize = (element: HTMLImageElement | HTMLVideoElement): { width: number; height: number } => {
+const getElementSize = (element: HTMLImageElement | HTMLVideoElement): Size => {
   if (element instanceof HTMLVideoElement) {
     return { width: element.clientWidth, height: element.clientHeight };
   }
   return { width: element.width, height: element.height };
 };
 
-export const getNaturalSize = (element: HTMLImageElement | HTMLVideoElement): { width: number; height: number } => {
+export const getNaturalSize = (element: HTMLImageElement | HTMLVideoElement): Size => {
   if (element instanceof HTMLVideoElement) {
     return { width: element.videoWidth, height: element.videoHeight };
   }
@@ -56,3 +74,38 @@ export const getContentMetrics = (element: HTMLImageElement | HTMLVideoElement):
     offsetY: (client.height - contentHeight) / 2,
   };
 };
+
+export function mapNormalizedToContent(point: Point, sizeOrMetrics: Size | ContentMetrics): Point {
+  if ('contentWidth' in sizeOrMetrics) {
+    return {
+      x: point.x * sizeOrMetrics.contentWidth + sizeOrMetrics.offsetX,
+      y: point.y * sizeOrMetrics.contentHeight + sizeOrMetrics.offsetY,
+    };
+  }
+  return {
+    x: point.x * sizeOrMetrics.width,
+    y: point.y * sizeOrMetrics.height,
+  };
+}
+
+export type Rect = {
+  top: number;
+  left: number;
+  width: number;
+  height: number;
+};
+
+export function mapNormalizedRectToContent(
+  topLeft: Point,
+  bottomRight: Point,
+  sizeOrMetrics: Size | ContentMetrics,
+): Rect {
+  const tl = mapNormalizedToContent(topLeft, sizeOrMetrics);
+  const br = mapNormalizedToContent(bottomRight, sizeOrMetrics);
+  return {
+    top: tl.y,
+    left: tl.x,
+    width: br.x - tl.x,
+    height: br.y - tl.y,
+  };
+}

web/src/lib/utils/ocr-utils.spec.ts

@@ -1,5 +1,5 @@
 import type { OcrBoundingBox } from '$lib/stores/ocr.svelte';
-import type { ContentMetrics } from '$lib/utils/container-utils';
+import type { Size } from '$lib/utils/container-utils';
 import { getOcrBoundingBoxes } from '$lib/utils/ocr-utils';
 
 describe('getOcrBoundingBoxes', () => {
@@ -21,9 +21,9 @@ describe('getOcrBoundingBoxes', () => {
         text: 'hello',
       },
     ];
-    const metrics: ContentMetrics = { contentWidth: 1000, contentHeight: 500, offsetX: 0, offsetY: 0 };
+    const imageSize: Size = { width: 1000, height: 500 };
 
-    const boxes = getOcrBoundingBoxes(ocrData, metrics);
+    const boxes = getOcrBoundingBoxes(ocrData, imageSize);
 
     expect(boxes).toHaveLength(1);
     expect(boxes[0].id).toBe('box1');
@@ -37,7 +37,7 @@ describe('getOcrBoundingBoxes', () => {
     ]);
   });
 
-  it('should apply offsets for letterboxed images', () => {
+  it('should map full-image box to full display area', () => {
     const ocrData: OcrBoundingBox[] = [
       {
         id: 'box1',
@@ -55,21 +55,20 @@ describe('getOcrBoundingBoxes', () => {
         text: 'test',
       },
     ];
-    const metrics: ContentMetrics = { contentWidth: 600, contentHeight: 400, offsetX: 100, offsetY: 50 };
+    const imageSize: Size = { width: 600, height: 400 };
 
-    const boxes = getOcrBoundingBoxes(ocrData, metrics);
+    const boxes = getOcrBoundingBoxes(ocrData, imageSize);
 
     expect(boxes[0].points).toEqual([
-      { x: 100, y: 50 },
-      { x: 700, y: 50 },
-      { x: 700, y: 450 },
-      { x: 100, y: 450 },
+      { x: 0, y: 0 },
+      { x: 600, y: 0 },
+      { x: 600, y: 400 },
+      { x: 0, y: 400 },
     ]);
   });
 
   it('should return empty array for empty input', () => {
-    const metrics: ContentMetrics = { contentWidth: 800, contentHeight: 600, offsetX: 0, offsetY: 0 };
-    expect(getOcrBoundingBoxes([], metrics)).toEqual([]);
+    expect(getOcrBoundingBoxes([], { width: 800, height: 600 })).toEqual([]);
   });
 
   it('should handle multiple boxes', () => {
@@ -105,9 +104,9 @@ describe('getOcrBoundingBoxes', () => {
         text: 'second',
       },
     ];
-    const metrics: ContentMetrics = { contentWidth: 200, contentHeight: 200, offsetX: 0, offsetY: 0 };
+    const imageSize: Size = { width: 200, height: 200 };
 
-    const boxes = getOcrBoundingBoxes(ocrData, metrics);
+    const boxes = getOcrBoundingBoxes(ocrData, imageSize);
 
     expect(boxes).toHaveLength(2);
     expect(boxes[0].text).toBe('first');

web/src/lib/utils/ocr-utils.ts

@@ -1,23 +1,19 @@
 import type { OcrBoundingBox } from '$lib/stores/ocr.svelte';
-import type { ContentMetrics } from '$lib/utils/container-utils';
+import { mapNormalizedToContent, type Point, type Size } from '$lib/utils/container-utils';
 import { clamp } from 'lodash-es';
-
-export type Point = {
-  x: number;
-  y: number;
-};
+export type { Point } from '$lib/utils/container-utils';
 
 const distance = (p1: Point, p2: Point) => Math.hypot(p2.x - p1.x, p2.y - p1.y);
 
 export type VerticalMode = 'none' | 'cjk' | 'rotated';
 
-export interface OcrBox {
+export type OcrBox = {
   id: string;
   points: Point[];
   text: string;
   confidence: number;
   verticalMode: VerticalMode;
-}
+};
 
 const CJK_PATTERN =
   /[\u3000-\u303F\u3040-\u309F\u30A0-\u30FF\u3400-\u4DBF\u4E00-\u9FFF\uF900-\uFAFF\uAC00-\uD7AF\uFF00-\uFFEF]/;
@@ -38,7 +34,7 @@ const getVerticalMode = (width: number, height: number, text: string): VerticalM
  * @param points - Array of 4 corner points of the bounding box
  * @returns 4x4 matrix to transform the div with text onto the polygon defined by the corner points, and size to set on the source div.
  */
-export const calculateBoundingBoxMatrix = (points: Point[]): { matrix: number[]; width: number; height: number } => {
+export const calculateBoundingBoxMatrix = (points: Point[]): Size & { matrix: number[] } => {
   const [topLeft, topRight, bottomRight, bottomLeft] = points;
 
   const width = Math.max(distance(topLeft, topRight), distance(bottomLeft, bottomRight));
@@ -163,7 +159,7 @@ export const calculateFittedFontSize = (
   return clamp(Math.min(scaleFromWidth, scaleFromHeight), MIN_FONT_SIZE, MAX_FONT_SIZE);
 };
 
-export const getOcrBoundingBoxes = (ocrData: OcrBoundingBox[], metrics: ContentMetrics): OcrBox[] => {
+export const getOcrBoundingBoxes = (ocrData: OcrBoundingBox[], imageSize: Size): OcrBox[] => {
   const boxes: OcrBox[] = [];
   for (const ocr of ocrData) {
     const points = [
@@ -171,10 +167,7 @@ export const getOcrBoundingBoxes = (ocrData: OcrBoundingBox[], metrics: ContentM
       { x: ocr.x2, y: ocr.y2 },
       { x: ocr.x3, y: ocr.y3 },
       { x: ocr.x4, y: ocr.y4 },
-    ].map((point) => ({
-      x: point.x * metrics.contentWidth + metrics.offsetX,
-      y: point.y * metrics.contentHeight + metrics.offsetY,
-    }));
+    ].map((point) => mapNormalizedToContent(point, imageSize));
 
     const boxWidth = Math.max(distance(points[0], points[1]), distance(points[3], points[2]));
     const boxHeight = Math.max(distance(points[0], points[3]), distance(points[1], points[2]));
@@ -188,7 +181,7 @@ export const getOcrBoundingBoxes = (ocrData: OcrBoundingBox[], metrics: ContentM
     });
   }
 
-  const rowThreshold = metrics.contentHeight * 0.02;
+  const rowThreshold = imageSize.height * 0.02;
   boxes.sort((a, b) => {
     const yDifference = a.points[0].y - b.points[0].y;
     if (Math.abs(yDifference) < rowThreshold) {

web/src/lib/utils/people-utils.spec.ts

@@ -1,5 +1,5 @@
 import type { Faces } from '$lib/stores/people.store';
-import type { ContentMetrics } from '$lib/utils/container-utils';
+import type { Size } from '$lib/utils/container-utils';
 import { getBoundingBox } from '$lib/utils/people-utils';
 
 const makeFace = (overrides: Partial<Faces> = {}): Faces => ({
@@ -16,21 +16,21 @@ const makeFace = (overrides: Partial<Faces> = {}): Faces => ({
 describe('getBoundingBox', () => {
   it('should scale face coordinates to display dimensions', () => {
     const face = makeFace();
-    const metrics: ContentMetrics = { contentWidth: 800, contentHeight: 600, offsetX: 0, offsetY: 0 };
+    const imageSize: Size = { width: 800, height: 600 };
 
-    const boxes = getBoundingBox([face], metrics);
+    const boxes = getBoundingBox([face], imageSize);
 
     expect(boxes).toHaveLength(1);
     expect(boxes[0]).toEqual({
       id: 'face-1',
-      top: Math.round(600 * (750 / 3000)),
-      left: Math.round(800 * (1000 / 4000)),
-      width: Math.round(800 * (2000 / 4000) - 800 * (1000 / 4000)),
-      height: Math.round(600 * (1500 / 3000) - 600 * (750 / 3000)),
+      top: 600 * (750 / 3000),
+      left: 800 * (1000 / 4000),
+      width: 800 * (2000 / 4000) - 800 * (1000 / 4000),
+      height: 600 * (1500 / 3000) - 600 * (750 / 3000),
     });
   });
 
-  it('should apply offsets for letterboxed display', () => {
+  it('should map full-image face to full display area', () => {
     const face = makeFace({
       imageWidth: 1000,
       imageHeight: 1000,
@@ -39,49 +39,21 @@ describe('getBoundingBox', () => {
       boundingBoxX2: 1000,
       boundingBoxY2: 1000,
     });
-    const metrics: ContentMetrics = { contentWidth: 600, contentHeight: 600, offsetX: 100, offsetY: 0 };
+    const imageSize: Size = { width: 600, height: 600 };
 
-    const boxes = getBoundingBox([face], metrics);
+    const boxes = getBoundingBox([face], imageSize);
 
     expect(boxes[0]).toEqual({
       id: 'face-1',
       top: 0,
-      left: 100,
+      left: 0,
       width: 600,
       height: 600,
     });
   });
 
-  it('should handle zoom by pre-scaled metrics', () => {
-    const face = makeFace({
-      imageWidth: 1000,
-      imageHeight: 1000,
-      boundingBoxX1: 0,
-      boundingBoxY1: 0,
-      boundingBoxX2: 500,
-      boundingBoxY2: 500,
-    });
-    const metrics: ContentMetrics = {
-      contentWidth: 1600,
-      contentHeight: 1200,
-      offsetX: -200,
-      offsetY: -100,
-    };
-
-    const boxes = getBoundingBox([face], metrics);
-
-    expect(boxes[0]).toEqual({
-      id: 'face-1',
-      top: -100,
-      left: -200,
-      width: 800,
-      height: 600,
-    });
-  });
-
   it('should return empty array for empty faces', () => {
-    const metrics: ContentMetrics = { contentWidth: 800, contentHeight: 600, offsetX: 0, offsetY: 0 };
-    expect(getBoundingBox([], metrics)).toEqual([]);
+    expect(getBoundingBox([], { width: 800, height: 600 })).toEqual([]);
   });
 
   it('should handle multiple faces', () => {
@@ -89,9 +61,8 @@ describe('getBoundingBox', () => {
       makeFace({ id: 'face-1', boundingBoxX1: 0, boundingBoxY1: 0, boundingBoxX2: 1000, boundingBoxY2: 1000 }),
       makeFace({ id: 'face-2', boundingBoxX1: 2000, boundingBoxY1: 1500, boundingBoxX2: 3000, boundingBoxY2: 2500 }),
     ];
-    const metrics: ContentMetrics = { contentWidth: 800, contentHeight: 600, offsetX: 0, offsetY: 0 };
 
-    const boxes = getBoundingBox(faces, metrics);
+    const boxes = getBoundingBox(faces, { width: 800, height: 600 });
 
     expect(boxes).toHaveLength(2);
     expect(boxes[0].left).toBeLessThan(boxes[1].left);

web/src/lib/utils/people-utils.ts

@@ -1,37 +1,21 @@
 import type { Faces } from '$lib/stores/people.store';
 import { getAssetMediaUrl } from '$lib/utils';
-import type { ContentMetrics } from '$lib/utils/container-utils';
+import { mapNormalizedRectToContent, type Rect, type Size } from '$lib/utils/container-utils';
 import { AssetTypeEnum, type AssetFaceResponseDto } from '@immich/sdk';
 
-export interface BoundingBox {
-  id: string;
-  top: number;
-  left: number;
-  width: number;
-  height: number;
-}
+export type BoundingBox = Rect & { id: string };
 
-export const getBoundingBox = (faces: Faces[], metrics: ContentMetrics): BoundingBox[] => {
+export const getBoundingBox = (faces: Faces[], imageSize: Size): BoundingBox[] => {
   const boxes: BoundingBox[] = [];
 
   for (const face of faces) {
-    const scaleX = metrics.contentWidth / face.imageWidth;
-    const scaleY = metrics.contentHeight / face.imageHeight;
+    const rect = mapNormalizedRectToContent(
+      { x: face.boundingBoxX1 / face.imageWidth, y: face.boundingBoxY1 / face.imageHeight },
+      { x: face.boundingBoxX2 / face.imageWidth, y: face.boundingBoxY2 / face.imageHeight },
+      imageSize,
+    );
 
-    const coordinates = {
-      x1: scaleX * face.boundingBoxX1 + metrics.offsetX,
-      x2: scaleX * face.boundingBoxX2 + metrics.offsetX,
-      y1: scaleY * face.boundingBoxY1 + metrics.offsetY,
-      y2: scaleY * face.boundingBoxY2 + metrics.offsetY,
-    };
-
-    boxes.push({
-      id: face.id,
-      top: Math.round(coordinates.y1),
-      left: Math.round(coordinates.x1),
-      width: Math.round(coordinates.x2 - coordinates.x1),
-      height: Math.round(coordinates.y2 - coordinates.y1),
-    });
+    boxes.push({ id: face.id, ...rect });
   }
 
   return boxes;