feat(web): show ocr text boxes in panoramas (#25727)

web/src/lib/components/asset-viewer/asset-viewer.svelte

@@ -425,7 +425,6 @@
   const showOcrButton = $derived(
     $slideshowState === SlideshowState.None &&
       asset.type === AssetTypeEnum.Image &&
-      !(asset.exifInfo?.projectionType === 'EQUIRECTANGULAR') &&
       !assetViewerManager.isShowEditor &&
       ocrManager.hasOcrData,
   );

web/src/lib/components/asset-viewer/ocr-bounding-box.svelte

@@ -1,6 +1,6 @@
 <script lang="ts">
   import type { OcrBox } from '$lib/utils/ocr-utils';
-  import { calculateBoundingBoxDimensions } from '$lib/utils/ocr-utils';
+  import { calculateBoundingBoxMatrix } from '$lib/utils/ocr-utils';
 
   type Props = {
     ocrBox: OcrBox;
@@ -8,28 +8,19 @@
 
   let { ocrBox }: Props = $props();
 
-  const dimensions = $derived(calculateBoundingBoxDimensions(ocrBox.points));
+  const dimensions = $derived(calculateBoundingBoxMatrix(ocrBox.points));
 
-  const transform = $derived(
-    `translate(${dimensions.minX}px, ${dimensions.minY}px) rotate(${dimensions.rotation}deg) skew(${dimensions.skewX}deg, ${dimensions.skewY}deg)`,
-  );
-
-  const transformOrigin = $derived(
-    `${dimensions.centerX - dimensions.minX}px ${dimensions.centerY - dimensions.minY}px`,
+  const transform = $derived(`matrix3d(${dimensions.matrix.join(',')})`);
+  // Fits almost all strings within the box, depends on font family
+  const fontSize = $derived(
+    `max(var(--text-sm), min(var(--text-6xl), ${(1.4 * dimensions.width) / ocrBox.text.length}px))`,
   );
 </script>
 
-<div class="absolute group left-0 top-0 pointer-events-none">
-  <!-- Bounding box with CSS transforms -->
+<div class="absolute left-0 top-0">
   <div
-    class="absolute border-2 border-blue-500 bg-blue-500/10 cursor-pointer pointer-events-auto transition-all group-hover:bg-blue-500/30 group-hover:border-blue-600 group-hover:border-[3px]"
-    style="width: {dimensions.width}px; height: {dimensions.height}px; transform: {transform}; transform-origin: {transformOrigin};"
-  ></div>
-
-  <!-- Text overlay - always rendered but invisible, allows text selection and copy -->
-  <div
-    class="absolute flex items-center justify-center text-transparent text-sm px-2 py-1 pointer-events-auto cursor-text whitespace-pre-wrap wrap-break-word select-text group-hover:text-white group-hover:bg-black/75 group-hover:z-10"
-    style="width: {dimensions.width}px; height: {dimensions.height}px; transform: {transform}; transform-origin: {transformOrigin};"
+    class="absolute flex items-center justify-center text-transparent text-sm border-2 border-blue-500 bg-blue-500/10 px-2 py-1 pointer-events-auto cursor-text whitespace-pre-wrap wrap-break-word select-text transition-all hover:text-white hover:bg-black/60 hover:border-blue-600 hover:border-3"
+    style="font-size: {fontSize}; width: {dimensions.width}px; height: {dimensions.height}px; transform: {transform}; transform-origin: 0 0;"
   >
     {ocrBox.text}
   </div>

web/src/lib/components/asset-viewer/photo-sphere-viewer-adapter.svelte

@@ -2,8 +2,10 @@
   import { shortcuts } from '$lib/actions/shortcut';
   import AssetViewerEvents from '$lib/components/AssetViewerEvents.svelte';
   import { assetViewerManager } from '$lib/managers/asset-viewer-manager.svelte';
+  import { ocrManager, type OcrBoundingBox } from '$lib/stores/ocr.svelte';
   import { boundingBoxesArray, type Faces } from '$lib/stores/people.store';
   import { alwaysLoadOriginalFile } from '$lib/stores/preferences.store';
+  import { calculateBoundingBoxMatrix, getOcrBoundingBoxesAtSize, type Point } from '$lib/utils/ocr-utils';
   import {
     EquirectangularAdapter,
     Viewer,
@@ -27,6 +29,17 @@
     strokeLinejoin: 'round',
   };
 
+  // Adapted as well as possible from classlist 'border-2 border-blue-500 bg-blue-500/10 hover:border-blue-600 hover:border-3'
+  const OCR_BOX_SVG_STYLE = {
+    fill: 'var(--color-blue-500)',
+    fillOpacity: '0.1',
+    stroke: 'var(--color-blue-500)',
+    strokeWidth: '2px',
+  };
+
+  const OCR_TOOLTIP_HTML_CLASS =
+    'flex items-center justify-center text-white bg-black/50 cursor-text pointer-events-auto whitespace-pre-wrap wrap-break-word select-text';
+
   type Props = {
     panorama: string | { source: string };
     originalPanorama?: string | { source: string };
@@ -96,6 +109,59 @@
     }
   });
 
+  $effect(() => {
+    updateOcrBoxes(ocrManager.showOverlay, ocrManager.data);
+  });
+
+  /** Use updateOnly=true on zoom, pan, or resize. */
+  const updateOcrBoxes = (showOverlay: boolean, ocrData: OcrBoundingBox[], updateOnly = false) => {
+    if (!viewer || !viewer.state.textureData || !viewer.getPlugin(MarkersPlugin)) {
+      return;
+    }
+    const markersPlugin = viewer.getPlugin<MarkersPlugin>(MarkersPlugin);
+    if (!showOverlay) {
+      markersPlugin.clearMarkers();
+      return;
+    }
+    if (!updateOnly) {
+      markersPlugin.clearMarkers();
+    }
+
+    const boxes = getOcrBoundingBoxesAtSize(ocrData, {
+      width: viewer.state.textureData.panoData.croppedWidth,
+      height: viewer.state.textureData.panoData.croppedHeight,
+    });
+
+    for (const [index, box] of boxes.entries()) {
+      const points = box.points.map((p) => texturePointToViewerPoint(viewer, p));
+      const { matrix, width, height } = calculateBoundingBoxMatrix(points);
+
+      const fontSize = (1.4 * width) / box.text.length; // fits almost all strings within the box, depends on font family
+      const transform = `matrix3d(${matrix.join(',')})`;
+      const content = `<div class="${OCR_TOOLTIP_HTML_CLASS}" style="font-size: ${fontSize}px; width: ${width}px; height: ${height}px; transform: ${transform}; transform-origin: 0 0;">${box.text}</div>`;
+
+      if (updateOnly) {
+        markersPlugin.updateMarker({
+          id: `box_${index}`,
+          polygonPixels: box.points.map((b) => [b.x, b.y]),
+          tooltip: { content },
+        });
+      } else {
+        markersPlugin.addMarker({
+          id: `box_${index}`,
+          polygonPixels: box.points.map((b) => [b.x, b.y]),
+          svgStyle: OCR_BOX_SVG_STYLE,
+          tooltip: { content, trigger: 'click' },
+        });
+      }
+    }
+  };
+
+  const texturePointToViewerPoint = (viewer: Viewer, point: Point) => {
+    const spherical = viewer.dataHelper.textureCoordsToSphericalCoords({ textureX: point.x, textureY: point.y });
+    return viewer.dataHelper.sphericalCoordsToViewerCoords(spherical);
+  };
+
   const onZoom = () => {
     viewer?.animate({ zoom: assetViewerManager.zoom > 1 ? 50 : 83.3, speed: 250 });
   };
@@ -160,7 +226,20 @@
       viewer.addEventListener(events.ZoomUpdatedEvent.type, zoomHandler, { passive: true });
     }
 
-    return () => viewer.removeEventListener(events.ZoomUpdatedEvent.type, zoomHandler);
+    const onReadyHandler = () => updateOcrBoxes(ocrManager.showOverlay, ocrManager.data, false);
+    const updateHandler = () => updateOcrBoxes(ocrManager.showOverlay, ocrManager.data, true);
+    viewer.addEventListener(events.ReadyEvent.type, onReadyHandler);
+    viewer.addEventListener(events.PositionUpdatedEvent.type, updateHandler);
+    viewer.addEventListener(events.SizeUpdatedEvent.type, updateHandler);
+    viewer.addEventListener(events.ZoomUpdatedEvent.type, updateHandler, { passive: true });
+
+    return () => {
+      viewer.removeEventListener(events.ReadyEvent.type, onReadyHandler);
+      viewer.removeEventListener(events.PositionUpdatedEvent.type, updateHandler);
+      viewer.removeEventListener(events.SizeUpdatedEvent.type, updateHandler);
+      viewer.removeEventListener(events.ZoomUpdatedEvent.type, updateHandler);
+      viewer.removeEventListener(events.ZoomUpdatedEvent.type, zoomHandler);
+    };
   });
 
   onDestroy(() => {
@@ -176,3 +255,25 @@
 
 <svelte:document use:shortcuts={[{ shortcut: { key: 'z' }, onShortcut: onZoom, preventDefault: true }]} />
 <div class="h-full w-full mb-0" bind:this={container}></div>
+
+<style>
+  /* Reset the default tooltip styling */
+  :global(.psv-tooltip) {
+    top: 0 !important;
+    left: 0 !important;
+    background: none;
+    box-shadow: none;
+    width: 0;
+    height: 0;
+  }
+
+  :global(.psv-tooltip-content) {
+    font: var(--font-normal);
+    padding: 0;
+    text-shadow: none;
+  }
+
+  :global(.psv-tooltip-arrow) {
+    display: none;
+  }
+</style>