feat(web): image-relative overlays with zoom support for faces and OCR

Split out from #26636 (adaptive image loading). Leverages the
ContentMetrics extraction from #26310.

Moves face bounding boxes and OCR overlays from viewport-relative
to image-relative coordinates.

New features: zoom with face editor open, zoom with OCR boxes
visible, accurate face hover hit-testing at any zoom level.

Bug fixes: face tagging on stacked assets, faces loading when browsing
stacks, face editor auto-close on last face deletion, zoomTarget was
null, disablePointer option name mismatch.
This commit is contained in:
midzelis
2026-03-07 21:14:45 +00:00
parent 6fdf99ca16
commit 39d8c6d048
13 changed files with 848 additions and 209 deletions
@@ -284,7 +284,11 @@ const createDefaultOwner = (ownerId: string) => {
* Convert a TimelineAssetConfig to a full AssetResponseDto
* This matches the response from GET /api/assets/:id
*/
export function toAssetResponseDto(asset: MockTimelineAsset, owner?: UserResponseDto): AssetResponseDto {
export function toAssetResponseDto(
asset: MockTimelineAsset,
owner?: UserResponseDto,
overrides?: Partial<Pick<AssetResponseDto, 'people' | 'unassignedFaces'>>,
): AssetResponseDto {
const now = new Date().toISOString();
// Default owner if not provided
@@ -338,8 +342,8 @@ export function toAssetResponseDto(asset: MockTimelineAsset, owner?: UserRespons
exifInfo,
livePhotoVideoId: asset.livePhotoVideoId,
tags: [],
people: [],
unassignedFaces: [],
people: overrides?.people ?? [],
unassignedFaces: overrides?.unassignedFaces ?? [],
stack: asset.stack,
isOffline: false,
hasMetadata: true,
@@ -1,3 +1,9 @@
import {
type AssetFaceResponseDto,
type AssetFaceWithoutPersonResponseDto,
type AssetResponseDto,
type PersonWithFacesResponseDto,
} from '@immich/sdk';
import { BrowserContext } from '@playwright/test';
import { randomThumbnail } from 'src/ui/generators/timeline';
@@ -125,3 +131,117 @@ export const setupFaceEditorMockApiRoutes = async (
});
});
};
export type MockFaceSpec = {
personId: string;
personName: string;
faceId: string;
boundingBoxX1: number;
boundingBoxY1: number;
boundingBoxX2: number;
boundingBoxY2: number;
};
export const createMockFaceData = (
faceSpecs: MockFaceSpec[],
imageWidth: number,
imageHeight: number,
): { people: PersonWithFacesResponseDto[]; unassignedFaces: AssetFaceWithoutPersonResponseDto[] } => {
const people: PersonWithFacesResponseDto[] = faceSpecs.map((spec) => ({
id: spec.personId,
name: spec.personName,
birthDate: null,
isHidden: false,
thumbnailPath: `/upload/thumbs/${spec.personId}.jpeg`,
updatedAt: new Date().toISOString(),
faces: [
{
id: spec.faceId,
imageWidth,
imageHeight,
boundingBoxX1: spec.boundingBoxX1,
boundingBoxY1: spec.boundingBoxY1,
boundingBoxX2: spec.boundingBoxX2,
boundingBoxY2: spec.boundingBoxY2,
},
],
}));
return { people, unassignedFaces: [] };
};
export const setupFaceOverlayMockApiRoutes = async (
context: BrowserContext,
assetDto: AssetResponseDto,
faceSpecs: MockFaceSpec[],
) => {
const faceResponseMap = new Map<string, AssetFaceResponseDto>();
for (const spec of faceSpecs) {
faceResponseMap.set(spec.faceId, {
id: spec.faceId,
imageWidth: assetDto.width ?? 3000,
imageHeight: assetDto.height ?? 4000,
boundingBoxX1: spec.boundingBoxX1,
boundingBoxY1: spec.boundingBoxY1,
boundingBoxX2: spec.boundingBoxX2,
boundingBoxY2: spec.boundingBoxY2,
person: {
id: spec.personId,
name: spec.personName,
birthDate: null,
isHidden: false,
thumbnailPath: `/upload/thumbs/${spec.personId}.jpeg`,
updatedAt: new Date().toISOString(),
},
});
}
await context.route(`**/api/assets/${assetDto.id}`, async (route, request) => {
if (request.method() !== 'GET') {
return route.fallback();
}
return route.fulfill({
status: 200,
contentType: 'application/json',
json: assetDto,
});
});
await context.route(`**/api/faces?id=${assetDto.id}`, async (route, request) => {
if (request.method() !== 'GET') {
return route.fallback();
}
return route.fulfill({
status: 200,
contentType: 'application/json',
json: [...faceResponseMap.values()],
});
});
await context.route('**/api/faces/*', async (route, request) => {
if (request.method() !== 'DELETE') {
return route.fallback();
}
const url = new URL(request.url());
const faceId = url.pathname.split('/').at(-1);
if (faceId) {
faceResponseMap.delete(faceId);
}
return route.fulfill({
status: 200,
contentType: 'text/plain',
body: 'OK',
});
});
await context.route('**/api/people/*/thumbnail', async (route) => {
if (!route.request().serviceWorker()) {
return route.continue();
}
return route.fulfill({
status: 200,
headers: { 'content-type': 'image/jpeg' },
body: await randomThumbnail('person-thumb', 1),
});
});
};
+55
View File
@@ -0,0 +1,55 @@
import { faker } from '@faker-js/faker';
import type { AssetOcrResponseDto } from '@immich/sdk';
import { BrowserContext } from '@playwright/test';
export type MockOcrBox = {
text: string;
x1: number;
y1: number;
x2: number;
y2: number;
x3: number;
y3: number;
x4: number;
y4: number;
};
export const createMockOcrData = (assetId: string, boxes: MockOcrBox[]): AssetOcrResponseDto[] => {
return boxes.map((box) => ({
id: faker.string.uuid(),
assetId,
x1: box.x1,
y1: box.y1,
x2: box.x2,
y2: box.y2,
x3: box.x3,
y3: box.y3,
x4: box.x4,
y4: box.y4,
boxScore: 0.95,
textScore: 0.9,
text: box.text,
}));
};
export const setupOcrMockApiRoutes = async (
context: BrowserContext,
ocrDataByAssetId: Map<string, AssetOcrResponseDto[]>,
) => {
await context.route('**/assets/*/ocr', async (route, request) => {
if (request.method() !== 'GET') {
return route.fallback();
}
const url = new URL(request.url());
const segments = url.pathname.split('/');
const assetIdIndex = segments.indexOf('assets') + 1;
const assetId = segments[assetIdIndex];
const ocrData = ocrDataByAssetId.get(assetId) ?? [];
return route.fulfill({
status: 200,
contentType: 'application/json',
json: ocrData,
});
});
};
@@ -0,0 +1,196 @@
import { expect, test } from '@playwright/test';
import { toAssetResponseDto } from 'src/ui/generators/timeline';
import {
createMockFaceData,
createMockPeople,
type MockFaceSpec,
setupFaceEditorMockApiRoutes,
setupFaceOverlayMockApiRoutes,
} from 'src/ui/mock-network/face-editor-network';
import { assetViewerUtils } from '../timeline/utils';
import { ensureDetailPanelVisible, setupAssetViewerFixture } from './utils';
test.describe.configure({ mode: 'parallel' });
const FACE_SPECS: MockFaceSpec[] = [
{
personId: 'person-alice',
personName: 'Alice Johnson',
faceId: 'face-alice',
boundingBoxX1: 1000,
boundingBoxY1: 500,
boundingBoxX2: 1500,
boundingBoxY2: 1200,
},
{
personId: 'person-bob',
personName: 'Bob Smith',
faceId: 'face-bob',
boundingBoxX1: 2000,
boundingBoxY1: 800,
boundingBoxX2: 2400,
boundingBoxY2: 1600,
},
];
test.describe('face overlay bounding boxes', () => {
const fixture = setupAssetViewerFixture(901);
const mockPeople = createMockPeople(4);
test.beforeEach(async ({ context }) => {
const faceData = createMockFaceData(
FACE_SPECS,
fixture.primaryAssetDto.width ?? 3000,
fixture.primaryAssetDto.height ?? 4000,
);
const assetDtoWithFaces = toAssetResponseDto(fixture.primaryAsset, undefined, faceData);
await setupFaceOverlayMockApiRoutes(context, assetDtoWithFaces, FACE_SPECS);
await setupFaceEditorMockApiRoutes(context, mockPeople, { requests: [] });
});
test('face overlay divs render with correct aria labels', async ({ page }) => {
await page.goto(`/photos/${fixture.primaryAsset.id}`);
await assetViewerUtils.waitForViewerLoad(page, fixture.primaryAsset);
const aliceOverlay = page.getByLabel('Person: Alice Johnson');
const bobOverlay = page.getByLabel('Person: Bob Smith');
await expect(aliceOverlay).toBeVisible();
await expect(bobOverlay).toBeVisible();
});
test('face overlay shows border on hover', async ({ page }) => {
await page.goto(`/photos/${fixture.primaryAsset.id}`);
await assetViewerUtils.waitForViewerLoad(page, fixture.primaryAsset);
const aliceOverlay = page.getByLabel('Person: Alice Johnson');
await expect(aliceOverlay).toBeVisible();
await expect(aliceOverlay).not.toHaveClass(/border-solid/);
await aliceOverlay.hover();
await expect(aliceOverlay).toHaveClass(/border-solid/);
await expect(aliceOverlay).toHaveClass(/border-white/);
await expect(aliceOverlay).toHaveClass(/border-3/);
});
test('face name tooltip appears on hover', async ({ page }) => {
await page.goto(`/photos/${fixture.primaryAsset.id}`);
await assetViewerUtils.waitForViewerLoad(page, fixture.primaryAsset);
const aliceOverlay = page.getByLabel('Person: Alice Johnson');
await expect(aliceOverlay).toBeVisible();
await aliceOverlay.hover();
const nameTooltip = aliceOverlay.locator('div', { hasText: 'Alice Johnson' });
await expect(nameTooltip).toBeVisible();
});
test('face overlays hidden in face edit mode', async ({ page }) => {
await page.goto(`/photos/${fixture.primaryAsset.id}`);
await assetViewerUtils.waitForViewerLoad(page, fixture.primaryAsset);
const aliceOverlay = page.getByLabel('Person: Alice Johnson');
await expect(aliceOverlay).toBeVisible();
await ensureDetailPanelVisible(page);
await page.getByLabel('Tag people').click();
await page.locator('#face-selector').waitFor({ state: 'visible' });
await expect(aliceOverlay).toBeHidden();
});
});
test.describe('zoom and face editor interaction', () => {
const fixture = setupAssetViewerFixture(902);
const mockPeople = createMockPeople(4);
test.beforeEach(async ({ context }) => {
const faceData = createMockFaceData(
FACE_SPECS,
fixture.primaryAssetDto.width ?? 3000,
fixture.primaryAssetDto.height ?? 4000,
);
const assetDtoWithFaces = toAssetResponseDto(fixture.primaryAsset, undefined, faceData);
await setupFaceOverlayMockApiRoutes(context, assetDtoWithFaces, FACE_SPECS);
await setupFaceEditorMockApiRoutes(context, mockPeople, { requests: [] });
});
test('zoom is preserved when entering face edit mode', async ({ page }) => {
await page.goto(`/photos/${fixture.primaryAsset.id}`);
await assetViewerUtils.waitForViewerLoad(page, fixture.primaryAsset);
const { width, height } = page.viewportSize()!;
await page.mouse.move(width / 2, height / 2);
await page.mouse.wheel(0, -1);
await page.waitForTimeout(300);
const zoomedTransform = await page.locator('[data-viewer-content] img[draggable="false"]').evaluate((element) => {
return getComputedStyle(element.closest('[style*="transform"]') ?? element).transform;
});
const isZoomed = zoomedTransform !== 'none' && zoomedTransform !== '';
await ensureDetailPanelVisible(page);
await page.getByLabel('Tag people').click();
await page.locator('#face-selector').waitFor({ state: 'visible' });
await expect(page.locator('#face-editor')).toBeVisible();
if (isZoomed) {
const afterTransform = await page.locator('[data-viewer-content] img[draggable="false"]').evaluate((element) => {
return getComputedStyle(element.closest('[style*="transform"]') ?? element).transform;
});
expect(afterTransform).not.toBe('none');
}
});
});
test.describe('face removal auto-close', () => {
const fixture = setupAssetViewerFixture(903);
const singleFaceSpec: MockFaceSpec[] = [
{
personId: 'person-solo',
personName: 'Solo Person',
faceId: 'face-solo',
boundingBoxX1: 1000,
boundingBoxY1: 500,
boundingBoxX2: 1500,
boundingBoxY2: 1200,
},
];
test.beforeEach(async ({ context }) => {
const faceData = createMockFaceData(
singleFaceSpec,
fixture.primaryAssetDto.width ?? 3000,
fixture.primaryAssetDto.height ?? 4000,
);
const assetDtoWithFaces = toAssetResponseDto(fixture.primaryAsset, undefined, faceData);
await setupFaceOverlayMockApiRoutes(context, assetDtoWithFaces, singleFaceSpec);
});
test('person side panel closes when last face is removed', async ({ page }) => {
await page.goto(`/photos/${fixture.primaryAsset.id}`);
await assetViewerUtils.waitForViewerLoad(page, fixture.primaryAsset);
await ensureDetailPanelVisible(page);
const editPeopleButton = page.locator('#detail-panel').getByLabel('Edit people');
await expect(editPeopleButton).toBeVisible();
await editPeopleButton.click();
const personName = page.locator('text=Solo Person');
await expect(personName.first()).toBeVisible({ timeout: 5000 });
const deleteButton = page.getByLabel('Delete face');
await expect(deleteButton).toBeVisible();
await deleteButton.click();
const confirmButton = page.getByRole('button', { name: /confirm/i });
await expect(confirmButton).toBeVisible();
await confirmButton.click();
await expect(page.locator('text=Edit faces')).toBeHidden({ timeout: 5000 });
});
});
@@ -0,0 +1,152 @@
import type { AssetOcrResponseDto, AssetResponseDto } from '@immich/sdk';
import { expect, test } from '@playwright/test';
import { toAssetResponseDto } from 'src/ui/generators/timeline';
import {
createMockStack,
createMockStackAsset,
MockStack,
setupBrokenAssetMockApiRoutes,
} from 'src/ui/mock-network/broken-asset-network';
import { createMockOcrData, setupOcrMockApiRoutes } from 'src/ui/mock-network/ocr-network';
import { assetViewerUtils } from '../timeline/utils';
import { setupAssetViewerFixture } from './utils';
test.describe.configure({ mode: 'parallel' });
const PRIMARY_OCR_BOXES = [
{ text: 'Hello World', x1: 0.1, y1: 0.1, x2: 0.4, y2: 0.1, x3: 0.4, y3: 0.15, x4: 0.1, y4: 0.15 },
{ text: 'Immich Photo', x1: 0.2, y1: 0.3, x2: 0.6, y2: 0.3, x3: 0.6, y3: 0.36, x4: 0.2, y4: 0.36 },
];
const SECONDARY_OCR_BOXES = [
{ text: 'Second Asset Text', x1: 0.15, y1: 0.2, x2: 0.55, y2: 0.2, x3: 0.55, y3: 0.26, x4: 0.15, y4: 0.26 },
];
test.describe('OCR bounding boxes', () => {
const fixture = setupAssetViewerFixture(920);
test.beforeEach(async ({ context }) => {
const primaryAssetDto = toAssetResponseDto(fixture.primaryAsset);
const ocrDataByAssetId = new Map<string, AssetOcrResponseDto[]>([
[primaryAssetDto.id, createMockOcrData(primaryAssetDto.id, PRIMARY_OCR_BOXES)],
]);
await setupOcrMockApiRoutes(context, ocrDataByAssetId);
});
test('OCR bounding boxes appear when clicking OCR button', async ({ page }) => {
await page.goto(`/photos/${fixture.primaryAsset.id}`);
await assetViewerUtils.waitForViewerLoad(page, fixture.primaryAsset);
const ocrButton = page.getByLabel('Text recognition');
await expect(ocrButton).toBeVisible();
await ocrButton.click();
const ocrBoxes = page.locator('[data-viewer-content] .border-blue-500');
await expect(ocrBoxes).toHaveCount(2);
await expect(ocrBoxes.nth(0)).toContainText('Hello World');
await expect(ocrBoxes.nth(1)).toContainText('Immich Photo');
});
test('OCR bounding boxes toggle off on second click', async ({ page }) => {
await page.goto(`/photos/${fixture.primaryAsset.id}`);
await assetViewerUtils.waitForViewerLoad(page, fixture.primaryAsset);
const ocrButton = page.getByLabel('Text recognition');
await ocrButton.click();
await expect(page.locator('[data-viewer-content] .border-blue-500').first()).toBeVisible();
await ocrButton.click();
await expect(page.locator('[data-viewer-content] .border-blue-500')).toHaveCount(0);
});
});
test.describe('OCR with stacked assets', () => {
const fixture = setupAssetViewerFixture(921);
let mockStack: MockStack;
let primaryAssetDto: AssetResponseDto;
let secondAssetDto: AssetResponseDto;
test.beforeAll(async () => {
primaryAssetDto = toAssetResponseDto(fixture.primaryAsset);
secondAssetDto = createMockStackAsset(fixture.adminUserId);
secondAssetDto.originalFileName = 'second-ocr-asset.jpg';
mockStack = createMockStack(primaryAssetDto, [secondAssetDto], new Set());
});
test.beforeEach(async ({ context }) => {
await setupBrokenAssetMockApiRoutes(context, mockStack);
const ocrDataByAssetId = new Map<string, AssetOcrResponseDto[]>([
[primaryAssetDto.id, createMockOcrData(primaryAssetDto.id, PRIMARY_OCR_BOXES)],
[secondAssetDto.id, createMockOcrData(secondAssetDto.id, SECONDARY_OCR_BOXES)],
]);
await setupOcrMockApiRoutes(context, ocrDataByAssetId);
});
test('different OCR boxes shown for different stacked assets', async ({ page }) => {
await page.goto(`/photos/${fixture.primaryAsset.id}`);
await assetViewerUtils.waitForViewerLoad(page, fixture.primaryAsset);
const ocrButton = page.getByLabel('Text recognition');
await expect(ocrButton).toBeVisible();
await ocrButton.click();
const ocrBoxes = page.locator('[data-viewer-content] .border-blue-500');
await expect(ocrBoxes).toHaveCount(2);
await expect(ocrBoxes.nth(0)).toContainText('Hello World');
const stackThumbnails = page.locator('#stack-slideshow [data-asset]');
await expect(stackThumbnails).toHaveCount(2);
await stackThumbnails.nth(1).click();
// refreshOcr() clears showOverlay when switching assets, so re-enable it
await expect(ocrBoxes).toHaveCount(0);
await expect(ocrButton).toBeVisible();
await ocrButton.click();
await expect(ocrBoxes).toHaveCount(1);
await expect(ocrBoxes.first()).toContainText('Second Asset Text');
});
});
test.describe('OCR boxes and zoom', () => {
const fixture = setupAssetViewerFixture(922);
test.beforeEach(async ({ context }) => {
const primaryAssetDto = toAssetResponseDto(fixture.primaryAsset);
const ocrDataByAssetId = new Map<string, AssetOcrResponseDto[]>([
[primaryAssetDto.id, createMockOcrData(primaryAssetDto.id, PRIMARY_OCR_BOXES)],
]);
await setupOcrMockApiRoutes(context, ocrDataByAssetId);
});
test('OCR boxes scale with zoom', async ({ page }) => {
await page.goto(`/photos/${fixture.primaryAsset.id}`);
await assetViewerUtils.waitForViewerLoad(page, fixture.primaryAsset);
const ocrButton = page.getByLabel('Text recognition');
await expect(ocrButton).toBeVisible();
await ocrButton.click();
const ocrBox = page.locator('[data-viewer-content] .border-blue-500').first();
await expect(ocrBox).toBeVisible();
const initialBox = await ocrBox.boundingBox();
expect(initialBox).toBeTruthy();
const { width, height } = page.viewportSize()!;
await page.mouse.move(width / 2, height / 2);
await page.mouse.wheel(0, -3);
await page.waitForTimeout(500);
const zoomedBox = await ocrBox.boundingBox();
expect(zoomedBox).toBeTruthy();
expect(zoomedBox!.width).toBeGreaterThan(initialBox!.width);
expect(zoomedBox!.height).toBeGreaterThan(initialBox!.height);
});
});
@@ -0,0 +1,100 @@
import { type AssetResponseDto } from '@immich/sdk';
import { expect, test } from '@playwright/test';
import { toAssetResponseDto } from 'src/ui/generators/timeline';
import {
createMockStack,
createMockStackAsset,
MockStack,
setupBrokenAssetMockApiRoutes,
} from 'src/ui/mock-network/broken-asset-network';
import {
createMockPeople,
FaceCreateCapture,
MockPerson,
setupFaceEditorMockApiRoutes,
} from 'src/ui/mock-network/face-editor-network';
import { assetViewerUtils } from '../timeline/utils';
import { ensureDetailPanelVisible, setupAssetViewerFixture } from './utils';
test.describe.configure({ mode: 'parallel' });
test.describe('stack face-tag selection preservation', () => {
const fixture = setupAssetViewerFixture(910);
let mockStack: MockStack;
let primaryAssetDto: AssetResponseDto;
let secondAssetDto: AssetResponseDto;
let mockPeople: MockPerson[];
let faceCreateCapture: FaceCreateCapture;
test.beforeAll(async () => {
primaryAssetDto = toAssetResponseDto(fixture.primaryAsset);
secondAssetDto = createMockStackAsset(fixture.adminUserId);
secondAssetDto.originalFileName = 'second-stacked-asset.jpg';
mockStack = createMockStack(primaryAssetDto, [secondAssetDto], new Set());
mockPeople = createMockPeople(3);
});
test.beforeEach(async ({ context }) => {
faceCreateCapture = { requests: [] };
await setupBrokenAssetMockApiRoutes(context, mockStack);
await setupFaceEditorMockApiRoutes(context, mockPeople, faceCreateCapture);
});
test('selected stacked asset is preserved after tagging a face', async ({ page }) => {
await page.goto(`/photos/${fixture.primaryAsset.id}`);
await assetViewerUtils.waitForViewerLoad(page, fixture.primaryAsset);
const stackSlideshow = page.locator('#stack-slideshow');
await expect(stackSlideshow).toBeVisible();
const stackThumbnails = stackSlideshow.locator('[data-asset]');
await expect(stackThumbnails).toHaveCount(2);
await stackThumbnails.nth(1).click();
await ensureDetailPanelVisible(page);
await expect(page.locator('#detail-panel')).toContainText('second-stacked-asset.jpg');
await page.getByLabel('Tag people').click();
await page.locator('#face-selector').waitFor({ state: 'visible' });
await page.locator('#face-selector').getByText(mockPeople[0].name).click();
const confirmButton = page.getByRole('button', { name: /confirm/i });
await expect(confirmButton).toBeVisible();
await confirmButton.click();
await expect(page.locator('#face-selector')).toBeHidden();
expect(faceCreateCapture.requests).toHaveLength(1);
expect(faceCreateCapture.requests[0].assetId).toBe(secondAssetDto.id);
await expect(page.locator('#detail-panel')).toContainText('second-stacked-asset.jpg');
const selectedThumbnail = stackSlideshow.locator(`[data-asset="${secondAssetDto.id}"]`);
await expect(selectedThumbnail).toBeVisible();
});
test('primary asset stays selected after tagging a face without switching', async ({ page }) => {
await page.goto(`/photos/${fixture.primaryAsset.id}`);
await assetViewerUtils.waitForViewerLoad(page, fixture.primaryAsset);
await ensureDetailPanelVisible(page);
await expect(page.locator('#detail-panel')).toContainText(primaryAssetDto.originalFileName);
await page.getByLabel('Tag people').click();
await page.locator('#face-selector').waitFor({ state: 'visible' });
await page.locator('#face-selector').getByText(mockPeople[0].name).click();
const confirmButton = page.getByRole('button', { name: /confirm/i });
await expect(confirmButton).toBeVisible();
await confirmButton.click();
await expect(page.locator('#face-selector')).toBeHidden();
expect(faceCreateCapture.requests).toHaveLength(1);
expect(faceCreateCapture.requests[0].assetId).toBe(primaryAssetDto.id);
await expect(page.locator('#detail-panel')).toContainText(primaryAssetDto.originalFileName);
});
});