feat: image editing (#24155)

This commit is contained in:
Brandon Wees
2026-01-09 17:59:52 -05:00
committed by GitHub
parent 76241a7b2b
commit e8c80d88a5
141 changed files with 7836 additions and 1634 deletions
+12
View File
@@ -157,6 +157,18 @@ const checkOtherAccess = async (access: AccessRepository, request: OtherAccessRe
return await access.asset.checkOwnerAccess(auth.user.id, ids, auth.session?.hasElevatedPermission);
}
case Permission.AssetEditGet: {
return await access.asset.checkOwnerAccess(auth.user.id, ids, auth.session?.hasElevatedPermission);
}
case Permission.AssetEditCreate: {
return await access.asset.checkOwnerAccess(auth.user.id, ids, auth.session?.hasElevatedPermission);
}
case Permission.AssetEditDelete: {
return await access.asset.checkOwnerAccess(auth.user.id, ids, auth.session?.hasElevatedPermission);
}
case Permission.AlbumRead: {
const isOwner = await access.album.checkOwnerAccess(auth.user.id, ids);
const isShared = await access.album.checkSharedAlbumAccess(
+29 -1
View File
@@ -1,9 +1,10 @@
import { BadRequestException } from '@nestjs/common';
import { GeneratedImageType, StorageCore } from 'src/cores/storage.core';
import { AssetFile } from 'src/database';
import { AssetFile, Exif } from 'src/database';
import { BulkIdErrorReason, BulkIdResponseDto } from 'src/dtos/asset-ids.response.dto';
import { UploadFieldName } from 'src/dtos/asset-media.dto';
import { AuthDto } from 'src/dtos/auth.dto';
import { ExifResponseDto } from 'src/dtos/exif.dto';
import { AssetFileType, AssetType, AssetVisibility, Permission } from 'src/enum';
import { AuthRequest } from 'src/middleware/auth.guard';
import { AccessRepository } from 'src/repositories/access.repository';
@@ -22,6 +23,10 @@ export const getAssetFiles = (files: AssetFile[]) => ({
previewFile: getAssetFile(files, AssetFileType.Preview),
thumbnailFile: getAssetFile(files, AssetFileType.Thumbnail),
sidecarFile: getAssetFile(files, AssetFileType.Sidecar),
editedFullsizeFile: getAssetFile(files, AssetFileType.FullSizeEdited),
editedPreviewFile: getAssetFile(files, AssetFileType.PreviewEdited),
editedThumbnailFile: getAssetFile(files, AssetFileType.ThumbnailEdited),
});
export const addAssets = async (
@@ -199,3 +204,26 @@ export const asUploadRequest = (request: AuthRequest, file: Express.Multer.File)
file: mapToUploadFile(file as ImmichFile),
};
};
const isFlipped = (orientation?: string | null) => {
const value = Number(orientation);
return value && [5, 6, 7, 8, -90, 90].includes(value);
};
export const getDimensions = (exifInfo: ExifResponseDto | Exif) => {
const { exifImageWidth: width, exifImageHeight: height } = exifInfo;
if (!width || !height) {
return { width: 0, height: 0 };
}
if (isFlipped(exifInfo.orientation)) {
return { width: height, height: width };
}
return { width, height };
};
export const isPanorama = (asset: { exifInfo?: Exif | null; originalFileName: string }) => {
return asset.exifInfo?.projectionType === 'EQUIRECTANGULAR' || asset.originalFileName.toLowerCase().endsWith('.insp');
};
+24 -4
View File
@@ -1,4 +1,5 @@
import {
AliasedRawBuilder,
DeduplicateJoinsPlugin,
Expression,
ExpressionBuilder,
@@ -16,6 +17,7 @@ import { jsonArrayFrom, jsonObjectFrom } from 'kysely/helpers/postgres';
import { parse } from 'pg-connection-string';
import postgres, { Notice, PostgresError } from 'postgres';
import { columns, Exif, lockableProperties, LockableProperty, Person } from 'src/database';
import { AssetEditActionItem } from 'src/dtos/editing.dto';
import { AssetFileType, AssetVisibility, DatabaseExtension, DatabaseSslMode } from 'src/enum';
import { AssetSearchBuilderOptions } from 'src/repositories/search.repository';
import { DB } from 'src/schema';
@@ -180,13 +182,14 @@ export function withSmartSearch<O>(qb: SelectQueryBuilder<DB, 'asset', O>) {
.select((eb) => toJson(eb, 'smart_search').as('smartSearch'));
}
export function withFaces(eb: ExpressionBuilder<DB, 'asset'>, withDeletedFace?: boolean) {
export function withFaces(eb: ExpressionBuilder<DB, 'asset'>, withHidden?: boolean, withDeletedFace?: boolean) {
return jsonArrayFrom(
eb
.selectFrom('asset_face')
.selectAll('asset_face')
.whereRef('asset_face.assetId', '=', 'asset.id')
.$if(!withDeletedFace, (qb) => qb.where('asset_face.deletedAt', 'is', null)),
.$if(!withDeletedFace, (qb) => qb.where('asset_face.deletedAt', 'is', null))
.$if(!withHidden, (qb) => qb.where('asset_face.isVisible', '=', true)),
).as('faces');
}
@@ -208,7 +211,11 @@ export function withFilePath(eb: ExpressionBuilder<DB, 'asset'>, type: AssetFile
.where('asset_file.type', '=', type);
}
export function withFacesAndPeople(eb: ExpressionBuilder<DB, 'asset'>, withDeletedFace?: boolean) {
export function withFacesAndPeople(
eb: ExpressionBuilder<DB, 'asset'>,
withHidden?: boolean,
withDeletedFace?: boolean,
) {
return jsonArrayFrom(
eb
.selectFrom('asset_face')
@@ -220,7 +227,8 @@ export function withFacesAndPeople(eb: ExpressionBuilder<DB, 'asset'>, withDelet
.selectAll('asset_face')
.select((eb) => eb.table('person').$castTo<Person>().as('person'))
.whereRef('asset_face.assetId', '=', 'asset.id')
.$if(!withDeletedFace, (qb) => qb.where('asset_face.deletedAt', 'is', null)),
.$if(!withDeletedFace, (qb) => qb.where('asset_face.deletedAt', 'is', null))
.$if(!withHidden, (qb) => qb.where('asset_face.isVisible', 'is', true)),
).as('faces');
}
@@ -232,6 +240,7 @@ export function hasPeople<O>(qb: SelectQueryBuilder<DB, 'asset', O>, personIds:
.select('assetId')
.where('personId', '=', anyUuid(personIds!))
.where('deletedAt', 'is', null)
.where('isVisible', 'is', true)
.groupBy('assetId')
.having((eb) => eb.fn.count('personId').distinct(), '=', personIds.length)
.as('has_people'),
@@ -346,6 +355,17 @@ export const tokenizeForSearch = (text: string): string[] => {
return tokens;
};
// needed to properly type the return with the EditActionItem discriminated union type
type AliasedEditActions = AliasedRawBuilder<AssetEditActionItem[], 'edits'>;
export function withEdits(eb: ExpressionBuilder<DB, 'asset'>): AliasedEditActions {
return jsonArrayFrom(
eb
.selectFrom('asset_edit')
.select(['asset_edit.action', 'asset_edit.parameters'])
.whereRef('asset_edit.assetId', '=', 'asset.id'),
).as('edits') as AliasedEditActions;
}
const joinDeduplicationPlugin = new DeduplicateJoinsPlugin();
/** TODO: This should only be used for search-related queries, not as a general purpose query builder */
+505
View File
@@ -0,0 +1,505 @@
import { AssetFace } from 'src/database';
import { AssetOcrResponseDto } from 'src/dtos/ocr.dto';
import { SourceType } from 'src/enum';
import { boundingBoxOverlap, checkFaceVisibility, checkOcrVisibility } from 'src/utils/editor';
import { describe, expect, it } from 'vitest';
describe('boundingBoxOverlap', () => {
it('should return 1 for identical boxes', () => {
const box = { x1: 0, y1: 0, x2: 100, y2: 100 };
expect(boundingBoxOverlap(box, box)).toBe(1);
});
it('should return 0 for non-overlapping boxes', () => {
const boxA = { x1: 0, y1: 0, x2: 100, y2: 100 };
const boxB = { x1: 200, y1: 200, x2: 300, y2: 300 };
expect(boundingBoxOverlap(boxA, boxB)).toBe(0);
});
it('should return 0.5 for 50% overlap', () => {
const boxA = { x1: 0, y1: 0, x2: 100, y2: 100 };
const boxB = { x1: 50, y1: 0, x2: 150, y2: 100 };
expect(boundingBoxOverlap(boxA, boxB)).toBe(0.5);
});
it('should return 0.25 for 25% overlap', () => {
const boxA = { x1: 0, y1: 0, x2: 100, y2: 100 };
const boxB = { x1: 50, y1: 50, x2: 150, y2: 150 };
expect(boundingBoxOverlap(boxA, boxB)).toBe(0.25);
});
it('should return 1 when boxA is fully contained in boxB', () => {
const boxA = { x1: 25, y1: 25, x2: 75, y2: 75 };
const boxB = { x1: 0, y1: 0, x2: 100, y2: 100 };
expect(boundingBoxOverlap(boxA, boxB)).toBe(1);
});
it('should handle partial containment correctly', () => {
const boxA = { x1: 0, y1: 0, x2: 100, y2: 100 };
const boxB = { x1: 25, y1: 25, x2: 75, y2: 75 };
// boxB is fully inside boxA, so overlap area is 50*50=2500, boxA area is 10000
expect(boundingBoxOverlap(boxA, boxB)).toBe(0.25);
});
it('should handle boxes that touch at edges (no overlap)', () => {
const boxA = { x1: 0, y1: 0, x2: 100, y2: 100 };
const boxB = { x1: 100, y1: 0, x2: 200, y2: 100 };
expect(boundingBoxOverlap(boxA, boxB)).toBe(0);
});
it('should handle vertical partial overlap', () => {
const boxA = { x1: 0, y1: 0, x2: 100, y2: 100 };
const boxB = { x1: 0, y1: 50, x2: 100, y2: 150 };
expect(boundingBoxOverlap(boxA, boxB)).toBe(0.5);
});
});
const createFace = (params: Partial<AssetFace> = {}): AssetFace => ({
id: 'face-id',
deletedAt: null,
assetId: 'asset-id',
boundingBoxX1: 100,
boundingBoxX2: 200,
boundingBoxY1: 100,
boundingBoxY2: 200,
imageWidth: 1000,
imageHeight: 1000,
personId: null,
sourceType: SourceType.MachineLearning,
person: null,
updatedAt: new Date(),
updateId: 'update-id',
isVisible: true,
...params,
});
describe('checkFaceVisibility', () => {
const assetDimensions = { width: 1000, height: 1000 };
it('should return only non-visible faces when no crop is provided', () => {
const faces = [
createFace({ id: 'face-1', isVisible: true }),
createFace({ id: 'face-2', isVisible: false }),
createFace({ id: 'face-3', isVisible: false }),
];
const result = checkFaceVisibility(faces, assetDimensions);
expect(result.visible).toHaveLength(2);
expect(result.hidden).toHaveLength(0);
expect(result.visible.map((f) => f.id)).toEqual(['face-2', 'face-3']);
});
it('should return all faces as visible when all are marked not visible and no crop provided', () => {
const faces = [createFace({ id: 'face-1', isVisible: false }), createFace({ id: 'face-2', isVisible: false })];
const result = checkFaceVisibility(faces, assetDimensions);
expect(result.visible).toHaveLength(2);
expect(result.hidden).toHaveLength(0);
});
it('should return empty visible array when all faces are already visible and no crop provided', () => {
const faces = [createFace({ id: 'face-1', isVisible: true }), createFace({ id: 'face-2', isVisible: true })];
const result = checkFaceVisibility(faces, assetDimensions);
expect(result.visible).toHaveLength(0);
expect(result.hidden).toHaveLength(0);
});
it('should return empty arrays when no faces provided', () => {
const result = checkFaceVisibility([], assetDimensions);
expect(result.visible).toHaveLength(0);
expect(result.hidden).toHaveLength(0);
});
it('should mark face as visible when fully inside crop area', () => {
const faces = [createFace({ boundingBoxX1: 100, boundingBoxY1: 100, boundingBoxX2: 200, boundingBoxY2: 200 })];
const crop = { x1: 0, y1: 0, x2: 500, y2: 500 };
const result = checkFaceVisibility(faces, assetDimensions, crop);
expect(result.visible).toHaveLength(1);
expect(result.hidden).toHaveLength(0);
});
it('should mark face as hidden when fully outside crop area', () => {
const faces = [createFace({ boundingBoxX1: 600, boundingBoxY1: 600, boundingBoxX2: 700, boundingBoxY2: 700 })];
const crop = { x1: 0, y1: 0, x2: 500, y2: 500 };
const result = checkFaceVisibility(faces, assetDimensions, crop);
expect(result.visible).toHaveLength(0);
expect(result.hidden).toHaveLength(1);
});
it('should mark face as visible when at least 50% overlaps with crop', () => {
// Face spans 100-200 (100px), crop starts at 150, so 50% overlap
const faces = [createFace({ boundingBoxX1: 100, boundingBoxY1: 100, boundingBoxX2: 200, boundingBoxY2: 200 })];
const crop = { x1: 150, y1: 100, x2: 500, y2: 500 };
const result = checkFaceVisibility(faces, assetDimensions, crop);
expect(result.visible).toHaveLength(1);
expect(result.hidden).toHaveLength(0);
});
it('should mark face as hidden when less than 50% overlaps with crop', () => {
// Face spans 100-200 (100px), crop starts at 160, so 40% overlap
const faces = [createFace({ boundingBoxX1: 100, boundingBoxY1: 100, boundingBoxX2: 200, boundingBoxY2: 200 })];
const crop = { x1: 160, y1: 100, x2: 500, y2: 500 };
const result = checkFaceVisibility(faces, assetDimensions, crop);
expect(result.visible).toHaveLength(0);
expect(result.hidden).toHaveLength(1);
});
it('should correctly categorize multiple faces', () => {
const faces = [
createFace({ id: 'face-inside', boundingBoxX1: 100, boundingBoxY1: 100, boundingBoxX2: 200, boundingBoxY2: 200 }),
createFace({
id: 'face-outside',
boundingBoxX1: 800,
boundingBoxY1: 800,
boundingBoxX2: 900,
boundingBoxY2: 900,
}),
// face-partial: 400-500 overlaps with crop (100x100=10000 overlap, face is 200x200=40000, so 25% - hidden)
createFace({
id: 'face-partial',
boundingBoxX1: 400,
boundingBoxY1: 400,
boundingBoxX2: 600,
boundingBoxY2: 600,
}),
];
const crop = { x1: 0, y1: 0, x2: 500, y2: 500 };
const result = checkFaceVisibility(faces, assetDimensions, crop);
// face-inside is fully visible, face-partial has 25% overlap (hidden), face-outside is fully hidden
expect(result.visible).toHaveLength(1);
expect(result.hidden).toHaveLength(2);
expect(result.visible.map((f) => f.id)).toContain('face-inside');
expect(result.hidden.map((f) => f.id)).toContain('face-partial');
expect(result.hidden.map((f) => f.id)).toContain('face-outside');
});
it('should handle face coordinates scaled to different image dimensions', () => {
// Face stored at 50-100 in a 500x500 image, scaled to 1000x1000 becomes 100-200
const faces = [
createFace({
boundingBoxX1: 50,
boundingBoxY1: 50,
boundingBoxX2: 100,
boundingBoxY2: 100,
imageWidth: 500,
imageHeight: 500,
}),
];
const crop = { x1: 0, y1: 0, x2: 200, y2: 200 };
const result = checkFaceVisibility(faces, assetDimensions, crop);
expect(result.visible).toHaveLength(1);
expect(result.hidden).toHaveLength(0);
});
it('should categorize based on crop overlap when crop is provided, regardless of isVisible property', () => {
const faces = [
createFace({
id: 'face-inside-visible',
boundingBoxX1: 100,
boundingBoxY1: 100,
boundingBoxX2: 200,
boundingBoxY2: 200,
isVisible: true,
}),
createFace({
id: 'face-inside-not-visible',
boundingBoxX1: 250,
boundingBoxY1: 250,
boundingBoxX2: 350,
boundingBoxY2: 350,
isVisible: false,
}),
createFace({
id: 'face-outside-visible',
boundingBoxX1: 800,
boundingBoxY1: 800,
boundingBoxX2: 900,
boundingBoxY2: 900,
isVisible: true,
}),
createFace({
id: 'face-outside-not-visible',
boundingBoxX1: 700,
boundingBoxY1: 700,
boundingBoxX2: 800,
boundingBoxY2: 800,
isVisible: false,
}),
];
const crop = { x1: 0, y1: 0, x2: 500, y2: 500 };
const result = checkFaceVisibility(faces, assetDimensions, crop);
// When crop is provided, only overlap matters, not isVisible property
expect(result.visible).toHaveLength(2);
expect(result.hidden).toHaveLength(2);
expect(result.visible.map((f) => f.id)).toContain('face-inside-visible');
expect(result.visible.map((f) => f.id)).toContain('face-inside-not-visible');
expect(result.hidden.map((f) => f.id)).toContain('face-outside-visible');
expect(result.hidden.map((f) => f.id)).toContain('face-outside-not-visible');
});
it('should handle mixed visibility states with partial overlap and crop', () => {
const faces = [
createFace({
id: 'face-partial-50',
boundingBoxX1: 100,
boundingBoxY1: 100,
boundingBoxX2: 200,
boundingBoxY2: 200,
isVisible: true,
}),
createFace({
id: 'face-partial-40',
boundingBoxX1: 100,
boundingBoxY1: 100,
boundingBoxX2: 200,
boundingBoxY2: 200,
isVisible: false,
}),
];
const crop1 = { x1: 150, y1: 100, x2: 500, y2: 500 }; // 50% overlap
const crop2 = { x1: 160, y1: 100, x2: 500, y2: 500 }; // 40% overlap
const result1 = checkFaceVisibility([faces[0]], assetDimensions, crop1);
const result2 = checkFaceVisibility([faces[1]], assetDimensions, crop2);
// 50% overlap should be visible
expect(result1.visible).toHaveLength(1);
expect(result1.hidden).toHaveLength(0);
// 40% overlap should be hidden
expect(result2.visible).toHaveLength(0);
expect(result2.hidden).toHaveLength(1);
});
});
const createOcr = (
params: Partial<AssetOcrResponseDto & { isVisible: boolean }> = {},
): AssetOcrResponseDto & { isVisible: boolean } => ({
id: 'ocr-id',
assetId: 'asset-id',
x1: 0.1,
y1: 0.1,
x2: 0.2,
y2: 0.1,
x3: 0.2,
y3: 0.2,
x4: 0.1,
y4: 0.2,
boxScore: 0.9,
textScore: 0.9,
text: 'Sample Text',
isVisible: true,
...params,
});
describe('checkOcrVisibility', () => {
const assetDimensions = { width: 1000, height: 1000 };
it('should return only non-visible OCR entries when no crop is provided', () => {
const ocrs = [
createOcr({ id: 'ocr-1', isVisible: true }),
createOcr({ id: 'ocr-2', isVisible: false }),
createOcr({ id: 'ocr-3', isVisible: false }),
];
const result = checkOcrVisibility(ocrs, assetDimensions);
expect(result.visible).toHaveLength(2);
expect(result.hidden).toHaveLength(0);
expect(result.visible.map((o) => o.id)).toEqual(['ocr-2', 'ocr-3']);
});
it('should return all OCR entries as visible when all are marked not visible and no crop provided', () => {
const ocrs = [createOcr({ id: 'ocr-1', isVisible: false }), createOcr({ id: 'ocr-2', isVisible: false })];
const result = checkOcrVisibility(ocrs, assetDimensions);
expect(result.visible).toHaveLength(2);
expect(result.hidden).toHaveLength(0);
});
it('should return empty visible array when all OCR entries are already visible and no crop provided', () => {
const ocrs = [createOcr({ id: 'ocr-1', isVisible: true }), createOcr({ id: 'ocr-2', isVisible: true })];
const result = checkOcrVisibility(ocrs, assetDimensions);
expect(result.visible).toHaveLength(0);
expect(result.hidden).toHaveLength(0);
});
it('should return empty arrays when no OCR entries provided', () => {
const result = checkOcrVisibility([], assetDimensions);
expect(result.visible).toHaveLength(0);
expect(result.hidden).toHaveLength(0);
});
it('should mark OCR as visible when fully inside crop area', () => {
// OCR box at normalized coords 0.1-0.2 = 100-200px in 1000x1000 image
const ocrs = [createOcr()];
const crop = { x1: 0, y1: 0, x2: 500, y2: 500 };
const result = checkOcrVisibility(ocrs, assetDimensions, crop);
expect(result.visible).toHaveLength(1);
expect(result.hidden).toHaveLength(0);
});
it('should mark OCR as hidden when fully outside crop area', () => {
// OCR box at normalized coords 0.8-0.9 = 800-900px
const ocrs = [createOcr({ x1: 0.8, y1: 0.8, x2: 0.9, y2: 0.8, x3: 0.9, y3: 0.9, x4: 0.8, y4: 0.9 })];
const crop = { x1: 0, y1: 0, x2: 500, y2: 500 };
const result = checkOcrVisibility(ocrs, assetDimensions, crop);
expect(result.visible).toHaveLength(0);
expect(result.hidden).toHaveLength(1);
});
it('should mark OCR as visible when at least 50% overlaps with crop', () => {
// OCR at 100-200px (0.1-0.2 normalized), crop starts at 150
const ocrs = [createOcr()];
const crop = { x1: 150, y1: 100, x2: 500, y2: 500 };
const result = checkOcrVisibility(ocrs, assetDimensions, crop);
expect(result.visible).toHaveLength(1);
expect(result.hidden).toHaveLength(0);
});
it('should mark OCR as hidden when less than 50% overlaps with crop', () => {
// OCR at 100-200px, crop starts at 160 = 40% overlap
const ocrs = [createOcr()];
const crop = { x1: 160, y1: 100, x2: 500, y2: 500 };
const result = checkOcrVisibility(ocrs, assetDimensions, crop);
expect(result.visible).toHaveLength(0);
expect(result.hidden).toHaveLength(1);
});
it('should correctly categorize multiple OCR entries', () => {
const ocrs = [
createOcr({ id: 'ocr-inside', x1: 0.1, y1: 0.1, x2: 0.2, y2: 0.1, x3: 0.2, y3: 0.2, x4: 0.1, y4: 0.2 }),
createOcr({ id: 'ocr-outside', x1: 0.8, y1: 0.8, x2: 0.9, y2: 0.8, x3: 0.9, y3: 0.9, x4: 0.8, y4: 0.9 }),
];
const crop = { x1: 0, y1: 0, x2: 500, y2: 500 };
const result = checkOcrVisibility(ocrs, assetDimensions, crop);
expect(result.visible).toHaveLength(1);
expect(result.hidden).toHaveLength(1);
expect(result.visible[0].id).toBe('ocr-inside');
expect(result.hidden[0].id).toBe('ocr-outside');
});
it('should handle rotated/skewed OCR polygons by using bounding box', () => {
// Rotated rectangle - the function should compute the bounding box correctly
const ocrs = [
createOcr({
id: 'ocr-rotated',
x1: 0.15,
y1: 0.1, // top
x2: 0.2,
y2: 0.15, // right
x3: 0.15,
y3: 0.2, // bottom
x4: 0.1,
y4: 0.15, // left
}),
];
const crop = { x1: 0, y1: 0, x2: 500, y2: 500 };
const result = checkOcrVisibility(ocrs, assetDimensions, crop);
expect(result.visible).toHaveLength(1);
expect(result.hidden).toHaveLength(0);
});
it('should handle different asset dimensions', () => {
const smallDimensions = { width: 500, height: 500 };
// OCR at 0.1-0.2 normalized = 50-100px in 500x500 image
const ocrs = [createOcr()];
const crop = { x1: 0, y1: 0, x2: 200, y2: 200 };
const result = checkOcrVisibility(ocrs, smallDimensions, crop);
expect(result.visible).toHaveLength(1);
expect(result.hidden).toHaveLength(0);
});
it('should categorize based on crop overlap when crop is provided, regardless of isVisible property', () => {
const ocrs = [
createOcr({ id: 'ocr-inside-visible', isVisible: true }), // Inside crop, already visible
createOcr({ id: 'ocr-inside-not-visible', isVisible: false }), // Inside crop, not visible
createOcr({
id: 'ocr-outside-visible',
x1: 0.8,
y1: 0.8,
x2: 0.9,
y2: 0.8,
x3: 0.9,
y3: 0.9,
x4: 0.8,
y4: 0.9,
isVisible: true,
}), // Outside crop, already visible
createOcr({
id: 'ocr-outside-not-visible',
x1: 0.8,
y1: 0.8,
x2: 0.9,
y2: 0.8,
x3: 0.9,
y3: 0.9,
x4: 0.8,
y4: 0.9,
isVisible: false,
}), // Outside crop, not visible
];
const crop = { x1: 0, y1: 0, x2: 500, y2: 500 };
const result = checkOcrVisibility(ocrs, assetDimensions, crop);
// When crop is provided, only overlap matters, not isVisible property
expect(result.visible).toHaveLength(2);
expect(result.hidden).toHaveLength(2);
expect(result.visible.map((o) => o.id)).toContain('ocr-inside-visible');
expect(result.visible.map((o) => o.id)).toContain('ocr-inside-not-visible');
expect(result.hidden.map((o) => o.id)).toContain('ocr-outside-visible');
expect(result.hidden.map((o) => o.id)).toContain('ocr-outside-not-visible');
});
it('should handle mixed visibility states with partial overlap and crop', () => {
const ocrs = [
createOcr({ id: 'ocr-partial-50', isVisible: true }), // 50% overlap
createOcr({ id: 'ocr-partial-40', isVisible: false }), // 40% overlap
];
const crop1 = { x1: 150, y1: 100, x2: 500, y2: 500 }; // 50% overlap
const crop2 = { x1: 160, y1: 100, x2: 500, y2: 500 }; // 40% overlap
const result1 = checkOcrVisibility([ocrs[0]], assetDimensions, crop1);
const result2 = checkOcrVisibility([ocrs[1]], assetDimensions, crop2);
// 50% overlap should be visible
expect(result1.visible).toHaveLength(1);
expect(result1.hidden).toHaveLength(0);
// 40% overlap should be hidden
expect(result2.visible).toHaveLength(0);
expect(result2.hidden).toHaveLength(1);
});
});
+107
View File
@@ -0,0 +1,107 @@
import { AssetFace } from 'src/database';
import { AssetOcrResponseDto } from 'src/dtos/ocr.dto';
import { ImageDimensions } from 'src/types';
type BoundingBox = {
x1: number;
y1: number;
x2: number;
y2: number;
};
export const boundingBoxOverlap = (boxA: BoundingBox, boxB: BoundingBox) => {
const overlapX1 = Math.max(boxA.x1, boxB.x1);
const overlapY1 = Math.max(boxA.y1, boxB.y1);
const overlapX2 = Math.min(boxA.x2, boxB.x2);
const overlapY2 = Math.min(boxA.y2, boxB.y2);
const overlapArea = Math.max(0, overlapX2 - overlapX1) * Math.max(0, overlapY2 - overlapY1);
const faceArea = (boxA.x2 - boxA.x1) * (boxA.y2 - boxA.y1);
return overlapArea / faceArea;
};
const scale = (box: BoundingBox, target: ImageDimensions, source?: ImageDimensions) => {
const { width: sourceWidth = 1, height: sourceHeight = 1 } = source ?? {};
return {
x1: (box.x1 / sourceWidth) * target.width,
y1: (box.y1 / sourceHeight) * target.height,
x2: (box.x2 / sourceWidth) * target.width,
y2: (box.y2 / sourceHeight) * target.height,
};
};
export const checkFaceVisibility = (
faces: AssetFace[],
originalAssetDimensions: ImageDimensions,
crop?: BoundingBox,
): { visible: AssetFace[]; hidden: AssetFace[] } => {
if (!crop) {
return {
visible: faces.filter((face) => !face.isVisible),
hidden: [],
};
}
const status = faces.map((face) => {
const scaledFace = scale(
{
x1: face.boundingBoxX1,
y1: face.boundingBoxY1,
x2: face.boundingBoxX2,
y2: face.boundingBoxY2,
},
originalAssetDimensions,
{ width: face.imageWidth, height: face.imageHeight },
);
const overlapPercentage = boundingBoxOverlap(scaledFace, crop);
return {
face,
isVisible: overlapPercentage >= 0.5,
};
});
return {
visible: status.filter((s) => s.isVisible).map((s) => s.face),
hidden: status.filter((s) => !s.isVisible).map((s) => s.face),
};
};
export const checkOcrVisibility = (
ocrs: (AssetOcrResponseDto & { isVisible: boolean })[],
originalAssetDimensions: ImageDimensions,
crop?: BoundingBox,
): { visible: AssetOcrResponseDto[]; hidden: AssetOcrResponseDto[] } => {
if (!crop) {
return {
visible: ocrs.filter((ocr) => !ocr.isVisible),
hidden: [],
};
}
const status = ocrs.map((ocr) => {
const ocrBox = scale(
{
x1: Math.min(ocr.x1, ocr.x2, ocr.x3, ocr.x4),
y1: Math.min(ocr.y1, ocr.y2, ocr.y3, ocr.y4),
x2: Math.max(ocr.x1, ocr.x2, ocr.x3, ocr.x4),
y2: Math.max(ocr.y1, ocr.y2, ocr.y3, ocr.y4),
},
originalAssetDimensions,
);
const overlapPercentage = boundingBoxOverlap(ocrBox, crop);
return {
ocr,
isVisible: overlapPercentage >= 0.5,
};
});
return {
visible: status.filter((s) => s.isVisible).map((s) => s.ocr),
hidden: status.filter((s) => !s.isVisible).map((s) => s.ocr),
};
};
+293
View File
@@ -0,0 +1,293 @@
import { AssetEditAction, AssetEditActionItem, MirrorAxis } from 'src/dtos/editing.dto';
import { AssetOcrResponseDto } from 'src/dtos/ocr.dto';
import { transformFaceBoundingBox, transformOcrBoundingBox } from 'src/utils/transform';
import { describe, expect, it } from 'vitest';
describe('transformFaceBoundingBox', () => {
const baseFace = {
boundingBoxX1: 100,
boundingBoxY1: 100,
boundingBoxX2: 200,
boundingBoxY2: 200,
imageWidth: 1000,
imageHeight: 800,
};
const baseDimensions = { width: 1000, height: 800 };
describe('with no edits', () => {
it('should return unchanged bounding box', () => {
const result = transformFaceBoundingBox(baseFace, [], baseDimensions);
expect(result).toEqual(baseFace);
});
});
describe('with crop edit', () => {
it('should adjust bounding box for crop offset', () => {
const edits: AssetEditActionItem[] = [
{ action: AssetEditAction.Crop, parameters: { x: 50, y: 50, width: 400, height: 300 } },
];
const result = transformFaceBoundingBox(baseFace, edits, baseDimensions);
expect(result.boundingBoxX1).toBe(50);
expect(result.boundingBoxY1).toBe(50);
expect(result.boundingBoxX2).toBe(150);
expect(result.boundingBoxY2).toBe(150);
expect(result.imageWidth).toBe(400);
expect(result.imageHeight).toBe(300);
});
it('should handle face partially outside crop area', () => {
const edits: AssetEditActionItem[] = [
{ action: AssetEditAction.Crop, parameters: { x: 150, y: 150, width: 400, height: 300 } },
];
const result = transformFaceBoundingBox(baseFace, edits, baseDimensions);
expect(result.boundingBoxX1).toBe(-50);
expect(result.boundingBoxY1).toBe(-50);
expect(result.boundingBoxX2).toBe(50);
expect(result.boundingBoxY2).toBe(50);
});
});
describe('with rotate edit', () => {
it('should rotate 90 degrees clockwise', () => {
const edits: AssetEditActionItem[] = [{ action: AssetEditAction.Rotate, parameters: { angle: 90 } }];
const result = transformFaceBoundingBox(baseFace, edits, baseDimensions);
expect(result.imageWidth).toBe(800);
expect(result.imageHeight).toBe(1000);
expect(result.boundingBoxX1).toBe(600);
expect(result.boundingBoxY1).toBe(100);
expect(result.boundingBoxX2).toBe(700);
expect(result.boundingBoxY2).toBe(200);
});
it('should rotate 180 degrees', () => {
const edits: AssetEditActionItem[] = [{ action: AssetEditAction.Rotate, parameters: { angle: 180 } }];
const result = transformFaceBoundingBox(baseFace, edits, baseDimensions);
expect(result.imageWidth).toBe(1000);
expect(result.imageHeight).toBe(800);
expect(result.boundingBoxX1).toBe(800);
expect(result.boundingBoxY1).toBe(600);
expect(result.boundingBoxX2).toBe(900);
expect(result.boundingBoxY2).toBe(700);
});
it('should rotate 270 degrees', () => {
const edits: AssetEditActionItem[] = [{ action: AssetEditAction.Rotate, parameters: { angle: 270 } }];
const result = transformFaceBoundingBox(baseFace, edits, baseDimensions);
expect(result.imageWidth).toBe(800);
expect(result.imageHeight).toBe(1000);
});
});
describe('with mirror edit', () => {
it('should mirror horizontally', () => {
const edits: AssetEditActionItem[] = [
{ action: AssetEditAction.Mirror, parameters: { axis: MirrorAxis.Horizontal } },
];
const result = transformFaceBoundingBox(baseFace, edits, baseDimensions);
expect(result.boundingBoxX1).toBe(800);
expect(result.boundingBoxY1).toBe(100);
expect(result.boundingBoxX2).toBe(900);
expect(result.boundingBoxY2).toBe(200);
expect(result.imageWidth).toBe(1000);
expect(result.imageHeight).toBe(800);
});
it('should mirror vertically', () => {
const edits: AssetEditActionItem[] = [
{ action: AssetEditAction.Mirror, parameters: { axis: MirrorAxis.Vertical } },
];
const result = transformFaceBoundingBox(baseFace, edits, baseDimensions);
expect(result.boundingBoxX1).toBe(100);
expect(result.boundingBoxY1).toBe(600);
expect(result.boundingBoxX2).toBe(200);
expect(result.boundingBoxY2).toBe(700);
expect(result.imageWidth).toBe(1000);
expect(result.imageHeight).toBe(800);
});
});
describe('with combined edits', () => {
it('should apply crop then rotate', () => {
const edits: AssetEditActionItem[] = [
{ action: AssetEditAction.Crop, parameters: { x: 50, y: 50, width: 400, height: 300 } },
{ action: AssetEditAction.Rotate, parameters: { angle: 90 } },
];
const result = transformFaceBoundingBox(baseFace, edits, baseDimensions);
expect(result.imageWidth).toBe(300);
expect(result.imageHeight).toBe(400);
});
it('should apply crop then mirror', () => {
const edits: AssetEditActionItem[] = [
{ action: AssetEditAction.Crop, parameters: { x: 0, y: 0, width: 500, height: 400 } },
{ action: AssetEditAction.Mirror, parameters: { axis: MirrorAxis.Vertical } },
];
const result = transformFaceBoundingBox(baseFace, edits, baseDimensions);
expect(result.boundingBoxX1).toBe(100);
expect(result.boundingBoxX2).toBe(200);
expect(result.boundingBoxY1).toBe(200);
expect(result.boundingBoxY2).toBe(300);
});
});
describe('with scaled dimensions', () => {
it('should scale face to match different image dimensions', () => {
const scaledDimensions = { width: 500, height: 400 }; // Half the original size
const edits: AssetEditActionItem[] = [
{ action: AssetEditAction.Crop, parameters: { x: 50, y: 50, width: 200, height: 150 } },
];
const result = transformFaceBoundingBox(baseFace, edits, scaledDimensions);
expect(result.boundingBoxX1).toBe(0);
expect(result.boundingBoxY1).toBe(0);
expect(result.boundingBoxX2).toBe(50);
expect(result.boundingBoxY2).toBe(50);
});
});
});
describe('transformOcrBoundingBox', () => {
const baseOcr: AssetOcrResponseDto = {
id: 'ocr-1',
assetId: 'asset-1',
x1: 0.1,
y1: 0.1,
x2: 0.2,
y2: 0.1,
x3: 0.2,
y3: 0.2,
x4: 0.1,
y4: 0.2,
boxScore: 0.9,
textScore: 0.85,
text: 'Test OCR',
};
const baseDimensions = { width: 1000, height: 800 };
describe('with no edits', () => {
it('should return unchanged bounding box', () => {
const result = transformOcrBoundingBox(baseOcr, [], baseDimensions);
expect(result).toEqual(baseOcr);
});
});
describe('with crop edit', () => {
it('should adjust normalized coordinates for crop', () => {
const edits: AssetEditActionItem[] = [
{ action: AssetEditAction.Crop, parameters: { x: 100, y: 80, width: 400, height: 320 } },
];
const result = transformOcrBoundingBox(baseOcr, edits, baseDimensions);
// Original OCR: (0.1,0.1)-(0.2,0.2) on 1000x800 = (100,80)-(200,160)
// After crop offset (100,80): (0,0)-(100,80)
// Normalized to 400x320: (0,0)-(0.25,0.25)
expect(result.x1).toBeCloseTo(0, 5);
expect(result.y1).toBeCloseTo(0, 5);
expect(result.x2).toBeCloseTo(0.25, 5);
expect(result.y2).toBeCloseTo(0, 5);
expect(result.x3).toBeCloseTo(0.25, 5);
expect(result.y3).toBeCloseTo(0.25, 5);
expect(result.x4).toBeCloseTo(0, 5);
expect(result.y4).toBeCloseTo(0.25, 5);
});
});
describe('with rotate edit', () => {
it('should rotate normalized coordinates 90 degrees and reorder points', () => {
const edits: AssetEditActionItem[] = [{ action: AssetEditAction.Rotate, parameters: { angle: 90 } }];
const result = transformOcrBoundingBox(baseOcr, edits, baseDimensions);
expect(result.id).toBe(baseOcr.id);
expect(result.text).toBe(baseOcr.text);
expect(result.x1).toBeCloseTo(0.8, 5);
expect(result.y1).toBeCloseTo(0.1, 5);
expect(result.x2).toBeCloseTo(0.9, 5);
expect(result.y2).toBeCloseTo(0.1, 5);
expect(result.x3).toBeCloseTo(0.9, 5);
expect(result.y3).toBeCloseTo(0.2, 5);
expect(result.x4).toBeCloseTo(0.8, 5);
expect(result.y4).toBeCloseTo(0.2, 5);
});
it('should rotate 180 degrees and reorder points', () => {
const edits: AssetEditActionItem[] = [{ action: AssetEditAction.Rotate, parameters: { angle: 180 } }];
const result = transformOcrBoundingBox(baseOcr, edits, baseDimensions);
expect(result.x1).toBeCloseTo(0.8, 5);
expect(result.y1).toBeCloseTo(0.8, 5);
expect(result.x2).toBeCloseTo(0.9, 5);
expect(result.y2).toBeCloseTo(0.8, 5);
expect(result.x3).toBeCloseTo(0.9, 5);
expect(result.y3).toBeCloseTo(0.9, 5);
expect(result.x4).toBeCloseTo(0.8, 5);
expect(result.y4).toBeCloseTo(0.9, 5);
});
it('should rotate 270 degrees and reorder points', () => {
const edits: AssetEditActionItem[] = [{ action: AssetEditAction.Rotate, parameters: { angle: 270 } }];
const result = transformOcrBoundingBox(baseOcr, edits, baseDimensions);
expect(result.id).toBe(baseOcr.id);
expect(result.text).toBe(baseOcr.text);
expect(result.x1).toBeCloseTo(0.1, 5);
expect(result.y1).toBeCloseTo(0.8, 5);
expect(result.x2).toBeCloseTo(0.2, 5);
expect(result.y2).toBeCloseTo(0.8, 5);
expect(result.x3).toBeCloseTo(0.2, 5);
expect(result.y3).toBeCloseTo(0.9, 5);
expect(result.x4).toBeCloseTo(0.1, 5);
expect(result.y4).toBeCloseTo(0.9, 5);
});
});
describe('with mirror edit', () => {
it('should mirror horizontally', () => {
const edits: AssetEditActionItem[] = [
{ action: AssetEditAction.Mirror, parameters: { axis: MirrorAxis.Horizontal } },
];
const result = transformOcrBoundingBox(baseOcr, edits, baseDimensions);
expect(result.x1).toBeCloseTo(0.9, 5);
expect(result.y1).toBeCloseTo(0.1, 5);
});
it('should mirror vertically', () => {
const edits: AssetEditActionItem[] = [
{ action: AssetEditAction.Mirror, parameters: { axis: MirrorAxis.Vertical } },
];
const result = transformOcrBoundingBox(baseOcr, edits, baseDimensions);
expect(result.x1).toBeCloseTo(0.1, 5);
expect(result.y1).toBeCloseTo(0.9, 5);
});
});
describe('with combined edits', () => {
it('should preserve OCR metadata through transforms', () => {
const edits: AssetEditActionItem[] = [
{ action: AssetEditAction.Crop, parameters: { x: 0, y: 0, width: 500, height: 400 } },
{ action: AssetEditAction.Rotate, parameters: { angle: 90 } },
];
const result = transformOcrBoundingBox(baseOcr, edits, baseDimensions);
expect(result.id).toBe(baseOcr.id);
expect(result.assetId).toBe(baseOcr.assetId);
expect(result.boxScore).toBe(baseOcr.boxScore);
expect(result.textScore).toBe(baseOcr.textScore);
expect(result.text).toBe(baseOcr.text);
});
});
});
+227
View File
@@ -0,0 +1,227 @@
import { AssetEditAction, AssetEditActionItem } from 'src/dtos/editing.dto';
import { AssetOcrResponseDto } from 'src/dtos/ocr.dto';
import { ImageDimensions } from 'src/types';
import { applyToPoint, compose, flipX, flipY, identity, Matrix, rotate, scale, translate } from 'transformation-matrix';
export const getOutputDimensions = (
edits: AssetEditActionItem[],
startingDimensions: ImageDimensions,
): ImageDimensions => {
let { width, height } = startingDimensions;
const crop = edits.find((edit) => edit.action === AssetEditAction.Crop);
if (crop) {
width = crop.parameters.width;
height = crop.parameters.height;
}
for (const edit of edits) {
if (edit.action === AssetEditAction.Rotate) {
const angleDegrees = edit.parameters.angle;
if (angleDegrees === 90 || angleDegrees === 270) {
[width, height] = [height, width];
}
}
}
return { width, height };
};
export const createAffineMatrix = (
edits: AssetEditActionItem[],
scalingParameters?: {
pointSpace: ImageDimensions;
targetSpace: ImageDimensions;
},
): Matrix => {
let scalingMatrix: Matrix = identity();
if (scalingParameters) {
const { pointSpace, targetSpace } = scalingParameters;
const scaleX = targetSpace.width / pointSpace.width;
scalingMatrix = scale(scaleX);
}
return compose(
scalingMatrix,
...edits.map((edit) => {
switch (edit.action) {
case 'rotate': {
const angleInRadians = (-edit.parameters.angle * Math.PI) / 180;
return rotate(angleInRadians);
}
case 'mirror': {
return edit.parameters.axis === 'horizontal' ? flipY() : flipX();
}
default: {
return identity();
}
}
}),
);
};
type Point = { x: number; y: number };
type TransformState = {
points: Point[];
currentWidth: number;
currentHeight: number;
};
/**
* Transforms an array of points through a series of edit operations (crop, rotate, mirror).
* Points should be in absolute pixel coordinates relative to the starting dimensions.
*/
const transformPoints = (
points: Point[],
edits: AssetEditActionItem[],
startingDimensions: ImageDimensions,
): TransformState => {
let currentWidth = startingDimensions.width;
let currentHeight = startingDimensions.height;
let transformedPoints = [...points];
// Handle crop first
const crop = edits.find((edit) => edit.action === 'crop');
if (crop) {
const { x: cropX, y: cropY, width: cropWidth, height: cropHeight } = crop.parameters;
transformedPoints = transformedPoints.map((p) => ({
x: p.x - cropX,
y: p.y - cropY,
}));
currentWidth = cropWidth;
currentHeight = cropHeight;
}
// Apply rotate and mirror transforms
for (const edit of edits) {
let matrix: Matrix = identity();
if (edit.action === 'rotate') {
const angleDegrees = edit.parameters.angle;
const angleRadians = (angleDegrees * Math.PI) / 180;
const newWidth = angleDegrees === 90 || angleDegrees === 270 ? currentHeight : currentWidth;
const newHeight = angleDegrees === 90 || angleDegrees === 270 ? currentWidth : currentHeight;
matrix = compose(
translate(newWidth / 2, newHeight / 2),
rotate(angleRadians),
translate(-currentWidth / 2, -currentHeight / 2),
);
currentWidth = newWidth;
currentHeight = newHeight;
} else if (edit.action === 'mirror') {
matrix = compose(
translate(currentWidth / 2, currentHeight / 2),
edit.parameters.axis === 'horizontal' ? flipY() : flipX(),
translate(-currentWidth / 2, -currentHeight / 2),
);
} else {
// Skip non-affine transformations
continue;
}
transformedPoints = transformedPoints.map((p) => applyToPoint(matrix, p));
}
return {
points: transformedPoints,
currentWidth,
currentHeight,
};
};
type FaceBoundingBox = {
boundingBoxX1: number;
boundingBoxX2: number;
boundingBoxY1: number;
boundingBoxY2: number;
imageWidth: number;
imageHeight: number;
};
export const transformFaceBoundingBox = (
box: FaceBoundingBox,
edits: AssetEditActionItem[],
imageDimensions: ImageDimensions,
): FaceBoundingBox => {
if (edits.length === 0) {
return box;
}
const scaleX = imageDimensions.width / box.imageWidth;
const scaleY = imageDimensions.height / box.imageHeight;
const points: Point[] = [
{ x: box.boundingBoxX1 * scaleX, y: box.boundingBoxY1 * scaleY },
{ x: box.boundingBoxX2 * scaleX, y: box.boundingBoxY2 * scaleY },
];
const { points: transformedPoints, currentWidth, currentHeight } = transformPoints(points, edits, imageDimensions);
// Ensure x1,y1 is top-left and x2,y2 is bottom-right
const [p1, p2] = transformedPoints;
return {
boundingBoxX1: Math.min(p1.x, p2.x),
boundingBoxY1: Math.min(p1.y, p2.y),
boundingBoxX2: Math.max(p1.x, p2.x),
boundingBoxY2: Math.max(p1.y, p2.y),
imageWidth: currentWidth,
imageHeight: currentHeight,
};
};
const reorderQuadPointsForRotation = (points: Point[], rotationDegrees: number): Point[] => {
const [p1, p2, p3, p4] = points;
switch (rotationDegrees) {
case 90: {
return [p4, p1, p2, p3];
}
case 180: {
return [p3, p4, p1, p2];
}
case 270: {
return [p2, p3, p4, p1];
}
default: {
return points;
}
}
};
export const transformOcrBoundingBox = (
box: AssetOcrResponseDto,
edits: AssetEditActionItem[],
imageDimensions: ImageDimensions,
): AssetOcrResponseDto => {
if (edits.length === 0) {
return box;
}
const points: Point[] = [
{ x: box.x1 * imageDimensions.width, y: box.y1 * imageDimensions.height },
{ x: box.x2 * imageDimensions.width, y: box.y2 * imageDimensions.height },
{ x: box.x3 * imageDimensions.width, y: box.y3 * imageDimensions.height },
{ x: box.x4 * imageDimensions.width, y: box.y4 * imageDimensions.height },
];
const { points: transformedPoints, currentWidth, currentHeight } = transformPoints(points, edits, imageDimensions);
// Reorder points to maintain semantic ordering (topLeft, topRight, bottomRight, bottomLeft)
const netRotation = edits.find((e) => e.action == AssetEditAction.Rotate)?.parameters.angle ?? 0 % 360;
const reorderedPoints = reorderQuadPointsForRotation(transformedPoints, netRotation);
const [p1, p2, p3, p4] = reorderedPoints;
return {
...box,
x1: p1.x / currentWidth,
y1: p1.y / currentHeight,
x2: p2.x / currentWidth,
y2: p2.y / currentHeight,
x3: p3.x / currentWidth,
y3: p3.y / currentHeight,
x4: p4.x / currentWidth,
y4: p4.y / currentHeight,
};
};