mirror of
https://github.com/immich-app/immich.git
synced 2026-05-18 03:10:24 +03:00
feat: image editing (#24155)
This commit is contained in:
@@ -157,6 +157,18 @@ const checkOtherAccess = async (access: AccessRepository, request: OtherAccessRe
|
||||
return await access.asset.checkOwnerAccess(auth.user.id, ids, auth.session?.hasElevatedPermission);
|
||||
}
|
||||
|
||||
case Permission.AssetEditGet: {
|
||||
return await access.asset.checkOwnerAccess(auth.user.id, ids, auth.session?.hasElevatedPermission);
|
||||
}
|
||||
|
||||
case Permission.AssetEditCreate: {
|
||||
return await access.asset.checkOwnerAccess(auth.user.id, ids, auth.session?.hasElevatedPermission);
|
||||
}
|
||||
|
||||
case Permission.AssetEditDelete: {
|
||||
return await access.asset.checkOwnerAccess(auth.user.id, ids, auth.session?.hasElevatedPermission);
|
||||
}
|
||||
|
||||
case Permission.AlbumRead: {
|
||||
const isOwner = await access.album.checkOwnerAccess(auth.user.id, ids);
|
||||
const isShared = await access.album.checkSharedAlbumAccess(
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
import { GeneratedImageType, StorageCore } from 'src/cores/storage.core';
|
||||
import { AssetFile } from 'src/database';
|
||||
import { AssetFile, Exif } from 'src/database';
|
||||
import { BulkIdErrorReason, BulkIdResponseDto } from 'src/dtos/asset-ids.response.dto';
|
||||
import { UploadFieldName } from 'src/dtos/asset-media.dto';
|
||||
import { AuthDto } from 'src/dtos/auth.dto';
|
||||
import { ExifResponseDto } from 'src/dtos/exif.dto';
|
||||
import { AssetFileType, AssetType, AssetVisibility, Permission } from 'src/enum';
|
||||
import { AuthRequest } from 'src/middleware/auth.guard';
|
||||
import { AccessRepository } from 'src/repositories/access.repository';
|
||||
@@ -22,6 +23,10 @@ export const getAssetFiles = (files: AssetFile[]) => ({
|
||||
previewFile: getAssetFile(files, AssetFileType.Preview),
|
||||
thumbnailFile: getAssetFile(files, AssetFileType.Thumbnail),
|
||||
sidecarFile: getAssetFile(files, AssetFileType.Sidecar),
|
||||
|
||||
editedFullsizeFile: getAssetFile(files, AssetFileType.FullSizeEdited),
|
||||
editedPreviewFile: getAssetFile(files, AssetFileType.PreviewEdited),
|
||||
editedThumbnailFile: getAssetFile(files, AssetFileType.ThumbnailEdited),
|
||||
});
|
||||
|
||||
export const addAssets = async (
|
||||
@@ -199,3 +204,26 @@ export const asUploadRequest = (request: AuthRequest, file: Express.Multer.File)
|
||||
file: mapToUploadFile(file as ImmichFile),
|
||||
};
|
||||
};
|
||||
|
||||
const isFlipped = (orientation?: string | null) => {
|
||||
const value = Number(orientation);
|
||||
return value && [5, 6, 7, 8, -90, 90].includes(value);
|
||||
};
|
||||
|
||||
export const getDimensions = (exifInfo: ExifResponseDto | Exif) => {
|
||||
const { exifImageWidth: width, exifImageHeight: height } = exifInfo;
|
||||
|
||||
if (!width || !height) {
|
||||
return { width: 0, height: 0 };
|
||||
}
|
||||
|
||||
if (isFlipped(exifInfo.orientation)) {
|
||||
return { width: height, height: width };
|
||||
}
|
||||
|
||||
return { width, height };
|
||||
};
|
||||
|
||||
export const isPanorama = (asset: { exifInfo?: Exif | null; originalFileName: string }) => {
|
||||
return asset.exifInfo?.projectionType === 'EQUIRECTANGULAR' || asset.originalFileName.toLowerCase().endsWith('.insp');
|
||||
};
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import {
|
||||
AliasedRawBuilder,
|
||||
DeduplicateJoinsPlugin,
|
||||
Expression,
|
||||
ExpressionBuilder,
|
||||
@@ -16,6 +17,7 @@ import { jsonArrayFrom, jsonObjectFrom } from 'kysely/helpers/postgres';
|
||||
import { parse } from 'pg-connection-string';
|
||||
import postgres, { Notice, PostgresError } from 'postgres';
|
||||
import { columns, Exif, lockableProperties, LockableProperty, Person } from 'src/database';
|
||||
import { AssetEditActionItem } from 'src/dtos/editing.dto';
|
||||
import { AssetFileType, AssetVisibility, DatabaseExtension, DatabaseSslMode } from 'src/enum';
|
||||
import { AssetSearchBuilderOptions } from 'src/repositories/search.repository';
|
||||
import { DB } from 'src/schema';
|
||||
@@ -180,13 +182,14 @@ export function withSmartSearch<O>(qb: SelectQueryBuilder<DB, 'asset', O>) {
|
||||
.select((eb) => toJson(eb, 'smart_search').as('smartSearch'));
|
||||
}
|
||||
|
||||
export function withFaces(eb: ExpressionBuilder<DB, 'asset'>, withDeletedFace?: boolean) {
|
||||
export function withFaces(eb: ExpressionBuilder<DB, 'asset'>, withHidden?: boolean, withDeletedFace?: boolean) {
|
||||
return jsonArrayFrom(
|
||||
eb
|
||||
.selectFrom('asset_face')
|
||||
.selectAll('asset_face')
|
||||
.whereRef('asset_face.assetId', '=', 'asset.id')
|
||||
.$if(!withDeletedFace, (qb) => qb.where('asset_face.deletedAt', 'is', null)),
|
||||
.$if(!withDeletedFace, (qb) => qb.where('asset_face.deletedAt', 'is', null))
|
||||
.$if(!withHidden, (qb) => qb.where('asset_face.isVisible', '=', true)),
|
||||
).as('faces');
|
||||
}
|
||||
|
||||
@@ -208,7 +211,11 @@ export function withFilePath(eb: ExpressionBuilder<DB, 'asset'>, type: AssetFile
|
||||
.where('asset_file.type', '=', type);
|
||||
}
|
||||
|
||||
export function withFacesAndPeople(eb: ExpressionBuilder<DB, 'asset'>, withDeletedFace?: boolean) {
|
||||
export function withFacesAndPeople(
|
||||
eb: ExpressionBuilder<DB, 'asset'>,
|
||||
withHidden?: boolean,
|
||||
withDeletedFace?: boolean,
|
||||
) {
|
||||
return jsonArrayFrom(
|
||||
eb
|
||||
.selectFrom('asset_face')
|
||||
@@ -220,7 +227,8 @@ export function withFacesAndPeople(eb: ExpressionBuilder<DB, 'asset'>, withDelet
|
||||
.selectAll('asset_face')
|
||||
.select((eb) => eb.table('person').$castTo<Person>().as('person'))
|
||||
.whereRef('asset_face.assetId', '=', 'asset.id')
|
||||
.$if(!withDeletedFace, (qb) => qb.where('asset_face.deletedAt', 'is', null)),
|
||||
.$if(!withDeletedFace, (qb) => qb.where('asset_face.deletedAt', 'is', null))
|
||||
.$if(!withHidden, (qb) => qb.where('asset_face.isVisible', 'is', true)),
|
||||
).as('faces');
|
||||
}
|
||||
|
||||
@@ -232,6 +240,7 @@ export function hasPeople<O>(qb: SelectQueryBuilder<DB, 'asset', O>, personIds:
|
||||
.select('assetId')
|
||||
.where('personId', '=', anyUuid(personIds!))
|
||||
.where('deletedAt', 'is', null)
|
||||
.where('isVisible', 'is', true)
|
||||
.groupBy('assetId')
|
||||
.having((eb) => eb.fn.count('personId').distinct(), '=', personIds.length)
|
||||
.as('has_people'),
|
||||
@@ -346,6 +355,17 @@ export const tokenizeForSearch = (text: string): string[] => {
|
||||
return tokens;
|
||||
};
|
||||
|
||||
// needed to properly type the return with the EditActionItem discriminated union type
|
||||
type AliasedEditActions = AliasedRawBuilder<AssetEditActionItem[], 'edits'>;
|
||||
export function withEdits(eb: ExpressionBuilder<DB, 'asset'>): AliasedEditActions {
|
||||
return jsonArrayFrom(
|
||||
eb
|
||||
.selectFrom('asset_edit')
|
||||
.select(['asset_edit.action', 'asset_edit.parameters'])
|
||||
.whereRef('asset_edit.assetId', '=', 'asset.id'),
|
||||
).as('edits') as AliasedEditActions;
|
||||
}
|
||||
|
||||
const joinDeduplicationPlugin = new DeduplicateJoinsPlugin();
|
||||
/** TODO: This should only be used for search-related queries, not as a general purpose query builder */
|
||||
|
||||
|
||||
@@ -0,0 +1,505 @@
|
||||
import { AssetFace } from 'src/database';
|
||||
import { AssetOcrResponseDto } from 'src/dtos/ocr.dto';
|
||||
import { SourceType } from 'src/enum';
|
||||
import { boundingBoxOverlap, checkFaceVisibility, checkOcrVisibility } from 'src/utils/editor';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
describe('boundingBoxOverlap', () => {
|
||||
it('should return 1 for identical boxes', () => {
|
||||
const box = { x1: 0, y1: 0, x2: 100, y2: 100 };
|
||||
expect(boundingBoxOverlap(box, box)).toBe(1);
|
||||
});
|
||||
|
||||
it('should return 0 for non-overlapping boxes', () => {
|
||||
const boxA = { x1: 0, y1: 0, x2: 100, y2: 100 };
|
||||
const boxB = { x1: 200, y1: 200, x2: 300, y2: 300 };
|
||||
expect(boundingBoxOverlap(boxA, boxB)).toBe(0);
|
||||
});
|
||||
|
||||
it('should return 0.5 for 50% overlap', () => {
|
||||
const boxA = { x1: 0, y1: 0, x2: 100, y2: 100 };
|
||||
const boxB = { x1: 50, y1: 0, x2: 150, y2: 100 };
|
||||
expect(boundingBoxOverlap(boxA, boxB)).toBe(0.5);
|
||||
});
|
||||
|
||||
it('should return 0.25 for 25% overlap', () => {
|
||||
const boxA = { x1: 0, y1: 0, x2: 100, y2: 100 };
|
||||
const boxB = { x1: 50, y1: 50, x2: 150, y2: 150 };
|
||||
expect(boundingBoxOverlap(boxA, boxB)).toBe(0.25);
|
||||
});
|
||||
|
||||
it('should return 1 when boxA is fully contained in boxB', () => {
|
||||
const boxA = { x1: 25, y1: 25, x2: 75, y2: 75 };
|
||||
const boxB = { x1: 0, y1: 0, x2: 100, y2: 100 };
|
||||
expect(boundingBoxOverlap(boxA, boxB)).toBe(1);
|
||||
});
|
||||
|
||||
it('should handle partial containment correctly', () => {
|
||||
const boxA = { x1: 0, y1: 0, x2: 100, y2: 100 };
|
||||
const boxB = { x1: 25, y1: 25, x2: 75, y2: 75 };
|
||||
// boxB is fully inside boxA, so overlap area is 50*50=2500, boxA area is 10000
|
||||
expect(boundingBoxOverlap(boxA, boxB)).toBe(0.25);
|
||||
});
|
||||
|
||||
it('should handle boxes that touch at edges (no overlap)', () => {
|
||||
const boxA = { x1: 0, y1: 0, x2: 100, y2: 100 };
|
||||
const boxB = { x1: 100, y1: 0, x2: 200, y2: 100 };
|
||||
expect(boundingBoxOverlap(boxA, boxB)).toBe(0);
|
||||
});
|
||||
|
||||
it('should handle vertical partial overlap', () => {
|
||||
const boxA = { x1: 0, y1: 0, x2: 100, y2: 100 };
|
||||
const boxB = { x1: 0, y1: 50, x2: 100, y2: 150 };
|
||||
expect(boundingBoxOverlap(boxA, boxB)).toBe(0.5);
|
||||
});
|
||||
});
|
||||
|
||||
const createFace = (params: Partial<AssetFace> = {}): AssetFace => ({
|
||||
id: 'face-id',
|
||||
deletedAt: null,
|
||||
assetId: 'asset-id',
|
||||
boundingBoxX1: 100,
|
||||
boundingBoxX2: 200,
|
||||
boundingBoxY1: 100,
|
||||
boundingBoxY2: 200,
|
||||
imageWidth: 1000,
|
||||
imageHeight: 1000,
|
||||
personId: null,
|
||||
sourceType: SourceType.MachineLearning,
|
||||
person: null,
|
||||
updatedAt: new Date(),
|
||||
updateId: 'update-id',
|
||||
isVisible: true,
|
||||
...params,
|
||||
});
|
||||
|
||||
describe('checkFaceVisibility', () => {
|
||||
const assetDimensions = { width: 1000, height: 1000 };
|
||||
|
||||
it('should return only non-visible faces when no crop is provided', () => {
|
||||
const faces = [
|
||||
createFace({ id: 'face-1', isVisible: true }),
|
||||
createFace({ id: 'face-2', isVisible: false }),
|
||||
createFace({ id: 'face-3', isVisible: false }),
|
||||
];
|
||||
const result = checkFaceVisibility(faces, assetDimensions);
|
||||
|
||||
expect(result.visible).toHaveLength(2);
|
||||
expect(result.hidden).toHaveLength(0);
|
||||
expect(result.visible.map((f) => f.id)).toEqual(['face-2', 'face-3']);
|
||||
});
|
||||
|
||||
it('should return all faces as visible when all are marked not visible and no crop provided', () => {
|
||||
const faces = [createFace({ id: 'face-1', isVisible: false }), createFace({ id: 'face-2', isVisible: false })];
|
||||
const result = checkFaceVisibility(faces, assetDimensions);
|
||||
|
||||
expect(result.visible).toHaveLength(2);
|
||||
expect(result.hidden).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('should return empty visible array when all faces are already visible and no crop provided', () => {
|
||||
const faces = [createFace({ id: 'face-1', isVisible: true }), createFace({ id: 'face-2', isVisible: true })];
|
||||
const result = checkFaceVisibility(faces, assetDimensions);
|
||||
|
||||
expect(result.visible).toHaveLength(0);
|
||||
expect(result.hidden).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('should return empty arrays when no faces provided', () => {
|
||||
const result = checkFaceVisibility([], assetDimensions);
|
||||
|
||||
expect(result.visible).toHaveLength(0);
|
||||
expect(result.hidden).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('should mark face as visible when fully inside crop area', () => {
|
||||
const faces = [createFace({ boundingBoxX1: 100, boundingBoxY1: 100, boundingBoxX2: 200, boundingBoxY2: 200 })];
|
||||
const crop = { x1: 0, y1: 0, x2: 500, y2: 500 };
|
||||
|
||||
const result = checkFaceVisibility(faces, assetDimensions, crop);
|
||||
|
||||
expect(result.visible).toHaveLength(1);
|
||||
expect(result.hidden).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('should mark face as hidden when fully outside crop area', () => {
|
||||
const faces = [createFace({ boundingBoxX1: 600, boundingBoxY1: 600, boundingBoxX2: 700, boundingBoxY2: 700 })];
|
||||
const crop = { x1: 0, y1: 0, x2: 500, y2: 500 };
|
||||
|
||||
const result = checkFaceVisibility(faces, assetDimensions, crop);
|
||||
|
||||
expect(result.visible).toHaveLength(0);
|
||||
expect(result.hidden).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('should mark face as visible when at least 50% overlaps with crop', () => {
|
||||
// Face spans 100-200 (100px), crop starts at 150, so 50% overlap
|
||||
const faces = [createFace({ boundingBoxX1: 100, boundingBoxY1: 100, boundingBoxX2: 200, boundingBoxY2: 200 })];
|
||||
const crop = { x1: 150, y1: 100, x2: 500, y2: 500 };
|
||||
|
||||
const result = checkFaceVisibility(faces, assetDimensions, crop);
|
||||
|
||||
expect(result.visible).toHaveLength(1);
|
||||
expect(result.hidden).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('should mark face as hidden when less than 50% overlaps with crop', () => {
|
||||
// Face spans 100-200 (100px), crop starts at 160, so 40% overlap
|
||||
const faces = [createFace({ boundingBoxX1: 100, boundingBoxY1: 100, boundingBoxX2: 200, boundingBoxY2: 200 })];
|
||||
const crop = { x1: 160, y1: 100, x2: 500, y2: 500 };
|
||||
|
||||
const result = checkFaceVisibility(faces, assetDimensions, crop);
|
||||
|
||||
expect(result.visible).toHaveLength(0);
|
||||
expect(result.hidden).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('should correctly categorize multiple faces', () => {
|
||||
const faces = [
|
||||
createFace({ id: 'face-inside', boundingBoxX1: 100, boundingBoxY1: 100, boundingBoxX2: 200, boundingBoxY2: 200 }),
|
||||
createFace({
|
||||
id: 'face-outside',
|
||||
boundingBoxX1: 800,
|
||||
boundingBoxY1: 800,
|
||||
boundingBoxX2: 900,
|
||||
boundingBoxY2: 900,
|
||||
}),
|
||||
// face-partial: 400-500 overlaps with crop (100x100=10000 overlap, face is 200x200=40000, so 25% - hidden)
|
||||
createFace({
|
||||
id: 'face-partial',
|
||||
boundingBoxX1: 400,
|
||||
boundingBoxY1: 400,
|
||||
boundingBoxX2: 600,
|
||||
boundingBoxY2: 600,
|
||||
}),
|
||||
];
|
||||
const crop = { x1: 0, y1: 0, x2: 500, y2: 500 };
|
||||
|
||||
const result = checkFaceVisibility(faces, assetDimensions, crop);
|
||||
|
||||
// face-inside is fully visible, face-partial has 25% overlap (hidden), face-outside is fully hidden
|
||||
expect(result.visible).toHaveLength(1);
|
||||
expect(result.hidden).toHaveLength(2);
|
||||
expect(result.visible.map((f) => f.id)).toContain('face-inside');
|
||||
expect(result.hidden.map((f) => f.id)).toContain('face-partial');
|
||||
expect(result.hidden.map((f) => f.id)).toContain('face-outside');
|
||||
});
|
||||
|
||||
it('should handle face coordinates scaled to different image dimensions', () => {
|
||||
// Face stored at 50-100 in a 500x500 image, scaled to 1000x1000 becomes 100-200
|
||||
const faces = [
|
||||
createFace({
|
||||
boundingBoxX1: 50,
|
||||
boundingBoxY1: 50,
|
||||
boundingBoxX2: 100,
|
||||
boundingBoxY2: 100,
|
||||
imageWidth: 500,
|
||||
imageHeight: 500,
|
||||
}),
|
||||
];
|
||||
const crop = { x1: 0, y1: 0, x2: 200, y2: 200 };
|
||||
|
||||
const result = checkFaceVisibility(faces, assetDimensions, crop);
|
||||
|
||||
expect(result.visible).toHaveLength(1);
|
||||
expect(result.hidden).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('should categorize based on crop overlap when crop is provided, regardless of isVisible property', () => {
|
||||
const faces = [
|
||||
createFace({
|
||||
id: 'face-inside-visible',
|
||||
boundingBoxX1: 100,
|
||||
boundingBoxY1: 100,
|
||||
boundingBoxX2: 200,
|
||||
boundingBoxY2: 200,
|
||||
isVisible: true,
|
||||
}),
|
||||
createFace({
|
||||
id: 'face-inside-not-visible',
|
||||
boundingBoxX1: 250,
|
||||
boundingBoxY1: 250,
|
||||
boundingBoxX2: 350,
|
||||
boundingBoxY2: 350,
|
||||
isVisible: false,
|
||||
}),
|
||||
createFace({
|
||||
id: 'face-outside-visible',
|
||||
boundingBoxX1: 800,
|
||||
boundingBoxY1: 800,
|
||||
boundingBoxX2: 900,
|
||||
boundingBoxY2: 900,
|
||||
isVisible: true,
|
||||
}),
|
||||
createFace({
|
||||
id: 'face-outside-not-visible',
|
||||
boundingBoxX1: 700,
|
||||
boundingBoxY1: 700,
|
||||
boundingBoxX2: 800,
|
||||
boundingBoxY2: 800,
|
||||
isVisible: false,
|
||||
}),
|
||||
];
|
||||
const crop = { x1: 0, y1: 0, x2: 500, y2: 500 };
|
||||
|
||||
const result = checkFaceVisibility(faces, assetDimensions, crop);
|
||||
|
||||
// When crop is provided, only overlap matters, not isVisible property
|
||||
expect(result.visible).toHaveLength(2);
|
||||
expect(result.hidden).toHaveLength(2);
|
||||
expect(result.visible.map((f) => f.id)).toContain('face-inside-visible');
|
||||
expect(result.visible.map((f) => f.id)).toContain('face-inside-not-visible');
|
||||
expect(result.hidden.map((f) => f.id)).toContain('face-outside-visible');
|
||||
expect(result.hidden.map((f) => f.id)).toContain('face-outside-not-visible');
|
||||
});
|
||||
|
||||
it('should handle mixed visibility states with partial overlap and crop', () => {
|
||||
const faces = [
|
||||
createFace({
|
||||
id: 'face-partial-50',
|
||||
boundingBoxX1: 100,
|
||||
boundingBoxY1: 100,
|
||||
boundingBoxX2: 200,
|
||||
boundingBoxY2: 200,
|
||||
isVisible: true,
|
||||
}),
|
||||
createFace({
|
||||
id: 'face-partial-40',
|
||||
boundingBoxX1: 100,
|
||||
boundingBoxY1: 100,
|
||||
boundingBoxX2: 200,
|
||||
boundingBoxY2: 200,
|
||||
isVisible: false,
|
||||
}),
|
||||
];
|
||||
const crop1 = { x1: 150, y1: 100, x2: 500, y2: 500 }; // 50% overlap
|
||||
const crop2 = { x1: 160, y1: 100, x2: 500, y2: 500 }; // 40% overlap
|
||||
|
||||
const result1 = checkFaceVisibility([faces[0]], assetDimensions, crop1);
|
||||
const result2 = checkFaceVisibility([faces[1]], assetDimensions, crop2);
|
||||
|
||||
// 50% overlap should be visible
|
||||
expect(result1.visible).toHaveLength(1);
|
||||
expect(result1.hidden).toHaveLength(0);
|
||||
|
||||
// 40% overlap should be hidden
|
||||
expect(result2.visible).toHaveLength(0);
|
||||
expect(result2.hidden).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
const createOcr = (
|
||||
params: Partial<AssetOcrResponseDto & { isVisible: boolean }> = {},
|
||||
): AssetOcrResponseDto & { isVisible: boolean } => ({
|
||||
id: 'ocr-id',
|
||||
assetId: 'asset-id',
|
||||
x1: 0.1,
|
||||
y1: 0.1,
|
||||
x2: 0.2,
|
||||
y2: 0.1,
|
||||
x3: 0.2,
|
||||
y3: 0.2,
|
||||
x4: 0.1,
|
||||
y4: 0.2,
|
||||
boxScore: 0.9,
|
||||
textScore: 0.9,
|
||||
text: 'Sample Text',
|
||||
isVisible: true,
|
||||
...params,
|
||||
});
|
||||
|
||||
describe('checkOcrVisibility', () => {
|
||||
const assetDimensions = { width: 1000, height: 1000 };
|
||||
|
||||
it('should return only non-visible OCR entries when no crop is provided', () => {
|
||||
const ocrs = [
|
||||
createOcr({ id: 'ocr-1', isVisible: true }),
|
||||
createOcr({ id: 'ocr-2', isVisible: false }),
|
||||
createOcr({ id: 'ocr-3', isVisible: false }),
|
||||
];
|
||||
const result = checkOcrVisibility(ocrs, assetDimensions);
|
||||
|
||||
expect(result.visible).toHaveLength(2);
|
||||
expect(result.hidden).toHaveLength(0);
|
||||
expect(result.visible.map((o) => o.id)).toEqual(['ocr-2', 'ocr-3']);
|
||||
});
|
||||
|
||||
it('should return all OCR entries as visible when all are marked not visible and no crop provided', () => {
|
||||
const ocrs = [createOcr({ id: 'ocr-1', isVisible: false }), createOcr({ id: 'ocr-2', isVisible: false })];
|
||||
const result = checkOcrVisibility(ocrs, assetDimensions);
|
||||
|
||||
expect(result.visible).toHaveLength(2);
|
||||
expect(result.hidden).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('should return empty visible array when all OCR entries are already visible and no crop provided', () => {
|
||||
const ocrs = [createOcr({ id: 'ocr-1', isVisible: true }), createOcr({ id: 'ocr-2', isVisible: true })];
|
||||
const result = checkOcrVisibility(ocrs, assetDimensions);
|
||||
|
||||
expect(result.visible).toHaveLength(0);
|
||||
expect(result.hidden).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('should return empty arrays when no OCR entries provided', () => {
|
||||
const result = checkOcrVisibility([], assetDimensions);
|
||||
|
||||
expect(result.visible).toHaveLength(0);
|
||||
expect(result.hidden).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('should mark OCR as visible when fully inside crop area', () => {
|
||||
// OCR box at normalized coords 0.1-0.2 = 100-200px in 1000x1000 image
|
||||
const ocrs = [createOcr()];
|
||||
const crop = { x1: 0, y1: 0, x2: 500, y2: 500 };
|
||||
|
||||
const result = checkOcrVisibility(ocrs, assetDimensions, crop);
|
||||
|
||||
expect(result.visible).toHaveLength(1);
|
||||
expect(result.hidden).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('should mark OCR as hidden when fully outside crop area', () => {
|
||||
// OCR box at normalized coords 0.8-0.9 = 800-900px
|
||||
const ocrs = [createOcr({ x1: 0.8, y1: 0.8, x2: 0.9, y2: 0.8, x3: 0.9, y3: 0.9, x4: 0.8, y4: 0.9 })];
|
||||
const crop = { x1: 0, y1: 0, x2: 500, y2: 500 };
|
||||
|
||||
const result = checkOcrVisibility(ocrs, assetDimensions, crop);
|
||||
|
||||
expect(result.visible).toHaveLength(0);
|
||||
expect(result.hidden).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('should mark OCR as visible when at least 50% overlaps with crop', () => {
|
||||
// OCR at 100-200px (0.1-0.2 normalized), crop starts at 150
|
||||
const ocrs = [createOcr()];
|
||||
const crop = { x1: 150, y1: 100, x2: 500, y2: 500 };
|
||||
|
||||
const result = checkOcrVisibility(ocrs, assetDimensions, crop);
|
||||
|
||||
expect(result.visible).toHaveLength(1);
|
||||
expect(result.hidden).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('should mark OCR as hidden when less than 50% overlaps with crop', () => {
|
||||
// OCR at 100-200px, crop starts at 160 = 40% overlap
|
||||
const ocrs = [createOcr()];
|
||||
const crop = { x1: 160, y1: 100, x2: 500, y2: 500 };
|
||||
|
||||
const result = checkOcrVisibility(ocrs, assetDimensions, crop);
|
||||
|
||||
expect(result.visible).toHaveLength(0);
|
||||
expect(result.hidden).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('should correctly categorize multiple OCR entries', () => {
|
||||
const ocrs = [
|
||||
createOcr({ id: 'ocr-inside', x1: 0.1, y1: 0.1, x2: 0.2, y2: 0.1, x3: 0.2, y3: 0.2, x4: 0.1, y4: 0.2 }),
|
||||
createOcr({ id: 'ocr-outside', x1: 0.8, y1: 0.8, x2: 0.9, y2: 0.8, x3: 0.9, y3: 0.9, x4: 0.8, y4: 0.9 }),
|
||||
];
|
||||
const crop = { x1: 0, y1: 0, x2: 500, y2: 500 };
|
||||
|
||||
const result = checkOcrVisibility(ocrs, assetDimensions, crop);
|
||||
|
||||
expect(result.visible).toHaveLength(1);
|
||||
expect(result.hidden).toHaveLength(1);
|
||||
expect(result.visible[0].id).toBe('ocr-inside');
|
||||
expect(result.hidden[0].id).toBe('ocr-outside');
|
||||
});
|
||||
|
||||
it('should handle rotated/skewed OCR polygons by using bounding box', () => {
|
||||
// Rotated rectangle - the function should compute the bounding box correctly
|
||||
const ocrs = [
|
||||
createOcr({
|
||||
id: 'ocr-rotated',
|
||||
x1: 0.15,
|
||||
y1: 0.1, // top
|
||||
x2: 0.2,
|
||||
y2: 0.15, // right
|
||||
x3: 0.15,
|
||||
y3: 0.2, // bottom
|
||||
x4: 0.1,
|
||||
y4: 0.15, // left
|
||||
}),
|
||||
];
|
||||
const crop = { x1: 0, y1: 0, x2: 500, y2: 500 };
|
||||
|
||||
const result = checkOcrVisibility(ocrs, assetDimensions, crop);
|
||||
|
||||
expect(result.visible).toHaveLength(1);
|
||||
expect(result.hidden).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('should handle different asset dimensions', () => {
|
||||
const smallDimensions = { width: 500, height: 500 };
|
||||
// OCR at 0.1-0.2 normalized = 50-100px in 500x500 image
|
||||
const ocrs = [createOcr()];
|
||||
const crop = { x1: 0, y1: 0, x2: 200, y2: 200 };
|
||||
|
||||
const result = checkOcrVisibility(ocrs, smallDimensions, crop);
|
||||
|
||||
expect(result.visible).toHaveLength(1);
|
||||
expect(result.hidden).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('should categorize based on crop overlap when crop is provided, regardless of isVisible property', () => {
|
||||
const ocrs = [
|
||||
createOcr({ id: 'ocr-inside-visible', isVisible: true }), // Inside crop, already visible
|
||||
createOcr({ id: 'ocr-inside-not-visible', isVisible: false }), // Inside crop, not visible
|
||||
createOcr({
|
||||
id: 'ocr-outside-visible',
|
||||
x1: 0.8,
|
||||
y1: 0.8,
|
||||
x2: 0.9,
|
||||
y2: 0.8,
|
||||
x3: 0.9,
|
||||
y3: 0.9,
|
||||
x4: 0.8,
|
||||
y4: 0.9,
|
||||
isVisible: true,
|
||||
}), // Outside crop, already visible
|
||||
createOcr({
|
||||
id: 'ocr-outside-not-visible',
|
||||
x1: 0.8,
|
||||
y1: 0.8,
|
||||
x2: 0.9,
|
||||
y2: 0.8,
|
||||
x3: 0.9,
|
||||
y3: 0.9,
|
||||
x4: 0.8,
|
||||
y4: 0.9,
|
||||
isVisible: false,
|
||||
}), // Outside crop, not visible
|
||||
];
|
||||
const crop = { x1: 0, y1: 0, x2: 500, y2: 500 };
|
||||
|
||||
const result = checkOcrVisibility(ocrs, assetDimensions, crop);
|
||||
|
||||
// When crop is provided, only overlap matters, not isVisible property
|
||||
expect(result.visible).toHaveLength(2);
|
||||
expect(result.hidden).toHaveLength(2);
|
||||
expect(result.visible.map((o) => o.id)).toContain('ocr-inside-visible');
|
||||
expect(result.visible.map((o) => o.id)).toContain('ocr-inside-not-visible');
|
||||
expect(result.hidden.map((o) => o.id)).toContain('ocr-outside-visible');
|
||||
expect(result.hidden.map((o) => o.id)).toContain('ocr-outside-not-visible');
|
||||
});
|
||||
|
||||
it('should handle mixed visibility states with partial overlap and crop', () => {
|
||||
const ocrs = [
|
||||
createOcr({ id: 'ocr-partial-50', isVisible: true }), // 50% overlap
|
||||
createOcr({ id: 'ocr-partial-40', isVisible: false }), // 40% overlap
|
||||
];
|
||||
const crop1 = { x1: 150, y1: 100, x2: 500, y2: 500 }; // 50% overlap
|
||||
const crop2 = { x1: 160, y1: 100, x2: 500, y2: 500 }; // 40% overlap
|
||||
|
||||
const result1 = checkOcrVisibility([ocrs[0]], assetDimensions, crop1);
|
||||
const result2 = checkOcrVisibility([ocrs[1]], assetDimensions, crop2);
|
||||
|
||||
// 50% overlap should be visible
|
||||
expect(result1.visible).toHaveLength(1);
|
||||
expect(result1.hidden).toHaveLength(0);
|
||||
|
||||
// 40% overlap should be hidden
|
||||
expect(result2.visible).toHaveLength(0);
|
||||
expect(result2.hidden).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,107 @@
|
||||
import { AssetFace } from 'src/database';
|
||||
import { AssetOcrResponseDto } from 'src/dtos/ocr.dto';
|
||||
import { ImageDimensions } from 'src/types';
|
||||
|
||||
type BoundingBox = {
|
||||
x1: number;
|
||||
y1: number;
|
||||
x2: number;
|
||||
y2: number;
|
||||
};
|
||||
|
||||
export const boundingBoxOverlap = (boxA: BoundingBox, boxB: BoundingBox) => {
|
||||
const overlapX1 = Math.max(boxA.x1, boxB.x1);
|
||||
const overlapY1 = Math.max(boxA.y1, boxB.y1);
|
||||
const overlapX2 = Math.min(boxA.x2, boxB.x2);
|
||||
const overlapY2 = Math.min(boxA.y2, boxB.y2);
|
||||
|
||||
const overlapArea = Math.max(0, overlapX2 - overlapX1) * Math.max(0, overlapY2 - overlapY1);
|
||||
const faceArea = (boxA.x2 - boxA.x1) * (boxA.y2 - boxA.y1);
|
||||
return overlapArea / faceArea;
|
||||
};
|
||||
|
||||
const scale = (box: BoundingBox, target: ImageDimensions, source?: ImageDimensions) => {
|
||||
const { width: sourceWidth = 1, height: sourceHeight = 1 } = source ?? {};
|
||||
|
||||
return {
|
||||
x1: (box.x1 / sourceWidth) * target.width,
|
||||
y1: (box.y1 / sourceHeight) * target.height,
|
||||
x2: (box.x2 / sourceWidth) * target.width,
|
||||
y2: (box.y2 / sourceHeight) * target.height,
|
||||
};
|
||||
};
|
||||
|
||||
export const checkFaceVisibility = (
|
||||
faces: AssetFace[],
|
||||
originalAssetDimensions: ImageDimensions,
|
||||
crop?: BoundingBox,
|
||||
): { visible: AssetFace[]; hidden: AssetFace[] } => {
|
||||
if (!crop) {
|
||||
return {
|
||||
visible: faces.filter((face) => !face.isVisible),
|
||||
hidden: [],
|
||||
};
|
||||
}
|
||||
|
||||
const status = faces.map((face) => {
|
||||
const scaledFace = scale(
|
||||
{
|
||||
x1: face.boundingBoxX1,
|
||||
y1: face.boundingBoxY1,
|
||||
x2: face.boundingBoxX2,
|
||||
y2: face.boundingBoxY2,
|
||||
},
|
||||
originalAssetDimensions,
|
||||
{ width: face.imageWidth, height: face.imageHeight },
|
||||
);
|
||||
|
||||
const overlapPercentage = boundingBoxOverlap(scaledFace, crop);
|
||||
|
||||
return {
|
||||
face,
|
||||
isVisible: overlapPercentage >= 0.5,
|
||||
};
|
||||
});
|
||||
|
||||
return {
|
||||
visible: status.filter((s) => s.isVisible).map((s) => s.face),
|
||||
hidden: status.filter((s) => !s.isVisible).map((s) => s.face),
|
||||
};
|
||||
};
|
||||
|
||||
export const checkOcrVisibility = (
|
||||
ocrs: (AssetOcrResponseDto & { isVisible: boolean })[],
|
||||
originalAssetDimensions: ImageDimensions,
|
||||
crop?: BoundingBox,
|
||||
): { visible: AssetOcrResponseDto[]; hidden: AssetOcrResponseDto[] } => {
|
||||
if (!crop) {
|
||||
return {
|
||||
visible: ocrs.filter((ocr) => !ocr.isVisible),
|
||||
hidden: [],
|
||||
};
|
||||
}
|
||||
|
||||
const status = ocrs.map((ocr) => {
|
||||
const ocrBox = scale(
|
||||
{
|
||||
x1: Math.min(ocr.x1, ocr.x2, ocr.x3, ocr.x4),
|
||||
y1: Math.min(ocr.y1, ocr.y2, ocr.y3, ocr.y4),
|
||||
x2: Math.max(ocr.x1, ocr.x2, ocr.x3, ocr.x4),
|
||||
y2: Math.max(ocr.y1, ocr.y2, ocr.y3, ocr.y4),
|
||||
},
|
||||
originalAssetDimensions,
|
||||
);
|
||||
|
||||
const overlapPercentage = boundingBoxOverlap(ocrBox, crop);
|
||||
|
||||
return {
|
||||
ocr,
|
||||
isVisible: overlapPercentage >= 0.5,
|
||||
};
|
||||
});
|
||||
|
||||
return {
|
||||
visible: status.filter((s) => s.isVisible).map((s) => s.ocr),
|
||||
hidden: status.filter((s) => !s.isVisible).map((s) => s.ocr),
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,293 @@
|
||||
import { AssetEditAction, AssetEditActionItem, MirrorAxis } from 'src/dtos/editing.dto';
|
||||
import { AssetOcrResponseDto } from 'src/dtos/ocr.dto';
|
||||
import { transformFaceBoundingBox, transformOcrBoundingBox } from 'src/utils/transform';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
describe('transformFaceBoundingBox', () => {
|
||||
const baseFace = {
|
||||
boundingBoxX1: 100,
|
||||
boundingBoxY1: 100,
|
||||
boundingBoxX2: 200,
|
||||
boundingBoxY2: 200,
|
||||
imageWidth: 1000,
|
||||
imageHeight: 800,
|
||||
};
|
||||
|
||||
const baseDimensions = { width: 1000, height: 800 };
|
||||
|
||||
describe('with no edits', () => {
|
||||
it('should return unchanged bounding box', () => {
|
||||
const result = transformFaceBoundingBox(baseFace, [], baseDimensions);
|
||||
expect(result).toEqual(baseFace);
|
||||
});
|
||||
});
|
||||
|
||||
describe('with crop edit', () => {
|
||||
it('should adjust bounding box for crop offset', () => {
|
||||
const edits: AssetEditActionItem[] = [
|
||||
{ action: AssetEditAction.Crop, parameters: { x: 50, y: 50, width: 400, height: 300 } },
|
||||
];
|
||||
const result = transformFaceBoundingBox(baseFace, edits, baseDimensions);
|
||||
|
||||
expect(result.boundingBoxX1).toBe(50);
|
||||
expect(result.boundingBoxY1).toBe(50);
|
||||
expect(result.boundingBoxX2).toBe(150);
|
||||
expect(result.boundingBoxY2).toBe(150);
|
||||
expect(result.imageWidth).toBe(400);
|
||||
expect(result.imageHeight).toBe(300);
|
||||
});
|
||||
|
||||
it('should handle face partially outside crop area', () => {
|
||||
const edits: AssetEditActionItem[] = [
|
||||
{ action: AssetEditAction.Crop, parameters: { x: 150, y: 150, width: 400, height: 300 } },
|
||||
];
|
||||
const result = transformFaceBoundingBox(baseFace, edits, baseDimensions);
|
||||
|
||||
expect(result.boundingBoxX1).toBe(-50);
|
||||
expect(result.boundingBoxY1).toBe(-50);
|
||||
expect(result.boundingBoxX2).toBe(50);
|
||||
expect(result.boundingBoxY2).toBe(50);
|
||||
});
|
||||
});
|
||||
|
||||
describe('with rotate edit', () => {
|
||||
it('should rotate 90 degrees clockwise', () => {
|
||||
const edits: AssetEditActionItem[] = [{ action: AssetEditAction.Rotate, parameters: { angle: 90 } }];
|
||||
const result = transformFaceBoundingBox(baseFace, edits, baseDimensions);
|
||||
|
||||
expect(result.imageWidth).toBe(800);
|
||||
expect(result.imageHeight).toBe(1000);
|
||||
|
||||
expect(result.boundingBoxX1).toBe(600);
|
||||
expect(result.boundingBoxY1).toBe(100);
|
||||
expect(result.boundingBoxX2).toBe(700);
|
||||
expect(result.boundingBoxY2).toBe(200);
|
||||
});
|
||||
|
||||
it('should rotate 180 degrees', () => {
|
||||
const edits: AssetEditActionItem[] = [{ action: AssetEditAction.Rotate, parameters: { angle: 180 } }];
|
||||
const result = transformFaceBoundingBox(baseFace, edits, baseDimensions);
|
||||
|
||||
expect(result.imageWidth).toBe(1000);
|
||||
expect(result.imageHeight).toBe(800);
|
||||
|
||||
expect(result.boundingBoxX1).toBe(800);
|
||||
expect(result.boundingBoxY1).toBe(600);
|
||||
expect(result.boundingBoxX2).toBe(900);
|
||||
expect(result.boundingBoxY2).toBe(700);
|
||||
});
|
||||
|
||||
it('should rotate 270 degrees', () => {
|
||||
const edits: AssetEditActionItem[] = [{ action: AssetEditAction.Rotate, parameters: { angle: 270 } }];
|
||||
const result = transformFaceBoundingBox(baseFace, edits, baseDimensions);
|
||||
|
||||
expect(result.imageWidth).toBe(800);
|
||||
expect(result.imageHeight).toBe(1000);
|
||||
});
|
||||
});
|
||||
|
||||
describe('with mirror edit', () => {
|
||||
it('should mirror horizontally', () => {
|
||||
const edits: AssetEditActionItem[] = [
|
||||
{ action: AssetEditAction.Mirror, parameters: { axis: MirrorAxis.Horizontal } },
|
||||
];
|
||||
const result = transformFaceBoundingBox(baseFace, edits, baseDimensions);
|
||||
|
||||
expect(result.boundingBoxX1).toBe(800);
|
||||
expect(result.boundingBoxY1).toBe(100);
|
||||
expect(result.boundingBoxX2).toBe(900);
|
||||
expect(result.boundingBoxY2).toBe(200);
|
||||
expect(result.imageWidth).toBe(1000);
|
||||
expect(result.imageHeight).toBe(800);
|
||||
});
|
||||
|
||||
it('should mirror vertically', () => {
|
||||
const edits: AssetEditActionItem[] = [
|
||||
{ action: AssetEditAction.Mirror, parameters: { axis: MirrorAxis.Vertical } },
|
||||
];
|
||||
const result = transformFaceBoundingBox(baseFace, edits, baseDimensions);
|
||||
|
||||
expect(result.boundingBoxX1).toBe(100);
|
||||
expect(result.boundingBoxY1).toBe(600);
|
||||
expect(result.boundingBoxX2).toBe(200);
|
||||
expect(result.boundingBoxY2).toBe(700);
|
||||
expect(result.imageWidth).toBe(1000);
|
||||
expect(result.imageHeight).toBe(800);
|
||||
});
|
||||
});
|
||||
|
||||
describe('with combined edits', () => {
|
||||
it('should apply crop then rotate', () => {
|
||||
const edits: AssetEditActionItem[] = [
|
||||
{ action: AssetEditAction.Crop, parameters: { x: 50, y: 50, width: 400, height: 300 } },
|
||||
{ action: AssetEditAction.Rotate, parameters: { angle: 90 } },
|
||||
];
|
||||
const result = transformFaceBoundingBox(baseFace, edits, baseDimensions);
|
||||
|
||||
expect(result.imageWidth).toBe(300);
|
||||
expect(result.imageHeight).toBe(400);
|
||||
});
|
||||
|
||||
it('should apply crop then mirror', () => {
|
||||
const edits: AssetEditActionItem[] = [
|
||||
{ action: AssetEditAction.Crop, parameters: { x: 0, y: 0, width: 500, height: 400 } },
|
||||
{ action: AssetEditAction.Mirror, parameters: { axis: MirrorAxis.Vertical } },
|
||||
];
|
||||
const result = transformFaceBoundingBox(baseFace, edits, baseDimensions);
|
||||
|
||||
expect(result.boundingBoxX1).toBe(100);
|
||||
expect(result.boundingBoxX2).toBe(200);
|
||||
expect(result.boundingBoxY1).toBe(200);
|
||||
expect(result.boundingBoxY2).toBe(300);
|
||||
});
|
||||
});
|
||||
|
||||
describe('with scaled dimensions', () => {
|
||||
it('should scale face to match different image dimensions', () => {
|
||||
const scaledDimensions = { width: 500, height: 400 }; // Half the original size
|
||||
const edits: AssetEditActionItem[] = [
|
||||
{ action: AssetEditAction.Crop, parameters: { x: 50, y: 50, width: 200, height: 150 } },
|
||||
];
|
||||
const result = transformFaceBoundingBox(baseFace, edits, scaledDimensions);
|
||||
|
||||
expect(result.boundingBoxX1).toBe(0);
|
||||
expect(result.boundingBoxY1).toBe(0);
|
||||
expect(result.boundingBoxX2).toBe(50);
|
||||
expect(result.boundingBoxY2).toBe(50);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('transformOcrBoundingBox', () => {
|
||||
const baseOcr: AssetOcrResponseDto = {
|
||||
id: 'ocr-1',
|
||||
assetId: 'asset-1',
|
||||
x1: 0.1,
|
||||
y1: 0.1,
|
||||
x2: 0.2,
|
||||
y2: 0.1,
|
||||
x3: 0.2,
|
||||
y3: 0.2,
|
||||
x4: 0.1,
|
||||
y4: 0.2,
|
||||
boxScore: 0.9,
|
||||
textScore: 0.85,
|
||||
text: 'Test OCR',
|
||||
};
|
||||
|
||||
const baseDimensions = { width: 1000, height: 800 };
|
||||
|
||||
describe('with no edits', () => {
|
||||
it('should return unchanged bounding box', () => {
|
||||
const result = transformOcrBoundingBox(baseOcr, [], baseDimensions);
|
||||
expect(result).toEqual(baseOcr);
|
||||
});
|
||||
});
|
||||
|
||||
describe('with crop edit', () => {
|
||||
it('should adjust normalized coordinates for crop', () => {
|
||||
const edits: AssetEditActionItem[] = [
|
||||
{ action: AssetEditAction.Crop, parameters: { x: 100, y: 80, width: 400, height: 320 } },
|
||||
];
|
||||
const result = transformOcrBoundingBox(baseOcr, edits, baseDimensions);
|
||||
|
||||
// Original OCR: (0.1,0.1)-(0.2,0.2) on 1000x800 = (100,80)-(200,160)
|
||||
// After crop offset (100,80): (0,0)-(100,80)
|
||||
// Normalized to 400x320: (0,0)-(0.25,0.25)
|
||||
expect(result.x1).toBeCloseTo(0, 5);
|
||||
expect(result.y1).toBeCloseTo(0, 5);
|
||||
expect(result.x2).toBeCloseTo(0.25, 5);
|
||||
expect(result.y2).toBeCloseTo(0, 5);
|
||||
expect(result.x3).toBeCloseTo(0.25, 5);
|
||||
expect(result.y3).toBeCloseTo(0.25, 5);
|
||||
expect(result.x4).toBeCloseTo(0, 5);
|
||||
expect(result.y4).toBeCloseTo(0.25, 5);
|
||||
});
|
||||
});
|
||||
|
||||
describe('with rotate edit', () => {
|
||||
it('should rotate normalized coordinates 90 degrees and reorder points', () => {
|
||||
const edits: AssetEditActionItem[] = [{ action: AssetEditAction.Rotate, parameters: { angle: 90 } }];
|
||||
const result = transformOcrBoundingBox(baseOcr, edits, baseDimensions);
|
||||
|
||||
expect(result.id).toBe(baseOcr.id);
|
||||
expect(result.text).toBe(baseOcr.text);
|
||||
expect(result.x1).toBeCloseTo(0.8, 5);
|
||||
expect(result.y1).toBeCloseTo(0.1, 5);
|
||||
expect(result.x2).toBeCloseTo(0.9, 5);
|
||||
expect(result.y2).toBeCloseTo(0.1, 5);
|
||||
expect(result.x3).toBeCloseTo(0.9, 5);
|
||||
expect(result.y3).toBeCloseTo(0.2, 5);
|
||||
expect(result.x4).toBeCloseTo(0.8, 5);
|
||||
expect(result.y4).toBeCloseTo(0.2, 5);
|
||||
});
|
||||
|
||||
it('should rotate 180 degrees and reorder points', () => {
|
||||
const edits: AssetEditActionItem[] = [{ action: AssetEditAction.Rotate, parameters: { angle: 180 } }];
|
||||
const result = transformOcrBoundingBox(baseOcr, edits, baseDimensions);
|
||||
|
||||
expect(result.x1).toBeCloseTo(0.8, 5);
|
||||
expect(result.y1).toBeCloseTo(0.8, 5);
|
||||
expect(result.x2).toBeCloseTo(0.9, 5);
|
||||
expect(result.y2).toBeCloseTo(0.8, 5);
|
||||
expect(result.x3).toBeCloseTo(0.9, 5);
|
||||
expect(result.y3).toBeCloseTo(0.9, 5);
|
||||
expect(result.x4).toBeCloseTo(0.8, 5);
|
||||
expect(result.y4).toBeCloseTo(0.9, 5);
|
||||
});
|
||||
|
||||
it('should rotate 270 degrees and reorder points', () => {
|
||||
const edits: AssetEditActionItem[] = [{ action: AssetEditAction.Rotate, parameters: { angle: 270 } }];
|
||||
const result = transformOcrBoundingBox(baseOcr, edits, baseDimensions);
|
||||
|
||||
expect(result.id).toBe(baseOcr.id);
|
||||
expect(result.text).toBe(baseOcr.text);
|
||||
expect(result.x1).toBeCloseTo(0.1, 5);
|
||||
expect(result.y1).toBeCloseTo(0.8, 5);
|
||||
expect(result.x2).toBeCloseTo(0.2, 5);
|
||||
expect(result.y2).toBeCloseTo(0.8, 5);
|
||||
expect(result.x3).toBeCloseTo(0.2, 5);
|
||||
expect(result.y3).toBeCloseTo(0.9, 5);
|
||||
expect(result.x4).toBeCloseTo(0.1, 5);
|
||||
expect(result.y4).toBeCloseTo(0.9, 5);
|
||||
});
|
||||
});
|
||||
|
||||
describe('with mirror edit', () => {
|
||||
it('should mirror horizontally', () => {
|
||||
const edits: AssetEditActionItem[] = [
|
||||
{ action: AssetEditAction.Mirror, parameters: { axis: MirrorAxis.Horizontal } },
|
||||
];
|
||||
const result = transformOcrBoundingBox(baseOcr, edits, baseDimensions);
|
||||
|
||||
expect(result.x1).toBeCloseTo(0.9, 5);
|
||||
expect(result.y1).toBeCloseTo(0.1, 5);
|
||||
});
|
||||
|
||||
it('should mirror vertically', () => {
|
||||
const edits: AssetEditActionItem[] = [
|
||||
{ action: AssetEditAction.Mirror, parameters: { axis: MirrorAxis.Vertical } },
|
||||
];
|
||||
const result = transformOcrBoundingBox(baseOcr, edits, baseDimensions);
|
||||
|
||||
expect(result.x1).toBeCloseTo(0.1, 5);
|
||||
expect(result.y1).toBeCloseTo(0.9, 5);
|
||||
});
|
||||
});
|
||||
|
||||
describe('with combined edits', () => {
|
||||
it('should preserve OCR metadata through transforms', () => {
|
||||
const edits: AssetEditActionItem[] = [
|
||||
{ action: AssetEditAction.Crop, parameters: { x: 0, y: 0, width: 500, height: 400 } },
|
||||
{ action: AssetEditAction.Rotate, parameters: { angle: 90 } },
|
||||
];
|
||||
const result = transformOcrBoundingBox(baseOcr, edits, baseDimensions);
|
||||
|
||||
expect(result.id).toBe(baseOcr.id);
|
||||
expect(result.assetId).toBe(baseOcr.assetId);
|
||||
expect(result.boxScore).toBe(baseOcr.boxScore);
|
||||
expect(result.textScore).toBe(baseOcr.textScore);
|
||||
expect(result.text).toBe(baseOcr.text);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,227 @@
|
||||
import { AssetEditAction, AssetEditActionItem } from 'src/dtos/editing.dto';
|
||||
import { AssetOcrResponseDto } from 'src/dtos/ocr.dto';
|
||||
import { ImageDimensions } from 'src/types';
|
||||
import { applyToPoint, compose, flipX, flipY, identity, Matrix, rotate, scale, translate } from 'transformation-matrix';
|
||||
|
||||
export const getOutputDimensions = (
|
||||
edits: AssetEditActionItem[],
|
||||
startingDimensions: ImageDimensions,
|
||||
): ImageDimensions => {
|
||||
let { width, height } = startingDimensions;
|
||||
|
||||
const crop = edits.find((edit) => edit.action === AssetEditAction.Crop);
|
||||
if (crop) {
|
||||
width = crop.parameters.width;
|
||||
height = crop.parameters.height;
|
||||
}
|
||||
|
||||
for (const edit of edits) {
|
||||
if (edit.action === AssetEditAction.Rotate) {
|
||||
const angleDegrees = edit.parameters.angle;
|
||||
if (angleDegrees === 90 || angleDegrees === 270) {
|
||||
[width, height] = [height, width];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { width, height };
|
||||
};
|
||||
|
||||
export const createAffineMatrix = (
|
||||
edits: AssetEditActionItem[],
|
||||
scalingParameters?: {
|
||||
pointSpace: ImageDimensions;
|
||||
targetSpace: ImageDimensions;
|
||||
},
|
||||
): Matrix => {
|
||||
let scalingMatrix: Matrix = identity();
|
||||
|
||||
if (scalingParameters) {
|
||||
const { pointSpace, targetSpace } = scalingParameters;
|
||||
const scaleX = targetSpace.width / pointSpace.width;
|
||||
scalingMatrix = scale(scaleX);
|
||||
}
|
||||
|
||||
return compose(
|
||||
scalingMatrix,
|
||||
...edits.map((edit) => {
|
||||
switch (edit.action) {
|
||||
case 'rotate': {
|
||||
const angleInRadians = (-edit.parameters.angle * Math.PI) / 180;
|
||||
return rotate(angleInRadians);
|
||||
}
|
||||
case 'mirror': {
|
||||
return edit.parameters.axis === 'horizontal' ? flipY() : flipX();
|
||||
}
|
||||
default: {
|
||||
return identity();
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
};
|
||||
|
||||
type Point = { x: number; y: number };
|
||||
|
||||
type TransformState = {
|
||||
points: Point[];
|
||||
currentWidth: number;
|
||||
currentHeight: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* Transforms an array of points through a series of edit operations (crop, rotate, mirror).
|
||||
* Points should be in absolute pixel coordinates relative to the starting dimensions.
|
||||
*/
|
||||
const transformPoints = (
|
||||
points: Point[],
|
||||
edits: AssetEditActionItem[],
|
||||
startingDimensions: ImageDimensions,
|
||||
): TransformState => {
|
||||
let currentWidth = startingDimensions.width;
|
||||
let currentHeight = startingDimensions.height;
|
||||
let transformedPoints = [...points];
|
||||
|
||||
// Handle crop first
|
||||
const crop = edits.find((edit) => edit.action === 'crop');
|
||||
if (crop) {
|
||||
const { x: cropX, y: cropY, width: cropWidth, height: cropHeight } = crop.parameters;
|
||||
transformedPoints = transformedPoints.map((p) => ({
|
||||
x: p.x - cropX,
|
||||
y: p.y - cropY,
|
||||
}));
|
||||
currentWidth = cropWidth;
|
||||
currentHeight = cropHeight;
|
||||
}
|
||||
|
||||
// Apply rotate and mirror transforms
|
||||
for (const edit of edits) {
|
||||
let matrix: Matrix = identity();
|
||||
if (edit.action === 'rotate') {
|
||||
const angleDegrees = edit.parameters.angle;
|
||||
const angleRadians = (angleDegrees * Math.PI) / 180;
|
||||
const newWidth = angleDegrees === 90 || angleDegrees === 270 ? currentHeight : currentWidth;
|
||||
const newHeight = angleDegrees === 90 || angleDegrees === 270 ? currentWidth : currentHeight;
|
||||
|
||||
matrix = compose(
|
||||
translate(newWidth / 2, newHeight / 2),
|
||||
rotate(angleRadians),
|
||||
translate(-currentWidth / 2, -currentHeight / 2),
|
||||
);
|
||||
|
||||
currentWidth = newWidth;
|
||||
currentHeight = newHeight;
|
||||
} else if (edit.action === 'mirror') {
|
||||
matrix = compose(
|
||||
translate(currentWidth / 2, currentHeight / 2),
|
||||
edit.parameters.axis === 'horizontal' ? flipY() : flipX(),
|
||||
translate(-currentWidth / 2, -currentHeight / 2),
|
||||
);
|
||||
} else {
|
||||
// Skip non-affine transformations
|
||||
continue;
|
||||
}
|
||||
|
||||
transformedPoints = transformedPoints.map((p) => applyToPoint(matrix, p));
|
||||
}
|
||||
|
||||
return {
|
||||
points: transformedPoints,
|
||||
currentWidth,
|
||||
currentHeight,
|
||||
};
|
||||
};
|
||||
|
||||
type FaceBoundingBox = {
|
||||
boundingBoxX1: number;
|
||||
boundingBoxX2: number;
|
||||
boundingBoxY1: number;
|
||||
boundingBoxY2: number;
|
||||
imageWidth: number;
|
||||
imageHeight: number;
|
||||
};
|
||||
|
||||
export const transformFaceBoundingBox = (
|
||||
box: FaceBoundingBox,
|
||||
edits: AssetEditActionItem[],
|
||||
imageDimensions: ImageDimensions,
|
||||
): FaceBoundingBox => {
|
||||
if (edits.length === 0) {
|
||||
return box;
|
||||
}
|
||||
|
||||
const scaleX = imageDimensions.width / box.imageWidth;
|
||||
const scaleY = imageDimensions.height / box.imageHeight;
|
||||
|
||||
const points: Point[] = [
|
||||
{ x: box.boundingBoxX1 * scaleX, y: box.boundingBoxY1 * scaleY },
|
||||
{ x: box.boundingBoxX2 * scaleX, y: box.boundingBoxY2 * scaleY },
|
||||
];
|
||||
|
||||
const { points: transformedPoints, currentWidth, currentHeight } = transformPoints(points, edits, imageDimensions);
|
||||
|
||||
// Ensure x1,y1 is top-left and x2,y2 is bottom-right
|
||||
const [p1, p2] = transformedPoints;
|
||||
return {
|
||||
boundingBoxX1: Math.min(p1.x, p2.x),
|
||||
boundingBoxY1: Math.min(p1.y, p2.y),
|
||||
boundingBoxX2: Math.max(p1.x, p2.x),
|
||||
boundingBoxY2: Math.max(p1.y, p2.y),
|
||||
imageWidth: currentWidth,
|
||||
imageHeight: currentHeight,
|
||||
};
|
||||
};
|
||||
|
||||
const reorderQuadPointsForRotation = (points: Point[], rotationDegrees: number): Point[] => {
|
||||
const [p1, p2, p3, p4] = points;
|
||||
switch (rotationDegrees) {
|
||||
case 90: {
|
||||
return [p4, p1, p2, p3];
|
||||
}
|
||||
case 180: {
|
||||
return [p3, p4, p1, p2];
|
||||
}
|
||||
case 270: {
|
||||
return [p2, p3, p4, p1];
|
||||
}
|
||||
default: {
|
||||
return points;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
export const transformOcrBoundingBox = (
|
||||
box: AssetOcrResponseDto,
|
||||
edits: AssetEditActionItem[],
|
||||
imageDimensions: ImageDimensions,
|
||||
): AssetOcrResponseDto => {
|
||||
if (edits.length === 0) {
|
||||
return box;
|
||||
}
|
||||
|
||||
const points: Point[] = [
|
||||
{ x: box.x1 * imageDimensions.width, y: box.y1 * imageDimensions.height },
|
||||
{ x: box.x2 * imageDimensions.width, y: box.y2 * imageDimensions.height },
|
||||
{ x: box.x3 * imageDimensions.width, y: box.y3 * imageDimensions.height },
|
||||
{ x: box.x4 * imageDimensions.width, y: box.y4 * imageDimensions.height },
|
||||
];
|
||||
|
||||
const { points: transformedPoints, currentWidth, currentHeight } = transformPoints(points, edits, imageDimensions);
|
||||
|
||||
// Reorder points to maintain semantic ordering (topLeft, topRight, bottomRight, bottomLeft)
|
||||
const netRotation = edits.find((e) => e.action == AssetEditAction.Rotate)?.parameters.angle ?? 0 % 360;
|
||||
const reorderedPoints = reorderQuadPointsForRotation(transformedPoints, netRotation);
|
||||
|
||||
const [p1, p2, p3, p4] = reorderedPoints;
|
||||
return {
|
||||
...box,
|
||||
x1: p1.x / currentWidth,
|
||||
y1: p1.y / currentHeight,
|
||||
x2: p2.x / currentWidth,
|
||||
y2: p2.y / currentHeight,
|
||||
x3: p3.x / currentWidth,
|
||||
y3: p3.y / currentHeight,
|
||||
x4: p4.x / currentWidth,
|
||||
y4: p4.y / currentHeight,
|
||||
};
|
||||
};
|
||||
Reference in New Issue
Block a user