feat: image editing (#24155)

This commit is contained in:
Brandon Wees
2026-01-09 17:59:52 -05:00
committed by GitHub
parent 76241a7b2b
commit e8c80d88a5
141 changed files with 7836 additions and 1634 deletions
+1
View File
@@ -236,6 +236,7 @@ export const defaults = Object.freeze<SystemConfig>({
[QueueName.Notification]: { concurrency: 5 },
[QueueName.Ocr]: { concurrency: 1 },
[QueueName.Workflow]: { concurrency: 5 },
[QueueName.Editor]: { concurrency: 2 },
},
logging: {
enabled: true,
@@ -33,6 +33,7 @@ import {
CheckExistingAssetsDto,
UploadFieldName,
} from 'src/dtos/asset-media.dto';
import { AssetDownloadOriginalDto } from 'src/dtos/asset.dto';
import { AuthDto } from 'src/dtos/auth.dto';
import { ApiTag, ImmichHeader, Permission, RouteKey } from 'src/enum';
import { AssetUploadInterceptor } from 'src/middleware/asset-upload.interceptor';
@@ -104,10 +105,11 @@ export class AssetMediaController {
async downloadAsset(
@Auth() auth: AuthDto,
@Param() { id }: UUIDParamDto,
@Query() dto: AssetDownloadOriginalDto,
@Res() res: Response,
@Next() next: NextFunction,
) {
await sendFile(res, next, () => this.service.downloadOriginal(auth, id), this.logger);
await sendFile(res, next, () => this.service.downloadOriginal(auth, id, dto), this.logger);
}
@Put(':id/original')
@@ -292,6 +292,64 @@ describe(AssetController.name, () => {
});
});
describe('PUT /assets/:id/edits', () => {
it('should be an authenticated route', async () => {
await request(ctx.getHttpServer()).put(`/assets/${factory.uuid()}/edits`).send({ edits: [] });
expect(ctx.authenticate).toHaveBeenCalled();
});
it('should accept valid edits and pass to service correctly', async () => {
const edits = [
{
action: 'crop',
parameters: {
x: 0,
y: 0,
width: 100,
height: 100,
},
},
];
const assetId = factory.uuid();
const { status } = await request(ctx.getHttpServer()).put(`/assets/${assetId}/edits`).send({
edits,
});
expect(service.editAsset).toHaveBeenCalledWith(undefined, assetId, { edits });
expect(status).toBe(200);
});
it('should require a valid id', async () => {
const { status, body } = await request(ctx.getHttpServer())
.put(`/assets/123/edits`)
.send({
edits: [
{
action: 'crop',
parameters: {
x: 0,
y: 0,
width: 100,
height: 100,
},
},
],
});
expect(status).toBe(400);
expect(body).toEqual(factory.responses.badRequest(expect.arrayContaining(['id must be a UUID'])));
});
it('should require at least one edit', async () => {
const { status, body } = await request(ctx.getHttpServer())
.put(`/assets/${factory.uuid()}/edits`)
.send({ edits: [] });
expect(status).toBe(400);
expect(body).toEqual(factory.responses.badRequest(['edits must contain at least 1 elements']));
});
});
describe('DELETE /assets/:id/metadata/:key', () => {
it('should be an authenticated route', async () => {
await request(ctx.getHttpServer()).delete(`/assets/${factory.uuid()}/metadata/mobile-app`);
@@ -20,6 +20,7 @@ import {
UpdateAssetDto,
} from 'src/dtos/asset.dto';
import { AuthDto } from 'src/dtos/auth.dto';
import { AssetEditActionListDto, AssetEditsDto } from 'src/dtos/editing.dto';
import { AssetOcrResponseDto } from 'src/dtos/ocr.dto';
import { ApiTag, Permission, RouteKey } from 'src/enum';
import { Auth, Authenticated } from 'src/middleware/auth.guard';
@@ -226,4 +227,42 @@ export class AssetController {
deleteAssetMetadata(@Auth() auth: AuthDto, @Param() { id, key }: AssetMetadataRouteParams): Promise<void> {
return this.service.deleteMetadataByKey(auth, id, key);
}
@Get(':id/edits')
@Authenticated({ permission: Permission.AssetEditGet })
@Endpoint({
summary: 'Retrieve edits for an existing asset',
description: 'Retrieve a series of edit actions (crop, rotate, mirror) associated with the specified asset.',
history: new HistoryBuilder().added('v2.5.0').beta('v2.5.0'),
})
getAssetEdits(@Auth() auth: AuthDto, @Param() { id }: UUIDParamDto): Promise<AssetEditsDto> {
return this.service.getAssetEdits(auth, id);
}
@Put(':id/edits')
@Authenticated({ permission: Permission.AssetEditCreate })
@Endpoint({
summary: 'Apply edits to an existing asset',
description: 'Apply a series of edit actions (crop, rotate, mirror) to the specified asset.',
history: new HistoryBuilder().added('v2.5.0').beta('v2.5.0'),
})
editAsset(
@Auth() auth: AuthDto,
@Param() { id }: UUIDParamDto,
@Body() dto: AssetEditActionListDto,
): Promise<AssetEditsDto> {
return this.service.editAsset(auth, id, dto);
}
@Delete(':id/edits')
@Authenticated({ permission: Permission.AssetEditDelete })
@HttpCode(HttpStatus.NO_CONTENT)
@Endpoint({
summary: 'Remove edits from an existing asset',
description: 'Removes all edit actions (crop, rotate, mirror) associated with the specified asset.',
history: new HistoryBuilder().added('v2.5.0').beta('v2.5.0'),
})
removeAssetEdits(@Auth() auth: AuthDto, @Param() { id }: UUIDParamDto): Promise<void> {
return this.service.removeAssetEdits(auth, id);
}
}
+7 -1
View File
@@ -24,7 +24,13 @@ export interface MoveRequest {
};
}
export type GeneratedImageType = AssetPathType.Preview | AssetPathType.Thumbnail | AssetPathType.FullSize;
export type GeneratedImageType =
| AssetPathType.Preview
| AssetPathType.Thumbnail
| AssetPathType.FullSize
| AssetPathType.EditedPreview
| AssetPathType.EditedThumbnail
| AssetPathType.EditedFullSize;
export type GeneratedAssetType = GeneratedImageType | AssetPathType.EncodedVideo;
export type ThumbnailPathEntity = { id: string; ownerId: string };
+5
View File
@@ -272,6 +272,7 @@ export type AssetFace = {
person?: Person | null;
updatedAt: Date;
updateId: string;
isVisible: boolean;
};
export type Plugin = Selectable<PluginTable>;
@@ -340,6 +341,8 @@ export const columns = {
'asset.originalPath',
'asset.ownerId',
'asset.type',
'asset.width',
'asset.height',
],
assetFiles: ['asset_file.id', 'asset_file.path', 'asset_file.type'],
authUser: ['user.id', 'user.name', 'user.email', 'user.isAdmin', 'user.quotaUsageInBytes', 'user.quotaSizeInBytes'],
@@ -390,6 +393,8 @@ export const columns = {
'asset.livePhotoVideoId',
'asset.stackId',
'asset.libraryId',
'asset.width',
'asset.height',
],
syncAlbumUser: ['album_user.albumId as albumId', 'album_user.userId as userId', 'album_user.role'],
syncStack: ['stack.id', 'stack.createdAt', 'stack.updatedAt', 'stack.primaryAssetId', 'stack.ownerId'],
+3
View File
@@ -19,6 +19,9 @@ export enum AssetMediaSize {
export class AssetMediaOptionsDto {
@ValidateEnum({ enum: AssetMediaSize, name: 'AssetMediaSize', optional: true })
size?: AssetMediaSize;
@ValidateBoolean({ optional: true, default: false })
edited?: boolean;
}
export enum UploadFieldName {
+21 -3
View File
@@ -3,6 +3,7 @@ import { Selectable } from 'kysely';
import { AssetFace, AssetFile, Exif, Stack, Tag, User } from 'src/database';
import { HistoryBuilder, Property } from 'src/decorators';
import { AuthDto } from 'src/dtos/auth.dto';
import { AssetEditActionItem } from 'src/dtos/editing.dto';
import { ExifResponseDto, mapExif } from 'src/dtos/exif.dto';
import {
AssetFaceWithoutPersonResponseDto,
@@ -13,6 +14,8 @@ import {
import { TagResponseDto, mapTag } from 'src/dtos/tag.dto';
import { UserResponseDto, mapUser } from 'src/dtos/user.dto';
import { AssetStatus, AssetType, AssetVisibility } from 'src/enum';
import { ImageDimensions } from 'src/types';
import { getDimensions } from 'src/utils/asset.util';
import { hexOrBufferToBase64 } from 'src/utils/bytes';
import { mimeTypes } from 'src/utils/mime-types';
import { ValidateEnum } from 'src/validation';
@@ -34,6 +37,8 @@ export class SanitizedAssetResponseDto {
duration!: string;
livePhotoVideoId?: string | null;
hasMetadata!: boolean;
width!: number | null;
height!: number | null;
}
export class AssetResponseDto extends SanitizedAssetResponseDto {
@@ -107,6 +112,7 @@ export type MapAsset = {
deviceId: string;
duplicateId: string | null;
duration: string | null;
edits?: AssetEditActionItem[];
encodedVideoPath: string | null;
exifInfo?: Selectable<Exif> | null;
faces?: AssetFace[];
@@ -129,6 +135,8 @@ export type MapAsset = {
tags?: Tag[];
thumbhash: Buffer<ArrayBufferLike> | null;
type: AssetType;
width: number | null;
height: number | null;
};
export class AssetStackResponseDto {
@@ -147,7 +155,11 @@ export type AssetMapOptions = {
};
// TODO: this is inefficient
const peopleWithFaces = (faces?: AssetFace[]): PersonWithFacesResponseDto[] => {
const peopleWithFaces = (
faces?: AssetFace[],
edits?: AssetEditActionItem[],
assetDimensions?: ImageDimensions,
): PersonWithFacesResponseDto[] => {
const result: PersonWithFacesResponseDto[] = [];
if (faces) {
for (const face of faces) {
@@ -156,7 +168,7 @@ const peopleWithFaces = (faces?: AssetFace[]): PersonWithFacesResponseDto[] => {
if (existingPersonEntry) {
existingPersonEntry.faces.push(face);
} else {
result.push({ ...mapPerson(face.person!), faces: [mapFacesWithoutPerson(face)] });
result.push({ ...mapPerson(face.person!), faces: [mapFacesWithoutPerson(face, edits, assetDimensions)] });
}
}
}
@@ -190,10 +202,14 @@ export function mapAsset(entity: MapAsset, options: AssetMapOptions = {}): Asset
duration: entity.duration ?? '0:00:00.00000',
livePhotoVideoId: entity.livePhotoVideoId,
hasMetadata: false,
width: entity.width,
height: entity.height,
};
return sanitizedAssetResponse as AssetResponseDto;
}
const assetDimensions = entity.exifInfo ? getDimensions(entity.exifInfo) : undefined;
return {
id: entity.id,
createdAt: entity.createdAt,
@@ -219,7 +235,7 @@ export function mapAsset(entity: MapAsset, options: AssetMapOptions = {}): Asset
exifInfo: entity.exifInfo ? mapExif(entity.exifInfo) : undefined,
livePhotoVideoId: entity.livePhotoVideoId,
tags: entity.tags?.map((tag) => mapTag(tag)),
people: peopleWithFaces(entity.faces),
people: peopleWithFaces(entity.faces, entity.edits, assetDimensions),
unassignedFaces: entity.faces?.filter((face) => !face.person).map((a) => mapFacesWithoutPerson(a)),
checksum: hexOrBufferToBase64(entity.checksum)!,
stack: withStack ? mapStack(entity) : undefined,
@@ -227,5 +243,7 @@ export function mapAsset(entity: MapAsset, options: AssetMapOptions = {}): Asset
hasMetadata: true,
duplicateId: entity.duplicateId,
resized: true,
width: entity.width,
height: entity.height,
};
}
+5
View File
@@ -228,6 +228,11 @@ export class AssetCopyDto {
favorite?: boolean;
}
export class AssetDownloadOriginalDto {
@ValidateBoolean({ optional: true, default: false })
edited?: boolean;
}
export const mapStats = (stats: AssetStats): AssetStatsResponseDto => {
return {
images: stats[AssetType.Image],
+125
View File
@@ -0,0 +1,125 @@
import { ApiExtraModels, ApiProperty, getSchemaPath } from '@nestjs/swagger';
import { ClassConstructor, plainToInstance, Transform, Type } from 'class-transformer';
import { ArrayMinSize, IsEnum, IsInt, Min, ValidateNested } from 'class-validator';
import { IsAxisAlignedRotation, IsUniqueEditActions, ValidateUUID } from 'src/validation';
export enum AssetEditAction {
Crop = 'crop',
Rotate = 'rotate',
Mirror = 'mirror',
}
export enum MirrorAxis {
Horizontal = 'horizontal',
Vertical = 'vertical',
}
export class CropParameters {
@IsInt()
@Min(0)
@ApiProperty({ description: 'Top-Left X coordinate of crop' })
x!: number;
@IsInt()
@Min(0)
@ApiProperty({ description: 'Top-Left Y coordinate of crop' })
y!: number;
@IsInt()
@Min(1)
@ApiProperty({ description: 'Width of the crop' })
width!: number;
@IsInt()
@Min(1)
@ApiProperty({ description: 'Height of the crop' })
height!: number;
}
export class RotateParameters {
@IsAxisAlignedRotation()
@ApiProperty({ description: 'Rotation angle in degrees' })
angle!: number;
}
export class MirrorParameters {
@IsEnum(MirrorAxis)
@ApiProperty({ enum: MirrorAxis, enumName: 'MirrorAxis', description: 'Axis to mirror along' })
axis!: MirrorAxis;
}
class AssetEditActionBase {
@IsEnum(AssetEditAction)
@ApiProperty({ enum: AssetEditAction, enumName: 'AssetEditAction' })
action!: AssetEditAction;
}
export class AssetEditActionCrop extends AssetEditActionBase {
@ValidateNested()
@Type(() => CropParameters)
@ApiProperty({ type: CropParameters })
parameters!: CropParameters;
}
export class AssetEditActionRotate extends AssetEditActionBase {
@ValidateNested()
@Type(() => RotateParameters)
@ApiProperty({ type: RotateParameters })
parameters!: RotateParameters;
}
export class AssetEditActionMirror extends AssetEditActionBase {
@ValidateNested()
@Type(() => MirrorParameters)
@ApiProperty({ type: MirrorParameters })
parameters!: MirrorParameters;
}
export type AssetEditActionItem =
| {
action: AssetEditAction.Crop;
parameters: CropParameters;
}
| {
action: AssetEditAction.Rotate;
parameters: RotateParameters;
}
| {
action: AssetEditAction.Mirror;
parameters: MirrorParameters;
};
export type AssetEditActionParameter = {
[AssetEditAction.Crop]: CropParameters;
[AssetEditAction.Rotate]: RotateParameters;
[AssetEditAction.Mirror]: MirrorParameters;
};
type AssetEditActions = AssetEditActionCrop | AssetEditActionRotate | AssetEditActionMirror;
const actionToClass: Record<AssetEditAction, ClassConstructor<AssetEditActions>> = {
[AssetEditAction.Crop]: AssetEditActionCrop,
[AssetEditAction.Rotate]: AssetEditActionRotate,
[AssetEditAction.Mirror]: AssetEditActionMirror,
} as const;
const getActionClass = (item: { action: AssetEditAction }): ClassConstructor<AssetEditActions> =>
actionToClass[item.action];
@ApiExtraModels(AssetEditActionRotate, AssetEditActionMirror, AssetEditActionCrop)
export class AssetEditActionListDto {
/** list of edits */
@ArrayMinSize(1)
@IsUniqueEditActions()
@ValidateNested({ each: true })
@Transform(({ value: edits }) =>
Array.isArray(edits) ? edits.map((item) => plainToInstance(getActionClass(item), item)) : edits,
)
@ApiProperty({ anyOf: Object.values(actionToClass).map((target) => ({ $ref: getSchemaPath(target) })) })
edits!: AssetEditActionItem[];
}
export class AssetEditsDto extends AssetEditActionListDto {
@ValidateUUID()
@ApiProperty()
assetId!: string;
}
+27 -16
View File
@@ -6,9 +6,12 @@ import { DateTime } from 'luxon';
import { AssetFace, Person } from 'src/database';
import { HistoryBuilder, Property } from 'src/decorators';
import { AuthDto } from 'src/dtos/auth.dto';
import { AssetEditActionItem } from 'src/dtos/editing.dto';
import { SourceType } from 'src/enum';
import { AssetFaceTable } from 'src/schema/tables/asset-face.table';
import { ImageDimensions } from 'src/types';
import { asDateString } from 'src/utils/date';
import { transformFaceBoundingBox } from 'src/utils/transform';
import {
IsDateStringFormat,
MaxDateString,
@@ -233,29 +236,37 @@ export function mapPerson(person: Person): PersonResponseDto {
};
}
export function mapFacesWithoutPerson(face: Selectable<AssetFaceTable>): AssetFaceWithoutPersonResponseDto {
export function mapFacesWithoutPerson(
face: Selectable<AssetFaceTable>,
edits?: AssetEditActionItem[],
assetDimensions?: ImageDimensions,
): AssetFaceWithoutPersonResponseDto {
return {
id: face.id,
imageHeight: face.imageHeight,
imageWidth: face.imageWidth,
boundingBoxX1: face.boundingBoxX1,
boundingBoxX2: face.boundingBoxX2,
boundingBoxY1: face.boundingBoxY1,
boundingBoxY2: face.boundingBoxY2,
...transformFaceBoundingBox(
{
boundingBoxX1: face.boundingBoxX1,
boundingBoxY1: face.boundingBoxY1,
boundingBoxX2: face.boundingBoxX2,
boundingBoxY2: face.boundingBoxY2,
imageWidth: face.imageWidth,
imageHeight: face.imageHeight,
},
edits ?? [],
assetDimensions ?? { width: face.imageWidth, height: face.imageHeight },
),
sourceType: face.sourceType,
};
}
export function mapFaces(face: AssetFace, auth: AuthDto): AssetFaceResponseDto {
export function mapFaces(
face: AssetFace,
auth: AuthDto,
edits?: AssetEditActionItem[],
assetDimensions?: ImageDimensions,
): AssetFaceResponseDto {
return {
id: face.id,
imageHeight: face.imageHeight,
imageWidth: face.imageWidth,
boundingBoxX1: face.boundingBoxX1,
boundingBoxX2: face.boundingBoxX2,
boundingBoxY1: face.boundingBoxY1,
boundingBoxY2: face.boundingBoxY2,
sourceType: face.sourceType,
...mapFacesWithoutPerson(face, edits, assetDimensions),
person: face.person?.ownerId === auth.user.id ? mapPerson(face.person) : null,
};
}
+3
View File
@@ -66,6 +66,9 @@ export class QueuesResponseLegacyDto implements Record<QueueName, QueueResponseL
@ApiProperty({ type: QueueResponseLegacyDto })
[QueueName.Workflow]!: QueueResponseLegacyDto;
@ApiProperty({ type: QueueResponseLegacyDto })
[QueueName.Editor]!: QueueResponseLegacyDto;
}
export const mapQueueLegacy = (response: QueueResponseDto): QueueResponseLegacyDto => {
+4
View File
@@ -117,6 +117,10 @@ export class SyncAssetV1 {
livePhotoVideoId!: string | null;
stackId!: string | null;
libraryId!: string | null;
@ApiProperty({ type: 'integer' })
width!: number | null;
@ApiProperty({ type: 'integer' })
height!: number | null;
}
@ExtraModel()
+6
View File
@@ -230,6 +230,12 @@ class SystemConfigJobDto implements Record<ConcurrentQueueName, JobSettingsDto>
@IsObject()
@Type(() => JobSettingsDto)
[QueueName.Workflow]!: JobSettingsDto;
@ApiProperty({ type: JobSettingsDto })
@ValidateNested()
@IsObject()
@Type(() => JobSettingsDto)
[QueueName.Editor]!: JobSettingsDto;
}
class SystemConfigLibraryScanDto {
+13
View File
@@ -45,6 +45,9 @@ export enum AssetFileType {
Preview = 'preview',
Thumbnail = 'thumbnail',
Sidecar = 'sidecar',
FullSizeEdited = 'fullsize_edited',
PreviewEdited = 'preview_edited',
ThumbnailEdited = 'thumbnail_edited',
}
export enum AlbumUserRole {
@@ -106,6 +109,11 @@ export enum Permission {
AssetUpload = 'asset.upload',
AssetReplace = 'asset.replace',
AssetCopy = 'asset.copy',
AssetDerive = 'asset.derive',
AssetEditGet = 'asset.edit.get',
AssetEditCreate = 'asset.edit.create',
AssetEditDelete = 'asset.edit.delete',
AlbumCreate = 'album.create',
AlbumRead = 'album.read',
@@ -358,6 +366,9 @@ export enum AssetPathType {
Original = 'original',
FullSize = 'fullsize',
Preview = 'preview',
EditedFullSize = 'edited_fullsize',
EditedPreview = 'edited_preview',
EditedThumbnail = 'edited_thumbnail',
Thumbnail = 'thumbnail',
EncodedVideo = 'encoded_video',
Sidecar = 'sidecar',
@@ -555,6 +566,7 @@ export enum QueueName {
BackupDatabase = 'backupDatabase',
Ocr = 'ocr',
Workflow = 'workflow',
Editor = 'editor',
}
export enum QueueJobStatus {
@@ -573,6 +585,7 @@ export enum JobName {
AssetDetectFaces = 'AssetDetectFaces',
AssetDetectDuplicatesQueueAll = 'AssetDetectDuplicatesQueueAll',
AssetDetectDuplicates = 'AssetDetectDuplicates',
AssetEditThumbnailGeneration = 'AssetEditThumbnailGeneration',
AssetEncodeVideoQueueAll = 'AssetEncodeVideoQueueAll',
AssetEncodeVideo = 'AssetEncodeVideo',
AssetEmptyTrash = 'AssetEmptyTrash',
@@ -0,0 +1,17 @@
-- NOTE: This file is auto generated by ./sql-generator
-- AssetEditRepository.replaceAll
begin
delete from "asset_edit"
where
"assetId" = $1
rollback
-- AssetEditRepository.getAll
select
"action",
"parameters"
from
"asset_edit"
where
"assetId" = $1
+35 -3
View File
@@ -105,7 +105,21 @@ select
where
"asset_file"."assetId" = "asset"."id"
) as agg
) as "files"
) as "files",
(
select
coalesce(json_agg(agg), '[]')
from
(
select
"asset_edit"."action",
"asset_edit"."parameters"
from
"asset_edit"
where
"asset_edit"."assetId" = "asset"."id"
) as agg
) as "edits"
from
"asset"
inner join "asset_job_status" on "asset_job_status"."assetId" = "asset"."id"
@@ -167,6 +181,20 @@ select
"asset_file"."assetId" = "asset"."id"
) as agg
) as "files",
(
select
coalesce(json_agg(agg), '[]')
from
(
select
"asset_edit"."action",
"asset_edit"."parameters"
from
"asset_edit"
where
"asset_edit"."assetId" = "asset"."id"
) as agg
) as "edits",
to_json("asset_exif") as "exifInfo"
from
"asset"
@@ -191,6 +219,8 @@ select
"asset"."originalPath",
"asset"."ownerId",
"asset"."type",
"asset"."width",
"asset"."height",
(
select
coalesce(json_agg(agg), '[]')
@@ -203,6 +233,7 @@ select
where
"asset_face"."assetId" = "asset"."id"
and "asset_face"."deletedAt" is null
and "asset_face"."isVisible" = $1
) as agg
) as "faces",
(
@@ -218,13 +249,13 @@ select
"asset_file"
where
"asset_file"."assetId" = "asset"."id"
and "asset_file"."type" = $1
and "asset_file"."type" = $2
) as agg
) as "files"
from
"asset"
where
"asset"."id" = $2
"asset"."id" = $3
-- AssetJobRepository.getLockedPropertiesForMetadataExtraction
select
@@ -402,6 +433,7 @@ select
where
"asset_face"."assetId" = "asset"."id"
and "asset_face"."deletedAt" is null
and "asset_face"."isVisible" is true
) as agg
) as "faces",
(
+4 -7
View File
@@ -182,6 +182,7 @@ select
where
"asset_face"."assetId" = "asset"."id"
and "asset_face"."deletedAt" is null
and "asset_face"."isVisible" is true
) as agg
) as "faces",
(
@@ -383,14 +384,10 @@ with
"asset_exif"."projectionType",
coalesce(
case
when asset_exif."exifImageHeight" = 0
or asset_exif."exifImageWidth" = 0 then 1
when "asset_exif"."orientation" in ('5', '6', '7', '8', '-90', '90') then round(
asset_exif."exifImageHeight"::numeric / asset_exif."exifImageWidth"::numeric,
3
)
when asset."height" = 0
or asset."width" = 0 then 1
else round(
asset_exif."exifImageWidth"::numeric / asset_exif."exifImageHeight"::numeric,
asset."width"::numeric / asset."height"::numeric,
3
)
end,
+10
View File
@@ -15,6 +15,7 @@ from
"asset_ocr"
where
"asset_ocr"."assetId" = $1
and "asset_ocr"."isVisible" = $2
-- OcrRepository.upsert
with
@@ -66,3 +67,12 @@ with
)
select
1 as "dummy"
-- OcrRepository.updateOcrVisibilities
begin
update "ocr_search"
set
"text" = $1
where
"assetId" = $2
commit
+7 -1
View File
@@ -35,6 +35,7 @@ from
where
"person"."ownerId" = $1
and "asset_face"."deletedAt" is null
and "asset_face"."isVisible" is true
and "person"."isHidden" = $2
group by
"person"."id"
@@ -63,6 +64,7 @@ from
left join "asset_face" on "asset_face"."personId" = "person"."id"
where
"asset_face"."deletedAt" is null
and "asset_face"."isVisible" is true
group by
"person"."id"
having
@@ -89,6 +91,7 @@ from
where
"asset_face"."assetId" = $1
and "asset_face"."deletedAt" is null
and "asset_face"."isVisible" = $2
order by
"asset_face"."boundingBoxX1" asc
@@ -229,6 +232,7 @@ from
and "asset"."deletedAt" is null
where
"asset_face"."deletedAt" is null
and "asset_face"."isVisible" is true
-- PersonRepository.getNumberOfPeople
select
@@ -250,6 +254,7 @@ where
where
"asset_face"."personId" = "person"."id"
and "asset_face"."deletedAt" is null
and "asset_face"."isVisible" = $2
and exists (
select
from
@@ -260,7 +265,7 @@ where
and "asset"."deletedAt" is null
)
)
and "person"."ownerId" = $2
and "person"."ownerId" = $3
-- PersonRepository.refreshFaces
with
@@ -321,6 +326,7 @@ from
where
"asset_face"."personId" = $1
and "asset_face"."deletedAt" is null
and "asset_face"."isVisible" is true
-- PersonRepository.getLatestFaceDate
select
+14 -1
View File
@@ -69,6 +69,8 @@ select
"asset"."livePhotoVideoId",
"asset"."stackId",
"asset"."libraryId",
"asset"."width",
"asset"."height",
"album_asset"."updateId"
from
"album_asset" as "album_asset"
@@ -99,6 +101,8 @@ select
"asset"."livePhotoVideoId",
"asset"."stackId",
"asset"."libraryId",
"asset"."width",
"asset"."height",
"asset"."updateId"
from
"asset" as "asset"
@@ -134,7 +138,9 @@ select
"asset"."duration",
"asset"."livePhotoVideoId",
"asset"."stackId",
"asset"."libraryId"
"asset"."libraryId",
"asset"."width",
"asset"."height"
from
"album_asset" as "album_asset"
inner join "asset" on "asset"."id" = "album_asset"."assetId"
@@ -448,6 +454,8 @@ select
"asset"."livePhotoVideoId",
"asset"."stackId",
"asset"."libraryId",
"asset"."width",
"asset"."height",
"asset"."updateId"
from
"asset" as "asset"
@@ -536,6 +544,7 @@ where
"asset_face"."updateId" < $1
and "asset_face"."updateId" > $2
and "asset"."ownerId" = $3
and "asset_face"."isVisible" = $4
order by
"asset_face"."updateId" asc
@@ -740,6 +749,8 @@ select
"asset"."livePhotoVideoId",
"asset"."stackId",
"asset"."libraryId",
"asset"."width",
"asset"."height",
"asset"."updateId"
from
"asset" as "asset"
@@ -789,6 +800,8 @@ select
"asset"."livePhotoVideoId",
"asset"."stackId",
"asset"."libraryId",
"asset"."width",
"asset"."height",
"asset"."updateId"
from
"asset" as "asset"
@@ -0,0 +1,41 @@
import { Injectable } from '@nestjs/common';
import { Kysely } from 'kysely';
import { InjectKysely } from 'nestjs-kysely';
import { DummyValue, GenerateSql } from 'src/decorators';
import { AssetEditActionItem } from 'src/dtos/editing.dto';
import { DB } from 'src/schema';
@Injectable()
export class AssetEditRepository {
constructor(@InjectKysely() private db: Kysely<DB>) {}
@GenerateSql({
params: [DummyValue.UUID],
})
async replaceAll(assetId: string, edits: AssetEditActionItem[]): Promise<AssetEditActionItem[]> {
return await this.db.transaction().execute(async (trx) => {
await trx.deleteFrom('asset_edit').where('assetId', '=', assetId).execute();
if (edits.length > 0) {
return trx
.insertInto('asset_edit')
.values(edits.map((edit) => ({ assetId, ...edit })))
.returning(['action', 'parameters'])
.execute() as Promise<AssetEditActionItem[]>;
}
return [];
});
}
@GenerateSql({
params: [DummyValue.UUID],
})
async getAll(assetId: string): Promise<AssetEditActionItem[]> {
return this.db
.selectFrom('asset_edit')
.select(['action', 'parameters'])
.where('assetId', '=', assetId)
.execute() as Promise<AssetEditActionItem[]>;
}
}
@@ -11,6 +11,7 @@ import {
asUuid,
toJson,
withDefaultVisibility,
withEdits,
withExif,
withExifInner,
withFaces,
@@ -72,6 +73,7 @@ export class AssetJobRepository {
.selectFrom('asset')
.select(['asset.id', 'asset.thumbhash'])
.select(withFiles)
.select(withEdits)
.where('asset.deletedAt', 'is', null)
.where('asset.visibility', '!=', AssetVisibility.Hidden)
.$if(!force, (qb) =>
@@ -113,6 +115,7 @@ export class AssetJobRepository {
'asset.type',
])
.select(withFiles)
.select(withEdits)
.$call(withExifInner)
.where('asset.id', '=', id)
.executeTakeFirst();
@@ -200,7 +203,7 @@ export class AssetJobRepository {
.selectFrom('asset')
.select(['asset.id', 'asset.visibility'])
.$call(withExifInner)
.select((eb) => withFaces(eb, true))
.select((eb) => withFaces(eb, true, true))
.select((eb) => withFiles(eb, AssetFileType.Preview))
.where('asset.id', '=', id)
.executeTakeFirst();
+11 -6
View File
@@ -20,6 +20,7 @@ import {
truncatedDate,
unnest,
withDefaultVisibility,
withEdits,
withExif,
withFaces,
withFacesAndPeople,
@@ -112,6 +113,7 @@ interface GetByIdsRelations {
smartSearch?: boolean;
stack?: { assets?: boolean };
tags?: boolean;
edits?: boolean;
}
const distinctLocked = <T extends LockableProperty[] | null>(eb: ExpressionBuilder<DB, 'asset_exif'>, columns: T) =>
@@ -472,7 +474,10 @@ export class AssetRepository {
}
@GenerateSql({ params: [DummyValue.UUID] })
getById(id: string, { exifInfo, faces, files, library, owner, smartSearch, stack, tags }: GetByIdsRelations = {}) {
getById(
id: string,
{ exifInfo, faces, files, library, owner, smartSearch, stack, tags, edits }: GetByIdsRelations = {},
) {
return this.db
.selectFrom('asset')
.selectAll('asset')
@@ -509,6 +514,7 @@ export class AssetRepository {
)
.$if(!!files, (qb) => qb.select(withFiles))
.$if(!!tags, (qb) => qb.select(withTags))
.$if(!!edits, (qb) => qb.select(withEdits))
.limit(1)
.executeTakeFirst();
}
@@ -536,10 +542,11 @@ export class AssetRepository {
.selectAll('asset')
.$call(withExif)
.$call((qb) => qb.select(withFacesAndPeople))
.$call((qb) => qb.select(withEdits))
.executeTakeFirst();
}
return this.getById(asset.id, { exifInfo: true, faces: { person: true } });
return this.getById(asset.id, { exifInfo: true, faces: { person: true }, edits: true });
}
async remove(asset: { id: string }): Promise<void> {
@@ -696,11 +703,9 @@ export class AssetRepository {
.coalesce(
eb
.case()
.when(sql`asset_exif."exifImageHeight" = 0 or asset_exif."exifImageWidth" = 0`)
.when(sql`asset."height" = 0 or asset."width" = 0`)
.then(eb.lit(1))
.when('asset_exif.orientation', 'in', sql<string>`('5', '6', '7', '8', '-90', '90')`)
.then(sql`round(asset_exif."exifImageHeight"::numeric / asset_exif."exifImageWidth"::numeric, 3)`)
.else(sql`round(asset_exif."exifImageWidth"::numeric / asset_exif."exifImageHeight"::numeric, 3)`)
.else(sql`round(asset."width"::numeric / asset."height"::numeric, 3)`)
.end(),
eb.lit(1),
)
+2
View File
@@ -4,6 +4,7 @@ import { AlbumUserRepository } from 'src/repositories/album-user.repository';
import { AlbumRepository } from 'src/repositories/album.repository';
import { ApiKeyRepository } from 'src/repositories/api-key.repository';
import { AppRepository } from 'src/repositories/app.repository';
import { AssetEditRepository } from 'src/repositories/asset-edit.repository';
import { AssetJobRepository } from 'src/repositories/asset-job.repository';
import { AssetRepository } from 'src/repositories/asset.repository';
import { AuditRepository } from 'src/repositories/audit.repository';
@@ -59,6 +60,7 @@ export const repositories = [
ApiKeyRepository,
AppRepository,
AssetRepository,
AssetEditRepository,
AssetJobRepository,
ConfigRepository,
CronRepository,
@@ -0,0 +1,667 @@
import sharp from 'sharp';
import { AssetFace } from 'src/database';
import { AssetEditAction, MirrorAxis } from 'src/dtos/editing.dto';
import { AssetOcrResponseDto } from 'src/dtos/ocr.dto';
import { SourceType } from 'src/enum';
import { LoggingRepository } from 'src/repositories/logging.repository';
import { BoundingBox } from 'src/repositories/machine-learning.repository';
import { MediaRepository } from 'src/repositories/media.repository';
import { checkFaceVisibility, checkOcrVisibility } from 'src/utils/editor';
import { automock } from 'test/utils';
const getPixelColor = async (buffer: Buffer, x: number, y: number) => {
const metadata = await sharp(buffer).metadata();
const width = metadata.width!;
const { data } = await sharp(buffer).raw().toBuffer({ resolveWithObject: true });
const idx = (y * width + x) * 4;
return {
r: data[idx],
g: data[idx + 1],
b: data[idx + 2],
};
};
const buildTestQuadImage = async () => {
// build a 4 quadrant image for testing mirroring
const base = sharp({
create: { width: 1000, height: 1000, channels: 3, background: { r: 0, g: 0, b: 0 } },
}).png();
const tl = await sharp({
create: { width: 500, height: 500, channels: 3, background: { r: 255, g: 0, b: 0 } },
})
.png()
.toBuffer();
const tr = await sharp({
create: { width: 500, height: 500, channels: 3, background: { r: 0, g: 255, b: 0 } },
})
.png()
.toBuffer();
const bl = await sharp({
create: { width: 500, height: 500, channels: 3, background: { r: 0, g: 0, b: 255 } },
})
.png()
.toBuffer();
const br = await sharp({
create: { width: 500, height: 500, channels: 3, background: { r: 255, g: 255, b: 0 } },
})
.png()
.toBuffer();
const image = base.composite([
{ input: tl, left: 0, top: 0 }, // top-left
{ input: tr, left: 500, top: 0 }, // top-right
{ input: bl, left: 0, top: 500 }, // bottom-left
{ input: br, left: 500, top: 500 }, // bottom-right
]);
return image.png().toBuffer();
};
describe(MediaRepository.name, () => {
let sut: MediaRepository;
beforeEach(() => {
// eslint-disable-next-line no-sparse-arrays
sut = new MediaRepository(automock(LoggingRepository, { args: [, { getEnv: () => ({}) }], strict: false }));
});
describe('applyEdits (single actions)', () => {
it('should apply crop edit correctly', async () => {
const result = await sut['applyEdits'](
sharp({
create: {
width: 1000,
height: 1000,
channels: 4,
background: { r: 255, g: 0, b: 0, alpha: 0.5 },
},
}).png(),
[
{
action: AssetEditAction.Crop,
parameters: {
x: 100,
y: 200,
width: 700,
height: 300,
},
},
],
);
const metadata = await result.toBuffer().then((buf) => sharp(buf).metadata());
expect(metadata.width).toBe(700);
expect(metadata.height).toBe(300);
});
it('should apply rotate edit correctly', async () => {
const result = await sut['applyEdits'](
sharp({
create: {
width: 500,
height: 1000,
channels: 4,
background: { r: 255, g: 0, b: 0, alpha: 0.5 },
},
}).png(),
[
{
action: AssetEditAction.Rotate,
parameters: {
angle: 90,
},
},
],
);
const metadata = await result.toBuffer().then((buf) => sharp(buf).metadata());
expect(metadata.width).toBe(1000);
expect(metadata.height).toBe(500);
});
it('should apply mirror edit correctly', async () => {
const resultHorizontal = await sut['applyEdits'](sharp(await buildTestQuadImage()), [
{
action: AssetEditAction.Mirror,
parameters: {
axis: MirrorAxis.Horizontal,
},
},
]);
const bufferHorizontal = await resultHorizontal.toBuffer();
const metadataHorizontal = await resultHorizontal.metadata();
expect(metadataHorizontal.width).toBe(1000);
expect(metadataHorizontal.height).toBe(1000);
expect(await getPixelColor(bufferHorizontal, 10, 10)).toEqual({ r: 0, g: 255, b: 0 });
expect(await getPixelColor(bufferHorizontal, 990, 10)).toEqual({ r: 255, g: 0, b: 0 });
expect(await getPixelColor(bufferHorizontal, 10, 990)).toEqual({ r: 255, g: 255, b: 0 });
expect(await getPixelColor(bufferHorizontal, 990, 990)).toEqual({ r: 0, g: 0, b: 255 });
const resultVertical = await sut['applyEdits'](sharp(await buildTestQuadImage()), [
{
action: AssetEditAction.Mirror,
parameters: {
axis: MirrorAxis.Vertical,
},
},
]);
const bufferVertical = await resultVertical.toBuffer();
const metadataVertical = await resultVertical.metadata();
expect(metadataVertical.width).toBe(1000);
expect(metadataVertical.height).toBe(1000);
// top-left should now be bottom-left (blue)
expect(await getPixelColor(bufferVertical, 10, 10)).toEqual({ r: 0, g: 0, b: 255 });
// top-right should now be bottom-right (yellow)
expect(await getPixelColor(bufferVertical, 990, 10)).toEqual({ r: 255, g: 255, b: 0 });
// bottom-left should now be top-left (red)
expect(await getPixelColor(bufferVertical, 10, 990)).toEqual({ r: 255, g: 0, b: 0 });
// bottom-right should now be top-right (blue)
expect(await getPixelColor(bufferVertical, 990, 990)).toEqual({ r: 0, g: 255, b: 0 });
});
});
describe('applyEdits (multiple sequential edits)', () => {
it('should apply horizontal mirror then vertical mirror (equivalent to 180° rotation)', async () => {
const imageBuffer = await buildTestQuadImage();
const result = await sut['applyEdits'](sharp(imageBuffer), [
{ action: AssetEditAction.Mirror, parameters: { axis: MirrorAxis.Horizontal } },
{ action: AssetEditAction.Mirror, parameters: { axis: MirrorAxis.Vertical } },
]);
const buffer = await result.png().toBuffer();
const metadata = await sharp(buffer).metadata();
expect(metadata.width).toBe(1000);
expect(metadata.height).toBe(1000);
expect(await getPixelColor(buffer, 10, 10)).toEqual({ r: 255, g: 255, b: 0 });
expect(await getPixelColor(buffer, 990, 10)).toEqual({ r: 0, g: 0, b: 255 });
expect(await getPixelColor(buffer, 10, 990)).toEqual({ r: 0, g: 255, b: 0 });
expect(await getPixelColor(buffer, 990, 990)).toEqual({ r: 255, g: 0, b: 0 });
});
it('should apply rotate 90° then horizontal mirror', async () => {
const imageBuffer = await buildTestQuadImage();
const result = await sut['applyEdits'](sharp(imageBuffer), [
{ action: AssetEditAction.Rotate, parameters: { angle: 90 } },
{ action: AssetEditAction.Mirror, parameters: { axis: MirrorAxis.Horizontal } },
]);
const buffer = await result.png().toBuffer();
const metadata = await sharp(buffer).metadata();
expect(metadata.width).toBe(1000);
expect(metadata.height).toBe(1000);
expect(await getPixelColor(buffer, 10, 10)).toEqual({ r: 255, g: 0, b: 0 });
expect(await getPixelColor(buffer, 990, 10)).toEqual({ r: 0, g: 0, b: 255 });
expect(await getPixelColor(buffer, 10, 990)).toEqual({ r: 0, g: 255, b: 0 });
expect(await getPixelColor(buffer, 990, 990)).toEqual({ r: 255, g: 255, b: 0 });
});
it('should apply 180° rotation', async () => {
const imageBuffer = await buildTestQuadImage();
const result = await sut['applyEdits'](sharp(imageBuffer), [
{ action: AssetEditAction.Rotate, parameters: { angle: 180 } },
]);
const buffer = await result.png().toBuffer();
const metadata = await sharp(buffer).metadata();
expect(metadata.width).toBe(1000);
expect(metadata.height).toBe(1000);
expect(await getPixelColor(buffer, 10, 10)).toEqual({ r: 255, g: 255, b: 0 });
expect(await getPixelColor(buffer, 990, 10)).toEqual({ r: 0, g: 0, b: 255 });
expect(await getPixelColor(buffer, 10, 990)).toEqual({ r: 0, g: 255, b: 0 });
expect(await getPixelColor(buffer, 990, 990)).toEqual({ r: 255, g: 0, b: 0 });
});
it('should apply 270° rotations', async () => {
const imageBuffer = await buildTestQuadImage();
const result = await sut['applyEdits'](sharp(imageBuffer), [
{ action: AssetEditAction.Rotate, parameters: { angle: 270 } },
]);
const buffer = await result.png().toBuffer();
const metadata = await sharp(buffer).metadata();
expect(metadata.width).toBe(1000);
expect(metadata.height).toBe(1000);
expect(await getPixelColor(buffer, 10, 10)).toEqual({ r: 0, g: 255, b: 0 });
expect(await getPixelColor(buffer, 990, 10)).toEqual({ r: 255, g: 255, b: 0 });
expect(await getPixelColor(buffer, 10, 990)).toEqual({ r: 255, g: 0, b: 0 });
expect(await getPixelColor(buffer, 990, 990)).toEqual({ r: 0, g: 0, b: 255 });
});
it('should apply crop then rotate 90°', async () => {
const imageBuffer = await buildTestQuadImage();
const result = await sut['applyEdits'](sharp(imageBuffer), [
{ action: AssetEditAction.Crop, parameters: { x: 0, y: 0, width: 1000, height: 500 } },
{ action: AssetEditAction.Rotate, parameters: { angle: 90 } },
]);
const buffer = await result.png().toBuffer();
const metadata = await sharp(buffer).metadata();
expect(metadata.width).toBe(500);
expect(metadata.height).toBe(1000);
expect(await getPixelColor(buffer, 10, 10)).toEqual({ r: 255, g: 0, b: 0 });
expect(await getPixelColor(buffer, 10, 990)).toEqual({ r: 0, g: 255, b: 0 });
});
it('should apply rotate 90° then crop', async () => {
const imageBuffer = await buildTestQuadImage();
const result = await sut['applyEdits'](sharp(imageBuffer), [
{ action: AssetEditAction.Crop, parameters: { x: 0, y: 0, width: 500, height: 1000 } },
{ action: AssetEditAction.Rotate, parameters: { angle: 90 } },
]);
const buffer = await result.png().toBuffer();
const metadata = await sharp(buffer).metadata();
expect(metadata.width).toBe(1000);
expect(metadata.height).toBe(500);
expect(await getPixelColor(buffer, 10, 10)).toEqual({ r: 0, g: 0, b: 255 });
expect(await getPixelColor(buffer, 990, 10)).toEqual({ r: 255, g: 0, b: 0 });
});
it('should apply vertical mirror then horizontal mirror then rotate 90°', async () => {
const imageBuffer = await buildTestQuadImage();
const result = await sut['applyEdits'](sharp(imageBuffer), [
{ action: AssetEditAction.Mirror, parameters: { axis: MirrorAxis.Vertical } },
{ action: AssetEditAction.Mirror, parameters: { axis: MirrorAxis.Horizontal } },
{ action: AssetEditAction.Rotate, parameters: { angle: 90 } },
]);
const buffer = await result.png().toBuffer();
const metadata = await sharp(buffer).metadata();
expect(metadata.width).toBe(1000);
expect(metadata.height).toBe(1000);
expect(await getPixelColor(buffer, 10, 10)).toEqual({ r: 0, g: 255, b: 0 });
expect(await getPixelColor(buffer, 990, 10)).toEqual({ r: 255, g: 255, b: 0 });
expect(await getPixelColor(buffer, 10, 990)).toEqual({ r: 255, g: 0, b: 0 });
expect(await getPixelColor(buffer, 990, 990)).toEqual({ r: 0, g: 0, b: 255 });
});
it('should apply crop to single quadrant then mirror', async () => {
const imageBuffer = await buildTestQuadImage();
const result = await sut['applyEdits'](sharp(imageBuffer), [
{ action: AssetEditAction.Crop, parameters: { x: 0, y: 0, width: 500, height: 500 } },
{ action: AssetEditAction.Mirror, parameters: { axis: MirrorAxis.Horizontal } },
]);
const buffer = await result.png().toBuffer();
const metadata = await sharp(buffer).metadata();
expect(metadata.width).toBe(500);
expect(metadata.height).toBe(500);
expect(await getPixelColor(buffer, 10, 10)).toEqual({ r: 255, g: 0, b: 0 });
expect(await getPixelColor(buffer, 490, 10)).toEqual({ r: 255, g: 0, b: 0 });
expect(await getPixelColor(buffer, 10, 490)).toEqual({ r: 255, g: 0, b: 0 });
expect(await getPixelColor(buffer, 490, 490)).toEqual({ r: 255, g: 0, b: 0 });
});
it('should apply all operations: crop, rotate, mirror', async () => {
const imageBuffer = await buildTestQuadImage();
const result = await sut['applyEdits'](sharp(imageBuffer), [
{ action: AssetEditAction.Crop, parameters: { x: 0, y: 0, width: 500, height: 1000 } },
{ action: AssetEditAction.Rotate, parameters: { angle: 90 } },
{ action: AssetEditAction.Mirror, parameters: { axis: MirrorAxis.Horizontal } },
]);
const buffer = await result.png().toBuffer();
const metadata = await sharp(buffer).metadata();
expect(metadata.width).toBe(1000);
expect(metadata.height).toBe(500);
expect(await getPixelColor(buffer, 10, 10)).toEqual({ r: 255, g: 0, b: 0 });
expect(await getPixelColor(buffer, 990, 10)).toEqual({ r: 0, g: 0, b: 255 });
});
});
describe('checkFaceVisibility', () => {
const baseFace: AssetFace = {
id: 'face-1',
assetId: 'asset-1',
personId: 'person-1',
boundingBoxX1: 100,
boundingBoxY1: 100,
boundingBoxX2: 200,
boundingBoxY2: 200,
imageWidth: 1000,
imageHeight: 800,
sourceType: SourceType.MachineLearning,
isVisible: true,
updatedAt: new Date(),
deletedAt: null,
updateId: '',
};
const assetDimensions = { width: 1000, height: 800 };
describe('with no crop edit', () => {
it('should return only currently invisible faces when no crop is provided', () => {
const visibleFace = { ...baseFace, id: 'face-visible', isVisible: true };
const invisibleFace = { ...baseFace, id: 'face-invisible', isVisible: false };
const faces = [visibleFace, invisibleFace];
const result = checkFaceVisibility(faces, assetDimensions);
expect(result.visible).toEqual([invisibleFace]);
expect(result.hidden).toEqual([]);
});
it('should return empty arrays when all faces are already visible and no crop is provided', () => {
const faces = [baseFace];
const result = checkFaceVisibility(faces, assetDimensions);
expect(result.visible).toEqual([]);
expect(result.hidden).toEqual([]);
});
it('should return all faces when all are invisible and no crop is provided', () => {
const face1 = { ...baseFace, id: 'face-1', isVisible: false };
const face2 = { ...baseFace, id: 'face-2', isVisible: false };
const faces = [face1, face2];
const result = checkFaceVisibility(faces, assetDimensions);
expect(result.visible).toEqual([face1, face2]);
expect(result.hidden).toEqual([]);
});
});
describe('with crop edit', () => {
it('should mark face as visible when fully inside crop area', () => {
const crop: BoundingBox = { x1: 0, y1: 0, x2: 500, y2: 400 };
const faces = [baseFace];
const result = checkFaceVisibility(faces, assetDimensions, crop);
expect(result.visible).toEqual(faces);
expect(result.hidden).toEqual([]);
});
it('should mark face as visible when more than 50% inside crop area', () => {
const crop: BoundingBox = { x1: 150, y1: 150, x2: 650, y2: 550 };
// Face at (100,100)-(200,200), crop starts at (150,150)
// Overlap: (150,150)-(200,200) = 50x50 = 2500
// Face area: 100x100 = 10000
// Overlap percentage: 25% - should be hidden
const faces = [baseFace];
const result = checkFaceVisibility(faces, assetDimensions, crop);
expect(result.visible).toEqual([]);
expect(result.hidden).toEqual(faces);
});
it('should mark face as hidden when less than 50% inside crop area', () => {
const crop: BoundingBox = { x1: 250, y1: 250, x2: 750, y2: 650 };
// Face completely outside crop area
const faces = [baseFace];
const result = checkFaceVisibility(faces, assetDimensions, crop);
expect(result.visible).toEqual([]);
expect(result.hidden).toEqual(faces);
});
it('should mark face as hidden when completely outside crop area', () => {
const crop: BoundingBox = { x1: 500, y1: 500, x2: 700, y2: 700 };
const faces = [baseFace];
const result = checkFaceVisibility(faces, assetDimensions, crop);
expect(result.visible).toEqual([]);
expect(result.hidden).toEqual(faces);
});
it('should handle multiple faces with mixed visibility', () => {
const crop: BoundingBox = { x1: 0, y1: 0, x2: 300, y2: 300 };
const faceInside: AssetFace = {
...baseFace,
id: 'face-inside',
boundingBoxX1: 50,
boundingBoxY1: 50,
boundingBoxX2: 150,
boundingBoxY2: 150,
};
const faceOutside: AssetFace = {
...baseFace,
id: 'face-outside',
boundingBoxX1: 400,
boundingBoxY1: 400,
boundingBoxX2: 500,
boundingBoxY2: 500,
};
const faces = [faceInside, faceOutside];
const result = checkFaceVisibility(faces, assetDimensions, crop);
expect(result.visible).toEqual([faceInside]);
expect(result.hidden).toEqual([faceOutside]);
});
it('should handle face at exactly 50% overlap threshold', () => {
// Face at (0,0)-(100,100), crop at (50,0)-(150,100)
// Overlap: (50,0)-(100,100) = 50x100 = 5000
// Face area: 100x100 = 10000
// Overlap percentage: 50% - exactly at threshold, should be visible
const faceAtEdge: AssetFace = {
...baseFace,
id: 'face-edge',
boundingBoxX1: 0,
boundingBoxY1: 0,
boundingBoxX2: 100,
boundingBoxY2: 100,
};
const crop: BoundingBox = { x1: 50, y1: 0, x2: 150, y2: 100 };
const faces = [faceAtEdge];
const result = checkFaceVisibility(faces, assetDimensions, crop);
expect(result.visible).toEqual([faceAtEdge]);
expect(result.hidden).toEqual([]);
});
});
describe('with scaled dimensions', () => {
it('should handle faces when asset dimensions differ from face image dimensions', () => {
// Face stored at 1000x800 resolution, but displaying at 500x400
const scaledDimensions = { width: 500, height: 400 };
const crop: BoundingBox = { x1: 0, y1: 0, x2: 250, y2: 200 };
// Face at (100,100)-(200,200) on 1000x800
// Scaled to 500x400: (50,50)-(100,100)
// Crop at (0,0)-(250,200) - face is fully inside
const faces = [baseFace];
const result = checkFaceVisibility(faces, scaledDimensions, crop);
expect(result.visible).toEqual(faces);
expect(result.hidden).toEqual([]);
});
});
});
describe('checkOcrVisibility', () => {
const baseOcr: AssetOcrResponseDto & { isVisible: boolean } = {
id: 'ocr-1',
assetId: 'asset-1',
x1: 0.1,
y1: 0.1,
x2: 0.2,
y2: 0.1,
x3: 0.2,
y3: 0.2,
x4: 0.1,
y4: 0.2,
boxScore: 0.9,
textScore: 0.85,
text: 'Test OCR',
isVisible: false,
};
const assetDimensions = { width: 1000, height: 800 };
describe('with no crop edit', () => {
it('should return only currently invisible OCR items when no crop is provided', () => {
const visibleOcr = { ...baseOcr, id: 'ocr-visible', isVisible: true };
const invisibleOcr = { ...baseOcr, id: 'ocr-invisible', isVisible: false };
const ocrs = [visibleOcr, invisibleOcr];
const result = checkOcrVisibility(ocrs, assetDimensions);
expect(result.visible).toEqual([invisibleOcr]);
expect(result.hidden).toEqual([]);
});
it('should return empty arrays when all OCR items are already visible and no crop is provided', () => {
const visibleOcr = { ...baseOcr, isVisible: true };
const ocrs = [visibleOcr];
const result = checkOcrVisibility(ocrs, assetDimensions);
expect(result.visible).toEqual([]);
expect(result.hidden).toEqual([]);
});
it('should return all OCR items when all are invisible and no crop is provided', () => {
const ocr1 = { ...baseOcr, id: 'ocr-1', isVisible: false };
const ocr2 = { ...baseOcr, id: 'ocr-2', isVisible: false };
const ocrs = [ocr1, ocr2];
const result = checkOcrVisibility(ocrs, assetDimensions);
expect(result.visible).toEqual([ocr1, ocr2]);
expect(result.hidden).toEqual([]);
});
});
describe('with crop edit', () => {
it('should mark OCR as visible when fully inside crop area', () => {
const crop: BoundingBox = { x1: 0, y1: 0, x2: 500, y2: 400 };
// OCR box: (0.1,0.1)-(0.2,0.2) on 1000x800 = (100,80)-(200,160)
// Crop: (0,0)-(500,400) - OCR fully inside
const ocrs = [baseOcr];
const result = checkOcrVisibility(ocrs, assetDimensions, crop);
expect(result.visible).toEqual(ocrs);
expect(result.hidden).toEqual([]);
});
it('should mark OCR as hidden when completely outside crop area', () => {
const crop: BoundingBox = { x1: 500, y1: 500, x2: 700, y2: 700 };
// OCR box: (100,80)-(200,160) - completely outside crop
const ocrs = [baseOcr];
const result = checkOcrVisibility(ocrs, assetDimensions, crop);
expect(result.visible).toEqual([]);
expect(result.hidden).toEqual(ocrs);
});
it('should mark OCR as hidden when less than 50% inside crop area', () => {
const crop: BoundingBox = { x1: 150, y1: 120, x2: 650, y2: 520 };
// OCR box: (100,80)-(200,160)
// Crop: (150,120)-(650,520)
// Overlap: (150,120)-(200,160) = 50x40 = 2000
// OCR area: 100x80 = 8000
// Overlap percentage: 25% - should be hidden
const ocrs = [baseOcr];
const result = checkOcrVisibility(ocrs, assetDimensions, crop);
expect(result.visible).toEqual([]);
expect(result.hidden).toEqual(ocrs);
});
it('should handle multiple OCR items with mixed visibility', () => {
const crop: BoundingBox = { x1: 0, y1: 0, x2: 300, y2: 300 };
const ocrInside = {
...baseOcr,
id: 'ocr-inside',
};
const ocrOutside = {
...baseOcr,
id: 'ocr-outside',
x1: 0.5,
y1: 0.5,
x2: 0.6,
y2: 0.5,
x3: 0.6,
y3: 0.6,
x4: 0.5,
y4: 0.6,
};
const ocrs = [ocrInside, ocrOutside];
const result = checkOcrVisibility(ocrs, assetDimensions, crop);
expect(result.visible).toEqual([ocrInside]);
expect(result.hidden).toEqual([ocrOutside]);
});
it('should handle OCR boxes with rotated/skewed polygons', () => {
// OCR with a rotated bounding box (not axis-aligned)
const rotatedOcr = {
...baseOcr,
id: 'ocr-rotated',
x1: 0.15,
y1: 0.1,
x2: 0.25,
y2: 0.15,
x3: 0.2,
y3: 0.25,
x4: 0.1,
y4: 0.2,
};
const crop: BoundingBox = { x1: 0, y1: 0, x2: 300, y2: 300 };
const ocrs = [rotatedOcr];
const result = checkOcrVisibility(ocrs, assetDimensions, crop);
expect(result.visible).toEqual([rotatedOcr]);
expect(result.hidden).toEqual([]);
});
});
describe('visibility is only affected by crop (not rotate or mirror)', () => {
it('should keep all OCR items visible when there is no crop regardless of other transforms', () => {
// Rotate and mirror edits don't affect visibility - only crop does
// The visibility functions only take an optional crop parameter
const ocrs = [baseOcr];
// Without any crop, all OCR items remain visible
const result = checkOcrVisibility(ocrs, assetDimensions);
expect(result.visible).toEqual(ocrs);
expect(result.hidden).toEqual([]);
});
it('should only consider crop for visibility calculation', () => {
// Even if the image will be rotated/mirrored, visibility is determined
// solely by whether the OCR box overlaps with the crop area
const crop: BoundingBox = { x1: 0, y1: 0, x2: 300, y2: 300 };
const ocrInsideCrop = {
...baseOcr,
id: 'ocr-inside',
// OCR at (0.1,0.1)-(0.2,0.2) = (100,80)-(200,160) on 1000x800, inside crop
};
const ocrOutsideCrop = {
...baseOcr,
id: 'ocr-outside',
x1: 0.5,
y1: 0.5,
x2: 0.6,
y2: 0.5,
x3: 0.6,
y3: 0.6,
x4: 0.5,
y4: 0.6,
// OCR at (500,400)-(600,480) on 1000x800, outside crop
};
const ocrs = [ocrInsideCrop, ocrOutsideCrop];
const result = checkOcrVisibility(ocrs, assetDimensions, crop);
// OCR inside crop area is visible, OCR outside is hidden
// This is true regardless of any subsequent rotate/mirror operations
expect(result.visible).toEqual([ocrInsideCrop]);
expect(result.hidden).toEqual([ocrOutsideCrop]);
});
});
});
});
+53 -18
View File
@@ -7,6 +7,7 @@ import { Writable } from 'node:stream';
import sharp from 'sharp';
import { ORIENTATION_TO_SHARP_ROTATION } from 'src/constants';
import { Exif } from 'src/database';
import { AssetEditActionItem } from 'src/dtos/editing.dto';
import { Colorspace, LogLevel, RawExtractedFormat } from 'src/enum';
import { LoggingRepository } from 'src/repositories/logging.repository';
import {
@@ -19,6 +20,7 @@ import {
VideoInfo,
} from 'src/types';
import { handlePromiseError } from 'src/utils/misc';
import { createAffineMatrix } from 'src/utils/transform';
const probe = (input: string, options: string[]): Promise<FfprobeData> =>
new Promise((resolve, reject) =>
@@ -138,21 +140,48 @@ export class MediaRepository {
}
}
decodeImage(input: string | Buffer, options: DecodeToBufferOptions) {
return this.getImageDecodingPipeline(input, options).raw().toBuffer({ resolveWithObject: true });
async decodeImage(input: string | Buffer, options: DecodeToBufferOptions) {
const pipeline = await this.getImageDecodingPipeline(input, options);
return pipeline.raw().toBuffer({ resolveWithObject: true });
}
private async applyEdits(pipeline: sharp.Sharp, edits: AssetEditActionItem[]): Promise<sharp.Sharp> {
const affineEditOperations = edits.filter((edit) => edit.action !== 'crop');
const matrix = createAffineMatrix(affineEditOperations);
const crop = edits.find((edit) => edit.action === 'crop');
const dimensions = await pipeline.metadata();
if (crop) {
pipeline = pipeline.extract({
left: crop ? Math.round(crop.parameters.x) : 0,
top: crop ? Math.round(crop.parameters.y) : 0,
width: crop ? Math.round(crop.parameters.width) : dimensions.width || 0,
height: crop ? Math.round(crop.parameters.height) : dimensions.height || 0,
});
}
const { a, b, c, d } = matrix;
pipeline = pipeline.affine([
[a, b],
[c, d],
]);
return pipeline;
}
async generateThumbnail(input: string | Buffer, options: GenerateThumbnailOptions, output: string): Promise<void> {
await this.getImageDecodingPipeline(input, options)
.toFormat(options.format, {
quality: options.quality,
// this is default in libvips (except the threshold is 90), but we need to set it manually in sharp
chromaSubsampling: options.quality >= 80 ? '4:4:4' : '4:2:0',
})
.toFile(output);
const pipeline = await this.getImageDecodingPipeline(input, options);
const decoded = pipeline.toFormat(options.format, {
quality: options.quality,
// this is default in libvips (except the threshold is 90), but we need to set it manually in sharp
chromaSubsampling: options.quality >= 80 ? '4:4:4' : '4:2:0',
});
await decoded.toFile(output);
}
private getImageDecodingPipeline(input: string | Buffer, options: DecodeToBufferOptions) {
private async getImageDecodingPipeline(input: string | Buffer, options: DecodeToBufferOptions) {
let pipeline = sharp(input, {
// some invalid images can still be processed by sharp, but we want to fail on them by default to avoid crashes
failOn: options.processInvalidImages ? 'none' : 'error',
@@ -175,8 +204,8 @@ export class MediaRepository {
}
}
if (options.crop) {
pipeline = pipeline.extract(options.crop);
if (options.edits && options.edits.length > 0) {
pipeline = await this.applyEdits(pipeline, options.edits);
}
if (options.size !== undefined) {
@@ -186,14 +215,20 @@ export class MediaRepository {
}
async generateThumbhash(input: string | Buffer, options: GenerateThumbhashOptions): Promise<Buffer> {
const [{ rgbaToThumbHash }, { data, info }] = await Promise.all([
const [{ rgbaToThumbHash }, decodingPipeline] = await Promise.all([
import('thumbhash'),
sharp(input, options)
.resize(100, 100, { fit: 'inside', withoutEnlargement: true })
.raw()
.ensureAlpha()
.toBuffer({ resolveWithObject: true }),
this.getImageDecodingPipeline(input, {
colorspace: options.colorspace,
processInvalidImages: options.processInvalidImages,
raw: options.raw,
edits: options.edits,
}),
]);
const pipeline = decodingPipeline.resize(100, 100, { fit: 'inside', withoutEnlargement: true }).raw().ensureAlpha();
const { data, info } = await pipeline.toBuffer({ resolveWithObject: true });
return Buffer.from(rgbaToThumbHash(info.width, info.height, data));
}
+46 -2
View File
@@ -2,6 +2,7 @@ import { Injectable } from '@nestjs/common';
import { Insertable, Kysely, sql } from 'kysely';
import { InjectKysely } from 'nestjs-kysely';
import { DummyValue, GenerateSql } from 'src/decorators';
import { AssetOcrResponseDto } from 'src/dtos/ocr.dto';
import { DB } from 'src/schema';
import { AssetOcrTable } from 'src/schema/tables/asset-ocr.table';
@@ -15,8 +16,15 @@ export class OcrRepository {
}
@GenerateSql({ params: [DummyValue.UUID] })
getByAssetId(id: string) {
return this.db.selectFrom('asset_ocr').selectAll('asset_ocr').where('asset_ocr.assetId', '=', id).execute();
getByAssetId(id: string, options?: { isVisible?: boolean }) {
const isVisible = options === undefined ? true : options.isVisible;
return this.db
.selectFrom('asset_ocr')
.selectAll('asset_ocr')
.where('asset_ocr.assetId', '=', id)
.$if(isVisible !== undefined, (qb) => qb.where('asset_ocr.isVisible', '=', isVisible!))
.execute();
}
deleteAll() {
@@ -65,4 +73,40 @@ export class OcrRepository {
return query.selectNoFrom(sql`1`.as('dummy')).execute();
}
@GenerateSql({ params: [DummyValue.UUID, [], []] })
async updateOcrVisibilities(
assetId: string,
visible: AssetOcrResponseDto[],
hidden: AssetOcrResponseDto[],
): Promise<void> {
await this.db.transaction().execute(async (trx) => {
if (visible.length > 0) {
await trx
.updateTable('asset_ocr')
.set({ isVisible: true })
.where(
'asset_ocr.id',
'in',
visible.map((i) => i.id),
)
.execute();
}
if (hidden.length > 0) {
await trx
.updateTable('asset_ocr')
.set({ isVisible: false })
.where(
'asset_ocr.id',
'in',
hidden.map((i) => i.id),
)
.execute();
}
const searchText = visible.map((item) => item.text.trim()).join(' ');
await trx.updateTable('ocr_search').set({ text: searchText }).where('assetId', '=', assetId).execute();
});
}
}
+44 -1
View File
@@ -2,6 +2,7 @@ import { Injectable } from '@nestjs/common';
import { ExpressionBuilder, Insertable, Kysely, NotNull, Selectable, sql, Updateable } from 'kysely';
import { jsonObjectFrom } from 'kysely/helpers/postgres';
import { InjectKysely } from 'nestjs-kysely';
import { AssetFace } from 'src/database';
import { Chunked, ChunkedArray, DummyValue, GenerateSql } from 'src/decorators';
import { AssetFileType, AssetVisibility, SourceType } from 'src/enum';
import { DB } from 'src/schema';
@@ -121,6 +122,7 @@ export class PersonRepository {
.$if(!!options.sourceType, (qb) => qb.where('asset_face.sourceType', '=', options.sourceType!))
.$if(!!options.assetId, (qb) => qb.where('asset_face.assetId', '=', options.assetId!))
.where('asset_face.deletedAt', 'is', null)
.where('asset_face.isVisible', 'is', true)
.stream();
}
@@ -160,6 +162,7 @@ export class PersonRepository {
)
.where('person.ownerId', '=', userId)
.where('asset_face.deletedAt', 'is', null)
.where('asset_face.isVisible', 'is', true)
.orderBy('person.isHidden', 'asc')
.orderBy('person.isFavorite', 'desc')
.having((eb) =>
@@ -208,19 +211,23 @@ export class PersonRepository {
.selectAll('person')
.leftJoin('asset_face', 'asset_face.personId', 'person.id')
.where('asset_face.deletedAt', 'is', null)
.where('asset_face.isVisible', 'is', true)
.having((eb) => eb.fn.count('asset_face.assetId'), '=', 0)
.groupBy('person.id')
.execute();
}
@GenerateSql({ params: [DummyValue.UUID] })
getFaces(assetId: string) {
getFaces(assetId: string, options?: { isVisible?: boolean }) {
const isVisible = options === undefined ? true : options.isVisible;
return this.db
.selectFrom('asset_face')
.selectAll('asset_face')
.select(withPerson)
.where('asset_face.assetId', '=', assetId)
.where('asset_face.deletedAt', 'is', null)
.$if(isVisible !== undefined, (qb) => qb.where('asset_face.isVisible', '=', isVisible!))
.orderBy('asset_face.boundingBoxX1', 'asc')
.execute();
}
@@ -350,6 +357,7 @@ export class PersonRepository {
)
.select((eb) => eb.fn.count(eb.fn('distinct', ['asset.id'])).as('count'))
.where('asset_face.deletedAt', 'is', null)
.where('asset_face.isVisible', 'is', true)
.executeTakeFirst();
return {
@@ -368,6 +376,7 @@ export class PersonRepository {
.selectFrom('asset_face')
.whereRef('asset_face.personId', '=', 'person.id')
.where('asset_face.deletedAt', 'is', null)
.where('asset_face.isVisible', '=', true)
.where((eb) =>
eb.exists((eb) =>
eb
@@ -495,6 +504,7 @@ export class PersonRepository {
.selectAll('asset_face')
.where('asset_face.personId', '=', personId)
.where('asset_face.deletedAt', 'is', null)
.where('asset_face.isVisible', 'is', true)
.executeTakeFirst();
}
@@ -539,4 +549,37 @@ export class PersonRepository {
}
return this.db.selectFrom('person').select(['id', 'thumbnailPath']).where('id', 'in', ids).execute();
}
@GenerateSql({ params: [[], []] })
async updateVisibility(visible: AssetFace[], hidden: AssetFace[]): Promise<void> {
if (visible.length === 0 && hidden.length === 0) {
return;
}
await this.db.transaction().execute(async (trx) => {
if (visible.length > 0) {
await trx
.updateTable('asset_face')
.set({ isVisible: true })
.where(
'asset_face.id',
'in',
visible.map(({ id }) => id),
)
.execute();
}
if (hidden.length > 0) {
await trx
.updateTable('asset_face')
.set({ isVisible: false })
.where(
'asset_face.id',
'in',
hidden.map(({ id }) => id),
)
.execute();
}
});
}
}
@@ -483,6 +483,7 @@ class AssetFaceSync extends BaseSync {
])
.leftJoin('asset', 'asset.id', 'asset_face.assetId')
.where('asset.ownerId', '=', options.userId)
.where('asset_face.isVisible', '=', true)
.stream();
}
}
@@ -37,6 +37,7 @@ export interface ClientEventMap {
AssetUploadReadyV1: [{ asset: SyncAssetV1; exif: SyncAssetExifV1 }];
AppRestartV1: [AppRestartEvent];
AssetEditReadyV1: [{ assetId: string }];
}
export type AuthFn = (client: Socket) => Promise<AuthDto>;
+3
View File
@@ -28,6 +28,7 @@ import { AlbumUserTable } from 'src/schema/tables/album-user.table';
import { AlbumTable } from 'src/schema/tables/album.table';
import { ApiKeyTable } from 'src/schema/tables/api-key.table';
import { AssetAuditTable } from 'src/schema/tables/asset-audit.table';
import { AssetEditTable } from 'src/schema/tables/asset-edit.table';
import { AssetExifTable } from 'src/schema/tables/asset-exif.table';
import { AssetFaceAuditTable } from 'src/schema/tables/asset-face-audit.table';
import { AssetFaceTable } from 'src/schema/tables/asset-face.table';
@@ -86,6 +87,7 @@ export class ImmichDatabase {
AlbumTable,
ApiKeyTable,
AssetAuditTable,
AssetEditTable,
AssetFaceTable,
AssetFaceAuditTable,
AssetMetadataTable,
@@ -179,6 +181,7 @@ export interface DB {
asset: AssetTable;
asset_audit: AssetAuditTable;
asset_edit: AssetEditTable;
asset_exif: AssetExifTable;
asset_face: AssetFaceTable;
asset_face_audit: AssetFaceAuditTable;
@@ -0,0 +1,28 @@
import { Kysely, sql } from 'kysely';
export async function up(db: Kysely<any>): Promise<void> {
await sql`ALTER TABLE "asset" ADD COLUMN "width" integer;`.execute(db);
await sql`ALTER TABLE "asset" ADD COLUMN "height" integer;`.execute(db);
// Populate width and height from exif data with orientation-aware swapping
await sql`
UPDATE "asset"
SET
"width" = CASE
WHEN "asset_exif"."orientation" IN ('5', '6', '7', '8', '-90', '90') THEN "asset_exif"."exifImageHeight"
ELSE "asset_exif"."exifImageWidth"
END,
"height" = CASE
WHEN "asset_exif"."orientation" IN ('5', '6', '7', '8', '-90', '90') THEN "asset_exif"."exifImageWidth"
ELSE "asset_exif"."exifImageHeight"
END
FROM "asset_exif"
WHERE "asset"."id" = "asset_exif"."assetId"
AND ("asset_exif"."exifImageWidth" IS NOT NULL OR "asset_exif"."exifImageHeight" IS NOT NULL)
`.execute(db);
}
export async function down(db: Kysely<any>): Promise<void> {
await sql`ALTER TABLE "asset" DROP COLUMN "width";`.execute(db);
await sql`ALTER TABLE "asset" DROP COLUMN "height";`.execute(db);
}
@@ -0,0 +1,22 @@
import { Kysely, sql } from 'kysely';
export async function up(db: Kysely<any>): Promise<void> {
await sql`
CREATE TABLE "asset_edit" (
"id" uuid NOT NULL DEFAULT uuid_generate_v4(),
"assetId" uuid NOT NULL,
"action" varchar NOT NULL,
"parameters" jsonb NOT NULL
);
`.execute(db);
await sql`ALTER TABLE "asset_edit" ADD CONSTRAINT "asset_edit_pkey" PRIMARY KEY ("id");`.execute(db);
await sql`ALTER TABLE "asset_edit" ADD CONSTRAINT "asset_edit_assetId_fkey" FOREIGN KEY ("assetId") REFERENCES "asset" ("id") ON UPDATE CASCADE ON DELETE CASCADE;`.execute(
db,
);
await sql`CREATE INDEX "asset_edit_assetId_idx" ON "asset_edit" ("assetId")`.execute(db);
}
export async function down(db: Kysely<any>): Promise<void> {
await sql`DROP TABLE IF EXISTS "asset_edit";`.execute(db);
}
@@ -0,0 +1,11 @@
import { Kysely, sql } from 'kysely';
export async function up(db: Kysely<any>): Promise<void> {
await sql`ALTER TABLE "asset_ocr" ADD COLUMN "isVisible" boolean NOT NULL DEFAULT TRUE`.execute(db);
await sql`ALTER TABLE "asset_face" ADD COLUMN "isVisible" boolean NOT NULL DEFAULT TRUE`.execute(db);
}
export async function down(db: Kysely<any>): Promise<void> {
await sql`ALTER TABLE "asset_ocr" DROP COLUMN "isVisible";`.execute(db);
await sql`ALTER TABLE "asset_face" DROP COLUMN "isVisible";`.execute(db);
}
@@ -0,0 +1,17 @@
import { AssetEditAction, AssetEditActionParameter } from 'src/dtos/editing.dto';
import { AssetTable } from 'src/schema/tables/asset.table';
import { Column, ForeignKeyColumn, Generated, PrimaryGeneratedColumn } from 'src/sql-tools';
export class AssetEditTable<T extends AssetEditAction = AssetEditAction> {
@PrimaryGeneratedColumn()
id!: Generated<string>;
@ForeignKeyColumn(() => AssetTable, { onDelete: 'CASCADE', onUpdate: 'CASCADE', nullable: false })
assetId!: string;
@Column()
action!: T;
@Column({ type: 'jsonb' })
parameters!: AssetEditActionParameter[T];
}
@@ -78,4 +78,7 @@ export class AssetFaceTable {
@UpdateIdColumn()
updateId!: Generated<string>;
@Column({ type: 'boolean', default: true })
isVisible!: Generated<boolean>;
}
@@ -42,4 +42,7 @@ export class AssetOcrTable {
@Column({ type: 'text' })
text!: string;
@Column({ type: 'boolean', default: true })
isVisible!: Generated<boolean>;
}
+6
View File
@@ -137,4 +137,10 @@ export class AssetTable {
@Column({ enum: asset_visibility_enum, default: AssetVisibility.Timeline })
visibility!: Generated<AssetVisibility>;
@Column({ type: 'integer', nullable: true })
width!: number | null;
@Column({ type: 'integer', nullable: true })
height!: number | null;
}
+104 -4
View File
@@ -489,7 +489,7 @@ describe(AssetMediaService.name, () => {
describe('downloadOriginal', () => {
it('should require the asset.download permission', async () => {
await expect(sut.downloadOriginal(authStub.admin, 'asset-1')).rejects.toBeInstanceOf(BadRequestException);
await expect(sut.downloadOriginal(authStub.admin, 'asset-1', {})).rejects.toBeInstanceOf(BadRequestException);
expect(mocks.access.asset.checkOwnerAccess).toHaveBeenCalledWith(
authStub.admin.user.id,
@@ -503,16 +503,16 @@ describe(AssetMediaService.name, () => {
it('should throw an error if the asset is not found', async () => {
mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1']));
await expect(sut.downloadOriginal(authStub.admin, 'asset-1')).rejects.toBeInstanceOf(NotFoundException);
await expect(sut.downloadOriginal(authStub.admin, 'asset-1', {})).rejects.toBeInstanceOf(NotFoundException);
expect(mocks.asset.getById).toHaveBeenCalledWith('asset-1', { files: true });
expect(mocks.asset.getById).toHaveBeenCalledWith('asset-1', { files: true, edits: true });
});
it('should download a file', async () => {
mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1']));
mocks.asset.getById.mockResolvedValue(assetStub.image);
await expect(sut.downloadOriginal(authStub.admin, 'asset-1')).resolves.toEqual(
await expect(sut.downloadOriginal(authStub.admin, 'asset-1', {})).resolves.toEqual(
new ImmichFileResponse({
path: '/original/path.jpg',
fileName: 'asset-id.jpg',
@@ -521,6 +521,104 @@ describe(AssetMediaService.name, () => {
}),
);
});
it('should download edited file by default when edits exist', async () => {
const editedAsset = {
...assetStub.withCropEdit,
files: [
...assetStub.withCropEdit.files,
{
id: 'edited-file',
type: AssetFileType.FullSizeEdited,
path: '/uploads/user-id/fullsize/edited.jpg',
} as AssetFile,
],
};
mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1']));
mocks.asset.getById.mockResolvedValue(editedAsset);
await expect(sut.downloadOriginal(authStub.admin, 'asset-1', { edited: true })).resolves.toEqual(
new ImmichFileResponse({
path: '/uploads/user-id/fullsize/edited.jpg',
fileName: 'asset-id.jpg',
contentType: 'image/jpeg',
cacheControl: CacheControl.PrivateWithCache,
}),
);
});
it('should download edited file when edited=true', async () => {
const editedAsset = {
...assetStub.withCropEdit,
files: [
...assetStub.withCropEdit.files,
{
id: 'edited-file',
type: AssetFileType.FullSizeEdited,
path: '/uploads/user-id/fullsize/edited.jpg',
} as AssetFile,
],
};
mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1']));
mocks.asset.getById.mockResolvedValue(editedAsset);
await expect(sut.downloadOriginal(authStub.admin, 'asset-1', { edited: true })).resolves.toEqual(
new ImmichFileResponse({
path: '/uploads/user-id/fullsize/edited.jpg',
fileName: 'asset-id.jpg',
contentType: 'image/jpeg',
cacheControl: CacheControl.PrivateWithCache,
}),
);
});
it('should download original file when edited=false', async () => {
const editedAsset = {
...assetStub.withCropEdit,
files: [
...assetStub.withCropEdit.files,
{
id: 'edited-file',
type: AssetFileType.FullSizeEdited,
path: '/uploads/user-id/fullsize/edited.jpg',
} as AssetFile,
],
};
mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1']));
mocks.asset.getById.mockResolvedValue(editedAsset);
await expect(sut.downloadOriginal(authStub.admin, 'asset-1', { edited: false })).resolves.toEqual(
new ImmichFileResponse({
path: '/original/path.jpg',
fileName: 'asset-id.jpg',
contentType: 'image/jpeg',
cacheControl: CacheControl.PrivateWithCache,
}),
);
});
it('should download original file when no edits exist', async () => {
mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1']));
mocks.asset.getById.mockResolvedValue(assetStub.image);
await expect(sut.downloadOriginal(authStub.admin, 'asset-1', { edited: true })).resolves.toEqual(
new ImmichFileResponse({
path: '/original/path.jpg',
fileName: 'asset-id.jpg',
contentType: 'image/jpeg',
cacheControl: CacheControl.PrivateWithCache,
}),
);
});
it('should throw a not found when edits exist but no edited file available', async () => {
mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1']));
mocks.asset.getById.mockResolvedValue(assetStub.withCropEdit);
await expect(sut.downloadOriginal(authStub.admin, 'asset-1', { edited: true })).rejects.toBeInstanceOf(
NotFoundException,
);
});
});
describe('viewThumbnail', () => {
@@ -620,6 +718,8 @@ describe(AssetMediaService.name, () => {
}),
);
});
// TODO: Edited asset tests
});
describe('playbackVideo', () => {
+28 -4
View File
@@ -20,6 +20,7 @@ import {
CheckExistingAssetsDto,
UploadFieldName,
} from 'src/dtos/asset-media.dto';
import { AssetDownloadOriginalDto } from 'src/dtos/asset.dto';
import { AuthDto } from 'src/dtos/auth.dto';
import {
AssetFileType,
@@ -193,11 +194,26 @@ export class AssetMediaService extends BaseService {
}
}
async downloadOriginal(auth: AuthDto, id: string): Promise<ImmichFileResponse> {
async downloadOriginal(auth: AuthDto, id: string, dto: AssetDownloadOriginalDto): Promise<ImmichFileResponse> {
await this.requireAccess({ auth, permission: Permission.AssetDownload, ids: [id] });
const asset = await this.findOrFail(id);
if (asset.edits!.length > 0 && (dto.edited ?? false)) {
const { editedFullsizeFile } = getAssetFiles(asset.files ?? []);
if (!editedFullsizeFile) {
throw new NotFoundException('Edited asset media not found');
}
return new ImmichFileResponse({
path: editedFullsizeFile.path,
fileName: getFileNameWithoutExtension(asset.originalFileName) + getFilenameExtension(editedFullsizeFile.path),
contentType: mimeTypes.lookup(editedFullsizeFile.path),
cacheControl: CacheControl.PrivateWithCache,
});
}
return new ImmichFileResponse({
path: asset.originalPath,
fileName: asset.originalFileName,
@@ -216,12 +232,20 @@ export class AssetMediaService extends BaseService {
const asset = await this.findOrFail(id);
const size = dto.size ?? AssetMediaSize.THUMBNAIL;
const { thumbnailFile, previewFile, fullsizeFile } = getAssetFiles(asset.files ?? []);
const files = getAssetFiles(asset.files ?? []);
const requestingEdited = (dto.edited ?? false) && asset.edits!.length > 0;
const { fullsizeFile, previewFile, thumbnailFile } = {
fullsizeFile: requestingEdited ? files.editedFullsizeFile : files.fullsizeFile,
previewFile: requestingEdited ? files.editedPreviewFile : files.previewFile,
thumbnailFile: requestingEdited ? files.editedThumbnailFile : files.thumbnailFile,
};
let filepath = previewFile?.path;
if (size === AssetMediaSize.THUMBNAIL && thumbnailFile) {
filepath = thumbnailFile.path;
} else if (size === AssetMediaSize.FULLSIZE) {
if (mimeTypes.isWebSupportedImage(asset.originalPath)) {
if (mimeTypes.isWebSupportedImage(asset.originalPath) && !dto.edited) {
// use original file for web supported images
return { targetSize: 'original' };
}
@@ -465,7 +489,7 @@ export class AssetMediaService extends BaseService {
}
private async findOrFail(id: string) {
const asset = await this.assetRepository.getById(id, { files: true });
const asset = await this.assetRepository.getById(id, { files: true, edits: true });
if (!asset) {
throw new NotFoundException('Asset not found');
}
+2 -1
View File
@@ -704,6 +704,7 @@ describe(AssetService.name, () => {
mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1']));
mocks.ocr.getByAssetId.mockResolvedValue([ocr1, ocr2]);
mocks.asset.getById.mockResolvedValue(assetStub.image);
await expect(sut.getOcr(authStub.admin, 'asset-1')).resolves.toEqual([ocr1, ocr2]);
@@ -718,7 +719,7 @@ describe(AssetService.name, () => {
it('should return empty array when no OCR data exists', async () => {
mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1']));
mocks.ocr.getByAssetId.mockResolvedValue([]);
mocks.asset.getById.mockResolvedValue(assetStub.image);
await expect(sut.getOcr(authStub.admin, 'asset-1')).resolves.toEqual([]);
expect(mocks.ocr.getByAssetId).toHaveBeenCalledWith('asset-1');
+117 -6
View File
@@ -21,13 +21,32 @@ import {
mapStats,
} from 'src/dtos/asset.dto';
import { AuthDto } from 'src/dtos/auth.dto';
import { AssetEditAction, AssetEditActionListDto, AssetEditsDto } from 'src/dtos/editing.dto';
import { AssetOcrResponseDto } from 'src/dtos/ocr.dto';
import { AssetFileType, AssetStatus, AssetVisibility, JobName, JobStatus, Permission, QueueName } from 'src/enum';
import {
AssetFileType,
AssetStatus,
AssetType,
AssetVisibility,
JobName,
JobStatus,
Permission,
QueueName,
} from 'src/enum';
import { BaseService } from 'src/services/base.service';
import { JobItem, JobOf } from 'src/types';
import { requireElevatedPermission } from 'src/utils/access';
import { getAssetFiles, getMyPartnerIds, onAfterUnlink, onBeforeLink, onBeforeUnlink } from 'src/utils/asset.util';
import {
getAssetFiles,
getDimensions,
getMyPartnerIds,
isPanorama,
onAfterUnlink,
onBeforeLink,
onBeforeUnlink,
} from 'src/utils/asset.util';
import { updateLockedColumns } from 'src/utils/database';
import { transformOcrBoundingBox } from 'src/utils/transform';
@Injectable()
export class AssetService extends BaseService {
@@ -62,6 +81,7 @@ export class AssetService extends BaseService {
owner: true,
faces: { person: true },
stack: { assets: true },
edits: true,
tags: true,
});
@@ -339,11 +359,19 @@ export class AssetService extends BaseService {
}
}
const { fullsizeFile, previewFile, thumbnailFile, sidecarFile } = getAssetFiles(asset.files ?? []);
const files = [thumbnailFile?.path, previewFile?.path, fullsizeFile?.path, asset.encodedVideoPath];
const assetFiles = getAssetFiles(asset.files ?? []);
const files = [
assetFiles.thumbnailFile?.path,
assetFiles.previewFile?.path,
assetFiles.fullsizeFile?.path,
assetFiles.editedFullsizeFile?.path,
assetFiles.editedPreviewFile?.path,
assetFiles.editedThumbnailFile?.path,
asset.encodedVideoPath,
];
if (deleteOnDisk && !asset.isOffline) {
files.push(sidecarFile?.path, asset.originalPath);
files.push(assetFiles.sidecarFile?.path, asset.originalPath);
}
await this.jobRepository.queue({ name: JobName.FileDelete, data: { files: files.filter(Boolean) } });
@@ -372,7 +400,16 @@ export class AssetService extends BaseService {
async getOcr(auth: AuthDto, id: string): Promise<AssetOcrResponseDto[]> {
await this.requireAccess({ auth, permission: Permission.AssetRead, ids: [id] });
return this.ocrRepository.getByAssetId(id);
const ocr = await this.ocrRepository.getByAssetId(id);
const asset = await this.assetRepository.getById(id, { exifInfo: true, edits: true });
if (!asset || !asset.exifInfo || !asset.edits) {
throw new BadRequestException('Asset not found');
}
const dimensions = getDimensions(asset.exifInfo);
return ocr.map((item) => transformOcrBoundingBox(item, asset.edits!, dimensions));
}
async upsertBulkMetadata(auth: AuthDto, dto: AssetMetadataBulkUpsertDto): Promise<AssetMetadataBulkResponseDto[]> {
@@ -478,4 +515,78 @@ export class AssetService extends BaseService {
await this.jobRepository.queue({ name: JobName.SidecarWrite, data: { id } });
}
}
async getAssetEdits(auth: AuthDto, id: string): Promise<AssetEditsDto> {
await this.requireAccess({ auth, permission: Permission.AssetRead, ids: [id] });
const edits = await this.assetEditRepository.getAll(id);
return {
assetId: id,
edits,
};
}
async editAsset(auth: AuthDto, id: string, dto: AssetEditActionListDto): Promise<AssetEditsDto> {
await this.requireAccess({ auth, permission: Permission.AssetEditCreate, ids: [id] });
const asset = await this.assetRepository.getById(id, { exifInfo: true });
if (!asset) {
throw new BadRequestException('Asset not found');
}
if (asset.type !== AssetType.Image) {
throw new BadRequestException('Only images can be edited');
}
if (asset.livePhotoVideoId) {
throw new BadRequestException('Editing live photos is not supported');
}
if (isPanorama(asset)) {
throw new BadRequestException('Editing panorama images is not supported');
}
if (asset.originalPath?.toLowerCase().endsWith('.gif')) {
throw new BadRequestException('Editing GIF images is not supported');
}
if (asset.originalPath?.toLowerCase().endsWith('.svg')) {
throw new BadRequestException('Editing SVG images is not supported');
}
// check that crop parameters will not go out of bounds
const { width: assetWidth, height: assetHeight } = getDimensions(asset.exifInfo!);
if (!assetWidth || !assetHeight) {
throw new BadRequestException('Asset dimensions are not available for editing');
}
const crop = dto.edits.find((e) => e.action === AssetEditAction.Crop)?.parameters;
if (crop) {
const { x, y, width, height } = crop;
if (x + width > assetWidth || y + height > assetHeight) {
throw new BadRequestException('Crop parameters are out of bounds');
}
}
const newEdits = await this.assetEditRepository.replaceAll(id, dto.edits);
await this.jobRepository.queue({ name: JobName.AssetEditThumbnailGeneration, data: { id } });
// Return the asset and its applied edits
return {
assetId: id,
edits: newEdits,
};
}
async removeAssetEdits(auth: AuthDto, id: string): Promise<void> {
await this.requireAccess({ auth, permission: Permission.AssetEditDelete, ids: [id] });
const asset = await this.assetRepository.getById(id);
if (!asset) {
throw new BadRequestException('Asset not found');
}
await this.assetEditRepository.replaceAll(id, []);
await this.jobRepository.queue({ name: JobName.AssetEditThumbnailGeneration, data: { id } });
}
}
+3
View File
@@ -11,6 +11,7 @@ import { AlbumUserRepository } from 'src/repositories/album-user.repository';
import { AlbumRepository } from 'src/repositories/album.repository';
import { ApiKeyRepository } from 'src/repositories/api-key.repository';
import { AppRepository } from 'src/repositories/app.repository';
import { AssetEditRepository } from 'src/repositories/asset-edit.repository';
import { AssetJobRepository } from 'src/repositories/asset-job.repository';
import { AssetRepository } from 'src/repositories/asset.repository';
import { AuditRepository } from 'src/repositories/audit.repository';
@@ -69,6 +70,7 @@ export const BASE_SERVICE_DEPENDENCIES = [
ApiKeyRepository,
AppRepository,
AssetRepository,
AssetEditRepository,
AssetJobRepository,
AuditRepository,
ConfigRepository,
@@ -127,6 +129,7 @@ export class BaseService {
protected apiKeyRepository: ApiKeyRepository,
protected appRepository: AppRepository,
protected assetRepository: AssetRepository,
protected assetEditRepository: AssetEditRepository,
protected assetJobRepository: AssetJobRepository,
protected auditRepository: AuditRepository,
protected configRepository: ConfigRepository,
+12
View File
@@ -96,6 +96,16 @@ export class JobService extends BaseService {
break;
}
case JobName.AssetEditThumbnailGeneration: {
const asset = await this.assetRepository.getById(item.data.id);
if (asset) {
this.websocketRepository.clientSend('AssetEditReadyV1', asset.ownerId, { assetId: item.data.id });
}
break;
}
case JobName.AssetGenerateThumbnails: {
if (!item.data.notify && item.data.source !== 'upload') {
break;
@@ -141,6 +151,8 @@ export class JobService extends BaseService {
livePhotoVideoId: asset.livePhotoVideoId,
stackId: asset.stackId,
libraryId: asset.libraryId,
width: asset.width,
height: asset.height,
},
exif: {
assetId: exif.assetId,
+443 -54
View File
@@ -18,13 +18,17 @@ import {
} from 'src/enum';
import { MediaService } from 'src/services/media.service';
import { JobCounts, RawImageInfo } from 'src/types';
import { assetStub } from 'test/fixtures/asset.stub';
import { assetStub, previewFile } from 'test/fixtures/asset.stub';
import { faceStub } from 'test/fixtures/face.stub';
import { probeStub } from 'test/fixtures/media.stub';
import { personStub, personThumbnailStub } from 'test/fixtures/person.stub';
import { systemConfigStub } from 'test/fixtures/system-config.stub';
import { makeStream, newTestService, ServiceMocks } from 'test/utils';
const fullsizeBuffer = Buffer.from('embedded image data');
const rawBuffer = Buffer.from('raw image data');
const extractedBuffer = Buffer.from('embedded image file');
describe(MediaService.name, () => {
let sut: MediaService;
let mocks: ServiceMocks;
@@ -160,6 +164,42 @@ describe(MediaService.name, () => {
expect(mocks.person.getAll).toHaveBeenCalledWith({ thumbnailPath: '' });
});
it('should queue assets with edits but missing edited thumbnails', async () => {
mocks.assetJob.streamForThumbnailJob.mockReturnValue(makeStream([assetStub.withCropEdit]));
mocks.person.getAll.mockReturnValue(makeStream());
await sut.handleQueueGenerateThumbnails({ force: false });
expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith(false);
expect(mocks.job.queueAll).toHaveBeenCalledWith([
{
name: JobName.AssetEditThumbnailGeneration,
data: { id: assetStub.withCropEdit.id },
},
]);
expect(mocks.person.getAll).toHaveBeenCalledWith({ thumbnailPath: '' });
});
it('should queue both regular and edited thumbnails for assets with edits when force is true', async () => {
mocks.assetJob.streamForThumbnailJob.mockReturnValue(makeStream([assetStub.withCropEdit]));
mocks.person.getAll.mockReturnValue(makeStream());
await sut.handleQueueGenerateThumbnails({ force: true });
expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith(true);
expect(mocks.job.queueAll).toHaveBeenCalledWith([
{
name: JobName.AssetGenerateThumbnails,
data: { id: assetStub.withCropEdit.id },
},
{
name: JobName.AssetEditThumbnailGeneration,
data: { id: assetStub.withCropEdit.id },
},
]);
expect(mocks.person.getAll).toHaveBeenCalledWith(undefined);
});
});
describe('handleQueueMigration', () => {
@@ -222,16 +262,12 @@ describe(MediaService.name, () => {
});
describe('handleGenerateThumbnails', () => {
let rawBuffer: Buffer;
let fullsizeBuffer: Buffer;
let extractedBuffer: Buffer;
let rawInfo: RawImageInfo;
beforeEach(() => {
fullsizeBuffer = Buffer.from('embedded image data');
rawBuffer = Buffer.from('raw image data');
extractedBuffer = Buffer.from('embedded image file');
rawInfo = { width: 100, height: 100, channels: 3 };
mocks.person.getFaces.mockResolvedValue([]);
mocks.ocr.getByAssetId.mockResolvedValue([]);
mocks.media.decodeImage.mockImplementation((input) =>
Promise.resolve(
typeof input === 'string'
@@ -281,7 +317,12 @@ describe(MediaService.name, () => {
await sut.handleGenerateThumbnails({ id: assetStub.image.id });
expect(mocks.storage.unlink).toHaveBeenCalledWith('/uploads/user-id/thumbs/path.jpg');
expect(mocks.job.queue).toHaveBeenCalledWith({
name: JobName.FileDelete,
data: {
files: expect.arrayContaining([previewFile.path]),
},
});
});
it('should generate P3 thumbnails for a wide gamut image', async () => {
@@ -313,6 +354,7 @@ describe(MediaService.name, () => {
quality: 80,
processInvalidImages: false,
raw: rawInfo,
edits: [],
},
expect.any(String),
);
@@ -325,6 +367,7 @@ describe(MediaService.name, () => {
quality: 80,
processInvalidImages: false,
raw: rawInfo,
edits: [],
},
expect.any(String),
);
@@ -334,6 +377,7 @@ describe(MediaService.name, () => {
colorspace: Colorspace.P3,
processInvalidImages: false,
raw: rawInfo,
edits: [],
});
expect(mocks.asset.upsertFiles).toHaveBeenCalledWith([
@@ -527,6 +571,7 @@ describe(MediaService.name, () => {
quality: 80,
processInvalidImages: false,
raw: rawInfo,
edits: [],
},
previewPath,
);
@@ -539,6 +584,7 @@ describe(MediaService.name, () => {
quality: 80,
processInvalidImages: false,
raw: rawInfo,
edits: [],
},
thumbnailPath,
);
@@ -572,6 +618,7 @@ describe(MediaService.name, () => {
quality: 80,
processInvalidImages: false,
raw: rawInfo,
edits: [],
},
previewPath,
);
@@ -584,6 +631,7 @@ describe(MediaService.name, () => {
quality: 80,
processInvalidImages: false,
raw: rawInfo,
edits: [],
},
thumbnailPath,
);
@@ -595,7 +643,12 @@ describe(MediaService.name, () => {
await sut.handleGenerateThumbnails({ id: assetStub.image.id });
expect(mocks.storage.unlink).toHaveBeenCalledWith('/uploads/user-id/webp/path.ext');
expect(mocks.job.queue).toHaveBeenCalledWith({
name: JobName.FileDelete,
data: {
files: expect.arrayContaining([previewFile.path]),
},
});
});
it('should extract embedded image if enabled and available', async () => {
@@ -641,7 +694,6 @@ describe(MediaService.name, () => {
processInvalidImages: false,
size: 1440,
});
expect(mocks.media.getImageDimensions).not.toHaveBeenCalled();
});
it('should resize original image if embedded image extraction is not enabled', async () => {
@@ -657,7 +709,6 @@ describe(MediaService.name, () => {
processInvalidImages: false,
size: 1440,
});
expect(mocks.media.getImageDimensions).not.toHaveBeenCalled();
});
it('should process invalid images if enabled', async () => {
@@ -691,7 +742,6 @@ describe(MediaService.name, () => {
expect.objectContaining({ processInvalidImages: false }),
);
expect(mocks.media.getImageDimensions).not.toHaveBeenCalled();
vi.unstubAllEnvs();
});
@@ -722,6 +772,7 @@ describe(MediaService.name, () => {
quality: 80,
processInvalidImages: false,
raw: rawInfo,
edits: [],
},
expect.any(String),
);
@@ -752,6 +803,7 @@ describe(MediaService.name, () => {
quality: 80,
processInvalidImages: false,
raw: rawInfo,
edits: [],
},
expect.any(String),
);
@@ -764,6 +816,7 @@ describe(MediaService.name, () => {
quality: 80,
processInvalidImages: false,
raw: rawInfo,
edits: [],
},
expect.any(String),
);
@@ -792,6 +845,7 @@ describe(MediaService.name, () => {
quality: 80,
processInvalidImages: false,
raw: rawInfo,
edits: [],
},
expect.any(String),
);
@@ -804,6 +858,7 @@ describe(MediaService.name, () => {
size: 1440,
processInvalidImages: false,
raw: rawInfo,
edits: [],
},
expect.any(String),
);
@@ -833,6 +888,7 @@ describe(MediaService.name, () => {
quality: 80,
processInvalidImages: false,
raw: rawInfo,
edits: [],
},
expect.any(String),
);
@@ -888,6 +944,7 @@ describe(MediaService.name, () => {
quality: 80,
processInvalidImages: false,
raw: rawInfo,
edits: [],
},
expect.any(String),
);
@@ -926,12 +983,166 @@ describe(MediaService.name, () => {
quality: 90,
processInvalidImages: false,
raw: rawInfo,
edits: [],
},
expect.any(String),
);
});
});
describe('handleAssetEditThumbnailGeneration', () => {
let rawInfo: RawImageInfo;
beforeEach(() => {
rawInfo = { width: 100, height: 100, channels: 3 };
mocks.person.getFaces.mockResolvedValue([]);
mocks.ocr.getByAssetId.mockResolvedValue([]);
mocks.media.decodeImage.mockImplementation((input) =>
Promise.resolve(
typeof input === 'string'
? { data: rawBuffer, info: rawInfo as OutputInfo } // string implies original file
: { data: fullsizeBuffer, info: rawInfo as OutputInfo }, // buffer implies embedded image extracted
),
);
});
it('should skip videos', async () => {
mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue(assetStub.video);
await expect(sut.handleAssetEditThumbnailGeneration({ id: assetStub.video.id })).resolves.toBe(JobStatus.Success);
expect(mocks.media.generateThumbnail).not.toHaveBeenCalled();
});
it('should upsert 3 edited files for edit jobs', async () => {
mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue({
...assetStub.withCropEdit,
});
const thumbhashBuffer = Buffer.from('a thumbhash', 'utf8');
mocks.media.generateThumbhash.mockResolvedValue(thumbhashBuffer);
mocks.person.getFaces.mockResolvedValue([]);
mocks.ocr.getByAssetId.mockResolvedValue([]);
await sut.handleAssetEditThumbnailGeneration({ id: assetStub.image.id });
expect(mocks.asset.upsertFiles).toHaveBeenCalledWith(
expect.arrayContaining([
expect.objectContaining({ type: AssetFileType.FullSizeEdited }),
expect.objectContaining({ type: AssetFileType.PreviewEdited }),
expect.objectContaining({ type: AssetFileType.ThumbnailEdited }),
]),
);
});
it('should apply edits when generating thumbnails', async () => {
mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue({
...assetStub.withCropEdit,
});
mocks.person.getFaces.mockResolvedValue([]);
mocks.ocr.getByAssetId.mockResolvedValue([]);
await sut.handleAssetEditThumbnailGeneration({ id: assetStub.image.id });
expect(mocks.media.generateThumbnail).toHaveBeenCalledWith(
rawBuffer,
expect.objectContaining({
edits: [
{
action: 'crop',
parameters: { height: 1152, width: 1512, x: 216, y: 1512 },
},
],
}),
expect.any(String),
);
});
it('should clean up edited files if an asset has no edits', async () => {
mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue({
...assetStub.withoutEdits,
});
const status = await sut.handleAssetEditThumbnailGeneration({ id: assetStub.image.id });
expect(mocks.job.queue).toHaveBeenCalledWith({
name: JobName.FileDelete,
data: {
files: expect.arrayContaining([
'/uploads/user-id/fullsize/path_edited.jpg',
'/uploads/user-id/preview/path_edited.jpg',
'/uploads/user-id/thumbnail/path_edited.jpg',
]),
},
});
expect(mocks.asset.deleteFiles).toHaveBeenCalledWith(
expect.arrayContaining([
expect.objectContaining({ path: '/uploads/user-id/preview/path_edited.jpg' }),
expect.objectContaining({ path: '/uploads/user-id/thumbnail/path_edited.jpg' }),
expect.objectContaining({ path: '/uploads/user-id/fullsize/path_edited.jpg' }),
]),
);
expect(status).toBe(JobStatus.Success);
expect(mocks.media.generateThumbnail).not.toHaveBeenCalled();
expect(mocks.asset.upsertFiles).not.toHaveBeenCalled();
});
it('should generate all 3 edited files if an asset has edits', async () => {
mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue({
...assetStub.withCropEdit,
});
mocks.person.getFaces.mockResolvedValue([]);
mocks.ocr.getByAssetId.mockResolvedValue([]);
await sut.handleAssetEditThumbnailGeneration({ id: assetStub.image.id });
expect(mocks.media.generateThumbnail).toHaveBeenCalledTimes(3);
expect(mocks.media.generateThumbnail).toHaveBeenCalledWith(
rawBuffer,
expect.anything(),
expect.stringContaining('edited_preview.jpeg'),
);
expect(mocks.media.generateThumbnail).toHaveBeenCalledWith(
rawBuffer,
expect.anything(),
expect.stringContaining('edited_thumbnail.webp'),
);
expect(mocks.media.generateThumbnail).toHaveBeenCalledWith(
rawBuffer,
expect.anything(),
expect.stringContaining('edited_fullsize.jpeg'),
);
});
it('should generate the original thumbhash if no edits exist', async () => {
mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue({
...assetStub.withoutEdits,
});
const thumbhashBuffer = Buffer.from('a thumbhash', 'utf8');
mocks.media.generateThumbhash.mockResolvedValue(thumbhashBuffer);
await sut.handleAssetEditThumbnailGeneration({ id: assetStub.image.id, source: 'upload' });
expect(mocks.media.generateThumbhash).toHaveBeenCalled();
});
it('should apply thumbhash if job source is edit and edits exist', async () => {
mocks.assetJob.getForGenerateThumbnailJob.mockResolvedValue({
...assetStub.withCropEdit,
});
const thumbhashBuffer = Buffer.from('a thumbhash', 'utf8');
mocks.media.generateThumbhash.mockResolvedValue(thumbhashBuffer);
mocks.person.getFaces.mockResolvedValue([]);
mocks.ocr.getByAssetId.mockResolvedValue([]);
await sut.handleAssetEditThumbnailGeneration({ id: assetStub.image.id });
expect(mocks.asset.update).toHaveBeenCalledWith(
expect.objectContaining({
thumbhash: thumbhashBuffer,
}),
);
});
});
describe('handleGeneratePersonThumbnail', () => {
it('should skip if machine learning is disabled', async () => {
mocks.systemMetadata.get.mockResolvedValue(systemConfigStub.machineLearningDisabled);
@@ -981,12 +1192,17 @@ describe(MediaService.name, () => {
colorspace: Colorspace.P3,
format: ImageFormat.Jpeg,
quality: 80,
crop: {
left: 238,
top: 163,
width: 274,
height: 274,
},
edits: [
{
action: 'crop',
parameters: {
height: 274,
width: 274,
x: 238,
y: 163,
},
},
],
raw: info,
processInvalidImages: false,
size: 250,
@@ -1020,12 +1236,17 @@ describe(MediaService.name, () => {
colorspace: Colorspace.P3,
format: ImageFormat.Jpeg,
quality: 80,
crop: {
left: 238,
top: 163,
width: 274,
height: 274,
},
edits: [
{
action: 'crop',
parameters: {
height: 274,
width: 274,
x: 238,
y: 163,
},
},
],
raw: info,
processInvalidImages: false,
size: 250,
@@ -1057,12 +1278,17 @@ describe(MediaService.name, () => {
colorspace: Colorspace.P3,
format: ImageFormat.Jpeg,
quality: 80,
crop: {
left: 0,
top: 85,
width: 510,
height: 510,
},
edits: [
{
action: 'crop',
parameters: {
height: 510,
width: 510,
x: 0,
y: 85,
},
},
],
raw: info,
processInvalidImages: false,
size: 250,
@@ -1094,12 +1320,17 @@ describe(MediaService.name, () => {
colorspace: Colorspace.P3,
format: ImageFormat.Jpeg,
quality: 80,
crop: {
left: 591,
top: 591,
width: 408,
height: 408,
},
edits: [
{
action: 'crop',
parameters: {
height: 408,
width: 408,
x: 591,
y: 591,
},
},
],
raw: info,
processInvalidImages: false,
size: 250,
@@ -1131,12 +1362,17 @@ describe(MediaService.name, () => {
colorspace: Colorspace.P3,
format: ImageFormat.Jpeg,
quality: 80,
crop: {
left: 0,
top: 62,
width: 412,
height: 412,
},
edits: [
{
action: 'crop',
parameters: {
height: 412,
width: 412,
x: 0,
y: 62,
},
},
],
raw: info,
processInvalidImages: false,
size: 250,
@@ -1168,12 +1404,17 @@ describe(MediaService.name, () => {
colorspace: Colorspace.P3,
format: ImageFormat.Jpeg,
quality: 80,
crop: {
left: 4485,
top: 94,
width: 138,
height: 138,
},
edits: [
{
action: 'crop',
parameters: {
height: 138,
width: 138,
x: 4485,
y: 94,
},
},
],
raw: info,
processInvalidImages: false,
size: 250,
@@ -1210,12 +1451,17 @@ describe(MediaService.name, () => {
colorspace: Colorspace.P3,
format: ImageFormat.Jpeg,
quality: 80,
crop: {
height: 844,
left: 388,
top: 730,
width: 844,
},
edits: [
{
action: 'crop',
parameters: {
height: 844,
width: 844,
x: 388,
y: 730,
},
},
],
raw: info,
processInvalidImages: false,
size: 250,
@@ -2999,4 +3245,147 @@ describe(MediaService.name, () => {
expect(sut.isSRGB({ profileDescription: 'sRGB', bitsPerSample: 16 } as Exif)).toEqual(true);
});
});
describe('syncFiles', () => {
it('should upsert new files when they do not exist', async () => {
const asset = {
id: 'asset-id',
files: [],
};
await sut['syncFiles'](asset, [
{ type: AssetFileType.Preview, newPath: '/new/preview.jpg' },
{ type: AssetFileType.Thumbnail, newPath: '/new/thumbnail.jpg' },
]);
expect(mocks.asset.upsertFiles).toHaveBeenCalledWith([
{ assetId: 'asset-id', path: '/new/preview.jpg', type: AssetFileType.Preview },
{ assetId: 'asset-id', path: '/new/thumbnail.jpg', type: AssetFileType.Thumbnail },
]);
expect(mocks.asset.deleteFiles).not.toHaveBeenCalled();
expect(mocks.job.queue).not.toHaveBeenCalled();
});
it('should replace existing files with new paths', async () => {
const asset = {
id: 'asset-id',
files: [
{ id: 'file-1', assetId: 'asset-id', type: AssetFileType.Preview, path: '/old/preview.jpg' },
{ id: 'file-2', assetId: 'asset-id', type: AssetFileType.Thumbnail, path: '/old/thumbnail.jpg' },
],
};
await sut['syncFiles'](asset, [
{ type: AssetFileType.Preview, newPath: '/new/preview.jpg' },
{ type: AssetFileType.Thumbnail, newPath: '/new/thumbnail.jpg' },
]);
expect(mocks.asset.upsertFiles).toHaveBeenCalledWith([
{ assetId: 'asset-id', path: '/new/preview.jpg', type: AssetFileType.Preview },
{ assetId: 'asset-id', path: '/new/thumbnail.jpg', type: AssetFileType.Thumbnail },
]);
expect(mocks.asset.deleteFiles).not.toHaveBeenCalled();
expect(mocks.job.queue).toHaveBeenCalledWith({
name: JobName.FileDelete,
data: { files: ['/old/preview.jpg', '/old/thumbnail.jpg'] },
});
});
it('should delete files when newPath is not provided', async () => {
const asset = {
id: 'asset-id',
files: [
{ id: 'file-1', assetId: 'asset-id', type: AssetFileType.Preview, path: '/old/preview.jpg' },
{ id: 'file-2', assetId: 'asset-id', type: AssetFileType.Thumbnail, path: '/old/thumbnail.jpg' },
],
};
await sut['syncFiles'](asset, [{ type: AssetFileType.Preview }, { type: AssetFileType.Thumbnail }]);
expect(mocks.asset.upsertFiles).not.toHaveBeenCalled();
expect(mocks.asset.deleteFiles).toHaveBeenCalledWith([
{ id: 'file-1', assetId: 'asset-id', type: AssetFileType.Preview, path: '/old/preview.jpg' },
{ id: 'file-2', assetId: 'asset-id', type: AssetFileType.Thumbnail, path: '/old/thumbnail.jpg' },
]);
expect(mocks.job.queue).toHaveBeenCalledWith({
name: JobName.FileDelete,
data: { files: ['/old/preview.jpg', '/old/thumbnail.jpg'] },
});
});
it('should not make changes when file paths already match', async () => {
const asset = {
id: 'asset-id',
files: [
{ id: 'file-1', assetId: 'asset-id', type: AssetFileType.Preview, path: '/same/preview.jpg' },
{ id: 'file-2', assetId: 'asset-id', type: AssetFileType.Thumbnail, path: '/same/thumbnail.jpg' },
],
};
await sut['syncFiles'](asset, [
{ type: AssetFileType.Preview, newPath: '/same/preview.jpg' },
{ type: AssetFileType.Thumbnail, newPath: '/same/thumbnail.jpg' },
]);
expect(mocks.asset.upsertFiles).not.toHaveBeenCalled();
expect(mocks.asset.deleteFiles).not.toHaveBeenCalled();
expect(mocks.job.queue).not.toHaveBeenCalled();
});
it('should handle mixed operations (upsert, replace, delete)', async () => {
const asset = {
id: 'asset-id',
files: [
{ id: 'file-1', assetId: 'asset-id', type: AssetFileType.Preview, path: '/old/preview.jpg' },
{ id: 'file-2', assetId: 'asset-id', type: AssetFileType.Thumbnail, path: '/old/thumbnail.jpg' },
],
};
await sut['syncFiles'](asset, [
{ type: AssetFileType.Preview, newPath: '/new/preview.jpg' }, // replace
{ type: AssetFileType.Thumbnail }, // delete
{ type: AssetFileType.FullSize, newPath: '/new/fullsize.jpg' }, // new
]);
expect(mocks.asset.upsertFiles).toHaveBeenCalledWith([
{ assetId: 'asset-id', path: '/new/preview.jpg', type: AssetFileType.Preview },
{ assetId: 'asset-id', path: '/new/fullsize.jpg', type: AssetFileType.FullSize },
]);
expect(mocks.asset.deleteFiles).toHaveBeenCalledWith([
{ id: 'file-2', assetId: 'asset-id', type: AssetFileType.Thumbnail, path: '/old/thumbnail.jpg' },
]);
expect(mocks.job.queue).toHaveBeenCalledWith({
name: JobName.FileDelete,
data: { files: ['/old/preview.jpg', '/old/thumbnail.jpg'] },
});
});
it('should handle empty file list', async () => {
const asset = {
id: 'asset-id',
files: [],
};
await sut['syncFiles'](asset, []);
expect(mocks.asset.upsertFiles).not.toHaveBeenCalled();
expect(mocks.asset.deleteFiles).not.toHaveBeenCalled();
expect(mocks.job.queue).not.toHaveBeenCalled();
});
it('should delete non-existent file types when newPath is not provided', async () => {
const asset = {
id: 'asset-id',
files: [{ id: 'file-1', assetId: 'asset-id', type: AssetFileType.Preview, path: '/old/preview.jpg' }],
};
await sut['syncFiles'](asset, [
{ type: AssetFileType.Thumbnail }, // file doesn't exist, newPath not provided
]);
expect(mocks.asset.upsertFiles).not.toHaveBeenCalled();
expect(mocks.asset.deleteFiles).not.toHaveBeenCalled();
expect(mocks.job.queue).not.toHaveBeenCalled();
});
});
});
+229 -80
View File
@@ -1,8 +1,10 @@
import { Injectable } from '@nestjs/common';
import { SystemConfig } from 'src/config';
import { FACE_THUMBNAIL_SIZE, JOBS_ASSET_PAGINATION_SIZE } from 'src/constants';
import { StorageCore, ThumbnailPathEntity } from 'src/cores/storage.core';
import { Exif } from 'src/database';
import { AssetFile, Exif } from 'src/database';
import { OnEvent, OnJob } from 'src/decorators';
import { AssetEditAction, CropParameters } from 'src/dtos/editing.dto';
import { SystemConfigFFmpegDto } from 'src/dtos/system-config.dto';
import {
AssetFileType,
@@ -24,12 +26,13 @@ import {
VideoCodec,
VideoContainer,
} from 'src/enum';
import { AssetJobRepository } from 'src/repositories/asset-job.repository';
import { BoundingBox } from 'src/repositories/machine-learning.repository';
import { BaseService } from 'src/services/base.service';
import {
AudioStreamInfo,
CropOptions,
DecodeToBufferOptions,
GenerateThumbnailOptions,
ImageDimensions,
JobItem,
JobOf,
@@ -37,16 +40,20 @@ import {
VideoInterfaces,
VideoStreamInfo,
} from 'src/types';
import { getAssetFiles } from 'src/utils/asset.util';
import { getAssetFiles, getDimensions } from 'src/utils/asset.util';
import { checkFaceVisibility, checkOcrVisibility } from 'src/utils/editor';
import { BaseConfig, ThumbnailConfig } from 'src/utils/media';
import { mimeTypes } from 'src/utils/mime-types';
import { clamp, isFaceImportEnabled, isFacialRecognitionEnabled } from 'src/utils/misc';
import { getOutputDimensions } from 'src/utils/transform';
interface UpsertFileOptions {
assetId: string;
type: AssetFileType;
path: string;
}
type ThumbnailAsset = NonNullable<Awaited<ReturnType<AssetJobRepository['getForGenerateThumbnailJob']>>>;
@Injectable()
export class MediaService extends BaseService {
videoInterfaces: VideoInterfaces = { dri: [], mali: false };
@@ -67,12 +74,19 @@ export class MediaService extends BaseService {
};
for await (const asset of this.assetJobRepository.streamForThumbnailJob(!!force)) {
const { previewFile, thumbnailFile } = getAssetFiles(asset.files);
const assetFiles = getAssetFiles(asset.files);
if (!previewFile || !thumbnailFile || !asset.thumbhash || force) {
if (!assetFiles.previewFile || !assetFiles.thumbnailFile || !asset.thumbhash || force) {
jobs.push({ name: JobName.AssetGenerateThumbnails, data: { id: asset.id } });
}
if (
asset.edits.length > 0 &&
(!assetFiles.editedPreviewFile || !assetFiles.editedThumbnailFile || !assetFiles.editedFullsizeFile || force)
) {
jobs.push({ name: JobName.AssetEditThumbnailGeneration, data: { id: asset.id } });
}
if (jobs.length >= JOBS_ASSET_PAGINATION_SIZE) {
await queueAll();
}
@@ -154,9 +168,45 @@ export class MediaService extends BaseService {
return JobStatus.Success;
}
@OnJob({ name: JobName.AssetEditThumbnailGeneration, queue: QueueName.Editor })
async handleAssetEditThumbnailGeneration({ id }: JobOf<JobName.AssetEditThumbnailGeneration>): Promise<JobStatus> {
const asset = await this.assetJobRepository.getForGenerateThumbnailJob(id);
if (!asset) {
this.logger.warn(`Thumbnail generation failed for asset ${id}: not found in database or missing metadata`);
return JobStatus.Failed;
}
const generated = await this.generateEditedThumbnails(asset);
let thumbhash: Buffer | undefined = generated?.thumbhash;
if (!thumbhash) {
const { image } = await this.getConfig({ withCache: true });
const extractedImage = await this.extractOriginalImage(asset, image);
const { info, data, colorspace } = extractedImage;
thumbhash = await this.mediaRepository.generateThumbhash(data, {
colorspace,
processInvalidImages: false,
raw: info,
edits: [],
});
}
if (!asset.thumbhash || Buffer.compare(asset.thumbhash, thumbhash) !== 0) {
await this.assetRepository.update({ id: asset.id, thumbhash });
}
const fullsizeDimensions = generated?.fullsizeDimensions ?? getDimensions(asset.exifInfo!);
await this.assetRepository.update({ id: asset.id, ...fullsizeDimensions });
return JobStatus.Success;
}
@OnJob({ name: JobName.AssetGenerateThumbnails, queue: QueueName.ThumbnailGeneration })
async handleGenerateThumbnails({ id }: JobOf<JobName.AssetGenerateThumbnails>): Promise<JobStatus> {
const asset = await this.assetJobRepository.getForGenerateThumbnailJob(id);
if (!asset) {
this.logger.warn(`Thumbnail generation failed for asset ${id}: not found in database or missing metadata`);
return JobStatus.Failed;
@@ -172,6 +222,7 @@ export class MediaService extends BaseService {
thumbnailPath: string;
fullsizePath?: string;
thumbhash: Buffer;
fullsizeDimensions?: ImageDimensions;
};
if (asset.type === AssetType.Video || asset.originalFileName.toLowerCase().endsWith('.gif')) {
this.logger.verbose(`Thumbnail generation for video ${id} ${asset.originalPath}`);
@@ -184,54 +235,19 @@ export class MediaService extends BaseService {
return JobStatus.Skipped;
}
const { previewFile, thumbnailFile, fullsizeFile } = getAssetFiles(asset.files);
const toUpsert: UpsertFileOptions[] = [];
if (previewFile?.path !== generated.previewPath) {
toUpsert.push({ assetId: asset.id, path: generated.previewPath, type: AssetFileType.Preview });
}
await this.syncFiles(asset, [
{ type: AssetFileType.Preview, newPath: generated.previewPath },
{ type: AssetFileType.Thumbnail, newPath: generated.thumbnailPath },
{ type: AssetFileType.FullSize, newPath: generated.fullsizePath },
]);
if (thumbnailFile?.path !== generated.thumbnailPath) {
toUpsert.push({ assetId: asset.id, path: generated.thumbnailPath, type: AssetFileType.Thumbnail });
}
const editiedGenerated = await this.generateEditedThumbnails(asset);
const thumbhash = editiedGenerated?.thumbhash || generated.thumbhash;
if (generated.fullsizePath && fullsizeFile?.path !== generated.fullsizePath) {
toUpsert.push({ assetId: asset.id, path: generated.fullsizePath, type: AssetFileType.FullSize });
if (!asset.thumbhash || Buffer.compare(asset.thumbhash, thumbhash) !== 0) {
await this.assetRepository.update({ id: asset.id, thumbhash });
}
if (toUpsert.length > 0) {
await this.assetRepository.upsertFiles(toUpsert);
}
const pathsToDelete: string[] = [];
if (previewFile && previewFile.path !== generated.previewPath) {
this.logger.debug(`Deleting old preview for asset ${asset.id}`);
pathsToDelete.push(previewFile.path);
}
if (thumbnailFile && thumbnailFile.path !== generated.thumbnailPath) {
this.logger.debug(`Deleting old thumbnail for asset ${asset.id}`);
pathsToDelete.push(thumbnailFile.path);
}
if (fullsizeFile && fullsizeFile.path !== generated.fullsizePath) {
this.logger.debug(`Deleting old fullsize preview image for asset ${asset.id}`);
pathsToDelete.push(fullsizeFile.path);
if (!generated.fullsizePath) {
// did not generate a new fullsize image, delete the existing record
await this.assetRepository.deleteFiles([fullsizeFile]);
}
}
if (pathsToDelete.length > 0) {
await Promise.all(pathsToDelete.map((path) => this.storageRepository.unlink(path)));
}
if (!asset.thumbhash || Buffer.compare(asset.thumbhash, generated.thumbhash) !== 0) {
await this.assetRepository.update({ id: asset.id, thumbhash: generated.thumbhash });
}
await this.assetRepository.upsertJobStatus({ assetId: asset.id, previewAt: new Date(), thumbnailAt: new Date() });
return JobStatus.Success;
}
@@ -258,27 +274,20 @@ export class MediaService extends BaseService {
return { info, data, colorspace };
}
private async generateImageThumbnails(asset: {
id: string;
ownerId: string;
originalFileName: string;
originalPath: string;
exifInfo: Exif;
}) {
const { image } = await this.getConfig({ withCache: true });
const previewPath = StorageCore.getImagePath(asset, AssetPathType.Preview, image.preview.format);
const thumbnailPath = StorageCore.getImagePath(asset, AssetPathType.Thumbnail, image.thumbnail.format);
this.storageCore.ensureFolders(previewPath);
// Handle embedded preview extraction for RAW files
private async extractOriginalImage(
asset: NonNullable<ThumbnailAsset>,
image: SystemConfig['image'],
useEdits = false,
) {
const extractEmbedded = image.extractEmbedded && mimeTypes.isRaw(asset.originalFileName);
const extracted = extractEmbedded ? await this.extractImage(asset.originalPath, image.preview.size) : null;
const generateFullsize =
(image.fullsize.enabled || asset.exifInfo.projectionType == 'EQUIRECTANGULAR') &&
!mimeTypes.isWebSupportedImage(asset.originalPath);
((image.fullsize.enabled || asset.exifInfo.projectionType === 'EQUIRECTANGULAR') &&
!mimeTypes.isWebSupportedImage(asset.originalPath)) ||
useEdits;
const convertFullsize = generateFullsize && (!extracted || !mimeTypes.isWebSupportedImage(` .${extracted.format}`));
const { info, data, colorspace } = await this.decodeImage(
const { data, info, colorspace } = await this.decodeImage(
extracted ? extracted.buffer : asset.originalPath,
// only specify orientation to extracted images which don't have EXIF orientation data
// or it can double rotate the image
@@ -286,20 +295,64 @@ export class MediaService extends BaseService {
convertFullsize ? undefined : image.preview.size,
);
return {
extracted,
data,
info,
colorspace,
convertFullsize,
generateFullsize,
};
}
private async generateImageThumbnails(asset: ThumbnailAsset, useEdits: boolean = false) {
const { image } = await this.getConfig({ withCache: true });
const previewPath = StorageCore.getImagePath(
asset,
useEdits ? AssetPathType.EditedPreview : AssetPathType.Preview,
image.preview.format,
);
const thumbnailPath = StorageCore.getImagePath(
asset,
useEdits ? AssetPathType.EditedThumbnail : AssetPathType.Thumbnail,
image.thumbnail.format,
);
this.storageCore.ensureFolders(previewPath);
// Handle embedded preview extraction for RAW files
const extractedImage = await this.extractOriginalImage(asset, image, useEdits);
const { info, data, colorspace, generateFullsize, convertFullsize, extracted } = extractedImage;
// generate final images
const thumbnailOptions = { colorspace, processInvalidImages: false, raw: info };
const thumbnailOptions = { colorspace, processInvalidImages: false, raw: info, edits: useEdits ? asset.edits : [] };
const promises = [
this.mediaRepository.generateThumbhash(data, thumbnailOptions),
this.mediaRepository.generateThumbnail(data, { ...image.thumbnail, ...thumbnailOptions }, thumbnailPath),
this.mediaRepository.generateThumbnail(data, { ...image.preview, ...thumbnailOptions }, previewPath),
this.mediaRepository.generateThumbnail(
data,
{ ...image.thumbnail, ...thumbnailOptions, edits: useEdits ? asset.edits : [] },
thumbnailPath,
),
this.mediaRepository.generateThumbnail(
data,
{ ...image.preview, ...thumbnailOptions, edits: useEdits ? asset.edits : [] },
previewPath,
),
];
let fullsizePath: string | undefined;
if (convertFullsize) {
// convert a new fullsize image from the same source as the thumbnail
fullsizePath = StorageCore.getImagePath(asset, AssetPathType.FullSize, image.fullsize.format);
const fullsizeOptions = { format: image.fullsize.format, quality: image.fullsize.quality, ...thumbnailOptions };
fullsizePath = StorageCore.getImagePath(
asset,
useEdits ? AssetPathType.EditedFullSize : AssetPathType.FullSize,
image.fullsize.format,
);
const fullsizeOptions = {
format: image.fullsize.format,
quality: image.fullsize.quality,
...thumbnailOptions,
};
promises.push(this.mediaRepository.generateThumbnail(data, fullsizeOptions, fullsizePath));
} else if (generateFullsize && extracted && extracted.format === RawExtractedFormat.Jpeg) {
fullsizePath = StorageCore.getImagePath(asset, AssetPathType.FullSize, extracted.format);
@@ -328,7 +381,10 @@ export class MediaService extends BaseService {
await Promise.all(promises);
}
return { previewPath, thumbnailPath, fullsizePath, thumbhash: outputs[0] as Buffer };
const decodedDimensions = { width: info.width, height: info.height };
const fullsizeDimensions = useEdits ? getOutputDimensions(asset.edits, decodedDimensions) : decodedDimensions;
return { previewPath, thumbnailPath, fullsizePath, thumbhash: outputs[0] as Buffer, fullsizeDimensions };
}
@OnJob({ name: JobName.PersonGenerateThumbnail, queue: QueueName.ThumbnailGeneration })
@@ -369,17 +425,22 @@ export class MediaService extends BaseService {
const thumbnailPath = StorageCore.getPersonThumbnailPath({ id, ownerId });
this.storageCore.ensureFolders(thumbnailPath);
const thumbnailOptions = {
const thumbnailOptions: GenerateThumbnailOptions = {
colorspace: image.colorspace,
format: ImageFormat.Jpeg,
raw: info,
quality: image.thumbnail.quality,
crop: this.getCrop(
{ old: { width: oldWidth, height: oldHeight }, new: { width: info.width, height: info.height } },
{ x1, y1, x2, y2 },
),
processInvalidImages: false,
size: FACE_THUMBNAIL_SIZE,
edits: [
{
action: AssetEditAction.Crop,
parameters: this.getCrop(
{ old: { width: oldWidth, height: oldHeight }, new: { width: info.width, height: info.height } },
{ x1, y1, x2, y2 },
),
},
],
};
await this.mediaRepository.generateThumbnail(decodedImage, thumbnailOptions, thumbnailPath);
@@ -388,7 +449,10 @@ export class MediaService extends BaseService {
return JobStatus.Success;
}
private getCrop(dims: { old: ImageDimensions; new: ImageDimensions }, { x1, y1, x2, y2 }: BoundingBox): CropOptions {
private getCrop(
dims: { old: ImageDimensions; new: ImageDimensions },
{ x1, y1, x2, y2 }: BoundingBox,
): CropParameters {
// face bounding boxes can spill outside the image dimensions
const clampedX1 = clamp(x1, 0, dims.old.width);
const clampedY1 = clamp(y1, 0, dims.old.height);
@@ -416,8 +480,8 @@ export class MediaService extends BaseService {
);
return {
left: middleX - newHalfSize,
top: middleY - newHalfSize,
x: middleX - newHalfSize,
y: middleY - newHalfSize,
width: newHalfSize * 2,
height: newHalfSize * 2,
};
@@ -454,7 +518,12 @@ export class MediaService extends BaseService {
processInvalidImages: process.env.IMMICH_PROCESS_INVALID_IMAGES === 'true',
});
return { previewPath, thumbnailPath, thumbhash };
return {
previewPath,
thumbnailPath,
thumbhash,
fullsizeDimensions: { width: mainVideoStream.width, height: mainVideoStream.height },
};
}
@OnJob({ name: JobName.AssetEncodeVideoQueueAll, queue: QueueName.VideoConversion })
@@ -707,4 +776,84 @@ export class MediaService extends BaseService {
return false;
}
}
private async syncFiles(
asset: { id: string; files: AssetFile[] },
files: { type: AssetFileType; newPath?: string }[],
) {
const toUpsert: UpsertFileOptions[] = [];
const pathsToDelete: string[] = [];
const toDelete: AssetFile[] = [];
for (const { type, newPath } of files) {
const existingFile = asset.files.find((file) => file.type === type);
// upsert new file path
if (newPath && existingFile?.path !== newPath) {
toUpsert.push({ assetId: asset.id, path: newPath, type });
// delete old file from disk
if (existingFile) {
this.logger.debug(`Deleting old ${type} image for asset ${asset.id} in favor of a replacement`);
pathsToDelete.push(existingFile.path);
}
}
// delete old file from disk and database
if (!newPath && existingFile) {
this.logger.debug(`Deleting old ${type} image for asset ${asset.id}`);
pathsToDelete.push(existingFile.path);
toDelete.push(existingFile);
}
}
if (toUpsert.length > 0) {
await this.assetRepository.upsertFiles(toUpsert);
}
if (toDelete.length > 0) {
await this.assetRepository.deleteFiles(toDelete);
}
if (pathsToDelete.length > 0) {
await this.jobRepository.queue({ name: JobName.FileDelete, data: { files: pathsToDelete } });
}
}
private async generateEditedThumbnails(asset: ThumbnailAsset) {
if (asset.type !== AssetType.Image) {
return;
}
const generated = asset.edits.length > 0 ? await this.generateImageThumbnails(asset, true) : undefined;
await this.syncFiles(asset, [
{ type: AssetFileType.PreviewEdited, newPath: generated?.previewPath },
{ type: AssetFileType.ThumbnailEdited, newPath: generated?.thumbnailPath },
{ type: AssetFileType.FullSizeEdited, newPath: generated?.fullsizePath },
]);
const crop = asset.edits.find((e) => e.action === AssetEditAction.Crop);
const cropBox = crop
? {
x1: crop.parameters.x,
y1: crop.parameters.y,
x2: crop.parameters.x + crop.parameters.width,
y2: crop.parameters.y + crop.parameters.height,
}
: undefined;
const originalDimensions = getDimensions(asset.exifInfo!);
const assetFaces = await this.personRepository.getFaces(asset.id, {});
const ocrData = await this.ocrRepository.getByAssetId(asset.id, {});
const faceStatuses = checkFaceVisibility(assetFaces, originalDimensions, cropBox);
await this.personRepository.updateVisibility(faceStatuses.visible, faceStatuses.hidden);
const ocrStatuses = checkOcrVisibility(ocrData, originalDimensions, cropBox);
await this.ocrRepository.updateOcrVisibilities(asset.id, ocrStatuses.visible, ocrStatuses.hidden);
return generated;
}
}
@@ -224,6 +224,8 @@ describe(MetadataService.name, () => {
fileCreatedAt: fileModifiedAt,
fileModifiedAt,
localDateTime: fileModifiedAt,
width: null,
height: null,
});
});
@@ -251,6 +253,8 @@ describe(MetadataService.name, () => {
fileCreatedAt,
fileModifiedAt,
localDateTime: fileCreatedAt,
width: null,
height: null,
});
});
@@ -297,6 +301,8 @@ describe(MetadataService.name, () => {
fileCreatedAt: assetStub.image.fileCreatedAt,
fileModifiedAt: assetStub.image.fileCreatedAt,
localDateTime: assetStub.image.fileCreatedAt,
width: null,
height: null,
});
});
@@ -327,6 +333,8 @@ describe(MetadataService.name, () => {
fileCreatedAt: assetStub.withLocation.fileCreatedAt,
fileModifiedAt: assetStub.withLocation.fileModifiedAt,
localDateTime: new Date('2023-02-22T05:06:29.716Z'),
width: null,
height: null,
});
});
@@ -357,6 +365,8 @@ describe(MetadataService.name, () => {
fileCreatedAt: assetStub.withLocation.fileCreatedAt,
fileModifiedAt: assetStub.withLocation.fileModifiedAt,
localDateTime: new Date('2023-02-22T05:06:29.716Z'),
width: null,
height: null,
});
});
@@ -1560,6 +1570,49 @@ describe(MetadataService.name, () => {
{ lockedPropertiesBehavior: 'skip' },
);
});
it('should properly set width/height for normal images', async () => {
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(assetStub.image);
mockReadTags({ ImageWidth: 1000, ImageHeight: 2000 });
await sut.handleMetadataExtraction({ id: assetStub.image.id });
expect(mocks.asset.update).toHaveBeenCalledWith(
expect.objectContaining({
width: 1000,
height: 2000,
}),
);
});
it('should properly swap asset width/height for rotated images', async () => {
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(assetStub.image);
mockReadTags({ ImageWidth: 1000, ImageHeight: 2000, Orientation: 6 });
await sut.handleMetadataExtraction({ id: assetStub.image.id });
expect(mocks.asset.update).toHaveBeenCalledWith(
expect.objectContaining({
width: 2000,
height: 1000,
}),
);
});
it('should not overwrite existing width/height if they already exist', async () => {
mocks.assetJob.getForMetadataExtraction.mockResolvedValue({
...assetStub.image,
width: 1920,
height: 1080,
});
mockReadTags({ ImageWidth: 1280, ImageHeight: 720 });
await sut.handleMetadataExtraction({ id: assetStub.image.id });
expect(mocks.asset.update).not.toHaveBeenCalledWith(
expect.objectContaining({
width: 1280,
height: 720,
}),
);
});
});
describe('handleQueueSidecar', () => {
+28 -7
View File
@@ -196,6 +196,15 @@ export class MetadataService extends BaseService {
await this.eventRepository.emit('AssetHide', { assetId: motionAsset.id, userId: motionAsset.ownerId });
}
private isOrientationSidewards(orientation: ExifOrientation | number): boolean {
return [
ExifOrientation.MirrorHorizontalRotate270CW,
ExifOrientation.Rotate90CW,
ExifOrientation.MirrorHorizontalRotate90CW,
ExifOrientation.Rotate270CW,
].includes(orientation);
}
@OnJob({ name: JobName.AssetExtractMetadataQueueAll, queue: QueueName.MetadataExtraction })
async handleQueueMetadataExtraction(job: JobOf<JobName.AssetExtractMetadataQueueAll>): Promise<JobStatus> {
const { force } = job;
@@ -289,6 +298,10 @@ export class MetadataService extends BaseService {
autoStackId: this.getAutoStackId(exifTags),
};
const isSidewards = exifTags.Orientation && this.isOrientationSidewards(exifTags.Orientation);
const assetWidth = isSidewards ? validate(height) : validate(width);
const assetHeight = isSidewards ? validate(width) : validate(height);
const promises: Promise<unknown>[] = [
this.assetRepository.upsertExif(exifData, { lockedPropertiesBehavior: 'skip' }),
this.assetRepository.update({
@@ -297,6 +310,11 @@ export class MetadataService extends BaseService {
localDateTime: dates.localDateTime,
fileCreatedAt: dates.dateTimeOriginal ?? undefined,
fileModifiedAt: stats.mtime,
// only update the dimensions if they don't already exist
// we don't want to overwrite width/height that are modified by edits
width: asset.width == null ? assetWidth : undefined,
height: asset.height == null ? assetHeight : undefined,
}),
this.applyTagList(asset, exifTags),
];
@@ -716,12 +734,7 @@ export class MetadataService extends BaseService {
return regionInfo;
}
const isSidewards = [
ExifOrientation.MirrorHorizontalRotate270CW,
ExifOrientation.Rotate90CW,
ExifOrientation.MirrorHorizontalRotate90CW,
ExifOrientation.Rotate270CW,
].includes(orientation);
const isSidewards = this.isOrientationSidewards(orientation);
// swap image dimensions in AppliedToDimensions if orientation is sidewards
const adjustedAppliedToDimensions = isSidewards
@@ -971,9 +984,17 @@ export class MetadataService extends BaseService {
private async getVideoTags(originalPath: string) {
const { videoStreams, format } = await this.mediaRepository.probe(originalPath);
const tags: Pick<ImmichTags, 'Duration' | 'Orientation'> = {};
const tags: Pick<ImmichTags, 'Duration' | 'Orientation' | 'ImageWidth' | 'ImageHeight'> = {};
if (videoStreams[0]) {
// Set video dimensions
if (videoStreams[0].width) {
tags.ImageWidth = videoStreams[0].width;
}
if (videoStreams[0].height) {
tags.ImageHeight = videoStreams[0].height;
}
switch (videoStreams[0].rotation) {
case -90: {
tags.Orientation = ExifOrientation.Rotate90CW;
@@ -354,6 +354,7 @@ describe(PersonService.name, () => {
it('should get the bounding boxes for an asset', async () => {
mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([faceStub.face1.assetId]));
mocks.person.getFaces.mockResolvedValue([faceStub.primaryFace1]);
mocks.asset.getById.mockResolvedValue(assetStub.image);
await expect(sut.getFacesById(authStub.admin, { id: faceStub.face1.assetId })).resolves.toStrictEqual([
mapFaces(faceStub.primaryFace1, authStub.admin),
]);
+5 -1
View File
@@ -40,6 +40,7 @@ import { AssetFaceTable } from 'src/schema/tables/asset-face.table';
import { FaceSearchTable } from 'src/schema/tables/face-search.table';
import { BaseService } from 'src/services/base.service';
import { JobItem, JobOf } from 'src/types';
import { getDimensions } from 'src/utils/asset.util';
import { ImmichFileResponse } from 'src/utils/file';
import { mimeTypes } from 'src/utils/mime-types';
import { isFacialRecognitionEnabled } from 'src/utils/misc';
@@ -126,7 +127,10 @@ export class PersonService extends BaseService {
async getFacesById(auth: AuthDto, dto: FaceDto): Promise<AssetFaceResponseDto[]> {
await this.requireAccess({ auth, permission: Permission.AssetRead, ids: [dto.id] });
const faces = await this.personRepository.getFaces(dto.id);
return faces.map((asset) => mapFaces(asset, auth));
const asset = await this.assetRepository.getById(dto.id, { edits: true, exifInfo: true });
const assetDimensions = getDimensions(asset!.exifInfo!);
return faces.map((face) => mapFaces(face, auth, asset!.edits!, assetDimensions));
}
async createNewFeaturePhoto(changeFeaturePhoto: string[]) {
+2 -1
View File
@@ -23,7 +23,7 @@ describe(QueueService.name, () => {
it('should update concurrency', () => {
sut.onConfigUpdate({ newConfig: defaults, oldConfig: {} as SystemConfig });
expect(mocks.job.setConcurrency).toHaveBeenCalledTimes(17);
expect(mocks.job.setConcurrency).toHaveBeenCalledTimes(18);
expect(mocks.job.setConcurrency).toHaveBeenNthCalledWith(5, QueueName.FacialRecognition, 1);
expect(mocks.job.setConcurrency).toHaveBeenNthCalledWith(7, QueueName.DuplicateDetection, 1);
expect(mocks.job.setConcurrency).toHaveBeenNthCalledWith(8, QueueName.BackgroundTask, 5);
@@ -77,6 +77,7 @@ describe(QueueService.name, () => {
[QueueName.BackupDatabase]: expected,
[QueueName.Ocr]: expected,
[QueueName.Workflow]: expected,
[QueueName.Editor]: expected,
});
});
});
@@ -41,6 +41,7 @@ const updatedConfig = Object.freeze<SystemConfig>({
[QueueName.Notification]: { concurrency: 5 },
[QueueName.Ocr]: { concurrency: 1 },
[QueueName.Workflow]: { concurrency: 5 },
[QueueName.Editor]: { concurrency: 2 },
},
backup: {
database: {
+7 -11
View File
@@ -3,6 +3,7 @@ import { VECTOR_EXTENSIONS } from 'src/constants';
import { Asset, AssetFile } from 'src/database';
import { UploadFieldName } from 'src/dtos/asset-media.dto';
import { AuthDto } from 'src/dtos/auth.dto';
import { AssetEditActionItem } from 'src/dtos/editing.dto';
import {
AssetOrder,
AssetType,
@@ -25,13 +26,6 @@ export type DeepPartial<T> = T extends object ? { [K in keyof T]?: DeepPartial<T
export type RepositoryInterface<T extends object> = Pick<T, keyof T>;
export interface CropOptions {
top: number;
left: number;
width: number;
height: number;
}
export interface FullsizeImageOptions {
format: ImageFormat;
quality: number;
@@ -52,9 +46,9 @@ export interface RawImageInfo {
interface DecodeImageOptions {
colorspace: string;
crop?: CropOptions;
processInvalidImages: boolean;
raw?: RawImageInfo;
edits?: AssetEditActionItem[];
}
export interface DecodeToBufferOptions extends DecodeImageOptions {
@@ -72,7 +66,6 @@ export type GenerateThumbhashFromBufferOptions = GenerateThumbhashOptions & { ra
export interface GenerateThumbnailsOptions {
colorspace: string;
crop?: CropOptions;
preview?: ImageOptions;
processInvalidImages: boolean;
thumbhash?: boolean;
@@ -186,7 +179,7 @@ export interface IDelayedJob extends IBaseJob {
delay?: number;
}
export type JobSource = 'upload' | 'sidecar-write' | 'copy';
export type JobSource = 'upload' | 'sidecar-write' | 'copy' | 'edit';
export interface IEntityJob extends IBaseJob {
id: string;
source?: JobSource;
@@ -385,7 +378,10 @@ export type JobItem =
| { name: JobName.Ocr; data: IEntityJob }
// Workflow
| { name: JobName.WorkflowRun; data: IWorkflowJob };
| { name: JobName.WorkflowRun; data: IWorkflowJob }
// Editor
| { name: JobName.AssetEditThumbnailGeneration; data: IEntityJob };
export type VectorExtension = (typeof VECTOR_EXTENSIONS)[number];
+12
View File
@@ -157,6 +157,18 @@ const checkOtherAccess = async (access: AccessRepository, request: OtherAccessRe
return await access.asset.checkOwnerAccess(auth.user.id, ids, auth.session?.hasElevatedPermission);
}
case Permission.AssetEditGet: {
return await access.asset.checkOwnerAccess(auth.user.id, ids, auth.session?.hasElevatedPermission);
}
case Permission.AssetEditCreate: {
return await access.asset.checkOwnerAccess(auth.user.id, ids, auth.session?.hasElevatedPermission);
}
case Permission.AssetEditDelete: {
return await access.asset.checkOwnerAccess(auth.user.id, ids, auth.session?.hasElevatedPermission);
}
case Permission.AlbumRead: {
const isOwner = await access.album.checkOwnerAccess(auth.user.id, ids);
const isShared = await access.album.checkSharedAlbumAccess(
+29 -1
View File
@@ -1,9 +1,10 @@
import { BadRequestException } from '@nestjs/common';
import { GeneratedImageType, StorageCore } from 'src/cores/storage.core';
import { AssetFile } from 'src/database';
import { AssetFile, Exif } from 'src/database';
import { BulkIdErrorReason, BulkIdResponseDto } from 'src/dtos/asset-ids.response.dto';
import { UploadFieldName } from 'src/dtos/asset-media.dto';
import { AuthDto } from 'src/dtos/auth.dto';
import { ExifResponseDto } from 'src/dtos/exif.dto';
import { AssetFileType, AssetType, AssetVisibility, Permission } from 'src/enum';
import { AuthRequest } from 'src/middleware/auth.guard';
import { AccessRepository } from 'src/repositories/access.repository';
@@ -22,6 +23,10 @@ export const getAssetFiles = (files: AssetFile[]) => ({
previewFile: getAssetFile(files, AssetFileType.Preview),
thumbnailFile: getAssetFile(files, AssetFileType.Thumbnail),
sidecarFile: getAssetFile(files, AssetFileType.Sidecar),
editedFullsizeFile: getAssetFile(files, AssetFileType.FullSizeEdited),
editedPreviewFile: getAssetFile(files, AssetFileType.PreviewEdited),
editedThumbnailFile: getAssetFile(files, AssetFileType.ThumbnailEdited),
});
export const addAssets = async (
@@ -199,3 +204,26 @@ export const asUploadRequest = (request: AuthRequest, file: Express.Multer.File)
file: mapToUploadFile(file as ImmichFile),
};
};
const isFlipped = (orientation?: string | null) => {
const value = Number(orientation);
return value && [5, 6, 7, 8, -90, 90].includes(value);
};
export const getDimensions = (exifInfo: ExifResponseDto | Exif) => {
const { exifImageWidth: width, exifImageHeight: height } = exifInfo;
if (!width || !height) {
return { width: 0, height: 0 };
}
if (isFlipped(exifInfo.orientation)) {
return { width: height, height: width };
}
return { width, height };
};
export const isPanorama = (asset: { exifInfo?: Exif | null; originalFileName: string }) => {
return asset.exifInfo?.projectionType === 'EQUIRECTANGULAR' || asset.originalFileName.toLowerCase().endsWith('.insp');
};
+24 -4
View File
@@ -1,4 +1,5 @@
import {
AliasedRawBuilder,
DeduplicateJoinsPlugin,
Expression,
ExpressionBuilder,
@@ -16,6 +17,7 @@ import { jsonArrayFrom, jsonObjectFrom } from 'kysely/helpers/postgres';
import { parse } from 'pg-connection-string';
import postgres, { Notice, PostgresError } from 'postgres';
import { columns, Exif, lockableProperties, LockableProperty, Person } from 'src/database';
import { AssetEditActionItem } from 'src/dtos/editing.dto';
import { AssetFileType, AssetVisibility, DatabaseExtension, DatabaseSslMode } from 'src/enum';
import { AssetSearchBuilderOptions } from 'src/repositories/search.repository';
import { DB } from 'src/schema';
@@ -180,13 +182,14 @@ export function withSmartSearch<O>(qb: SelectQueryBuilder<DB, 'asset', O>) {
.select((eb) => toJson(eb, 'smart_search').as('smartSearch'));
}
export function withFaces(eb: ExpressionBuilder<DB, 'asset'>, withDeletedFace?: boolean) {
export function withFaces(eb: ExpressionBuilder<DB, 'asset'>, withHidden?: boolean, withDeletedFace?: boolean) {
return jsonArrayFrom(
eb
.selectFrom('asset_face')
.selectAll('asset_face')
.whereRef('asset_face.assetId', '=', 'asset.id')
.$if(!withDeletedFace, (qb) => qb.where('asset_face.deletedAt', 'is', null)),
.$if(!withDeletedFace, (qb) => qb.where('asset_face.deletedAt', 'is', null))
.$if(!withHidden, (qb) => qb.where('asset_face.isVisible', '=', true)),
).as('faces');
}
@@ -208,7 +211,11 @@ export function withFilePath(eb: ExpressionBuilder<DB, 'asset'>, type: AssetFile
.where('asset_file.type', '=', type);
}
export function withFacesAndPeople(eb: ExpressionBuilder<DB, 'asset'>, withDeletedFace?: boolean) {
export function withFacesAndPeople(
eb: ExpressionBuilder<DB, 'asset'>,
withHidden?: boolean,
withDeletedFace?: boolean,
) {
return jsonArrayFrom(
eb
.selectFrom('asset_face')
@@ -220,7 +227,8 @@ export function withFacesAndPeople(eb: ExpressionBuilder<DB, 'asset'>, withDelet
.selectAll('asset_face')
.select((eb) => eb.table('person').$castTo<Person>().as('person'))
.whereRef('asset_face.assetId', '=', 'asset.id')
.$if(!withDeletedFace, (qb) => qb.where('asset_face.deletedAt', 'is', null)),
.$if(!withDeletedFace, (qb) => qb.where('asset_face.deletedAt', 'is', null))
.$if(!withHidden, (qb) => qb.where('asset_face.isVisible', 'is', true)),
).as('faces');
}
@@ -232,6 +240,7 @@ export function hasPeople<O>(qb: SelectQueryBuilder<DB, 'asset', O>, personIds:
.select('assetId')
.where('personId', '=', anyUuid(personIds!))
.where('deletedAt', 'is', null)
.where('isVisible', 'is', true)
.groupBy('assetId')
.having((eb) => eb.fn.count('personId').distinct(), '=', personIds.length)
.as('has_people'),
@@ -346,6 +355,17 @@ export const tokenizeForSearch = (text: string): string[] => {
return tokens;
};
// needed to properly type the return with the EditActionItem discriminated union type
type AliasedEditActions = AliasedRawBuilder<AssetEditActionItem[], 'edits'>;
export function withEdits(eb: ExpressionBuilder<DB, 'asset'>): AliasedEditActions {
return jsonArrayFrom(
eb
.selectFrom('asset_edit')
.select(['asset_edit.action', 'asset_edit.parameters'])
.whereRef('asset_edit.assetId', '=', 'asset.id'),
).as('edits') as AliasedEditActions;
}
const joinDeduplicationPlugin = new DeduplicateJoinsPlugin();
/** TODO: This should only be used for search-related queries, not as a general purpose query builder */
+505
View File
@@ -0,0 +1,505 @@
import { AssetFace } from 'src/database';
import { AssetOcrResponseDto } from 'src/dtos/ocr.dto';
import { SourceType } from 'src/enum';
import { boundingBoxOverlap, checkFaceVisibility, checkOcrVisibility } from 'src/utils/editor';
import { describe, expect, it } from 'vitest';
describe('boundingBoxOverlap', () => {
it('should return 1 for identical boxes', () => {
const box = { x1: 0, y1: 0, x2: 100, y2: 100 };
expect(boundingBoxOverlap(box, box)).toBe(1);
});
it('should return 0 for non-overlapping boxes', () => {
const boxA = { x1: 0, y1: 0, x2: 100, y2: 100 };
const boxB = { x1: 200, y1: 200, x2: 300, y2: 300 };
expect(boundingBoxOverlap(boxA, boxB)).toBe(0);
});
it('should return 0.5 for 50% overlap', () => {
const boxA = { x1: 0, y1: 0, x2: 100, y2: 100 };
const boxB = { x1: 50, y1: 0, x2: 150, y2: 100 };
expect(boundingBoxOverlap(boxA, boxB)).toBe(0.5);
});
it('should return 0.25 for 25% overlap', () => {
const boxA = { x1: 0, y1: 0, x2: 100, y2: 100 };
const boxB = { x1: 50, y1: 50, x2: 150, y2: 150 };
expect(boundingBoxOverlap(boxA, boxB)).toBe(0.25);
});
it('should return 1 when boxA is fully contained in boxB', () => {
const boxA = { x1: 25, y1: 25, x2: 75, y2: 75 };
const boxB = { x1: 0, y1: 0, x2: 100, y2: 100 };
expect(boundingBoxOverlap(boxA, boxB)).toBe(1);
});
it('should handle partial containment correctly', () => {
const boxA = { x1: 0, y1: 0, x2: 100, y2: 100 };
const boxB = { x1: 25, y1: 25, x2: 75, y2: 75 };
// boxB is fully inside boxA, so overlap area is 50*50=2500, boxA area is 10000
expect(boundingBoxOverlap(boxA, boxB)).toBe(0.25);
});
it('should handle boxes that touch at edges (no overlap)', () => {
const boxA = { x1: 0, y1: 0, x2: 100, y2: 100 };
const boxB = { x1: 100, y1: 0, x2: 200, y2: 100 };
expect(boundingBoxOverlap(boxA, boxB)).toBe(0);
});
it('should handle vertical partial overlap', () => {
const boxA = { x1: 0, y1: 0, x2: 100, y2: 100 };
const boxB = { x1: 0, y1: 50, x2: 100, y2: 150 };
expect(boundingBoxOverlap(boxA, boxB)).toBe(0.5);
});
});
const createFace = (params: Partial<AssetFace> = {}): AssetFace => ({
id: 'face-id',
deletedAt: null,
assetId: 'asset-id',
boundingBoxX1: 100,
boundingBoxX2: 200,
boundingBoxY1: 100,
boundingBoxY2: 200,
imageWidth: 1000,
imageHeight: 1000,
personId: null,
sourceType: SourceType.MachineLearning,
person: null,
updatedAt: new Date(),
updateId: 'update-id',
isVisible: true,
...params,
});
describe('checkFaceVisibility', () => {
const assetDimensions = { width: 1000, height: 1000 };
it('should return only non-visible faces when no crop is provided', () => {
const faces = [
createFace({ id: 'face-1', isVisible: true }),
createFace({ id: 'face-2', isVisible: false }),
createFace({ id: 'face-3', isVisible: false }),
];
const result = checkFaceVisibility(faces, assetDimensions);
expect(result.visible).toHaveLength(2);
expect(result.hidden).toHaveLength(0);
expect(result.visible.map((f) => f.id)).toEqual(['face-2', 'face-3']);
});
it('should return all faces as visible when all are marked not visible and no crop provided', () => {
const faces = [createFace({ id: 'face-1', isVisible: false }), createFace({ id: 'face-2', isVisible: false })];
const result = checkFaceVisibility(faces, assetDimensions);
expect(result.visible).toHaveLength(2);
expect(result.hidden).toHaveLength(0);
});
it('should return empty visible array when all faces are already visible and no crop provided', () => {
const faces = [createFace({ id: 'face-1', isVisible: true }), createFace({ id: 'face-2', isVisible: true })];
const result = checkFaceVisibility(faces, assetDimensions);
expect(result.visible).toHaveLength(0);
expect(result.hidden).toHaveLength(0);
});
it('should return empty arrays when no faces provided', () => {
const result = checkFaceVisibility([], assetDimensions);
expect(result.visible).toHaveLength(0);
expect(result.hidden).toHaveLength(0);
});
it('should mark face as visible when fully inside crop area', () => {
const faces = [createFace({ boundingBoxX1: 100, boundingBoxY1: 100, boundingBoxX2: 200, boundingBoxY2: 200 })];
const crop = { x1: 0, y1: 0, x2: 500, y2: 500 };
const result = checkFaceVisibility(faces, assetDimensions, crop);
expect(result.visible).toHaveLength(1);
expect(result.hidden).toHaveLength(0);
});
it('should mark face as hidden when fully outside crop area', () => {
const faces = [createFace({ boundingBoxX1: 600, boundingBoxY1: 600, boundingBoxX2: 700, boundingBoxY2: 700 })];
const crop = { x1: 0, y1: 0, x2: 500, y2: 500 };
const result = checkFaceVisibility(faces, assetDimensions, crop);
expect(result.visible).toHaveLength(0);
expect(result.hidden).toHaveLength(1);
});
it('should mark face as visible when at least 50% overlaps with crop', () => {
// Face spans 100-200 (100px), crop starts at 150, so 50% overlap
const faces = [createFace({ boundingBoxX1: 100, boundingBoxY1: 100, boundingBoxX2: 200, boundingBoxY2: 200 })];
const crop = { x1: 150, y1: 100, x2: 500, y2: 500 };
const result = checkFaceVisibility(faces, assetDimensions, crop);
expect(result.visible).toHaveLength(1);
expect(result.hidden).toHaveLength(0);
});
it('should mark face as hidden when less than 50% overlaps with crop', () => {
// Face spans 100-200 (100px), crop starts at 160, so 40% overlap
const faces = [createFace({ boundingBoxX1: 100, boundingBoxY1: 100, boundingBoxX2: 200, boundingBoxY2: 200 })];
const crop = { x1: 160, y1: 100, x2: 500, y2: 500 };
const result = checkFaceVisibility(faces, assetDimensions, crop);
expect(result.visible).toHaveLength(0);
expect(result.hidden).toHaveLength(1);
});
it('should correctly categorize multiple faces', () => {
const faces = [
createFace({ id: 'face-inside', boundingBoxX1: 100, boundingBoxY1: 100, boundingBoxX2: 200, boundingBoxY2: 200 }),
createFace({
id: 'face-outside',
boundingBoxX1: 800,
boundingBoxY1: 800,
boundingBoxX2: 900,
boundingBoxY2: 900,
}),
// face-partial: 400-500 overlaps with crop (100x100=10000 overlap, face is 200x200=40000, so 25% - hidden)
createFace({
id: 'face-partial',
boundingBoxX1: 400,
boundingBoxY1: 400,
boundingBoxX2: 600,
boundingBoxY2: 600,
}),
];
const crop = { x1: 0, y1: 0, x2: 500, y2: 500 };
const result = checkFaceVisibility(faces, assetDimensions, crop);
// face-inside is fully visible, face-partial has 25% overlap (hidden), face-outside is fully hidden
expect(result.visible).toHaveLength(1);
expect(result.hidden).toHaveLength(2);
expect(result.visible.map((f) => f.id)).toContain('face-inside');
expect(result.hidden.map((f) => f.id)).toContain('face-partial');
expect(result.hidden.map((f) => f.id)).toContain('face-outside');
});
it('should handle face coordinates scaled to different image dimensions', () => {
// Face stored at 50-100 in a 500x500 image, scaled to 1000x1000 becomes 100-200
const faces = [
createFace({
boundingBoxX1: 50,
boundingBoxY1: 50,
boundingBoxX2: 100,
boundingBoxY2: 100,
imageWidth: 500,
imageHeight: 500,
}),
];
const crop = { x1: 0, y1: 0, x2: 200, y2: 200 };
const result = checkFaceVisibility(faces, assetDimensions, crop);
expect(result.visible).toHaveLength(1);
expect(result.hidden).toHaveLength(0);
});
it('should categorize based on crop overlap when crop is provided, regardless of isVisible property', () => {
const faces = [
createFace({
id: 'face-inside-visible',
boundingBoxX1: 100,
boundingBoxY1: 100,
boundingBoxX2: 200,
boundingBoxY2: 200,
isVisible: true,
}),
createFace({
id: 'face-inside-not-visible',
boundingBoxX1: 250,
boundingBoxY1: 250,
boundingBoxX2: 350,
boundingBoxY2: 350,
isVisible: false,
}),
createFace({
id: 'face-outside-visible',
boundingBoxX1: 800,
boundingBoxY1: 800,
boundingBoxX2: 900,
boundingBoxY2: 900,
isVisible: true,
}),
createFace({
id: 'face-outside-not-visible',
boundingBoxX1: 700,
boundingBoxY1: 700,
boundingBoxX2: 800,
boundingBoxY2: 800,
isVisible: false,
}),
];
const crop = { x1: 0, y1: 0, x2: 500, y2: 500 };
const result = checkFaceVisibility(faces, assetDimensions, crop);
// When crop is provided, only overlap matters, not isVisible property
expect(result.visible).toHaveLength(2);
expect(result.hidden).toHaveLength(2);
expect(result.visible.map((f) => f.id)).toContain('face-inside-visible');
expect(result.visible.map((f) => f.id)).toContain('face-inside-not-visible');
expect(result.hidden.map((f) => f.id)).toContain('face-outside-visible');
expect(result.hidden.map((f) => f.id)).toContain('face-outside-not-visible');
});
it('should handle mixed visibility states with partial overlap and crop', () => {
const faces = [
createFace({
id: 'face-partial-50',
boundingBoxX1: 100,
boundingBoxY1: 100,
boundingBoxX2: 200,
boundingBoxY2: 200,
isVisible: true,
}),
createFace({
id: 'face-partial-40',
boundingBoxX1: 100,
boundingBoxY1: 100,
boundingBoxX2: 200,
boundingBoxY2: 200,
isVisible: false,
}),
];
const crop1 = { x1: 150, y1: 100, x2: 500, y2: 500 }; // 50% overlap
const crop2 = { x1: 160, y1: 100, x2: 500, y2: 500 }; // 40% overlap
const result1 = checkFaceVisibility([faces[0]], assetDimensions, crop1);
const result2 = checkFaceVisibility([faces[1]], assetDimensions, crop2);
// 50% overlap should be visible
expect(result1.visible).toHaveLength(1);
expect(result1.hidden).toHaveLength(0);
// 40% overlap should be hidden
expect(result2.visible).toHaveLength(0);
expect(result2.hidden).toHaveLength(1);
});
});
const createOcr = (
params: Partial<AssetOcrResponseDto & { isVisible: boolean }> = {},
): AssetOcrResponseDto & { isVisible: boolean } => ({
id: 'ocr-id',
assetId: 'asset-id',
x1: 0.1,
y1: 0.1,
x2: 0.2,
y2: 0.1,
x3: 0.2,
y3: 0.2,
x4: 0.1,
y4: 0.2,
boxScore: 0.9,
textScore: 0.9,
text: 'Sample Text',
isVisible: true,
...params,
});
describe('checkOcrVisibility', () => {
const assetDimensions = { width: 1000, height: 1000 };
it('should return only non-visible OCR entries when no crop is provided', () => {
const ocrs = [
createOcr({ id: 'ocr-1', isVisible: true }),
createOcr({ id: 'ocr-2', isVisible: false }),
createOcr({ id: 'ocr-3', isVisible: false }),
];
const result = checkOcrVisibility(ocrs, assetDimensions);
expect(result.visible).toHaveLength(2);
expect(result.hidden).toHaveLength(0);
expect(result.visible.map((o) => o.id)).toEqual(['ocr-2', 'ocr-3']);
});
it('should return all OCR entries as visible when all are marked not visible and no crop provided', () => {
const ocrs = [createOcr({ id: 'ocr-1', isVisible: false }), createOcr({ id: 'ocr-2', isVisible: false })];
const result = checkOcrVisibility(ocrs, assetDimensions);
expect(result.visible).toHaveLength(2);
expect(result.hidden).toHaveLength(0);
});
it('should return empty visible array when all OCR entries are already visible and no crop provided', () => {
const ocrs = [createOcr({ id: 'ocr-1', isVisible: true }), createOcr({ id: 'ocr-2', isVisible: true })];
const result = checkOcrVisibility(ocrs, assetDimensions);
expect(result.visible).toHaveLength(0);
expect(result.hidden).toHaveLength(0);
});
it('should return empty arrays when no OCR entries provided', () => {
const result = checkOcrVisibility([], assetDimensions);
expect(result.visible).toHaveLength(0);
expect(result.hidden).toHaveLength(0);
});
it('should mark OCR as visible when fully inside crop area', () => {
// OCR box at normalized coords 0.1-0.2 = 100-200px in 1000x1000 image
const ocrs = [createOcr()];
const crop = { x1: 0, y1: 0, x2: 500, y2: 500 };
const result = checkOcrVisibility(ocrs, assetDimensions, crop);
expect(result.visible).toHaveLength(1);
expect(result.hidden).toHaveLength(0);
});
it('should mark OCR as hidden when fully outside crop area', () => {
// OCR box at normalized coords 0.8-0.9 = 800-900px
const ocrs = [createOcr({ x1: 0.8, y1: 0.8, x2: 0.9, y2: 0.8, x3: 0.9, y3: 0.9, x4: 0.8, y4: 0.9 })];
const crop = { x1: 0, y1: 0, x2: 500, y2: 500 };
const result = checkOcrVisibility(ocrs, assetDimensions, crop);
expect(result.visible).toHaveLength(0);
expect(result.hidden).toHaveLength(1);
});
it('should mark OCR as visible when at least 50% overlaps with crop', () => {
// OCR at 100-200px (0.1-0.2 normalized), crop starts at 150
const ocrs = [createOcr()];
const crop = { x1: 150, y1: 100, x2: 500, y2: 500 };
const result = checkOcrVisibility(ocrs, assetDimensions, crop);
expect(result.visible).toHaveLength(1);
expect(result.hidden).toHaveLength(0);
});
it('should mark OCR as hidden when less than 50% overlaps with crop', () => {
// OCR at 100-200px, crop starts at 160 = 40% overlap
const ocrs = [createOcr()];
const crop = { x1: 160, y1: 100, x2: 500, y2: 500 };
const result = checkOcrVisibility(ocrs, assetDimensions, crop);
expect(result.visible).toHaveLength(0);
expect(result.hidden).toHaveLength(1);
});
it('should correctly categorize multiple OCR entries', () => {
const ocrs = [
createOcr({ id: 'ocr-inside', x1: 0.1, y1: 0.1, x2: 0.2, y2: 0.1, x3: 0.2, y3: 0.2, x4: 0.1, y4: 0.2 }),
createOcr({ id: 'ocr-outside', x1: 0.8, y1: 0.8, x2: 0.9, y2: 0.8, x3: 0.9, y3: 0.9, x4: 0.8, y4: 0.9 }),
];
const crop = { x1: 0, y1: 0, x2: 500, y2: 500 };
const result = checkOcrVisibility(ocrs, assetDimensions, crop);
expect(result.visible).toHaveLength(1);
expect(result.hidden).toHaveLength(1);
expect(result.visible[0].id).toBe('ocr-inside');
expect(result.hidden[0].id).toBe('ocr-outside');
});
it('should handle rotated/skewed OCR polygons by using bounding box', () => {
// Rotated rectangle - the function should compute the bounding box correctly
const ocrs = [
createOcr({
id: 'ocr-rotated',
x1: 0.15,
y1: 0.1, // top
x2: 0.2,
y2: 0.15, // right
x3: 0.15,
y3: 0.2, // bottom
x4: 0.1,
y4: 0.15, // left
}),
];
const crop = { x1: 0, y1: 0, x2: 500, y2: 500 };
const result = checkOcrVisibility(ocrs, assetDimensions, crop);
expect(result.visible).toHaveLength(1);
expect(result.hidden).toHaveLength(0);
});
it('should handle different asset dimensions', () => {
const smallDimensions = { width: 500, height: 500 };
// OCR at 0.1-0.2 normalized = 50-100px in 500x500 image
const ocrs = [createOcr()];
const crop = { x1: 0, y1: 0, x2: 200, y2: 200 };
const result = checkOcrVisibility(ocrs, smallDimensions, crop);
expect(result.visible).toHaveLength(1);
expect(result.hidden).toHaveLength(0);
});
it('should categorize based on crop overlap when crop is provided, regardless of isVisible property', () => {
const ocrs = [
createOcr({ id: 'ocr-inside-visible', isVisible: true }), // Inside crop, already visible
createOcr({ id: 'ocr-inside-not-visible', isVisible: false }), // Inside crop, not visible
createOcr({
id: 'ocr-outside-visible',
x1: 0.8,
y1: 0.8,
x2: 0.9,
y2: 0.8,
x3: 0.9,
y3: 0.9,
x4: 0.8,
y4: 0.9,
isVisible: true,
}), // Outside crop, already visible
createOcr({
id: 'ocr-outside-not-visible',
x1: 0.8,
y1: 0.8,
x2: 0.9,
y2: 0.8,
x3: 0.9,
y3: 0.9,
x4: 0.8,
y4: 0.9,
isVisible: false,
}), // Outside crop, not visible
];
const crop = { x1: 0, y1: 0, x2: 500, y2: 500 };
const result = checkOcrVisibility(ocrs, assetDimensions, crop);
// When crop is provided, only overlap matters, not isVisible property
expect(result.visible).toHaveLength(2);
expect(result.hidden).toHaveLength(2);
expect(result.visible.map((o) => o.id)).toContain('ocr-inside-visible');
expect(result.visible.map((o) => o.id)).toContain('ocr-inside-not-visible');
expect(result.hidden.map((o) => o.id)).toContain('ocr-outside-visible');
expect(result.hidden.map((o) => o.id)).toContain('ocr-outside-not-visible');
});
it('should handle mixed visibility states with partial overlap and crop', () => {
const ocrs = [
createOcr({ id: 'ocr-partial-50', isVisible: true }), // 50% overlap
createOcr({ id: 'ocr-partial-40', isVisible: false }), // 40% overlap
];
const crop1 = { x1: 150, y1: 100, x2: 500, y2: 500 }; // 50% overlap
const crop2 = { x1: 160, y1: 100, x2: 500, y2: 500 }; // 40% overlap
const result1 = checkOcrVisibility([ocrs[0]], assetDimensions, crop1);
const result2 = checkOcrVisibility([ocrs[1]], assetDimensions, crop2);
// 50% overlap should be visible
expect(result1.visible).toHaveLength(1);
expect(result1.hidden).toHaveLength(0);
// 40% overlap should be hidden
expect(result2.visible).toHaveLength(0);
expect(result2.hidden).toHaveLength(1);
});
});
+107
View File
@@ -0,0 +1,107 @@
import { AssetFace } from 'src/database';
import { AssetOcrResponseDto } from 'src/dtos/ocr.dto';
import { ImageDimensions } from 'src/types';
type BoundingBox = {
x1: number;
y1: number;
x2: number;
y2: number;
};
export const boundingBoxOverlap = (boxA: BoundingBox, boxB: BoundingBox) => {
const overlapX1 = Math.max(boxA.x1, boxB.x1);
const overlapY1 = Math.max(boxA.y1, boxB.y1);
const overlapX2 = Math.min(boxA.x2, boxB.x2);
const overlapY2 = Math.min(boxA.y2, boxB.y2);
const overlapArea = Math.max(0, overlapX2 - overlapX1) * Math.max(0, overlapY2 - overlapY1);
const faceArea = (boxA.x2 - boxA.x1) * (boxA.y2 - boxA.y1);
return overlapArea / faceArea;
};
const scale = (box: BoundingBox, target: ImageDimensions, source?: ImageDimensions) => {
const { width: sourceWidth = 1, height: sourceHeight = 1 } = source ?? {};
return {
x1: (box.x1 / sourceWidth) * target.width,
y1: (box.y1 / sourceHeight) * target.height,
x2: (box.x2 / sourceWidth) * target.width,
y2: (box.y2 / sourceHeight) * target.height,
};
};
export const checkFaceVisibility = (
faces: AssetFace[],
originalAssetDimensions: ImageDimensions,
crop?: BoundingBox,
): { visible: AssetFace[]; hidden: AssetFace[] } => {
if (!crop) {
return {
visible: faces.filter((face) => !face.isVisible),
hidden: [],
};
}
const status = faces.map((face) => {
const scaledFace = scale(
{
x1: face.boundingBoxX1,
y1: face.boundingBoxY1,
x2: face.boundingBoxX2,
y2: face.boundingBoxY2,
},
originalAssetDimensions,
{ width: face.imageWidth, height: face.imageHeight },
);
const overlapPercentage = boundingBoxOverlap(scaledFace, crop);
return {
face,
isVisible: overlapPercentage >= 0.5,
};
});
return {
visible: status.filter((s) => s.isVisible).map((s) => s.face),
hidden: status.filter((s) => !s.isVisible).map((s) => s.face),
};
};
export const checkOcrVisibility = (
ocrs: (AssetOcrResponseDto & { isVisible: boolean })[],
originalAssetDimensions: ImageDimensions,
crop?: BoundingBox,
): { visible: AssetOcrResponseDto[]; hidden: AssetOcrResponseDto[] } => {
if (!crop) {
return {
visible: ocrs.filter((ocr) => !ocr.isVisible),
hidden: [],
};
}
const status = ocrs.map((ocr) => {
const ocrBox = scale(
{
x1: Math.min(ocr.x1, ocr.x2, ocr.x3, ocr.x4),
y1: Math.min(ocr.y1, ocr.y2, ocr.y3, ocr.y4),
x2: Math.max(ocr.x1, ocr.x2, ocr.x3, ocr.x4),
y2: Math.max(ocr.y1, ocr.y2, ocr.y3, ocr.y4),
},
originalAssetDimensions,
);
const overlapPercentage = boundingBoxOverlap(ocrBox, crop);
return {
ocr,
isVisible: overlapPercentage >= 0.5,
};
});
return {
visible: status.filter((s) => s.isVisible).map((s) => s.ocr),
hidden: status.filter((s) => !s.isVisible).map((s) => s.ocr),
};
};
+293
View File
@@ -0,0 +1,293 @@
import { AssetEditAction, AssetEditActionItem, MirrorAxis } from 'src/dtos/editing.dto';
import { AssetOcrResponseDto } from 'src/dtos/ocr.dto';
import { transformFaceBoundingBox, transformOcrBoundingBox } from 'src/utils/transform';
import { describe, expect, it } from 'vitest';
describe('transformFaceBoundingBox', () => {
const baseFace = {
boundingBoxX1: 100,
boundingBoxY1: 100,
boundingBoxX2: 200,
boundingBoxY2: 200,
imageWidth: 1000,
imageHeight: 800,
};
const baseDimensions = { width: 1000, height: 800 };
describe('with no edits', () => {
it('should return unchanged bounding box', () => {
const result = transformFaceBoundingBox(baseFace, [], baseDimensions);
expect(result).toEqual(baseFace);
});
});
describe('with crop edit', () => {
it('should adjust bounding box for crop offset', () => {
const edits: AssetEditActionItem[] = [
{ action: AssetEditAction.Crop, parameters: { x: 50, y: 50, width: 400, height: 300 } },
];
const result = transformFaceBoundingBox(baseFace, edits, baseDimensions);
expect(result.boundingBoxX1).toBe(50);
expect(result.boundingBoxY1).toBe(50);
expect(result.boundingBoxX2).toBe(150);
expect(result.boundingBoxY2).toBe(150);
expect(result.imageWidth).toBe(400);
expect(result.imageHeight).toBe(300);
});
it('should handle face partially outside crop area', () => {
const edits: AssetEditActionItem[] = [
{ action: AssetEditAction.Crop, parameters: { x: 150, y: 150, width: 400, height: 300 } },
];
const result = transformFaceBoundingBox(baseFace, edits, baseDimensions);
expect(result.boundingBoxX1).toBe(-50);
expect(result.boundingBoxY1).toBe(-50);
expect(result.boundingBoxX2).toBe(50);
expect(result.boundingBoxY2).toBe(50);
});
});
describe('with rotate edit', () => {
it('should rotate 90 degrees clockwise', () => {
const edits: AssetEditActionItem[] = [{ action: AssetEditAction.Rotate, parameters: { angle: 90 } }];
const result = transformFaceBoundingBox(baseFace, edits, baseDimensions);
expect(result.imageWidth).toBe(800);
expect(result.imageHeight).toBe(1000);
expect(result.boundingBoxX1).toBe(600);
expect(result.boundingBoxY1).toBe(100);
expect(result.boundingBoxX2).toBe(700);
expect(result.boundingBoxY2).toBe(200);
});
it('should rotate 180 degrees', () => {
const edits: AssetEditActionItem[] = [{ action: AssetEditAction.Rotate, parameters: { angle: 180 } }];
const result = transformFaceBoundingBox(baseFace, edits, baseDimensions);
expect(result.imageWidth).toBe(1000);
expect(result.imageHeight).toBe(800);
expect(result.boundingBoxX1).toBe(800);
expect(result.boundingBoxY1).toBe(600);
expect(result.boundingBoxX2).toBe(900);
expect(result.boundingBoxY2).toBe(700);
});
it('should rotate 270 degrees', () => {
const edits: AssetEditActionItem[] = [{ action: AssetEditAction.Rotate, parameters: { angle: 270 } }];
const result = transformFaceBoundingBox(baseFace, edits, baseDimensions);
expect(result.imageWidth).toBe(800);
expect(result.imageHeight).toBe(1000);
});
});
describe('with mirror edit', () => {
it('should mirror horizontally', () => {
const edits: AssetEditActionItem[] = [
{ action: AssetEditAction.Mirror, parameters: { axis: MirrorAxis.Horizontal } },
];
const result = transformFaceBoundingBox(baseFace, edits, baseDimensions);
expect(result.boundingBoxX1).toBe(800);
expect(result.boundingBoxY1).toBe(100);
expect(result.boundingBoxX2).toBe(900);
expect(result.boundingBoxY2).toBe(200);
expect(result.imageWidth).toBe(1000);
expect(result.imageHeight).toBe(800);
});
it('should mirror vertically', () => {
const edits: AssetEditActionItem[] = [
{ action: AssetEditAction.Mirror, parameters: { axis: MirrorAxis.Vertical } },
];
const result = transformFaceBoundingBox(baseFace, edits, baseDimensions);
expect(result.boundingBoxX1).toBe(100);
expect(result.boundingBoxY1).toBe(600);
expect(result.boundingBoxX2).toBe(200);
expect(result.boundingBoxY2).toBe(700);
expect(result.imageWidth).toBe(1000);
expect(result.imageHeight).toBe(800);
});
});
describe('with combined edits', () => {
it('should apply crop then rotate', () => {
const edits: AssetEditActionItem[] = [
{ action: AssetEditAction.Crop, parameters: { x: 50, y: 50, width: 400, height: 300 } },
{ action: AssetEditAction.Rotate, parameters: { angle: 90 } },
];
const result = transformFaceBoundingBox(baseFace, edits, baseDimensions);
expect(result.imageWidth).toBe(300);
expect(result.imageHeight).toBe(400);
});
it('should apply crop then mirror', () => {
const edits: AssetEditActionItem[] = [
{ action: AssetEditAction.Crop, parameters: { x: 0, y: 0, width: 500, height: 400 } },
{ action: AssetEditAction.Mirror, parameters: { axis: MirrorAxis.Vertical } },
];
const result = transformFaceBoundingBox(baseFace, edits, baseDimensions);
expect(result.boundingBoxX1).toBe(100);
expect(result.boundingBoxX2).toBe(200);
expect(result.boundingBoxY1).toBe(200);
expect(result.boundingBoxY2).toBe(300);
});
});
describe('with scaled dimensions', () => {
it('should scale face to match different image dimensions', () => {
const scaledDimensions = { width: 500, height: 400 }; // Half the original size
const edits: AssetEditActionItem[] = [
{ action: AssetEditAction.Crop, parameters: { x: 50, y: 50, width: 200, height: 150 } },
];
const result = transformFaceBoundingBox(baseFace, edits, scaledDimensions);
expect(result.boundingBoxX1).toBe(0);
expect(result.boundingBoxY1).toBe(0);
expect(result.boundingBoxX2).toBe(50);
expect(result.boundingBoxY2).toBe(50);
});
});
});
describe('transformOcrBoundingBox', () => {
const baseOcr: AssetOcrResponseDto = {
id: 'ocr-1',
assetId: 'asset-1',
x1: 0.1,
y1: 0.1,
x2: 0.2,
y2: 0.1,
x3: 0.2,
y3: 0.2,
x4: 0.1,
y4: 0.2,
boxScore: 0.9,
textScore: 0.85,
text: 'Test OCR',
};
const baseDimensions = { width: 1000, height: 800 };
describe('with no edits', () => {
it('should return unchanged bounding box', () => {
const result = transformOcrBoundingBox(baseOcr, [], baseDimensions);
expect(result).toEqual(baseOcr);
});
});
describe('with crop edit', () => {
it('should adjust normalized coordinates for crop', () => {
const edits: AssetEditActionItem[] = [
{ action: AssetEditAction.Crop, parameters: { x: 100, y: 80, width: 400, height: 320 } },
];
const result = transformOcrBoundingBox(baseOcr, edits, baseDimensions);
// Original OCR: (0.1,0.1)-(0.2,0.2) on 1000x800 = (100,80)-(200,160)
// After crop offset (100,80): (0,0)-(100,80)
// Normalized to 400x320: (0,0)-(0.25,0.25)
expect(result.x1).toBeCloseTo(0, 5);
expect(result.y1).toBeCloseTo(0, 5);
expect(result.x2).toBeCloseTo(0.25, 5);
expect(result.y2).toBeCloseTo(0, 5);
expect(result.x3).toBeCloseTo(0.25, 5);
expect(result.y3).toBeCloseTo(0.25, 5);
expect(result.x4).toBeCloseTo(0, 5);
expect(result.y4).toBeCloseTo(0.25, 5);
});
});
describe('with rotate edit', () => {
it('should rotate normalized coordinates 90 degrees and reorder points', () => {
const edits: AssetEditActionItem[] = [{ action: AssetEditAction.Rotate, parameters: { angle: 90 } }];
const result = transformOcrBoundingBox(baseOcr, edits, baseDimensions);
expect(result.id).toBe(baseOcr.id);
expect(result.text).toBe(baseOcr.text);
expect(result.x1).toBeCloseTo(0.8, 5);
expect(result.y1).toBeCloseTo(0.1, 5);
expect(result.x2).toBeCloseTo(0.9, 5);
expect(result.y2).toBeCloseTo(0.1, 5);
expect(result.x3).toBeCloseTo(0.9, 5);
expect(result.y3).toBeCloseTo(0.2, 5);
expect(result.x4).toBeCloseTo(0.8, 5);
expect(result.y4).toBeCloseTo(0.2, 5);
});
it('should rotate 180 degrees and reorder points', () => {
const edits: AssetEditActionItem[] = [{ action: AssetEditAction.Rotate, parameters: { angle: 180 } }];
const result = transformOcrBoundingBox(baseOcr, edits, baseDimensions);
expect(result.x1).toBeCloseTo(0.8, 5);
expect(result.y1).toBeCloseTo(0.8, 5);
expect(result.x2).toBeCloseTo(0.9, 5);
expect(result.y2).toBeCloseTo(0.8, 5);
expect(result.x3).toBeCloseTo(0.9, 5);
expect(result.y3).toBeCloseTo(0.9, 5);
expect(result.x4).toBeCloseTo(0.8, 5);
expect(result.y4).toBeCloseTo(0.9, 5);
});
it('should rotate 270 degrees and reorder points', () => {
const edits: AssetEditActionItem[] = [{ action: AssetEditAction.Rotate, parameters: { angle: 270 } }];
const result = transformOcrBoundingBox(baseOcr, edits, baseDimensions);
expect(result.id).toBe(baseOcr.id);
expect(result.text).toBe(baseOcr.text);
expect(result.x1).toBeCloseTo(0.1, 5);
expect(result.y1).toBeCloseTo(0.8, 5);
expect(result.x2).toBeCloseTo(0.2, 5);
expect(result.y2).toBeCloseTo(0.8, 5);
expect(result.x3).toBeCloseTo(0.2, 5);
expect(result.y3).toBeCloseTo(0.9, 5);
expect(result.x4).toBeCloseTo(0.1, 5);
expect(result.y4).toBeCloseTo(0.9, 5);
});
});
describe('with mirror edit', () => {
it('should mirror horizontally', () => {
const edits: AssetEditActionItem[] = [
{ action: AssetEditAction.Mirror, parameters: { axis: MirrorAxis.Horizontal } },
];
const result = transformOcrBoundingBox(baseOcr, edits, baseDimensions);
expect(result.x1).toBeCloseTo(0.9, 5);
expect(result.y1).toBeCloseTo(0.1, 5);
});
it('should mirror vertically', () => {
const edits: AssetEditActionItem[] = [
{ action: AssetEditAction.Mirror, parameters: { axis: MirrorAxis.Vertical } },
];
const result = transformOcrBoundingBox(baseOcr, edits, baseDimensions);
expect(result.x1).toBeCloseTo(0.1, 5);
expect(result.y1).toBeCloseTo(0.9, 5);
});
});
describe('with combined edits', () => {
it('should preserve OCR metadata through transforms', () => {
const edits: AssetEditActionItem[] = [
{ action: AssetEditAction.Crop, parameters: { x: 0, y: 0, width: 500, height: 400 } },
{ action: AssetEditAction.Rotate, parameters: { angle: 90 } },
];
const result = transformOcrBoundingBox(baseOcr, edits, baseDimensions);
expect(result.id).toBe(baseOcr.id);
expect(result.assetId).toBe(baseOcr.assetId);
expect(result.boxScore).toBe(baseOcr.boxScore);
expect(result.textScore).toBe(baseOcr.textScore);
expect(result.text).toBe(baseOcr.text);
});
});
});
+227
View File
@@ -0,0 +1,227 @@
import { AssetEditAction, AssetEditActionItem } from 'src/dtos/editing.dto';
import { AssetOcrResponseDto } from 'src/dtos/ocr.dto';
import { ImageDimensions } from 'src/types';
import { applyToPoint, compose, flipX, flipY, identity, Matrix, rotate, scale, translate } from 'transformation-matrix';
export const getOutputDimensions = (
edits: AssetEditActionItem[],
startingDimensions: ImageDimensions,
): ImageDimensions => {
let { width, height } = startingDimensions;
const crop = edits.find((edit) => edit.action === AssetEditAction.Crop);
if (crop) {
width = crop.parameters.width;
height = crop.parameters.height;
}
for (const edit of edits) {
if (edit.action === AssetEditAction.Rotate) {
const angleDegrees = edit.parameters.angle;
if (angleDegrees === 90 || angleDegrees === 270) {
[width, height] = [height, width];
}
}
}
return { width, height };
};
export const createAffineMatrix = (
edits: AssetEditActionItem[],
scalingParameters?: {
pointSpace: ImageDimensions;
targetSpace: ImageDimensions;
},
): Matrix => {
let scalingMatrix: Matrix = identity();
if (scalingParameters) {
const { pointSpace, targetSpace } = scalingParameters;
const scaleX = targetSpace.width / pointSpace.width;
scalingMatrix = scale(scaleX);
}
return compose(
scalingMatrix,
...edits.map((edit) => {
switch (edit.action) {
case 'rotate': {
const angleInRadians = (-edit.parameters.angle * Math.PI) / 180;
return rotate(angleInRadians);
}
case 'mirror': {
return edit.parameters.axis === 'horizontal' ? flipY() : flipX();
}
default: {
return identity();
}
}
}),
);
};
type Point = { x: number; y: number };
type TransformState = {
points: Point[];
currentWidth: number;
currentHeight: number;
};
/**
* Transforms an array of points through a series of edit operations (crop, rotate, mirror).
* Points should be in absolute pixel coordinates relative to the starting dimensions.
*/
const transformPoints = (
points: Point[],
edits: AssetEditActionItem[],
startingDimensions: ImageDimensions,
): TransformState => {
let currentWidth = startingDimensions.width;
let currentHeight = startingDimensions.height;
let transformedPoints = [...points];
// Handle crop first
const crop = edits.find((edit) => edit.action === 'crop');
if (crop) {
const { x: cropX, y: cropY, width: cropWidth, height: cropHeight } = crop.parameters;
transformedPoints = transformedPoints.map((p) => ({
x: p.x - cropX,
y: p.y - cropY,
}));
currentWidth = cropWidth;
currentHeight = cropHeight;
}
// Apply rotate and mirror transforms
for (const edit of edits) {
let matrix: Matrix = identity();
if (edit.action === 'rotate') {
const angleDegrees = edit.parameters.angle;
const angleRadians = (angleDegrees * Math.PI) / 180;
const newWidth = angleDegrees === 90 || angleDegrees === 270 ? currentHeight : currentWidth;
const newHeight = angleDegrees === 90 || angleDegrees === 270 ? currentWidth : currentHeight;
matrix = compose(
translate(newWidth / 2, newHeight / 2),
rotate(angleRadians),
translate(-currentWidth / 2, -currentHeight / 2),
);
currentWidth = newWidth;
currentHeight = newHeight;
} else if (edit.action === 'mirror') {
matrix = compose(
translate(currentWidth / 2, currentHeight / 2),
edit.parameters.axis === 'horizontal' ? flipY() : flipX(),
translate(-currentWidth / 2, -currentHeight / 2),
);
} else {
// Skip non-affine transformations
continue;
}
transformedPoints = transformedPoints.map((p) => applyToPoint(matrix, p));
}
return {
points: transformedPoints,
currentWidth,
currentHeight,
};
};
type FaceBoundingBox = {
boundingBoxX1: number;
boundingBoxX2: number;
boundingBoxY1: number;
boundingBoxY2: number;
imageWidth: number;
imageHeight: number;
};
export const transformFaceBoundingBox = (
box: FaceBoundingBox,
edits: AssetEditActionItem[],
imageDimensions: ImageDimensions,
): FaceBoundingBox => {
if (edits.length === 0) {
return box;
}
const scaleX = imageDimensions.width / box.imageWidth;
const scaleY = imageDimensions.height / box.imageHeight;
const points: Point[] = [
{ x: box.boundingBoxX1 * scaleX, y: box.boundingBoxY1 * scaleY },
{ x: box.boundingBoxX2 * scaleX, y: box.boundingBoxY2 * scaleY },
];
const { points: transformedPoints, currentWidth, currentHeight } = transformPoints(points, edits, imageDimensions);
// Ensure x1,y1 is top-left and x2,y2 is bottom-right
const [p1, p2] = transformedPoints;
return {
boundingBoxX1: Math.min(p1.x, p2.x),
boundingBoxY1: Math.min(p1.y, p2.y),
boundingBoxX2: Math.max(p1.x, p2.x),
boundingBoxY2: Math.max(p1.y, p2.y),
imageWidth: currentWidth,
imageHeight: currentHeight,
};
};
const reorderQuadPointsForRotation = (points: Point[], rotationDegrees: number): Point[] => {
const [p1, p2, p3, p4] = points;
switch (rotationDegrees) {
case 90: {
return [p4, p1, p2, p3];
}
case 180: {
return [p3, p4, p1, p2];
}
case 270: {
return [p2, p3, p4, p1];
}
default: {
return points;
}
}
};
export const transformOcrBoundingBox = (
box: AssetOcrResponseDto,
edits: AssetEditActionItem[],
imageDimensions: ImageDimensions,
): AssetOcrResponseDto => {
if (edits.length === 0) {
return box;
}
const points: Point[] = [
{ x: box.x1 * imageDimensions.width, y: box.y1 * imageDimensions.height },
{ x: box.x2 * imageDimensions.width, y: box.y2 * imageDimensions.height },
{ x: box.x3 * imageDimensions.width, y: box.y3 * imageDimensions.height },
{ x: box.x4 * imageDimensions.width, y: box.y4 * imageDimensions.height },
];
const { points: transformedPoints, currentWidth, currentHeight } = transformPoints(points, edits, imageDimensions);
// Reorder points to maintain semantic ordering (topLeft, topRight, bottomRight, bottomLeft)
const netRotation = edits.find((e) => e.action == AssetEditAction.Rotate)?.parameters.angle ?? 0 % 360;
const reorderedPoints = reorderQuadPointsForRotation(transformedPoints, netRotation);
const [p1, p2, p3, p4] = reorderedPoints;
return {
...box,
x1: p1.x / currentWidth,
y1: p1.y / currentHeight,
x2: p2.x / currentWidth,
y2: p2.y / currentHeight,
x3: p3.x / currentWidth,
y3: p3.y / currentHeight,
x4: p4.x / currentWidth,
y4: p4.y / currentHeight,
};
};
+43
View File
@@ -81,6 +81,49 @@ export const ValidateUUID = (options?: UUIDOptions & ApiPropertyOptions) => {
);
};
export function IsAxisAlignedRotation() {
return ValidateBy(
{
name: 'isAxisAlignedRotation',
validator: {
validate(value: any) {
return [0, 90, 180, 270].includes(value);
},
defaultMessage: buildMessage(
(eachPrefix) => eachPrefix + '$property must be one of the following values: 0, 90, 180, 270',
{},
),
},
},
{},
);
}
@ValidatorConstraint({ name: 'uniqueEditActions' })
class UniqueEditActionsValidator implements ValidatorConstraintInterface {
validate(edits: { action: string; parameters?: unknown }[]): boolean {
if (!Array.isArray(edits)) {
return true;
}
const actionSet = new Set<string>();
for (const edit of edits) {
const key = edit.action === 'mirror' ? `${edit.action}-${JSON.stringify(edit.parameters)}` : edit.action;
if (actionSet.has(key)) {
return false;
}
actionSet.add(key);
}
return true;
}
defaultMessage(): string {
return 'Duplicate edit actions are not allowed';
}
}
export const IsUniqueEditActions = () => Validate(UniqueEditActionsValidator);
export class UUIDParamDto {
@IsNotEmpty()
@IsUUID('4')