merge: remote-tracking branch 'immich/main' into feat/integrity-checks-izzy

This commit is contained in:
izzy
2025-12-17 15:09:39 +00:00
parent 08e532170f
commit 0e75f38e4a
310 changed files with 18246 additions and 7046 deletions
+19 -24
View File
@@ -24,6 +24,18 @@ const fullsizeFile: AssetFile = {
path: '/uploads/user-id/fullsize/path.webp',
};
const sidecarFileWithExt: AssetFile = {
id: 'sidecar-with-ext',
type: AssetFileType.Sidecar,
path: '/original/path.ext.xmp',
};
const sidecarFileWithoutExt: AssetFile = {
id: 'sidecar-without-ext',
type: AssetFileType.Sidecar,
path: '/original/path.xmp',
};
const files: AssetFile[] = [fullsizeFile, previewFile, thumbnailFile];
export const stackStub = (stackId: string, assets: (MapAsset & { exifInfo: Exif })[]) => {
@@ -51,8 +63,8 @@ export const assetStub = {
fileCreatedAt: new Date('2022-06-19T23:41:36.910Z'),
originalPath: '/original/path.jpg',
originalFileName: 'IMG_123.jpg',
sidecarPath: null,
fileSizeInByte: 12_345,
files: [],
...asset,
}),
noResizePath: Object.freeze({
@@ -81,7 +93,6 @@ export const assetStub = {
sharedLinks: [],
faces: [],
exifInfo: {} as Exif,
sidecarPath: null,
deletedAt: null,
isExternal: false,
duplicateId: null,
@@ -117,7 +128,6 @@ export const assetStub = {
sharedLinks: [],
originalFileName: 'IMG_456.jpg',
faces: [],
sidecarPath: null,
isExternal: false,
exifInfo: {
fileSizeInByte: 123_000,
@@ -157,7 +167,6 @@ export const assetStub = {
sharedLinks: [],
originalFileName: 'asset-id.ext',
faces: [],
sidecarPath: null,
deletedAt: null,
duplicateId: null,
isOffline: false,
@@ -194,7 +203,6 @@ export const assetStub = {
originalFileName: 'asset-id.jpg',
faces: [],
deletedAt: null,
sidecarPath: null,
exifInfo: {
fileSizeInByte: 5000,
exifImageHeight: 1000,
@@ -243,7 +251,6 @@ export const assetStub = {
originalFileName: 'asset-id.jpg',
faces: [],
deletedAt: null,
sidecarPath: null,
exifInfo: {
fileSizeInByte: 5000,
exifImageHeight: 3840,
@@ -285,7 +292,6 @@ export const assetStub = {
sharedLinks: [],
originalFileName: 'asset-id.jpg',
faces: [],
sidecarPath: null,
exifInfo: {
fileSizeInByte: 5000,
exifImageHeight: 3840,
@@ -328,7 +334,6 @@ export const assetStub = {
sharedLinks: [],
originalFileName: 'asset-id.jpg',
faces: [],
sidecarPath: null,
exifInfo: {
fileSizeInByte: 5000,
exifImageHeight: 3840,
@@ -367,7 +372,6 @@ export const assetStub = {
originalFileName: 'asset-id.jpg',
faces: [],
deletedAt: null,
sidecarPath: null,
exifInfo: {
fileSizeInByte: 5000,
exifImageHeight: 3840,
@@ -409,7 +413,6 @@ export const assetStub = {
originalFileName: 'asset-id.jpg',
faces: [],
deletedAt: null,
sidecarPath: null,
exifInfo: {
fileSizeInByte: 5000,
} as Exif,
@@ -448,7 +451,6 @@ export const assetStub = {
sharedLinks: [],
originalFileName: 'asset-id.ext',
faces: [],
sidecarPath: null,
exifInfo: {
fileSizeInByte: 5000,
} as Exif,
@@ -490,7 +492,6 @@ export const assetStub = {
sharedLinks: [],
originalFileName: 'asset-id.ext',
faces: [],
sidecarPath: null,
exifInfo: {
fileSizeInByte: 5000,
} as Exif,
@@ -526,7 +527,6 @@ export const assetStub = {
livePhotoVideoId: null,
sharedLinks: [],
faces: [],
sidecarPath: null,
exifInfo: {
fileSizeInByte: 100_000,
exifImageHeight: 2160,
@@ -553,6 +553,7 @@ export const assetStub = {
fileSizeInByte: 100_000,
timeZone: `America/New_York`,
},
files: [] as AssetFile[],
libraryId: null,
visibility: AssetVisibility.Hidden,
} as MapAsset & { faces: AssetFace[]; files: AssetFile[]; exifInfo: Exif }),
@@ -573,7 +574,7 @@ export const assetStub = {
files,
faces: [] as AssetFace[],
visibility: AssetVisibility.Timeline,
} as MapAsset & { faces: AssetFace[] }),
} as MapAsset & { faces: AssetFace[]; files: AssetFile[] }),
livePhotoWithOriginalFileName: Object.freeze({
id: 'live-photo-still-asset',
@@ -589,10 +590,11 @@ export const assetStub = {
fileSizeInByte: 25_000,
timeZone: `America/New_York`,
},
files: [] as AssetFile[],
libraryId: null,
faces: [] as AssetFace[],
visibility: AssetVisibility.Timeline,
} as MapAsset & { faces: AssetFace[] }),
} as MapAsset & { faces: AssetFace[]; files: AssetFile[] }),
withLocation: Object.freeze({
id: 'asset-with-favorite-id',
@@ -605,7 +607,6 @@ export const assetStub = {
deviceId: 'device-id',
checksum: Buffer.from('file hash', 'utf8'),
originalPath: '/original/path.ext',
sidecarPath: null,
type: AssetType.Image,
files: [previewFile],
thumbhash: null,
@@ -652,7 +653,7 @@ export const assetStub = {
thumbhash: null,
checksum: Buffer.from('file hash', 'utf8'),
type: AssetType.Image,
files: [previewFile],
files: [previewFile, sidecarFileWithExt],
encodedVideoPath: null,
createdAt: new Date('2023-02-23T05:06:29.716Z'),
updatedAt: new Date('2023-02-23T05:06:29.716Z'),
@@ -665,7 +666,6 @@ export const assetStub = {
sharedLinks: [],
originalFileName: 'asset-id.ext',
faces: [],
sidecarPath: '/original/path.ext.xmp',
deletedAt: null,
duplicateId: null,
isOffline: false,
@@ -688,7 +688,7 @@ export const assetStub = {
thumbhash: null,
checksum: Buffer.from('file hash', 'utf8'),
type: AssetType.Image,
files: [previewFile],
files: [previewFile, sidecarFileWithoutExt],
encodedVideoPath: null,
createdAt: new Date('2023-02-23T05:06:29.716Z'),
updatedAt: new Date('2023-02-23T05:06:29.716Z'),
@@ -701,7 +701,6 @@ export const assetStub = {
sharedLinks: [],
originalFileName: 'asset-id.ext',
faces: [],
sidecarPath: '/original/path.xmp',
deletedAt: null,
duplicateId: null,
isOffline: false,
@@ -734,7 +733,6 @@ export const assetStub = {
livePhotoVideoId: null,
sharedLinks: [],
faces: [],
sidecarPath: null,
exifInfo: {
fileSizeInByte: 100_000,
} as Exif,
@@ -776,7 +774,6 @@ export const assetStub = {
originalFileName: 'photo.jpg',
faces: [],
deletedAt: null,
sidecarPath: null,
exifInfo: {
fileSizeInByte: 5000,
} as Exif,
@@ -812,7 +809,6 @@ export const assetStub = {
originalFileName: 'asset-id.dng',
faces: [],
deletedAt: null,
sidecarPath: null,
exifInfo: {
fileSizeInByte: 5000,
profileDescription: 'Adobe RGB',
@@ -853,7 +849,6 @@ export const assetStub = {
originalFileName: 'asset-id.hif',
faces: [],
deletedAt: null,
sidecarPath: null,
exifInfo: {
fileSizeInByte: 5000,
profileDescription: 'Adobe RGB',
@@ -1,14 +1,17 @@
import { Kysely } from 'kysely';
import { JobName, SharedLinkType } from 'src/enum';
import { AssetFileType, JobName, SharedLinkType } from 'src/enum';
import { AccessRepository } from 'src/repositories/access.repository';
import { AlbumRepository } from 'src/repositories/album.repository';
import { AssetJobRepository } from 'src/repositories/asset-job.repository';
import { AssetRepository } from 'src/repositories/asset.repository';
import { EventRepository } from 'src/repositories/event.repository';
import { JobRepository } from 'src/repositories/job.repository';
import { LoggingRepository } from 'src/repositories/logging.repository';
import { SharedLinkAssetRepository } from 'src/repositories/shared-link-asset.repository';
import { SharedLinkRepository } from 'src/repositories/shared-link.repository';
import { StackRepository } from 'src/repositories/stack.repository';
import { StorageRepository } from 'src/repositories/storage.repository';
import { UserRepository } from 'src/repositories/user.repository';
import { DB } from 'src/schema';
import { AssetService } from 'src/services/asset.service';
import { newMediumService } from 'test/medium.factory';
@@ -20,8 +23,16 @@ let defaultDatabase: Kysely<DB>;
const setup = (db?: Kysely<DB>) => {
return newMediumService(AssetService, {
database: db || defaultDatabase,
real: [AssetRepository, AlbumRepository, AccessRepository, SharedLinkAssetRepository, StackRepository],
mock: [LoggingRepository, JobRepository, StorageRepository],
real: [
AssetRepository,
AssetJobRepository,
AlbumRepository,
AccessRepository,
SharedLinkAssetRepository,
StackRepository,
UserRepository,
],
mock: [EventRepository, LoggingRepository, JobRepository, StorageRepository],
});
};
@@ -184,7 +195,15 @@ describe(AssetService.name, () => {
jobRepo.queue.mockResolvedValue();
const { user } = await ctx.newUser();
const { asset: oldAsset } = await ctx.newAsset({ ownerId: user.id, sidecarPath: '/path/to/my/sidecar.xmp' });
const { asset: oldAsset } = await ctx.newAsset({ ownerId: user.id });
await ctx.newAssetFile({
assetId: oldAsset.id,
path: '/path/to/my/sidecar.xmp',
type: AssetFileType.Sidecar,
});
const { asset: newAsset } = await ctx.newAsset({ ownerId: user.id });
await ctx.newExif({ assetId: oldAsset.id, description: 'foo' });
@@ -202,4 +221,51 @@ describe(AssetService.name, () => {
});
});
});
describe('delete', () => {
it('should delete asset', async () => {
const { sut, ctx } = setup();
ctx.getMock(EventRepository).emit.mockResolvedValue();
ctx.getMock(JobRepository).queue.mockResolvedValue();
const { user } = await ctx.newUser();
const { asset } = await ctx.newAsset({ ownerId: user.id });
const thumbnailPath = '/path/to/thumbnail.jpg';
const previewPath = '/path/to/preview.jpg';
const sidecarPath = '/path/to/sidecar.xmp';
await Promise.all([
ctx.newAssetFile({ assetId: asset.id, type: AssetFileType.Thumbnail, path: thumbnailPath }),
ctx.newAssetFile({ assetId: asset.id, type: AssetFileType.Preview, path: previewPath }),
ctx.newAssetFile({ assetId: asset.id, type: AssetFileType.Sidecar, path: sidecarPath }),
]);
await sut.handleAssetDeletion({ id: asset.id, deleteOnDisk: true });
expect(ctx.getMock(JobRepository).queue).toHaveBeenCalledWith({
name: JobName.FileDelete,
data: { files: [thumbnailPath, previewPath, sidecarPath, asset.originalPath] },
});
});
it('should not delete offline assets', async () => {
const { sut, ctx } = setup();
ctx.getMock(EventRepository).emit.mockResolvedValue();
ctx.getMock(JobRepository).queue.mockResolvedValue();
const { user } = await ctx.newUser();
const { asset } = await ctx.newAsset({ ownerId: user.id, isOffline: true });
const thumbnailPath = '/path/to/thumbnail.jpg';
const previewPath = '/path/to/preview.jpg';
await Promise.all([
ctx.newAssetFile({ assetId: asset.id, type: AssetFileType.Thumbnail, path: thumbnailPath }),
ctx.newAssetFile({ assetId: asset.id, type: AssetFileType.Preview, path: previewPath }),
ctx.newAssetFile({ assetId: asset.id, type: AssetFileType.Sidecar, path: `/path/to/sidecar.xmp` }),
]);
await sut.handleAssetDeletion({ id: asset.id, deleteOnDisk: true });
expect(ctx.getMock(JobRepository).queue).toHaveBeenCalledWith({
name: JobName.FileDelete,
data: { files: [thumbnailPath, previewPath] },
});
});
});
});
@@ -82,7 +82,11 @@ describe(MetadataService.name, () => {
process.env.TZ = serverTimeZone ?? undefined;
const { filePath } = await createTestFile(exifData);
mocks.assetJob.getForMetadataExtraction.mockResolvedValue({ id: 'asset-1', originalPath: filePath } as any);
mocks.assetJob.getForMetadataExtraction.mockResolvedValue({
id: 'asset-1',
originalPath: filePath,
files: [],
} as any);
await sut.handleMetadataExtraction({ id: 'asset-1' });
@@ -113,13 +113,13 @@ describe(WorkflowService.name, () => {
enabled: true,
filters: [
{
filterId: testFilterId,
pluginFilterId: testFilterId,
filterConfig: { key: 'value' },
},
],
actions: [
{
actionId: testActionId,
pluginActionId: testActionId,
actionConfig: { action: 'test' },
},
],
@@ -137,7 +137,7 @@ describe(WorkflowService.name, () => {
expect(workflow.filters[0]).toMatchObject({
id: expect.any(String),
workflowId: workflow.id,
filterId: testFilterId,
pluginFilterId: testFilterId,
filterConfig: { key: 'value' },
order: 0,
});
@@ -146,7 +146,7 @@ describe(WorkflowService.name, () => {
expect(workflow.actions[0]).toMatchObject({
id: expect.any(String),
workflowId: workflow.id,
actionId: testActionId,
pluginActionId: testActionId,
actionConfig: { action: 'test' },
order: 0,
});
@@ -163,7 +163,7 @@ describe(WorkflowService.name, () => {
name: 'invalid-workflow',
description: 'A workflow with invalid filter',
enabled: true,
filters: [{ filterId: factory.uuid(), filterConfig: { key: 'value' } }],
filters: [{ pluginFilterId: factory.uuid(), filterConfig: { key: 'value' } }],
actions: [],
}),
).rejects.toThrow('Invalid filter ID');
@@ -181,7 +181,7 @@ describe(WorkflowService.name, () => {
description: 'A workflow with invalid action',
enabled: true,
filters: [],
actions: [{ actionId: factory.uuid(), actionConfig: { action: 'test' } }],
actions: [{ pluginActionId: factory.uuid(), actionConfig: { action: 'test' } }],
}),
).rejects.toThrow('Invalid action ID');
});
@@ -220,7 +220,7 @@ describe(WorkflowService.name, () => {
name: 'invalid-context-workflow',
description: 'A workflow with context mismatch',
enabled: true,
filters: [{ filterId: result.filters[0].id }],
filters: [{ pluginFilterId: result.filters[0].id }],
actions: [],
}),
).rejects.toThrow('does not support asset context');
@@ -261,7 +261,7 @@ describe(WorkflowService.name, () => {
description: 'A workflow with context mismatch',
enabled: true,
filters: [],
actions: [{ actionId: result.actions[0].id }],
actions: [{ pluginActionId: result.actions[0].id }],
}),
).rejects.toThrow('does not support asset context');
});
@@ -277,13 +277,13 @@ describe(WorkflowService.name, () => {
description: 'A workflow with multiple filters and actions',
enabled: true,
filters: [
{ filterId: testFilterId, filterConfig: { step: 1 } },
{ filterId: testFilterId, filterConfig: { step: 2 } },
{ pluginFilterId: testFilterId, filterConfig: { step: 1 } },
{ pluginFilterId: testFilterId, filterConfig: { step: 2 } },
],
actions: [
{ actionId: testActionId, actionConfig: { step: 1 } },
{ actionId: testActionId, actionConfig: { step: 2 } },
{ actionId: testActionId, actionConfig: { step: 3 } },
{ pluginActionId: testActionId, actionConfig: { step: 1 } },
{ pluginActionId: testActionId, actionConfig: { step: 2 } },
{ pluginActionId: testActionId, actionConfig: { step: 3 } },
],
});
@@ -378,8 +378,8 @@ describe(WorkflowService.name, () => {
name: 'test-workflow',
description: 'A test workflow',
enabled: true,
filters: [{ filterId: testFilterId, filterConfig: { key: 'value' } }],
actions: [{ actionId: testActionId, actionConfig: { action: 'test' } }],
filters: [{ pluginFilterId: testFilterId, filterConfig: { key: 'value' } }],
actions: [{ pluginActionId: testActionId, actionConfig: { action: 'test' } }],
});
const workflow = await sut.get(auth, created.id);
@@ -461,14 +461,14 @@ describe(WorkflowService.name, () => {
name: 'test-workflow',
description: 'Test',
enabled: true,
filters: [{ filterId: testFilterId, filterConfig: { old: 'config' } }],
filters: [{ pluginFilterId: testFilterId, filterConfig: { old: 'config' } }],
actions: [],
});
const updated = await sut.update(auth, created.id, {
filters: [
{ filterId: testFilterId, filterConfig: { new: 'config' } },
{ filterId: testFilterId, filterConfig: { second: 'filter' } },
{ pluginFilterId: testFilterId, filterConfig: { new: 'config' } },
{ pluginFilterId: testFilterId, filterConfig: { second: 'filter' } },
],
});
@@ -488,13 +488,13 @@ describe(WorkflowService.name, () => {
description: 'Test',
enabled: true,
filters: [],
actions: [{ actionId: testActionId, actionConfig: { old: 'config' } }],
actions: [{ pluginActionId: testActionId, actionConfig: { old: 'config' } }],
});
const updated = await sut.update(auth, created.id, {
actions: [
{ actionId: testActionId, actionConfig: { new: 'config' } },
{ actionId: testActionId, actionConfig: { second: 'action' } },
{ pluginActionId: testActionId, actionConfig: { new: 'config' } },
{ pluginActionId: testActionId, actionConfig: { second: 'action' } },
],
});
@@ -513,7 +513,7 @@ describe(WorkflowService.name, () => {
name: 'test-workflow',
description: 'Test',
enabled: true,
filters: [{ filterId: testFilterId, filterConfig: { key: 'value' } }],
filters: [{ pluginFilterId: testFilterId, filterConfig: { key: 'value' } }],
actions: [],
});
@@ -588,7 +588,7 @@ describe(WorkflowService.name, () => {
await expect(
sut.update(auth, created.id, {
filters: [{ filterId: factory.uuid(), filterConfig: {} }],
filters: [{ pluginFilterId: factory.uuid(), filterConfig: {} }],
}),
).rejects.toThrow();
});
@@ -608,7 +608,7 @@ describe(WorkflowService.name, () => {
});
await expect(
sut.update(auth, created.id, { actions: [{ actionId: factory.uuid(), actionConfig: {} }] }),
sut.update(auth, created.id, { actions: [{ pluginActionId: factory.uuid(), actionConfig: {} }] }),
).rejects.toThrow();
});
});
@@ -643,8 +643,8 @@ describe(WorkflowService.name, () => {
name: 'test-workflow',
description: 'Test',
enabled: true,
filters: [{ filterId: testFilterId, filterConfig: {} }],
actions: [{ actionId: testActionId, actionConfig: {} }],
filters: [{ pluginFilterId: testFilterId, filterConfig: {} }],
actions: [{ pluginActionId: testActionId, actionConfig: {} }],
});
await sut.delete(auth, workflow.id);
@@ -176,7 +176,7 @@ describe(SyncRequestType.PartnerAssetExifsV1, () => {
const newResponse = await ctx.syncStream(auth, [SyncRequestType.PartnerAssetExifsV1]);
expect(newResponse).toEqual([
{
ack: expect.stringMatching(new RegExp(`${SyncEntityType.PartnerAssetExifBackfillV1}\\|.+?\\|.+`)),
ack: expect.stringMatching(new RegExp(String.raw`${SyncEntityType.PartnerAssetExifBackfillV1}\|.+?\|.+`)),
data: expect.objectContaining({
assetId: assetUser3.id,
}),
@@ -226,7 +226,7 @@ describe(SyncRequestType.PartnerAssetExifsV1, () => {
const newResponse = await ctx.syncStream(auth, [SyncRequestType.PartnerAssetExifsV1]);
expect(newResponse).toEqual([
{
ack: expect.stringMatching(new RegExp(`${SyncEntityType.PartnerAssetExifBackfillV1}\\|.+?\\|.+`)),
ack: expect.stringMatching(new RegExp(String.raw`${SyncEntityType.PartnerAssetExifBackfillV1}\|.+?\|.+`)),
data: expect.objectContaining({
assetId: assetUser3.id,
}),
@@ -10,6 +10,7 @@ export const newAssetRepositoryMock = (): Mocked<RepositoryInterface<AssetReposi
updateAllExif: vitest.fn(),
updateDateTimeOriginal: vitest.fn().mockResolvedValue([]),
upsertJobStatus: vitest.fn(),
getForCopy: vitest.fn(),
getByDayOfYear: vitest.fn(),
getByIds: vitest.fn().mockResolvedValue([]),
getByIdsWithAllRelationsButStacks: vitest.fn().mockResolvedValue([]),
@@ -36,6 +37,7 @@ export const newAssetRepositoryMock = (): Mocked<RepositoryInterface<AssetReposi
getChangedDeltaSync: vitest.fn(),
upsertFile: vitest.fn(),
upsertFiles: vitest.fn(),
deleteFile: vitest.fn(),
deleteFiles: vitest.fn(),
detectOfflineExternalAssets: vitest.fn(),
filterNewExternalAssetPaths: vitest.fn(),
@@ -38,6 +38,7 @@ export const makeMockWatcher =
return () => close();
}
// eslint-disable-next-line unicorn/consistent-function-scoping
return () => Promise.resolve();
};
+18 -6
View File
@@ -8,14 +8,22 @@ import {
Memory,
Partner,
Session,
SidecarWriteAsset,
User,
UserAdmin,
} from 'src/database';
import { MapAsset } from 'src/dtos/asset-response.dto';
import { AuthDto } from 'src/dtos/auth.dto';
import { QueueStatisticsDto } from 'src/dtos/queue.dto';
import { AssetStatus, AssetType, AssetVisibility, MemoryType, Permission, UserMetadataKey, UserStatus } from 'src/enum';
import {
AssetFileType,
AssetStatus,
AssetType,
AssetVisibility,
MemoryType,
Permission,
UserMetadataKey,
UserStatus,
} from 'src/enum';
import { OnThisDayData, UserMetadataItem } from 'src/types';
import { v4, v7 } from 'uuid';
@@ -237,7 +245,6 @@ const assetFactory = (asset: Partial<MapAsset> = {}) => ({
originalFileName: 'IMG_123.jpg',
originalPath: `/data/12/34/IMG_123.jpg`,
ownerId: newUuid(),
sidecarPath: null,
stackId: null,
thumbhash: null,
type: AssetType.Image,
@@ -312,12 +319,17 @@ const versionHistoryFactory = () => ({
version: '1.123.45',
});
const assetSidecarWriteFactory = (asset: Partial<SidecarWriteAsset> = {}) => ({
const assetSidecarWriteFactory = () => ({
id: newUuid(),
sidecarPath: '/path/to/original-path.jpg.xmp',
originalPath: '/path/to/original-path.jpg.xmp',
tags: [],
...asset,
files: [
{
id: newUuid(),
path: '/path/to/original-path.jpg.xmp',
type: AssetFileType.Sidecar,
},
],
});
const assetOcrFactory = (
@@ -0,0 +1,40 @@
import { Column, DatabaseSchema, Table } from 'src/sql-tools';
@Table()
export class Table1 {
@Column({ type: 'character varying', array: true, default: [] })
column1!: string[];
}
export const description = 'should register a table with a column with a default value (array)';
export const schema: DatabaseSchema = {
databaseName: 'postgres',
schemaName: 'public',
functions: [],
enums: [],
extensions: [],
parameters: [],
overrides: [],
tables: [
{
name: 'table1',
columns: [
{
name: 'column1',
tableName: 'table1',
type: 'character varying',
nullable: false,
isArray: true,
primary: false,
synchronize: true,
default: "'{}'",
},
],
indexes: [],
triggers: [],
constraints: [],
synchronize: true,
},
],
warnings: [],
};