move to medium tests

This commit is contained in:
Jonathan Jogenfors
2026-03-24 14:03:32 +01:00
parent 05b07ca233
commit 292fbef180
9 changed files with 309 additions and 231 deletions
+1 -166
View File
@@ -1,4 +1,4 @@
import { LibraryResponseDto, LoginResponseDto, SystemConfigDto, getAllLibraries } from '@immich/sdk';
import { LibraryResponseDto, LoginResponseDto, getAllLibraries } from '@immich/sdk';
import { cpSync, existsSync, rmSync, unlinkSync } from 'node:fs';
import { Socket } from 'socket.io-client';
import { userDto, uuidDto } from 'src/fixtures';
@@ -1317,171 +1317,6 @@ describe('/libraries', () => {
});
});
describe('watch', () => {
let config: SystemConfigDto;
let library: LibraryResponseDto;
let changeAssetId: string;
let deleteAssetId: string;
const watcherPath = `${testAssetDir}/temp/watcher-behavior`;
beforeAll(async () => {
config = await utils.getSystemConfig(admin.accessToken);
const changeAssetPath = `${watcherPath}/change.jpg`;
const deleteAssetPath = `${watcherPath}/delete.png`;
const changeIgnoredAssetPath = `${watcherPath}/@eaDir/change.png`;
const unlinkIgnoredAssetPath = `${watcherPath}/@eaDir/unlink.png`;
const changeAssetInternalPath = `${testAssetDirInternal}/temp/watcher-behavior/change.jpg`;
const deleteAssetInternalPath = `${testAssetDirInternal}/temp/watcher-behavior/delete.png`;
utils.createDirectory(watcherPath);
cpSync(`${testAssetDir}/albums/nature/el_torcal_rocks.jpg`, changeAssetPath);
await utimes(changeAssetPath, 447_775_200_000);
utils.createImageFile(deleteAssetPath);
utils.createImageFile(changeIgnoredAssetPath);
utils.createImageFile(unlinkIgnoredAssetPath);
library = await utils.createLibrary(admin.accessToken, {
ownerId: admin.userId,
importPaths: [`${testAssetDirInternal}/temp/watcher-behavior`],
exclusionPatterns: ['**/@eaDir/**'],
});
await utils.scan(admin.accessToken, library.id);
const { assets: changeAssets } = await utils.searchAssets(admin.accessToken, {
libraryId: library.id,
originalPath: changeAssetInternalPath,
});
expect(changeAssets.count).toBe(1);
changeAssetId = changeAssets.items[0].id;
const { assets: deleteAssets } = await utils.searchAssets(admin.accessToken, {
libraryId: library.id,
originalPath: deleteAssetInternalPath,
});
expect(deleteAssets.count).toBe(1);
deleteAssetId = deleteAssets.items[0].id;
const { status } = await request(app)
.put('/system-config')
.set('Authorization', `Bearer ${admin.accessToken}`)
.send({ ...config, library: { ...config.library, watch: { ...config.library.watch, enabled: true } } });
expect(status).toBe(200);
await utils.waitForWebsocketEvent({ event: 'libraryWatchEnabled', id: library.id });
});
afterAll(async () => {
await request(app).put('/system-config').set('Authorization', `Bearer ${admin.accessToken}`).send(config);
utils.removeDirectory(watcherPath);
});
it('should import a new file', async () => {
const addAssetPath = `${watcherPath}/add.png`;
const addAssetInternalPath = `${testAssetDirInternal}/temp/watcher-behavior/add.png`;
utils.createImageFile(addAssetPath);
await utils.waitForLibraryWatchEvent({ libraryId: library.id, event: 'add', path: addAssetInternalPath });
await utils.waitForQueueFinish(admin.accessToken, 'library');
await utils.waitForQueueFinish(admin.accessToken, 'sidecar');
await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction');
const { assets } = await utils.searchAssets(admin.accessToken, {
libraryId: library.id,
originalPath: addAssetInternalPath,
});
expect(assets.items).toEqual([expect.objectContaining({ originalPath: addAssetInternalPath })]);
const asset = await utils.getAssetInfo(admin.accessToken, assets.items[0].id);
expect(asset.originalPath).toBe(addAssetInternalPath);
expect(asset.exifInfo).not.toBe(null);
});
it('should detect a changed file', async () => {
const changeAssetPath = `${watcherPath}/change.jpg`;
const changeAssetInternalPath = `${testAssetDirInternal}/temp/watcher-behavior/change.jpg`;
cpSync(`${testAssetDir}/albums/nature/tanners_ridge.jpg`, changeAssetPath);
await utimes(changeAssetPath, 447_775_200_001);
await utils.waitForLibraryWatchEvent({ libraryId: library.id, event: 'change', path: changeAssetInternalPath });
await utils.waitForQueueFinish(admin.accessToken, 'library');
await utils.waitForQueueFinish(admin.accessToken, 'sidecar');
await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction');
const { assets } = await utils.searchAssets(admin.accessToken, {
libraryId: library.id,
originalPath: changeAssetInternalPath,
});
expect(assets.count).toBe(1);
expect(assets.items[0].id).toBe(changeAssetId);
const updatedAsset = await utils.getAssetInfo(admin.accessToken, assets.items[0].id);
expect(updatedAsset.originalPath).toBe(changeAssetInternalPath);
expect(updatedAsset.isOffline).toBe(false);
expect(updatedAsset.isTrashed).toBe(false);
});
it('should remove an asset when its file is deleted', async () => {
const deleteAssetPath = `${watcherPath}/delete.png`;
const deleteAssetInternalPath = `${testAssetDirInternal}/temp/watcher-behavior/delete.png`;
utils.removeImageFile(deleteAssetPath);
await utils.waitForLibraryWatchEvent({ libraryId: library.id, event: 'unlink', path: deleteAssetInternalPath });
await utils.waitForQueueFinish(admin.accessToken, 'library');
const { assets } = await utils.searchAssets(admin.accessToken, {
libraryId: library.id,
originalPath: deleteAssetInternalPath,
});
expect(assets.items).toEqual([]);
const { assets: withDeletedAssets } = await utils.searchAssets(admin.accessToken, {
libraryId: library.id,
withDeleted: true,
});
expect(withDeletedAssets.items.find((asset) => asset.id === deleteAssetId)).toBeUndefined();
});
it('should not fire a library watcher event for ignored directory add, change, or unlink activity', async () => {
const addIgnoredAssetPath = `${watcherPath}/@eaDir/add.png`;
const addIgnoredAssetInternalPath = `${testAssetDirInternal}/temp/watcher-behavior/@eaDir/add.png`;
const changeIgnoredAssetPath = `${watcherPath}/@eaDir/change.png`;
const changeIgnoredAssetInternalPath = `${testAssetDirInternal}/temp/watcher-behavior/@eaDir/change.png`;
const unlinkIgnoredAssetPath = `${watcherPath}/@eaDir/unlink.png`;
const unlinkIgnoredAssetInternalPath = `${testAssetDirInternal}/temp/watcher-behavior/@eaDir/unlink.png`;
const addWatchPromise = utils.waitForLibraryWatchEvent({
libraryId: library.id,
event: 'add',
path: addIgnoredAssetInternalPath,
timeout: 7000,
});
const changeWatchPromise = utils.waitForLibraryWatchEvent({
libraryId: library.id,
event: 'change',
path: changeIgnoredAssetInternalPath,
timeout: 7000,
});
const unlinkWatchPromise = utils.waitForLibraryWatchEvent({
libraryId: library.id,
event: 'unlink',
path: unlinkIgnoredAssetInternalPath,
timeout: 7000,
});
utils.createImageFile(addIgnoredAssetPath);
cpSync(`${testAssetDir}/albums/nature/tanners_ridge.jpg`, changeIgnoredAssetPath);
await utimes(changeIgnoredAssetPath, 447_775_200_001);
utils.removeImageFile(unlinkIgnoredAssetPath);
await Promise.all([
expect(addWatchPromise).rejects.toThrow('Timed out waiting for libraryWatchFired event'),
expect(changeWatchPromise).rejects.toThrow('Timed out waiting for libraryWatchFired event'),
expect(unlinkWatchPromise).rejects.toThrow('Timed out waiting for libraryWatchFired event'),
]);
});
});
describe('POST /libraries/:id/validate', () => {
it('should require authentication', async () => {
const { status, body } = await request(app).post(`/libraries/${uuidDto.notFound}/validate`).send({});
+1 -32
View File
@@ -76,21 +76,8 @@ import { playwrightDbHost, playwrightHost, playwriteBaseUrl } from '../playwrigh
export type { Emitter } from '@socket.io/component-emitter';
type CommandResponse = { stdout: string; stderr: string; exitCode: number | null };
type EventType =
| 'assetUpload'
| 'assetUpdate'
| 'assetDelete'
| 'userDelete'
| 'assetHidden'
| 'libraryWatchEnabled'
| 'libraryWatchFired';
type EventType = 'assetUpload' | 'assetUpdate' | 'assetDelete' | 'userDelete' | 'assetHidden';
type WaitOptions = { event: EventType; id?: string; total?: number; timeout?: number };
type LibraryWatchEventOptions = {
libraryId: string;
event: 'add' | 'change' | 'unlink';
path: string;
timeout?: number;
};
type AdminSetupOptions = { onboarding?: boolean };
type FileData = { bytes?: Buffer; filename: string };
@@ -141,8 +128,6 @@ const events: Record<EventType, Set<string>> = {
assetUpdate: new Set<string>(),
assetDelete: new Set<string>(),
userDelete: new Set<string>(),
libraryWatchEnabled: new Set<string>(),
libraryWatchFired: new Set<string>(),
};
const idCallbacks: Record<string, () => void> = {};
@@ -150,9 +135,6 @@ const countCallbacks: Record<string, { count: number; callback: () => void }> =
const execPromise = promisify(exec);
const getLibraryWatchEventKey = ({ libraryId, event, path }: Omit<LibraryWatchEventOptions, 'timeout'>) =>
`${libraryId}:${event}:${path}`;
const onEvent = ({ event, id }: { event: EventType; id: string }) => {
// console.log(`Received event: ${event} [id=${id}]`);
const set = events[event];
@@ -266,12 +248,6 @@ export const utils = {
.on('on_asset_hidden', (assetId: string) => onEvent({ event: 'assetHidden', id: assetId }))
.on('on_asset_delete', (assetId: string) => onEvent({ event: 'assetDelete', id: assetId }))
.on('on_user_delete', (userId: string) => onEvent({ event: 'userDelete', id: userId }))
.on('on_library_watch_enabled', (data: { libraryId: string }) =>
onEvent({ event: 'libraryWatchEnabled', id: data.libraryId }),
)
.on('on_library_watch_fired', (data: { libraryId: string; event: 'add' | 'change' | 'unlink'; path: string }) =>
onEvent({ event: 'libraryWatchFired', id: getLibraryWatchEventKey(data) }),
)
.connect();
});
},
@@ -324,13 +300,6 @@ export const utils = {
});
},
waitForLibraryWatchEvent: ({ libraryId, event, path, timeout }: LibraryWatchEventOptions): Promise<void> =>
utils.waitForWebsocketEvent({
event: 'libraryWatchFired',
id: getLibraryWatchEventKey({ libraryId, event, path }),
timeout,
}),
initSdk: () => {
setBaseUrl(app);
},
@@ -95,10 +95,6 @@ type EventMap = {
// websocket events
WebsocketConnect: [{ userId: string }];
// library events
LibraryWatchEnabled: [{ id: string }];
LibraryWatchFired: [{ libraryId: string; event: 'add' | 'change' | 'unlink'; path: string; ignored: boolean }];
};
export type AppRestartEvent = {
@@ -35,9 +35,6 @@ export interface ClientEventMap {
on_notification: [NotificationDto];
on_session_delete: [string];
on_library_watch_enabled: [{ libraryId: string }];
on_library_watch_fired: [{ libraryId: string; event: 'add' | 'change' | 'unlink'; path: string; ignored: boolean }];
AssetUploadReadyV1: [{ asset: SyncAssetV1; exif: SyncAssetExifV1 }];
AppRestartV1: [AppRestartEvent];
AssetEditReadyV1: [{ asset: SyncAssetV1; edit: SyncAssetEditV1[] }];
-16
View File
@@ -107,13 +107,6 @@ export class LibraryService extends BaseService {
const handler = async (event: string, path: string) => {
const ignored = !isSupportedFile(path);
await this.eventRepository.emit('LibraryWatchFired', {
libraryId: library.id,
event: event as 'add' | 'change',
path,
ignored,
});
if (ignored) {
this.logger.verbose(`Ignoring file ${event} event for ${path} in library ${library.id}`);
return;
@@ -127,13 +120,6 @@ export class LibraryService extends BaseService {
};
const deletionHandler = async (path: string) => {
await this.eventRepository.emit('LibraryWatchFired', {
libraryId: library.id,
event: 'unlink',
path,
ignored: false,
});
this.logger.debug(`File unlink event received for ${path} in library ${library.id}}`);
await this.jobRepository.queue({
name: JobName.LibraryRemoveAsset,
@@ -172,8 +158,6 @@ export class LibraryService extends BaseService {
// Wait for the watcher to initialize before returning
await ready$;
await this.eventRepository.emit('LibraryWatchEnabled', { id });
return true;
}
@@ -114,16 +114,6 @@ export class NotificationService extends BaseService {
this.websocketRepository.serverSend('ConfigUpdate', { oldConfig, newConfig });
}
@OnEvent({ name: 'LibraryWatchEnabled' })
onLibraryWatchEnabled({ id }: ArgOf<'LibraryWatchEnabled'>) {
this.websocketRepository.clientBroadcast('on_library_watch_enabled', { libraryId: id });
}
@OnEvent({ name: 'LibraryWatchFired' })
onLibraryWatchFired(event: ArgOf<'LibraryWatchFired'>) {
this.websocketRepository.clientBroadcast('on_library_watch_fired', event);
}
@OnEvent({ name: 'AppRestart' })
onAppRestart(state: ArgOf<'AppRestart'>) {
this.websocketRepository.clientBroadcast('AppRestartV1', {
+3
View File
@@ -30,6 +30,7 @@ import { DatabaseRepository } from 'src/repositories/database.repository';
import { EmailRepository } from 'src/repositories/email.repository';
import { EventRepository } from 'src/repositories/event.repository';
import { JobRepository } from 'src/repositories/job.repository';
import { LibraryRepository } from 'src/repositories/library.repository';
import { LoggingRepository } from 'src/repositories/logging.repository';
import { MachineLearningRepository } from 'src/repositories/machine-learning.repository';
import { MapRepository } from 'src/repositories/map.repository';
@@ -405,6 +406,7 @@ const newRealRepository = <T>(key: ClassConstructor<T>, db: Kysely<DB>): T => {
case AssetRepository:
case AssetEditRepository:
case AssetJobRepository:
case LibraryRepository:
case MemoryRepository:
case NotificationRepository:
case OcrRepository:
@@ -466,6 +468,7 @@ const newMockRepository = <T>(key: ClassConstructor<T>) => {
case AlbumRepository:
case AssetRepository:
case AssetJobRepository:
case LibraryRepository:
case ConfigRepository:
case CryptoRepository:
case MemoryRepository:
@@ -0,0 +1,135 @@
import { Kysely } from 'kysely';
import { mkdtemp, rm, unlink, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { LoggingRepository } from 'src/repositories/logging.repository';
import { StorageRepository } from 'src/repositories/storage.repository';
import { DB } from 'src/schema';
import { BaseService } from 'src/services/base.service';
import { newMediumService } from 'test/medium.factory';
import { getKyselyDB } from 'test/utils';
let defaultDatabase: Kysely<DB>;
const setup = (db?: Kysely<DB>) => {
const { ctx } = newMediumService(BaseService, {
database: db || defaultDatabase,
real: [],
mock: [LoggingRepository],
});
return { ctx, sut: ctx.get(StorageRepository) };
};
beforeAll(async () => {
defaultDatabase = await getKyselyDB();
});
const watchForEvent = (
sut: StorageRepository,
folder: string,
event: 'add' | 'change' | 'unlink',
action: () => Promise<void>,
): Promise<string> => {
return new Promise<string>((resolve, reject) => {
const timeout = setTimeout(() => {
void close().finally(() => reject(new Error(`Timed out waiting for ${event} event`)));
}, 10_000);
const onResolve = (path: string) => {
clearTimeout(timeout);
void close().finally(() => resolve(path));
};
const onReject = (error: Error) => {
clearTimeout(timeout);
void close().finally(() => reject(error));
};
const close = sut.watch(
[folder],
{
ignoreInitial: true,
usePolling: true,
interval: 50,
},
{
onReady: () => {
void action().catch((error) => onReject(error as Error));
},
onAdd: (path) => {
if (event === 'add') {
onResolve(path);
}
},
onChange: (path) => {
if (event === 'change') {
onResolve(path);
}
},
onUnlink: (path) => {
if (event === 'unlink') {
onResolve(path);
}
},
onError: (error) => onReject(error),
},
);
});
};
describe(StorageRepository.name, () => {
describe('watch', () => {
it('should fire create (add) events', async () => {
const { sut } = setup();
const folder = await mkdtemp(join(tmpdir(), 'immich-storage-watch-add-'));
const file = join(folder, 'created.jpg');
try {
const changedPath = await watchForEvent(sut, folder, 'add', async () => {
await writeFile(file, 'created');
});
expect(changedPath).toBe(file);
} finally {
await rm(folder, { recursive: true, force: true });
}
});
it('should fire change events', async () => {
const { sut } = setup();
const folder = await mkdtemp(join(tmpdir(), 'immich-storage-watch-change-'));
const file = join(folder, 'changed.jpg');
await writeFile(file, 'before');
try {
const changedPath = await watchForEvent(sut, folder, 'change', async () => {
await writeFile(file, 'after');
});
expect(changedPath).toBe(file);
} finally {
await rm(folder, { recursive: true, force: true });
}
});
it('should fire unlink events', async () => {
const { sut } = setup();
const folder = await mkdtemp(join(tmpdir(), 'immich-storage-watch-unlink-'));
const file = join(folder, 'deleted.jpg');
await writeFile(file, 'content');
try {
const changedPath = await watchForEvent(sut, folder, 'unlink', async () => {
await unlink(file);
});
expect(changedPath).toBe(file);
} finally {
await rm(folder, { recursive: true, force: true });
}
});
});
});
@@ -0,0 +1,169 @@
import { ChokidarOptions } from 'chokidar';
import { Kysely } from 'kysely';
import { JobName } from 'src/enum';
import { EventRepository } from 'src/repositories/event.repository';
import { JobRepository } from 'src/repositories/job.repository';
import { LibraryRepository } from 'src/repositories/library.repository';
import { LoggingRepository } from 'src/repositories/logging.repository';
import { StorageRepository, WatchEvents } from 'src/repositories/storage.repository';
import { DB } from 'src/schema';
import { LibraryService } from 'src/services/library.service';
import { newMediumService } from 'test/medium.factory';
import { getKyselyDB } from 'test/utils';
let defaultDatabase: Kysely<DB>;
const setup = (db?: Kysely<DB>) => {
return newMediumService(LibraryService, {
database: db || defaultDatabase,
real: [LibraryRepository],
mock: [EventRepository, JobRepository, LoggingRepository, StorageRepository],
});
};
beforeAll(async () => {
defaultDatabase = await getKyselyDB();
});
const closeWatcher = () => Promise.resolve();
const setWatchMode = (sut: LibraryService) => {
const service = sut as unknown as { lock: boolean; watchLibraries: boolean };
service.lock = true;
service.watchLibraries = true;
};
const makeMockWatcher =
(paths: { add?: string[]; change?: string[]; unlink?: string[] }) =>
(_paths: string[], options: ChokidarOptions, events: Partial<WatchEvents>): (() => Promise<void>) => {
const ignored = options.ignored as ((path: string) => boolean) | undefined;
events.onReady?.();
for (const path of paths.add ?? []) {
if (!ignored?.(path)) {
events.onAdd?.(path);
}
}
for (const path of paths.change ?? []) {
if (!ignored?.(path)) {
events.onChange?.(path);
}
}
for (const path of paths.unlink ?? []) {
if (!ignored?.(path)) {
events.onUnlink?.(path);
}
}
return closeWatcher;
};
describe(`${LibraryService.name} (watch, medium)`, () => {
it('should queue add and change events for supported files', async () => {
const { sut, ctx } = setup();
const storageRepo = ctx.getMock(StorageRepository);
const jobRepo = ctx.getMock(JobRepository);
jobRepo.queue.mockResolvedValue();
setWatchMode(sut);
const { user } = await ctx.newUser();
const library = await sut.create({
ownerId: user.id,
importPaths: ['/test-assets/temp/watcher-behavior'],
exclusionPatterns: ['**/@eaDir/**'],
});
storageRepo.watch.mockImplementation(
makeMockWatcher({
add: ['/test-assets/temp/watcher-behavior/add.png'],
change: ['/test-assets/temp/watcher-behavior/change.jpg'],
}),
);
await sut.watchAll();
expect(jobRepo.queue).toHaveBeenCalledWith({
name: JobName.LibrarySyncFiles,
data: { libraryId: library.id, paths: ['/test-assets/temp/watcher-behavior/add.png'] },
});
expect(jobRepo.queue).toHaveBeenCalledWith({
name: JobName.LibrarySyncFiles,
data: { libraryId: library.id, paths: ['/test-assets/temp/watcher-behavior/change.jpg'] },
});
await sut.onShutdown();
});
it('should queue unlink events for tracked files', async () => {
const { sut, ctx } = setup();
const storageRepo = ctx.getMock(StorageRepository);
const jobRepo = ctx.getMock(JobRepository);
jobRepo.queue.mockResolvedValue();
setWatchMode(sut);
const { user } = await ctx.newUser();
const library = await sut.create({
ownerId: user.id,
importPaths: ['/test-assets/temp/watcher-behavior'],
exclusionPatterns: ['**/@eaDir/**'],
});
storageRepo.watch.mockImplementation(
makeMockWatcher({
unlink: ['/test-assets/temp/watcher-behavior/delete.png'],
}),
);
await sut.watchAll();
expect(jobRepo.queue).toHaveBeenCalledWith({
name: JobName.LibraryRemoveAsset,
data: { libraryId: library.id, paths: ['/test-assets/temp/watcher-behavior/delete.png'] },
});
await sut.onShutdown();
});
it('should ignore add, change, and unlink events in excluded directories', async () => {
const { sut, ctx } = setup();
const storageRepo = ctx.getMock(StorageRepository);
const jobRepo = ctx.getMock(JobRepository);
jobRepo.queue.mockResolvedValue();
setWatchMode(sut);
await ctx.newUser().then(({ user }) =>
sut.create({
ownerId: user.id,
importPaths: ['/test-assets/temp/watcher-behavior'],
exclusionPatterns: ['**/@eaDir/**'],
}),
);
storageRepo.watch.mockImplementation(
makeMockWatcher({
add: ['/test-assets/temp/watcher-behavior/@eaDir/add.png'],
change: ['/test-assets/temp/watcher-behavior/@eaDir/change.png'],
unlink: ['/test-assets/temp/watcher-behavior/@eaDir/unlink.png'],
}),
);
await sut.watchAll();
expect(jobRepo.queue).not.toHaveBeenCalledWith(
expect.objectContaining({
name: JobName.LibrarySyncFiles,
}),
);
expect(jobRepo.queue).not.toHaveBeenCalledWith(
expect.objectContaining({
name: JobName.LibraryRemoveAsset,
}),
);
await sut.onShutdown();
});
});