diff --git a/e2e/src/specs/server/api/library.e2e-spec.ts b/e2e/src/specs/server/api/library.e2e-spec.ts index 4d67a84647..82bbce4b87 100644 --- a/e2e/src/specs/server/api/library.e2e-spec.ts +++ b/e2e/src/specs/server/api/library.e2e-spec.ts @@ -1,4 +1,4 @@ -import { LibraryResponseDto, LoginResponseDto, getAllLibraries } from '@immich/sdk'; +import { LibraryResponseDto, LoginResponseDto, SystemConfigDto, getAllLibraries } from '@immich/sdk'; import { cpSync, existsSync, rmSync, unlinkSync } from 'node:fs'; import { Socket } from 'socket.io-client'; import { userDto, uuidDto } from 'src/fixtures'; @@ -1317,6 +1317,171 @@ describe('/libraries', () => { }); }); + describe('watch', () => { + let config: SystemConfigDto; + let library: LibraryResponseDto; + let changeAssetId: string; + let deleteAssetId: string; + const watcherPath = `${testAssetDir}/temp/watcher-behavior`; + + beforeAll(async () => { + config = await utils.getSystemConfig(admin.accessToken); + + const changeAssetPath = `${watcherPath}/change.jpg`; + const deleteAssetPath = `${watcherPath}/delete.png`; + const changeIgnoredAssetPath = `${watcherPath}/@eaDir/change.png`; + const unlinkIgnoredAssetPath = `${watcherPath}/@eaDir/unlink.png`; + const changeAssetInternalPath = `${testAssetDirInternal}/temp/watcher-behavior/change.jpg`; + const deleteAssetInternalPath = `${testAssetDirInternal}/temp/watcher-behavior/delete.png`; + + utils.createDirectory(watcherPath); + cpSync(`${testAssetDir}/albums/nature/el_torcal_rocks.jpg`, changeAssetPath); + await utimes(changeAssetPath, 447_775_200_000); + utils.createImageFile(deleteAssetPath); + utils.createImageFile(changeIgnoredAssetPath); + utils.createImageFile(unlinkIgnoredAssetPath); + + library = await utils.createLibrary(admin.accessToken, { + ownerId: admin.userId, + importPaths: [`${testAssetDirInternal}/temp/watcher-behavior`], + exclusionPatterns: ['**/@eaDir/**'], + }); + + await utils.scan(admin.accessToken, library.id); + + const { assets: changeAssets } = await utils.searchAssets(admin.accessToken, { + libraryId: library.id, + originalPath: changeAssetInternalPath, + }); + expect(changeAssets.count).toBe(1); + changeAssetId = changeAssets.items[0].id; + + const { assets: deleteAssets } = await utils.searchAssets(admin.accessToken, { + libraryId: library.id, + originalPath: deleteAssetInternalPath, + }); + expect(deleteAssets.count).toBe(1); + deleteAssetId = deleteAssets.items[0].id; + + const { status } = await request(app) + .put('/system-config') + .set('Authorization', `Bearer ${admin.accessToken}`) + .send({ ...config, library: { ...config.library, watch: { ...config.library.watch, enabled: true } } }); + expect(status).toBe(200); + await utils.waitForWebsocketEvent({ event: 'libraryWatchEnabled', id: library.id }); + }); + + afterAll(async () => { + await request(app).put('/system-config').set('Authorization', `Bearer ${admin.accessToken}`).send(config); + utils.removeDirectory(watcherPath); + }); + + it('should import a new file', async () => { + const addAssetPath = `${watcherPath}/add.png`; + const addAssetInternalPath = `${testAssetDirInternal}/temp/watcher-behavior/add.png`; + utils.createImageFile(addAssetPath); + + await utils.waitForLibraryWatchEvent({ libraryId: library.id, event: 'add', path: addAssetInternalPath }); + await utils.waitForQueueFinish(admin.accessToken, 'library'); + await utils.waitForQueueFinish(admin.accessToken, 'sidecar'); + await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction'); + + const { assets } = await utils.searchAssets(admin.accessToken, { + libraryId: library.id, + originalPath: addAssetInternalPath, + }); + expect(assets.items).toEqual([expect.objectContaining({ originalPath: addAssetInternalPath })]); + + const asset = await utils.getAssetInfo(admin.accessToken, assets.items[0].id); + expect(asset.originalPath).toBe(addAssetInternalPath); + expect(asset.exifInfo).not.toBe(null); + }); + + it('should detect a changed file', async () => { + const changeAssetPath = `${watcherPath}/change.jpg`; + const changeAssetInternalPath = `${testAssetDirInternal}/temp/watcher-behavior/change.jpg`; + cpSync(`${testAssetDir}/albums/nature/tanners_ridge.jpg`, changeAssetPath); + await utimes(changeAssetPath, 447_775_200_001); + + await utils.waitForLibraryWatchEvent({ libraryId: library.id, event: 'change', path: changeAssetInternalPath }); + await utils.waitForQueueFinish(admin.accessToken, 'library'); + await utils.waitForQueueFinish(admin.accessToken, 'sidecar'); + await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction'); + + const { assets } = await utils.searchAssets(admin.accessToken, { + libraryId: library.id, + originalPath: changeAssetInternalPath, + }); + expect(assets.count).toBe(1); + expect(assets.items[0].id).toBe(changeAssetId); + + const updatedAsset = await utils.getAssetInfo(admin.accessToken, assets.items[0].id); + expect(updatedAsset.originalPath).toBe(changeAssetInternalPath); + expect(updatedAsset.isOffline).toBe(false); + expect(updatedAsset.isTrashed).toBe(false); + }); + + it('should remove an asset when its file is deleted', async () => { + const deleteAssetPath = `${watcherPath}/delete.png`; + const deleteAssetInternalPath = `${testAssetDirInternal}/temp/watcher-behavior/delete.png`; + utils.removeImageFile(deleteAssetPath); + + await utils.waitForLibraryWatchEvent({ libraryId: library.id, event: 'unlink', path: deleteAssetInternalPath }); + await utils.waitForQueueFinish(admin.accessToken, 'library'); + + const { assets } = await utils.searchAssets(admin.accessToken, { + libraryId: library.id, + originalPath: deleteAssetInternalPath, + }); + expect(assets.items).toEqual([]); + + const { assets: withDeletedAssets } = await utils.searchAssets(admin.accessToken, { + libraryId: library.id, + withDeleted: true, + }); + expect(withDeletedAssets.items.find((asset) => asset.id === deleteAssetId)).toBeUndefined(); + }); + + it('should not fire a library watcher event for ignored directory add, change, or unlink activity', async () => { + const addIgnoredAssetPath = `${watcherPath}/@eaDir/add.png`; + const addIgnoredAssetInternalPath = `${testAssetDirInternal}/temp/watcher-behavior/@eaDir/add.png`; + const changeIgnoredAssetPath = `${watcherPath}/@eaDir/change.png`; + const changeIgnoredAssetInternalPath = `${testAssetDirInternal}/temp/watcher-behavior/@eaDir/change.png`; + const unlinkIgnoredAssetPath = `${watcherPath}/@eaDir/unlink.png`; + const unlinkIgnoredAssetInternalPath = `${testAssetDirInternal}/temp/watcher-behavior/@eaDir/unlink.png`; + + const addWatchPromise = utils.waitForLibraryWatchEvent({ + libraryId: library.id, + event: 'add', + path: addIgnoredAssetInternalPath, + timeout: 7000, + }); + const changeWatchPromise = utils.waitForLibraryWatchEvent({ + libraryId: library.id, + event: 'change', + path: changeIgnoredAssetInternalPath, + timeout: 7000, + }); + const unlinkWatchPromise = utils.waitForLibraryWatchEvent({ + libraryId: library.id, + event: 'unlink', + path: unlinkIgnoredAssetInternalPath, + timeout: 7000, + }); + + utils.createImageFile(addIgnoredAssetPath); + cpSync(`${testAssetDir}/albums/nature/tanners_ridge.jpg`, changeIgnoredAssetPath); + await utimes(changeIgnoredAssetPath, 447_775_200_001); + utils.removeImageFile(unlinkIgnoredAssetPath); + + await Promise.all([ + expect(addWatchPromise).rejects.toThrow('Timed out waiting for libraryWatchFired event'), + expect(changeWatchPromise).rejects.toThrow('Timed out waiting for libraryWatchFired event'), + expect(unlinkWatchPromise).rejects.toThrow('Timed out waiting for libraryWatchFired event'), + ]); + }); + }); + describe('POST /libraries/:id/validate', () => { it('should require authentication', async () => { const { status, body } = await request(app).post(`/libraries/${uuidDto.notFound}/validate`).send({}); diff --git a/e2e/src/utils.ts b/e2e/src/utils.ts index a5567f0778..16639773d4 100644 --- a/e2e/src/utils.ts +++ b/e2e/src/utils.ts @@ -76,8 +76,21 @@ import { playwrightDbHost, playwrightHost, playwriteBaseUrl } from '../playwrigh export type { Emitter } from '@socket.io/component-emitter'; type CommandResponse = { stdout: string; stderr: string; exitCode: number | null }; -type EventType = 'assetUpload' | 'assetUpdate' | 'assetDelete' | 'userDelete' | 'assetHidden'; +type EventType = + | 'assetUpload' + | 'assetUpdate' + | 'assetDelete' + | 'userDelete' + | 'assetHidden' + | 'libraryWatchEnabled' + | 'libraryWatchFired'; type WaitOptions = { event: EventType; id?: string; total?: number; timeout?: number }; +type LibraryWatchEventOptions = { + libraryId: string; + event: 'add' | 'change' | 'unlink'; + path: string; + timeout?: number; +}; type AdminSetupOptions = { onboarding?: boolean }; type FileData = { bytes?: Buffer; filename: string }; @@ -128,6 +141,8 @@ const events: Record> = { assetUpdate: new Set(), assetDelete: new Set(), userDelete: new Set(), + libraryWatchEnabled: new Set(), + libraryWatchFired: new Set(), }; const idCallbacks: Record void> = {}; @@ -135,6 +150,9 @@ const countCallbacks: Record void }> = const execPromise = promisify(exec); +const getLibraryWatchEventKey = ({ libraryId, event, path }: Omit) => + `${libraryId}:${event}:${path}`; + const onEvent = ({ event, id }: { event: EventType; id: string }) => { // console.log(`Received event: ${event} [id=${id}]`); const set = events[event]; @@ -248,6 +266,12 @@ export const utils = { .on('on_asset_hidden', (assetId: string) => onEvent({ event: 'assetHidden', id: assetId })) .on('on_asset_delete', (assetId: string) => onEvent({ event: 'assetDelete', id: assetId })) .on('on_user_delete', (userId: string) => onEvent({ event: 'userDelete', id: userId })) + .on('on_library_watch_enabled', (data: { libraryId: string }) => + onEvent({ event: 'libraryWatchEnabled', id: data.libraryId }), + ) + .on('on_library_watch_fired', (data: { libraryId: string; event: 'add' | 'change' | 'unlink'; path: string }) => + onEvent({ event: 'libraryWatchFired', id: getLibraryWatchEventKey(data) }), + ) .connect(); }); }, @@ -300,6 +324,13 @@ export const utils = { }); }, + waitForLibraryWatchEvent: ({ libraryId, event, path, timeout }: LibraryWatchEventOptions): Promise => + utils.waitForWebsocketEvent({ + event: 'libraryWatchFired', + id: getLibraryWatchEventKey({ libraryId, event, path }), + timeout, + }), + initSdk: () => { setBaseUrl(app); }, diff --git a/server/src/repositories/event.repository.ts b/server/src/repositories/event.repository.ts index fbc281ccb3..2fc8f7008a 100644 --- a/server/src/repositories/event.repository.ts +++ b/server/src/repositories/event.repository.ts @@ -95,6 +95,10 @@ type EventMap = { // websocket events WebsocketConnect: [{ userId: string }]; + + // library events + LibraryWatchEnabled: [{ id: string }]; + LibraryWatchFired: [{ libraryId: string; event: 'add' | 'change' | 'unlink'; path: string; ignored: boolean }]; }; export type AppRestartEvent = { diff --git a/server/src/repositories/websocket.repository.ts b/server/src/repositories/websocket.repository.ts index 235d2f2a84..3a265897a0 100644 --- a/server/src/repositories/websocket.repository.ts +++ b/server/src/repositories/websocket.repository.ts @@ -35,6 +35,9 @@ export interface ClientEventMap { on_notification: [NotificationDto]; on_session_delete: [string]; + on_library_watch_enabled: [{ libraryId: string }]; + on_library_watch_fired: [{ libraryId: string; event: 'add' | 'change' | 'unlink'; path: string; ignored: boolean }]; + AssetUploadReadyV1: [{ asset: SyncAssetV1; exif: SyncAssetExifV1 }]; AppRestartV1: [AppRestartEvent]; AssetEditReadyV1: [{ asset: SyncAssetV1; edit: SyncAssetEditV1[] }]; diff --git a/server/src/services/library.service.ts b/server/src/services/library.service.ts index 841fa4743c..e88b0d8fd2 100644 --- a/server/src/services/library.service.ts +++ b/server/src/services/library.service.ts @@ -90,27 +90,50 @@ export class LibraryService extends BaseService { this.logger.log(`Starting to watch library ${library.id} with import path(s) ${library.importPaths}`); - const matcher = picomatch(`**/*{${mimeTypes.getSupportedFileExtensions().join(',')}}`, { - nocase: true, - ignore: library.exclusionPatterns, - }); + const supportedExtensions = mimeTypes.getSupportedFileExtensions().map((extension) => extension.toLowerCase()); + const exclusionPatterns = library.exclusionPatterns.flatMap((pattern) => + pattern.endsWith('/**') ? [pattern, pattern.slice(0, -3)] : [pattern], + ); + const excludeMatcher = picomatch(exclusionPatterns, { nocase: true }); + const isExcluded = (path: string) => excludeMatcher(path.replaceAll('\\', '/')); + const isSupportedFile = (path: string) => { + const normalizedPath = path.toLowerCase(); + return supportedExtensions.some((extension) => normalizedPath.endsWith(extension)); + }; let _resolve: () => void; const ready$ = new Promise((resolve) => (_resolve = resolve)); const handler = async (event: string, path: string) => { - if (matcher(path)) { - this.logger.debug(`File ${event} event received for ${path} in library ${library.id}}`); - await this.jobRepository.queue({ - name: JobName.LibrarySyncFiles, - data: { libraryId: library.id, paths: [path] }, - }); - } else { + const ignored = !isSupportedFile(path); + + await this.eventRepository.emit('LibraryWatchFired', { + libraryId: library.id, + event: event as 'add' | 'change', + path, + ignored, + }); + + if (ignored) { this.logger.verbose(`Ignoring file ${event} event for ${path} in library ${library.id}`); + return; } + + this.logger.debug(`File ${event} event received for ${path} in library ${library.id}}`); + await this.jobRepository.queue({ + name: JobName.LibrarySyncFiles, + data: { libraryId: library.id, paths: [path] }, + }); }; const deletionHandler = async (path: string) => { + await this.eventRepository.emit('LibraryWatchFired', { + libraryId: library.id, + event: 'unlink', + path, + ignored: false, + }); + this.logger.debug(`File unlink event received for ${path} in library ${library.id}}`); await this.jobRepository.queue({ name: JobName.LibraryRemoveAsset, @@ -123,6 +146,7 @@ export class LibraryService extends BaseService { { usePolling: false, ignoreInitial: true, + ignored: isExcluded, awaitWriteFinish: { stabilityThreshold: 5000, pollInterval: 1000, @@ -148,6 +172,8 @@ export class LibraryService extends BaseService { // Wait for the watcher to initialize before returning await ready$; + await this.eventRepository.emit('LibraryWatchEnabled', { id }); + return true; } diff --git a/server/src/services/notification.service.ts b/server/src/services/notification.service.ts index 9f11d19af7..a110943a42 100644 --- a/server/src/services/notification.service.ts +++ b/server/src/services/notification.service.ts @@ -114,6 +114,16 @@ export class NotificationService extends BaseService { this.websocketRepository.serverSend('ConfigUpdate', { oldConfig, newConfig }); } + @OnEvent({ name: 'LibraryWatchEnabled' }) + onLibraryWatchEnabled({ id }: ArgOf<'LibraryWatchEnabled'>) { + this.websocketRepository.clientBroadcast('on_library_watch_enabled', { libraryId: id }); + } + + @OnEvent({ name: 'LibraryWatchFired' }) + onLibraryWatchFired(event: ArgOf<'LibraryWatchFired'>) { + this.websocketRepository.clientBroadcast('on_library_watch_fired', event); + } + @OnEvent({ name: 'AppRestart' }) onAppRestart(state: ArgOf<'AppRestart'>) { this.websocketRepository.clientBroadcast('AppRestartV1', {