From 2903b2653b3f5e0402789a556a10f42f9b47ee7c Mon Sep 17 00:00:00 2001 From: Mert <101130780+mertalev@users.noreply.github.com> Date: Tue, 7 Apr 2026 15:58:03 -0400 Subject: [PATCH] fix(server): library import batch size (#27595) * lower batch size * update test --- server/src/decorators.ts | 15 ++++++++------- server/src/repositories/asset.repository.ts | 6 ++++-- server/src/services/library.service.spec.ts | 2 +- server/src/services/library.service.ts | 8 +------- 4 files changed, 14 insertions(+), 17 deletions(-) diff --git a/server/src/decorators.ts b/server/src/decorators.ts index 695adb4a36..a5c5bf38db 100644 --- a/server/src/decorators.ts +++ b/server/src/decorators.ts @@ -73,7 +73,8 @@ export function Chunked( const originalMethod = descriptor.value; const parameterIndex = options.paramIndex ?? 0; const chunkSize = options.chunkSize || DATABASE_PARAMETER_CHUNK_SIZE; - descriptor.value = async function (...arguments_: any[]) { + const mergeFn = options.mergeFn; + descriptor.value = function (...arguments_: any[]) { const argument = arguments_[parameterIndex]; // Early return if argument length is less than or equal to the chunk size. @@ -81,27 +82,27 @@ export function Chunked( (Array.isArray(argument) && argument.length <= chunkSize) || (argument instanceof Set && argument.size <= chunkSize) ) { - return await originalMethod.apply(this, arguments_); + return originalMethod.apply(this, arguments_); } return Promise.all( - chunks(argument, chunkSize).map(async (chunk) => { - return await Reflect.apply(originalMethod, this, [ + chunks(argument, chunkSize).map((chunk) => { + return Reflect.apply(originalMethod, this, [ ...arguments_.slice(0, parameterIndex), chunk, ...arguments_.slice(parameterIndex + 1), ]); }), - ).then((results) => (options.mergeFn ? options.mergeFn(results) : results)); + ).then((results) => (mergeFn ? mergeFn(results) : results)); }; }; } -export function ChunkedArray(options?: { paramIndex?: number }): MethodDecorator { +export function ChunkedArray(options?: { paramIndex?: number; chunkSize?: number }): MethodDecorator { return Chunked({ ...options, mergeFn: _.flatten }); } -export function ChunkedSet(options?: { paramIndex?: number }): MethodDecorator { +export function ChunkedSet(options?: { paramIndex?: number; chunkSize?: number }): MethodDecorator { return Chunked({ ...options, mergeFn: (args: Set[]) => setUnion(...args) }); } diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.ts index 2e1d02ef28..5876b934e5 100644 --- a/server/src/repositories/asset.repository.ts +++ b/server/src/repositories/asset.repository.ts @@ -380,8 +380,10 @@ export class AssetRepository { return this.db.insertInto('asset').values(asset).returningAll().executeTakeFirstOrThrow(); } - createAll(assets: Insertable[]) { - return this.db.insertInto('asset').values(assets).returningAll().execute(); + @ChunkedArray({ chunkSize: 4000 }) + async createAll(assets: Insertable[]) { + const ids = await this.db.insertInto('asset').values(assets).returning('id').execute(); + return ids.map(({ id }) => id); } @GenerateSql({ params: [DummyValue.UUID, { year: 2000, day: 1, month: 1 }] }) diff --git a/server/src/services/library.service.spec.ts b/server/src/services/library.service.spec.ts index d0c2d0a785..9a2ec00815 100644 --- a/server/src/services/library.service.spec.ts +++ b/server/src/services/library.service.spec.ts @@ -560,7 +560,7 @@ describe(LibraryService.name, () => { paths: ['/data/user1/photo.jpg'], }; - mocks.asset.createAll.mockResolvedValue([asset]); + mocks.asset.createAll.mockResolvedValue([asset.id]); mocks.library.get.mockResolvedValue(library); await expect(sut.handleSyncFiles(mockLibraryJob)).resolves.toBe(JobStatus.Success); diff --git a/server/src/services/library.service.ts b/server/src/services/library.service.ts index 9f2d69bab5..ce3c9ee662 100644 --- a/server/src/services/library.service.ts +++ b/server/src/services/library.service.ts @@ -266,13 +266,7 @@ export class LibraryService extends BaseService { ), ); - const assetIds: string[] = []; - - for (let i = 0; i < assetImports.length; i += 5000) { - // Chunk the imports to avoid the postgres limit of max parameters at once - const chunk = assetImports.slice(i, i + 5000); - await this.assetRepository.createAll(chunk).then((assets) => assetIds.push(...assets.map((asset) => asset.id))); - } + const assetIds = await this.assetRepository.createAll(assetImports); const progressMessage = job.progressCounter && job.totalAssets