fix(server): dedupe database backup jobs (#28341)

* fix(server): dedupe database backup jobs via jobId

#27268 shows backup jobs piling up in the queue across upgrades; one pending
backup is always enough.

* fix(tests): Avoid stale backup files from previous test runs being erroneously returned from createBackup

* fix(jobs): Use bullmq's deduplication over jobId to avoid failed jobs from blocking future executions.

---------

Co-authored-by: Robert Deaton <immich@rdeaton.space>

e2e/src/specs/maintenance/server/database-backups.e2e-spec.ts

@@ -2,7 +2,7 @@ import { LoginResponseDto, ManualJobName } from '@immich/sdk';
 import { errorDto } from 'src/responses';
 import { app, utils } from 'src/utils';
 import request from 'supertest';
-import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
 
 describe('/admin/database-backups', () => {
   let cookie: string | undefined;
@@ -13,6 +13,9 @@ describe('/admin/database-backups', () => {
     admin = await utils.adminSetup({
       onboarding: false,
     });
+  });
+
+  beforeEach(async () => {
     await utils.resetBackups(admin.accessToken);
   });
 

e2e/src/utils.ts

@@ -568,6 +568,8 @@ export const utils = {
       name: ManualJobName.BackupDatabase,
     });
 
+    await utils.waitForQueueFinish(accessToken, 'backupDatabase');
+
     return utils.poll(
       () => request(app).get('/admin/database-backups').set('Authorization', `Bearer ${accessToken}`),
       ({ status, body }) => status === 200 && body.backups.length === 1,

server/src/repositories/job.repository.ts

@@ -171,8 +171,8 @@ export class JobRepository {
         options: this.getJobOptions(item) || undefined,
       } as JobItem & { data: any; options: JobsOptions | undefined };
 
-      if (job.options?.jobId) {
+      if (job.options?.jobId || job.options?.deduplication) {
-        // need to use add() instead of addBulk() for jobId deduplication
+        // need to use add() instead of addBulk() for jobId/deduplication to take effect
         promises.push(this.getQueue(queueName).add(item.name, item.data, job.options));
       } else {
         itemsByQueue[queueName] = itemsByQueue[queueName] || [];
@@ -230,10 +230,13 @@ export class JobRepository {
         return { priority: 1 };
       }
       case JobName.FacialRecognitionQueueAll: {
-        return { jobId: JobName.FacialRecognitionQueueAll };
+        return { deduplication: { id: JobName.FacialRecognitionQueueAll } };
       }
       case JobName.VersionCheck: {
-        return { jobId: JobName.VersionCheck };
+        return { deduplication: { id: JobName.VersionCheck } };
+      }
+      case JobName.DatabaseBackup: {
+        return { deduplication: { id: JobName.DatabaseBackup } };
       }
       default: {
         return null;