feat: Use postgres as a queue

We've been keen to try this for a while as it means we can remove redis as a
dependency, which makes Immich easier to setup and run.

This replaces bullmq with a bespoke postgres queue. Jobs in the queue are
processed either immediately via triggers and notifications, or eventually if a
notification is missed.

server/src/app.module.ts

@@ -1,4 +1,3 @@
-import { BullModule } from '@nestjs/bullmq';
 import { Inject, Module, OnModuleDestroy, OnModuleInit, ValidationPipe } from '@nestjs/common';
 import { APP_FILTER, APP_GUARD, APP_INTERCEPTOR, APP_PIPE } from '@nestjs/core';
 import { ScheduleModule, SchedulerRegistry } from '@nestjs/schedule';
@@ -37,11 +36,9 @@ export const middleware = [
 ];
 
 const configRepository = new ConfigRepository();
-const { bull, cls, database, otel } = configRepository.getEnv();
+const { cls, database, otel } = configRepository.getEnv();
 
 const imports = [
-  BullModule.forRoot(bull.config),
-  BullModule.registerQueue(...bull.queues),
   ClsModule.forRoot(cls.config),
   OpenTelemetryModule.forRoot(otel),
   KyselyModule.forRoot(getKyselyConfig(database.config)),

server/src/db.d.ts

@@ -236,6 +236,30 @@ export interface GeodataPlaces {
   name: string;
 }
 
+export interface GraphileWorkerJobs {
+  id: Generated<string>;
+  task_identifier: string;
+  locked_at: Timestamp | null;
+  locked_by: string | null;
+  run_at: Timestamp | null;
+  attempts: number;
+  max_attempts: number;
+}
+
+export interface GraphileWorkerPrivateJobs {
+  id: Generated<string>;
+  task_id: string;
+  locked_at: Timestamp | null;
+  locked_by: string | null;
+  attempts: number;
+  max_attempts: number;
+}
+
+export interface GraphileWorkerPrivateTasks {
+  id: Generated<string>;
+  identifier: string;
+}
+
 export interface Libraries {
   createdAt: Generated<Timestamp>;
   deletedAt: Timestamp | null;
@@ -476,6 +500,9 @@ export interface DB {
   exif: Exif;
   face_search: FaceSearch;
   geodata_places: GeodataPlaces;
+  'graphile_worker.jobs': GraphileWorkerJobs;
+  'graphile_worker._private_jobs': GraphileWorkerPrivateJobs;
+  'graphile_worker._private_tasks': GraphileWorkerPrivateTasks;
   libraries: Libraries;
   memories: Memories;
   memories_assets_assets: MemoriesAssetsAssets;

server/src/dtos/env.dto.ts

@@ -157,34 +157,4 @@ export class EnvDto {
   @IsString()
   @Optional()
   NO_COLOR?: string;
-
-  @IsString()
-  @Optional()
-  REDIS_HOSTNAME?: string;
-
-  @IsInt()
-  @Optional()
-  @Type(() => Number)
-  REDIS_PORT?: number;
-
-  @IsInt()
-  @Optional()
-  @Type(() => Number)
-  REDIS_DBINDEX?: number;
-
-  @IsString()
-  @Optional()
-  REDIS_USERNAME?: string;
-
-  @IsString()
-  @Optional()
-  REDIS_PASSWORD?: string;
-
-  @IsString()
-  @Optional()
-  REDIS_SOCKET?: string;
-
-  @IsString()
-  @Optional()
-  REDIS_URL?: string;
 }

server/src/dtos/job.dto.ts

@@ -30,20 +30,15 @@ export class JobCountsDto {
   @ApiProperty({ type: 'integer' })
   active!: number;
   @ApiProperty({ type: 'integer' })
-  completed!: number;
-  @ApiProperty({ type: 'integer' })
-  failed!: number;
+  waiting!: number;
   @ApiProperty({ type: 'integer' })
   delayed!: number;
   @ApiProperty({ type: 'integer' })
-  waiting!: number;
-  @ApiProperty({ type: 'integer' })
-  paused!: number;
+  failed!: number;
 }
 
 export class QueueStatusDto {
-  isActive!: boolean;
-  isPaused!: boolean;
+  paused!: boolean;
 }
 
 export class JobStatusDto {

server/src/enum.ts

@@ -204,6 +204,7 @@ export enum SystemMetadataKey {
   SYSTEM_FLAGS = 'system-flags',
   VERSION_CHECK_STATE = 'version-check-state',
   LICENSE = 'license',
+  QUEUES_STATE = 'queues-state',
 }
 
 export enum UserMetadataKey {
@@ -533,10 +534,20 @@ export enum JobName {
 }
 
 export enum JobCommand {
+  // The behavior of start depends on the queue. Usually it is a request to
+  // reprocess everything associated with the queue from scratch.
   START = 'start',
+
+  // Pause prevents workers from processing jobs.
   PAUSE = 'pause',
+
+  // Resume allows workers to continue processing jobs.
   RESUME = 'resume',
-  EMPTY = 'empty',
+
+  // Clear removes all pending jobs.
+  CLEAR = 'clear',
+
+  // ClearFailed removes all failed jobs.
   CLEAR_FAILED = 'clear-failed',
 }
 

server/src/middleware/websocket.adapter.ts

@@ -1,9 +1,10 @@
 import { INestApplicationContext } from '@nestjs/common';
 import { IoAdapter } from '@nestjs/platform-socket.io';
-import { createAdapter } from '@socket.io/redis-adapter';
-import { Redis } from 'ioredis';
+import { createAdapter } from '@socket.io/postgres-adapter';
+import pg, { PoolConfig } from 'pg';
 import { ServerOptions } from 'socket.io';
 import { ConfigRepository } from 'src/repositories/config.repository';
+import { asPostgresConnectionConfig } from 'src/utils/database';
 
 export class WebSocketAdapter extends IoAdapter {
   constructor(private app: INestApplicationContext) {
@@ -11,11 +12,11 @@ export class WebSocketAdapter extends IoAdapter {
   }
 
   createIOServer(port: number, options?: ServerOptions): any {
-    const { redis } = this.app.get(ConfigRepository).getEnv();
     const server = super.createIOServer(port, options);
-    const pubClient = new Redis(redis);
-    const subClient = pubClient.duplicate();
-    server.adapter(createAdapter(pubClient, subClient));
+    const configRepository = new ConfigRepository();
+    const { database } = configRepository.getEnv();
+    const pool = new pg.Pool(asPostgresConnectionConfig(database.config) as PoolConfig);
+    server.adapter(createAdapter(pool));
     return server;
   }
 }

server/src/repositories/config.repository.spec.ts

@@ -26,38 +26,12 @@ const resetEnv = () => {
     'DB_SKIP_MIGRATIONS',
     'DB_VECTOR_EXTENSION',
 
-    'REDIS_HOSTNAME',
-    'REDIS_PORT',
-    'REDIS_DBINDEX',
-    'REDIS_USERNAME',
-    'REDIS_PASSWORD',
-    'REDIS_SOCKET',
-    'REDIS_URL',
-
     'NO_COLOR',
   ]) {
     delete process.env[env];
   }
 };
 
-const sentinelConfig = {
-  sentinels: [
-    {
-      host: 'redis-sentinel-node-0',
-      port: 26_379,
-    },
-    {
-      host: 'redis-sentinel-node-1',
-      port: 26_379,
-    },
-    {
-      host: 'redis-sentinel-node-2',
-      port: 26_379,
-    },
-  ],
-  name: 'redis-sentinel',
-};
-
 describe('getEnv', () => {
   beforeEach(() => {
     resetEnv();
@@ -108,34 +82,6 @@ describe('getEnv', () => {
     });
   });
 
-  describe('redis', () => {
-    it('should use defaults', () => {
-      const { redis } = getEnv();
-      expect(redis).toEqual({
-        host: 'redis',
-        port: 6379,
-        db: 0,
-        username: undefined,
-        password: undefined,
-        path: undefined,
-      });
-    });
-
-    it('should parse base64 encoded config, ignore other env', () => {
-      process.env.REDIS_URL = `ioredis://${Buffer.from(JSON.stringify(sentinelConfig)).toString('base64')}`;
-      process.env.REDIS_HOSTNAME = 'redis-host';
-      process.env.REDIS_USERNAME = 'redis-user';
-      process.env.REDIS_PASSWORD = 'redis-password';
-      const { redis } = getEnv();
-      expect(redis).toEqual(sentinelConfig);
-    });
-
-    it('should reject invalid json', () => {
-      process.env.REDIS_URL = `ioredis://${Buffer.from('{ "invalid json"').toString('base64')}`;
-      expect(() => getEnv()).toThrowError('Failed to decode redis options');
-    });
-  });
-
   describe('noColor', () => {
     beforeEach(() => {
       delete process.env.NO_COLOR;

server/src/repositories/config.repository.ts

@@ -1,25 +1,14 @@
-import { RegisterQueueOptions } from '@nestjs/bullmq';
 import { Inject, Injectable, Optional } from '@nestjs/common';
-import { QueueOptions } from 'bullmq';
 import { plainToInstance } from 'class-transformer';
 import { validateSync } from 'class-validator';
 import { Request, Response } from 'express';
-import { RedisOptions } from 'ioredis';
 import { CLS_ID, ClsModuleOptions } from 'nestjs-cls';
 import { OpenTelemetryModuleOptions } from 'nestjs-otel/lib/interfaces';
-import { join } from 'node:path';
+import { join, resolve } from 'node:path';
 import { citiesFile, excludePaths, IWorker } from 'src/constants';
 import { Telemetry } from 'src/decorators';
 import { EnvDto } from 'src/dtos/env.dto';
-import {
-  DatabaseExtension,
-  ImmichEnvironment,
-  ImmichHeader,
-  ImmichTelemetry,
-  ImmichWorker,
-  LogLevel,
-  QueueName,
-} from 'src/enum';
+import { DatabaseExtension, ImmichEnvironment, ImmichHeader, ImmichTelemetry, ImmichWorker, LogLevel } from 'src/enum';
 import { DatabaseConnectionParams, VectorExtension } from 'src/types';
 import { setDifference } from 'src/utils/set';
 
@@ -46,11 +35,6 @@ export interface EnvData {
     thirdPartySupportUrl?: string;
   };
 
-  bull: {
-    config: QueueOptions;
-    queues: RegisterQueueOptions[];
-  };
-
   cls: {
     config: ClsModuleOptions;
   };
@@ -87,8 +71,6 @@ export interface EnvData {
     };
   };
 
-  redis: RedisOptions;
-
   telemetry: {
     apiPort: number;
     microservicesPort: number;
@@ -149,28 +131,12 @@ const getEnv = (): EnvData => {
   const isProd = environment === ImmichEnvironment.PRODUCTION;
   const buildFolder = dto.IMMICH_BUILD_DATA || '/build';
   const folders = {
+    // eslint-disable-next-line unicorn/prefer-module
+    dist: resolve(`${__dirname}/..`),
     geodata: join(buildFolder, 'geodata'),
     web: join(buildFolder, 'www'),
   };
 
-  let redisConfig = {
-    host: dto.REDIS_HOSTNAME || 'redis',
-    port: dto.REDIS_PORT || 6379,
-    db: dto.REDIS_DBINDEX || 0,
-    username: dto.REDIS_USERNAME || undefined,
-    password: dto.REDIS_PASSWORD || undefined,
-    path: dto.REDIS_SOCKET || undefined,
-  };
-
-  const redisUrl = dto.REDIS_URL;
-  if (redisUrl && redisUrl.startsWith('ioredis://')) {
-    try {
-      redisConfig = JSON.parse(Buffer.from(redisUrl.slice(10), 'base64').toString());
-    } catch (error) {
-      throw new Error(`Failed to decode redis options: ${error}`);
-    }
-  }
-
   const includedTelemetries =
     dto.IMMICH_TELEMETRY_INCLUDE === 'all'
       ? new Set(Object.values(ImmichTelemetry))
@@ -218,19 +184,6 @@ const getEnv = (): EnvData => {
       thirdPartySupportUrl: dto.IMMICH_THIRD_PARTY_SUPPORT_URL,
     },
 
-    bull: {
-      config: {
-        prefix: 'immich_bull',
-        connection: { ...redisConfig },
-        defaultJobOptions: {
-          attempts: 3,
-          removeOnComplete: true,
-          removeOnFail: false,
-        },
-      },
-      queues: Object.values(QueueName).map((name) => ({ name })),
-    },
-
     cls: {
       config: {
         middleware: {
@@ -269,8 +222,6 @@ const getEnv = (): EnvData => {
       },
     },
 
-    redis: redisConfig,
-
     resourcePaths: {
       lockFile: join(buildFolder, 'build-lock.json'),
       geodata: {

server/src/repositories/event.repository.ts

@@ -64,6 +64,9 @@ type EventMap = {
   'assets.delete': [{ assetIds: string[]; userId: string }];
   'assets.restore': [{ assetIds: string[]; userId: string }];
 
+  'queue.pause': [QueueName];
+  'queue.resume': [QueueName];
+
   'job.start': [QueueName, JobItem];
   'job.failed': [{ job: JobItem; error: Error | any }];
 
@@ -85,7 +88,7 @@ type EventMap = {
   'websocket.connect': [{ userId: string }];
 };
 
-export const serverEvents = ['config.update'] as const;
+export const serverEvents = ['config.update', 'queue.pause', 'queue.resume'] as const;
 export type ServerEvents = (typeof serverEvents)[number];
 
 export type EmitEvent = keyof EventMap;

server/src/repositories/job.repository.ts

@@ -1,15 +1,20 @@
-import { getQueueToken } from '@nestjs/bullmq';
 import { Injectable } from '@nestjs/common';
 import { ModuleRef, Reflector } from '@nestjs/core';
-import { JobsOptions, Queue, Worker } from 'bullmq';
 import { ClassConstructor } from 'class-transformer';
-import { setTimeout } from 'node:timers/promises';
-import { JobConfig } from 'src/decorators';
-import { JobName, JobStatus, MetadataKey, QueueCleanType, QueueName } from 'src/enum';
+import { makeWorkerUtils, run, Runner, TaskSpec, WorkerUtils } from 'graphile-worker';
+import { Kysely } from 'kysely';
+import { DateTime, Duration } from 'luxon';
+import { InjectKysely } from 'nestjs-kysely';
+import pg, { PoolConfig } from 'pg';
+import { DB } from 'src/db';
+import { GenerateSql, JobConfig } from 'src/decorators';
+import { JobName, JobStatus, MetadataKey, QueueName, SystemMetadataKey } from 'src/enum';
 import { ConfigRepository } from 'src/repositories/config.repository';
 import { EventRepository } from 'src/repositories/event.repository';
 import { LoggingRepository } from 'src/repositories/logging.repository';
-import { IEntityJob, JobCounts, JobItem, JobOf, QueueStatus } from 'src/types';
+import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository';
+import { JobCounts, JobItem, JobOf, QueueStatus } from 'src/types';
+import { asPostgresConnectionConfig } from 'src/utils/database';
 import { getKeyByValue, getMethodNames, ImmichStartupError } from 'src/utils/misc';
 
 type JobMapItem = {
@@ -19,26 +24,38 @@ type JobMapItem = {
   label: string;
 };
 
+type QueueConfiguration = {
+  paused: boolean;
+  concurrency: number;
+};
+
 @Injectable()
 export class JobRepository {
-  private workers: Partial<Record<QueueName, Worker>> = {};
   private handlers: Partial<Record<JobName, JobMapItem>> = {};
 
+  // todo inject the pg pool
+  private pool?: pg.Pool;
+  // todo inject worker utils?
+  private workerUtils?: WorkerUtils;
+  private queueConfig: Record<string, QueueConfiguration> = {};
+  private runners: Record<string, Runner> = {};
+
   constructor(
-    private moduleRef: ModuleRef,
-    private configRepository: ConfigRepository,
-    private eventRepository: EventRepository,
+    @InjectKysely() private db: Kysely<DB>,
     private logger: LoggingRepository,
+    private moduleRef: ModuleRef,
+    private eventRepository: EventRepository,
+    private configRepository: ConfigRepository,
+    private systemMetadataRepository: SystemMetadataRepository,
   ) {
-    this.logger.setContext(JobRepository.name);
+    logger.setContext(JobRepository.name);
   }
 
-  setup(services: ClassConstructor<unknown>[]) {
+  async setup(services: ClassConstructor<unknown>[]) {
     const reflector = this.moduleRef.get(Reflector, { strict: false });
 
-    // discovery
-    for (const Service of services) {
-      const instance = this.moduleRef.get<any>(Service);
+    for (const service of services) {
+      const instance = this.moduleRef.get<any>(service);
       for (const methodName of getMethodNames(instance)) {
         const handler = instance[methodName];
         const config = reflector.get<JobConfig>(MetadataKey.JOB_CONFIG, handler);
@@ -47,7 +64,7 @@ export class JobRepository {
         }
 
         const { name: jobName, queue: queueName } = config;
-        const label = `${Service.name}.${handler.name}`;
+        const label = `${service.name}.${handler.name}`;
 
         // one handler per job
         if (this.handlers[jobName]) {
@@ -70,176 +87,217 @@ export class JobRepository {
       }
     }
 
-    // no missing handlers
-    for (const [jobKey, jobName] of Object.entries(JobName)) {
-      const item = this.handlers[jobName];
-      if (!item) {
-        const errorMessage = `Failed to find job handler for Job.${jobKey} ("${jobName}")`;
-        this.logger.error(
-          `${errorMessage}. Make sure to add the @OnJob({ name: JobName.${jobKey}, queue: QueueName.XYZ }) decorator for the new job.`,
-        );
-        throw new ImmichStartupError(errorMessage);
-      }
+    const { database } = this.configRepository.getEnv();
+    const pool = new pg.Pool({
+      ...asPostgresConnectionConfig(database.config),
+      max: 100,
+    } as PoolConfig);
+
+    // todo: remove debug info
+    setInterval(() => {
+      this.logger.log(`connections:
+        total: ${pool.totalCount}
+        idle: ${pool.idleCount}
+        waiting: ${pool.waitingCount}`);
+    }, 5000);
+
+    pool.on('connect', (client) => {
+      client.setMaxListeners(200);
+    });
+
+    this.pool = pool;
+
+    this.workerUtils = await makeWorkerUtils({ pgPool: pool });
+  }
+
+  async start(queueName: QueueName, concurrency?: number): Promise<void> {
+    if (concurrency) {
+      this.queueConfig[queueName] = {
+        ...this.queueConfig[queueName],
+        concurrency,
+      };
+    } else {
+      concurrency = this.queueConfig[queueName].concurrency;
+    }
+
+    if (this.queueConfig[queueName].paused) {
+      return;
+    }
+
+    await this.stop(queueName);
+    this.runners[queueName] = await run({
+      concurrency,
+      taskList: {
+        [queueName]: async (payload: unknown): Promise<void> => {
+          this.logger.log(`Job ${queueName} started with payload: ${JSON.stringify(payload)}`);
+          await this.eventRepository.emit('job.start', queueName, payload as JobItem);
+        },
+      },
+      pgPool: this.pool,
+    });
+  }
+
+  async stop(queueName: QueueName): Promise<void> {
+    const runner = this.runners[queueName];
+    if (runner) {
+      await runner.stop();
+      delete this.runners[queueName];
     }
   }
 
-  startWorkers() {
-    const { bull } = this.configRepository.getEnv();
-    for (const queueName of Object.values(QueueName)) {
-      this.logger.debug(`Starting worker for queue: ${queueName}`);
-      this.workers[queueName] = new Worker(
-        queueName,
-        (job) => this.eventRepository.emit('job.start', queueName, job as JobItem),
-        { ...bull.config, concurrency: 1 },
-      );
-    }
+  async pause(queueName: QueueName): Promise<void> {
+    await this.setState(queueName, true);
+    await this.stop(queueName);
   }
 
-  async run({ name, data }: JobItem) {
+  async resume(queueName: QueueName): Promise<void> {
+    await this.setState(queueName, false);
+    await this.start(queueName);
+  }
+
+  private async setState(queueName: QueueName, paused: boolean): Promise<void> {
+    const state = await this.systemMetadataRepository.get(SystemMetadataKey.QUEUES_STATE);
+    await this.systemMetadataRepository.set(SystemMetadataKey.QUEUES_STATE, {
+      ...state,
+      [queueName]: { paused },
+    });
+    this.queueConfig[queueName] = {
+      ...this.queueConfig[queueName],
+      paused,
+    };
+  }
+
+  // todo: we should consolidate queue and job names and have queues be
+  // homogenous.
+  //
+  // the reason there are multiple kinds of jobs per queue is so that
+  // concurrency settings apply to all of them. We could instead create a
+  // concept of "queue" groups, such that workers will run for groups of queues
+  // rather than just a single queue and achieve the same outcome.
+  private getQueueName(name: JobName) {
+    return (this.handlers[name] as JobMapItem).queueName;
+  }
+
+  async run({ name, data }: JobItem): Promise<JobStatus> {
     const item = this.handlers[name as JobName];
     if (!item) {
       this.logger.warn(`Skipping unknown job: "${name}"`);
       return JobStatus.SKIPPED;
     }
-
     return item.handler(data);
   }
 
-  setConcurrency(queueName: QueueName, concurrency: number) {
-    const worker = this.workers[queueName];
-    if (!worker) {
-      this.logger.warn(`Unable to set queue concurrency, worker not found: '${queueName}'`);
-      return;
-    }
-
-    worker.concurrency = concurrency;
-  }
-
-  async getQueueStatus(name: QueueName): Promise<QueueStatus> {
-    const queue = this.getQueue(name);
-
-    return {
-      isActive: !!(await queue.getActiveCount()),
-      isPaused: await queue.isPaused(),
-    };
-  }
-
-  pause(name: QueueName) {
-    return this.getQueue(name).pause();
-  }
-
-  resume(name: QueueName) {
-    return this.getQueue(name).resume();
-  }
-
-  empty(name: QueueName) {
-    return this.getQueue(name).drain();
-  }
-
-  clear(name: QueueName, type: QueueCleanType) {
-    return this.getQueue(name).clean(0, 1000, type);
-  }
-
-  getJobCounts(name: QueueName): Promise<JobCounts> {
-    return this.getQueue(name).getJobCounts(
-      'active',
-      'completed',
-      'failed',
-      'delayed',
-      'waiting',
-      'paused',
-    ) as unknown as Promise<JobCounts>;
-  }
-
-  private getQueueName(name: JobName) {
-    return (this.handlers[name] as JobMapItem).queueName;
+  async queue(item: JobItem): Promise<void> {
+    this.logger.log(`Queueing job: ${this.getQueueName(item.name)}, data: ${JSON.stringify(item)}`);
+    await this.workerUtils!.addJob(this.getQueueName(item.name), item, this.getJobOptions(item));
   }
 
   async queueAll(items: JobItem[]): Promise<void> {
-    if (items.length === 0) {
-      return;
-    }
-
-    const promises = [];
-    const itemsByQueue = {} as Record<string, (JobItem & { data: any; options: JobsOptions | undefined })[]>;
     for (const item of items) {
-      const queueName = this.getQueueName(item.name);
-      const job = {
-        name: item.name,
-        data: item.data || {},
-        options: this.getJobOptions(item) || undefined,
-      } as JobItem & { data: any; options: JobsOptions | undefined };
-
-      if (job.options?.jobId) {
-        // need to use add() instead of addBulk() for jobId deduplication
-        promises.push(this.getQueue(queueName).add(item.name, item.data, job.options));
-      } else {
-        itemsByQueue[queueName] = itemsByQueue[queueName] || [];
-        itemsByQueue[queueName].push(job);
-      }
-    }
-
-    for (const [queueName, jobs] of Object.entries(itemsByQueue)) {
-      const queue = this.getQueue(queueName as QueueName);
-      promises.push(queue.addBulk(jobs));
-    }
-
-    await Promise.all(promises);
-  }
-
-  async queue(item: JobItem): Promise<void> {
-    return this.queueAll([item]);
-  }
-
-  async waitForQueueCompletion(...queues: QueueName[]): Promise<void> {
-    let activeQueue: QueueStatus | undefined;
-    do {
-      const statuses = await Promise.all(queues.map((name) => this.getQueueStatus(name)));
-      activeQueue = statuses.find((status) => status.isActive);
-    } while (activeQueue);
-    {
-      this.logger.verbose(`Waiting for ${activeQueue} queue to stop...`);
-      await setTimeout(1000);
+      await this.queue(item);
     }
   }
 
-  private getJobOptions(item: JobItem): JobsOptions | null {
+  // todo: are we actually generating sql
+  async clear(name: QueueName): Promise<void> {
+    await this.db
+      .deleteFrom('graphile_worker._private_jobs')
+      .where(({ eb, selectFrom }) =>
+        eb('task_id', 'in', selectFrom('graphile_worker._private_tasks').select('id').where('identifier', '=', name)),
+      )
+      .execute();
+
+    const workers = await this.db
+      .selectFrom('graphile_worker.jobs')
+      .select('locked_by')
+      .where('locked_by', 'is not', null)
+      .distinct()
+      .execute();
+
+    // Potentially dangerous? It helps if jobs get stuck active though. The
+    // documentation says that stuck jobs will be unlocked automatically after 4
+    // hours. Though, it can be strange to click "clear" in the UI and see
+    // nothing happen. Especially as the UI is binary, such that new jobs cannot
+    // usually be scheduled unless both active and waiting are zero.
+    await this.workerUtils!.forceUnlockWorkers(workers.map((worker) => worker.locked_by!));
+  }
+
+  async clearFailed(name: QueueName): Promise<void> {
+    await this.db
+      .deleteFrom('graphile_worker._private_jobs')
+      .where(({ eb, selectFrom }) =>
+        eb(
+          'task_id',
+          'in',
+          selectFrom('graphile_worker._private_tasks')
+            .select('id')
+            .where((eb) => eb.and([eb('identifier', '=', name), eb('attempts', '>=', eb.ref('max_attempts'))])),
+        ),
+      )
+      .execute();
+  }
+
+  // todo: are we actually generating sql
+  @GenerateSql({ params: [] })
+  async getJobCounts(name: QueueName): Promise<JobCounts> {
+    return await this.db
+      .selectFrom('graphile_worker.jobs')
+      .select((eb) => [
+        eb.fn
+          .countAll<number>()
+          .filterWhere((eb) => eb.and([eb('task_identifier', '=', name), eb('locked_by', 'is not', null)]))
+          .as('active'),
+        eb.fn
+          .countAll<number>()
+          .filterWhere((eb) =>
+            eb.and([
+              eb('task_identifier', '=', name),
+              eb('locked_by', 'is', null),
+              eb('run_at', '<=', eb.fn<Date>('now')),
+            ]),
+          )
+          .as('waiting'),
+        eb.fn
+          .countAll<number>()
+          .filterWhere((eb) =>
+            eb.and([
+              eb('task_identifier', '=', name),
+              eb('locked_by', 'is', null),
+              eb('run_at', '>', eb.fn<Date>('now')),
+            ]),
+          )
+          .as('delayed'),
+        eb.fn
+          .countAll<number>()
+          .filterWhere((eb) => eb.and([eb('task_identifier', '=', name), eb('attempts', '>=', eb.ref('max_attempts'))]))
+          .as('failed'),
+      ])
+      .executeTakeFirstOrThrow();
+  }
+
+  async getQueueStatus(queueName: QueueName): Promise<QueueStatus> {
+    const state = await this.systemMetadataRepository.get(SystemMetadataKey.QUEUES_STATE);
+    return { paused: state?.[queueName]?.paused ?? false };
+  }
+
+  private getJobOptions(item: JobItem): TaskSpec | undefined {
     switch (item.name) {
       case JobName.NOTIFY_ALBUM_UPDATE: {
-        return { jobId: item.data.id, delay: item.data?.delay };
+        let runAt: Date | undefined;
+        if (item.data?.delay) {
+          runAt = DateTime.now().plus(Duration.fromMillis(item.data.delay)).toJSDate();
+        }
+        return { jobKey: item.data.id, runAt };
       }
       case JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE: {
-        return { jobId: item.data.id };
+        return { jobKey: QueueName.STORAGE_TEMPLATE_MIGRATION };
       }
       case JobName.GENERATE_PERSON_THUMBNAIL: {
         return { priority: 1 };
       }
       case JobName.QUEUE_FACIAL_RECOGNITION: {
-        return { jobId: JobName.QUEUE_FACIAL_RECOGNITION };
-      }
-      default: {
-        return null;
+        return { jobKey: JobName.QUEUE_FACIAL_RECOGNITION };
       }
     }
   }
-
-  private getQueue(queue: QueueName): Queue {
-    return this.moduleRef.get<Queue>(getQueueToken(queue), { strict: false });
-  }
-
-  public async removeJob(jobId: string, name: JobName): Promise<IEntityJob | undefined> {
-    const existingJob = await this.getQueue(this.getQueueName(name)).getJob(jobId);
-    if (!existingJob) {
-      return;
-    }
-    try {
-      await existingJob.remove();
-    } catch (error: any) {
-      if (error.message?.includes('Missing key for job')) {
-        return;
-      }
-      throw error;
-    }
-    return existingJob.data;
-  }
 }

server/src/repositories/telemetry.repository.ts

@@ -4,7 +4,6 @@ import { MetricOptions } from '@opentelemetry/api';
 import { AsyncLocalStorageContextManager } from '@opentelemetry/context-async-hooks';
 import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
 import { HttpInstrumentation } from '@opentelemetry/instrumentation-http';
-import { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis';
 import { NestInstrumentation } from '@opentelemetry/instrumentation-nestjs-core';
 import { PgInstrumentation } from '@opentelemetry/instrumentation-pg';
 import { resourceFromAttributes } from '@opentelemetry/resources';
@@ -68,12 +67,7 @@ export const bootstrapTelemetry = (port: number) => {
     }),
     metricReader: new PrometheusExporter({ port }),
     contextManager: new AsyncLocalStorageContextManager(),
-    instrumentations: [
-      new HttpInstrumentation(),
-      new IORedisInstrumentation(),
-      new NestInstrumentation(),
-      new PgInstrumentation(),
-    ],
+    instrumentations: [new HttpInstrumentation(), new NestInstrumentation(), new PgInstrumentation()],
     views: [
       {
         instrumentName: '*',

server/src/services/job.service.spec.ts

@@ -2,7 +2,7 @@ import { BadRequestException } from '@nestjs/common';
 import { defaults, SystemConfig } from 'src/config';
 import { ImmichWorker, JobCommand, JobName, JobStatus, QueueName } from 'src/enum';
 import { JobService } from 'src/services/job.service';
-import { JobItem } from 'src/types';
+import { JobCounts, JobItem } from 'src/types';
 import { assetStub } from 'test/fixtures/asset.stub';
 import { newTestService, ServiceMocks } from 'test/utils';
 
@@ -21,14 +21,14 @@ describe(JobService.name, () => {
   });
 
   describe('onConfigUpdate', () => {
-    it('should update concurrency', () => {
-      sut.onConfigUpdate({ newConfig: defaults, oldConfig: {} as SystemConfig });
+    it('should update concurrency', async () => {
+      await sut.onConfigUpdate({ newConfig: defaults, oldConfig: {} as SystemConfig });
 
-      expect(mocks.job.setConcurrency).toHaveBeenCalledTimes(15);
-      expect(mocks.job.setConcurrency).toHaveBeenNthCalledWith(5, QueueName.FACIAL_RECOGNITION, 1);
-      expect(mocks.job.setConcurrency).toHaveBeenNthCalledWith(7, QueueName.DUPLICATE_DETECTION, 1);
-      expect(mocks.job.setConcurrency).toHaveBeenNthCalledWith(8, QueueName.BACKGROUND_TASK, 5);
-      expect(mocks.job.setConcurrency).toHaveBeenNthCalledWith(9, QueueName.STORAGE_TEMPLATE_MIGRATION, 1);
+      expect(mocks.job.start).toHaveBeenCalledTimes(15);
+      expect(mocks.job.start).toHaveBeenNthCalledWith(5, QueueName.FACIAL_RECOGNITION, 1);
+      expect(mocks.job.start).toHaveBeenNthCalledWith(7, QueueName.DUPLICATE_DETECTION, 1);
+      expect(mocks.job.start).toHaveBeenNthCalledWith(8, QueueName.BACKGROUND_TASK, 5);
+      expect(mocks.job.start).toHaveBeenNthCalledWith(9, QueueName.STORAGE_TEMPLATE_MIGRATION, 1);
     });
   });
 
@@ -55,29 +55,20 @@ describe(JobService.name, () => {
     it('should get all job statuses', async () => {
       mocks.job.getJobCounts.mockResolvedValue({
         active: 1,
-        completed: 1,
-        failed: 1,
-        delayed: 1,
         waiting: 1,
-        paused: 1,
-      });
-      mocks.job.getQueueStatus.mockResolvedValue({
-        isActive: true,
-        isPaused: true,
+        delayed: 1,
+        failed: 1,
       });
 
       const expectedJobStatus = {
         jobCounts: {
           active: 1,
-          completed: 1,
+          waiting: 1,
           delayed: 1,
           failed: 1,
-          waiting: 1,
-          paused: 1,
         },
         queueStatus: {
-          isActive: true,
-          isPaused: true,
+          paused: true,
         },
       };
 
@@ -114,14 +105,20 @@ describe(JobService.name, () => {
       expect(mocks.job.resume).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION);
     });
 
-    it('should handle an empty command', async () => {
-      await sut.handleCommand(QueueName.METADATA_EXTRACTION, { command: JobCommand.EMPTY, force: false });
+    it('should handle a clear command', async () => {
+      await sut.handleCommand(QueueName.METADATA_EXTRACTION, { command: JobCommand.CLEAR, force: false });
 
-      expect(mocks.job.empty).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION);
+      expect(mocks.job.clear).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION);
+    });
+
+    it('should handle a clear-failed command', async () => {
+      await sut.handleCommand(QueueName.METADATA_EXTRACTION, { command: JobCommand.CLEAR_FAILED, force: false });
+
+      expect(mocks.job.clearFailed).toHaveBeenCalledWith(QueueName.METADATA_EXTRACTION);
     });
 
     it('should not start a job that is already running', async () => {
-      mocks.job.getQueueStatus.mockResolvedValue({ isActive: true, isPaused: false });
+      mocks.job.getJobCounts.mockResolvedValue({ active: 1 } as JobCounts);
 
       await expect(
         sut.handleCommand(QueueName.VIDEO_CONVERSION, { command: JobCommand.START, force: false }),
@@ -132,7 +129,7 @@ describe(JobService.name, () => {
     });
 
     it('should handle a start video conversion command', async () => {
-      mocks.job.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
+      mocks.job.getJobCounts.mockResolvedValue({ active: 0 } as JobCounts);
 
       await sut.handleCommand(QueueName.VIDEO_CONVERSION, { command: JobCommand.START, force: false });
 
@@ -140,7 +137,7 @@ describe(JobService.name, () => {
     });
 
     it('should handle a start storage template migration command', async () => {
-      mocks.job.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
+      mocks.job.getJobCounts.mockResolvedValue({ active: 0 } as JobCounts);
 
       await sut.handleCommand(QueueName.STORAGE_TEMPLATE_MIGRATION, { command: JobCommand.START, force: false });
 
@@ -148,7 +145,7 @@ describe(JobService.name, () => {
     });
 
     it('should handle a start smart search command', async () => {
-      mocks.job.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
+      mocks.job.getJobCounts.mockResolvedValue({ active: 0 } as JobCounts);
 
       await sut.handleCommand(QueueName.SMART_SEARCH, { command: JobCommand.START, force: false });
 
@@ -156,7 +153,7 @@ describe(JobService.name, () => {
     });
 
     it('should handle a start metadata extraction command', async () => {
-      mocks.job.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
+      mocks.job.getJobCounts.mockResolvedValue({ active: 0 } as JobCounts);
 
       await sut.handleCommand(QueueName.METADATA_EXTRACTION, { command: JobCommand.START, force: false });
 
@@ -164,7 +161,7 @@ describe(JobService.name, () => {
     });
 
     it('should handle a start sidecar command', async () => {
-      mocks.job.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
+      mocks.job.getJobCounts.mockResolvedValue({ active: 0 } as JobCounts);
 
       await sut.handleCommand(QueueName.SIDECAR, { command: JobCommand.START, force: false });
 
@@ -172,7 +169,7 @@ describe(JobService.name, () => {
     });
 
     it('should handle a start thumbnail generation command', async () => {
-      mocks.job.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
+      mocks.job.getJobCounts.mockResolvedValue({ active: 0 } as JobCounts);
 
       await sut.handleCommand(QueueName.THUMBNAIL_GENERATION, { command: JobCommand.START, force: false });
 
@@ -180,7 +177,7 @@ describe(JobService.name, () => {
     });
 
     it('should handle a start face detection command', async () => {
-      mocks.job.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
+      mocks.job.getJobCounts.mockResolvedValue({ active: 0 } as JobCounts);
 
       await sut.handleCommand(QueueName.FACE_DETECTION, { command: JobCommand.START, force: false });
 
@@ -188,7 +185,7 @@ describe(JobService.name, () => {
     });
 
     it('should handle a start facial recognition command', async () => {
-      mocks.job.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
+      mocks.job.getJobCounts.mockResolvedValue({ active: 0 } as JobCounts);
 
       await sut.handleCommand(QueueName.FACIAL_RECOGNITION, { command: JobCommand.START, force: false });
 
@@ -196,7 +193,7 @@ describe(JobService.name, () => {
     });
 
     it('should handle a start backup database command', async () => {
-      mocks.job.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
+      mocks.job.getJobCounts.mockResolvedValue({ active: 0 } as JobCounts);
 
       await sut.handleCommand(QueueName.BACKUP_DATABASE, { command: JobCommand.START, force: false });
 
@@ -204,7 +201,7 @@ describe(JobService.name, () => {
     });
 
     it('should throw a bad request when an invalid queue is used', async () => {
-      mocks.job.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
+      mocks.job.getJobCounts.mockResolvedValue({ active: 0 } as JobCounts);
 
       await expect(
         sut.handleCommand(QueueName.BACKGROUND_TASK, { command: JobCommand.START, force: false }),

server/src/services/job.service.ts

@@ -12,7 +12,6 @@ import {
   JobName,
   JobStatus,
   ManualJobName,
-  QueueCleanType,
   QueueName,
 } from 'src/enum';
 import { ArgOf, ArgsOf } from 'src/repositories/event.repository';
@@ -56,7 +55,7 @@ export class JobService extends BaseService {
   private services: ClassConstructor<unknown>[] = [];
 
   @OnEvent({ name: 'config.init', workers: [ImmichWorker.MICROSERVICES] })
-  onConfigInit({ newConfig: config }: ArgOf<'config.init'>) {
+  async onConfigInit({ newConfig: config }: ArgOf<'config.init'>) {
     this.logger.debug(`Updating queue concurrency settings`);
     for (const queueName of Object.values(QueueName)) {
       let concurrency = 1;
@@ -64,21 +63,18 @@ export class JobService extends BaseService {
         concurrency = config.job[queueName].concurrency;
       }
       this.logger.debug(`Setting ${queueName} concurrency to ${concurrency}`);
-      this.jobRepository.setConcurrency(queueName, concurrency);
+      await this.jobRepository.start(queueName, concurrency);
     }
   }
 
   @OnEvent({ name: 'config.update', server: true, workers: [ImmichWorker.MICROSERVICES] })
-  onConfigUpdate({ newConfig: config }: ArgOf<'config.update'>) {
-    this.onConfigInit({ newConfig: config });
+  async onConfigUpdate({ newConfig: config }: ArgOf<'config.update'>) {
+    await this.onConfigInit({ newConfig: config });
   }
 
   @OnEvent({ name: 'app.bootstrap', priority: BootstrapEventPriority.JobService })
-  onBootstrap() {
-    this.jobRepository.setup(this.services);
-    if (this.worker === ImmichWorker.MICROSERVICES) {
-      this.jobRepository.startWorkers();
-    }
+  async onBootstrap() {
+    await this.jobRepository.setup(this.services);
   }
 
   setServices(services: ClassConstructor<unknown>[]) {
@@ -97,25 +93,20 @@ export class JobService extends BaseService {
         await this.start(queueName, dto);
         break;
       }
-
       case JobCommand.PAUSE: {
-        await this.jobRepository.pause(queueName);
+        this.eventRepository.serverSend('queue.pause', queueName);
         break;
       }
-
       case JobCommand.RESUME: {
-        await this.jobRepository.resume(queueName);
+        this.eventRepository.serverSend('queue.resume', queueName);
         break;
       }
-
-      case JobCommand.EMPTY: {
-        await this.jobRepository.empty(queueName);
+      case JobCommand.CLEAR: {
+        await this.jobRepository.clear(queueName);
         break;
       }
-
       case JobCommand.CLEAR_FAILED: {
-        const failedJobs = await this.jobRepository.clear(queueName, QueueCleanType.FAILED);
-        this.logger.debug(`Cleared failed jobs: ${failedJobs}`);
+        await this.jobRepository.clearFailed(queueName);
         break;
       }
     }
@@ -141,9 +132,9 @@ export class JobService extends BaseService {
   }
 
   private async start(name: QueueName, { force }: JobCommandDto): Promise<void> {
-    const { isActive } = await this.jobRepository.getQueueStatus(name);
-    if (isActive) {
-      throw new BadRequestException(`Job is already running`);
+    const { active } = await this.jobRepository.getJobCounts(name);
+    if (active > 0) {
+      throw new BadRequestException(`Jobs are already running`);
     }
 
     this.telemetryRepository.jobs.addToCounter(`immich.queues.${snakeCase(name)}.started`, 1);
@@ -203,6 +194,16 @@ export class JobService extends BaseService {
     }
   }
 
+  @OnEvent({ name: 'queue.pause', server: true, workers: [ImmichWorker.MICROSERVICES] })
+  async pause(...[queueName]: ArgsOf<'queue.pause'>): Promise<void> {
+    await this.jobRepository.pause(queueName);
+  }
+
+  @OnEvent({ name: 'queue.resume', server: true, workers: [ImmichWorker.MICROSERVICES] })
+  async resume(...[queueName]: ArgsOf<'queue.resume'>): Promise<void> {
+    await this.jobRepository.resume(queueName);
+  }
+
   @OnEvent({ name: 'job.start' })
   async onJobStart(...[queueName, job]: ArgsOf<'job.start'>) {
     const queueMetric = `immich.queues.${snakeCase(queueName)}.active`;

server/src/services/metadata.service.spec.ts

@@ -67,16 +67,12 @@ describe(MetadataService.name, () => {
   });
 
   describe('onBootstrapEvent', () => {
-    it('should pause and resume queue during init', async () => {
-      mocks.job.pause.mockResolvedValue();
+    it('should init', async () => {
       mocks.map.init.mockResolvedValue();
-      mocks.job.resume.mockResolvedValue();
 
       await sut.onBootstrap();
 
-      expect(mocks.job.pause).toHaveBeenCalledTimes(1);
       expect(mocks.map.init).toHaveBeenCalledTimes(1);
-      expect(mocks.job.resume).toHaveBeenCalledTimes(1);
     });
   });
 

server/src/services/metadata.service.ts

@@ -121,9 +121,7 @@ export class MetadataService extends BaseService {
     this.logger.log('Initializing metadata service');
 
     try {
-      await this.jobRepository.pause(QueueName.METADATA_EXTRACTION);
       await this.databaseRepository.withLock(DatabaseLock.GeodataImport, () => this.mapRepository.init());
-      await this.jobRepository.resume(QueueName.METADATA_EXTRACTION);
 
       this.logger.log(`Initialized local reverse geocoder`);
     } catch (error: Error | any) {

server/src/services/notification.service.spec.ts

@@ -499,14 +499,13 @@ describe(NotificationService.name, () => {
     });
 
     it('should add new recipients for new images if job is already queued', async () => {
-      mocks.job.removeJob.mockResolvedValue({ id: '1', recipientIds: ['2', '3', '4'] } as INotifyAlbumUpdateJob);
       await sut.onAlbumUpdate({ id: '1', recipientIds: ['1', '2', '3'] } as INotifyAlbumUpdateJob);
       expect(mocks.job.queue).toHaveBeenCalledWith({
         name: JobName.NOTIFY_ALBUM_UPDATE,
         data: {
           id: '1',
           delay: 300_000,
-          recipientIds: ['1', '2', '3', '4'],
+          recipientIds: ['1', '2', '3'],
         },
       });
     });

server/src/services/notification.service.ts

@@ -196,14 +196,15 @@ export class NotificationService extends BaseService {
       data: { id, recipientIds, delay: NotificationService.albumUpdateEmailDelayMs },
     };
 
-    const previousJobData = await this.jobRepository.removeJob(id, JobName.NOTIFY_ALBUM_UPDATE);
-    if (previousJobData && this.isAlbumUpdateJob(previousJobData)) {
-      for (const id of previousJobData.recipientIds) {
-        if (!recipientIds.includes(id)) {
-          recipientIds.push(id);
-        }
-      }
-    }
+    // todo: https://github.com/immich-app/immich/pull/17879
+    // const previousJobData = await this.jobRepository.removeJob(id, JobName.NOTIFY_ALBUM_UPDATE);
+    // if (previousJobData && this.isAlbumUpdateJob(previousJobData)) {
+    //   for (const id of previousJobData.recipientIds) {
+    //     if (!recipientIds.includes(id)) {
+    //       recipientIds.push(id);
+    //     }
+    //   }
+    // }
     await this.jobRepository.queue(job);
   }
 

server/src/services/person.service.spec.ts

@@ -529,10 +529,8 @@ describe(PersonService.name, () => {
       mocks.job.getJobCounts.mockResolvedValue({
         active: 1,
         waiting: 0,
-        paused: 0,
-        completed: 0,
-        failed: 0,
         delayed: 0,
+        failed: 0,
       });
       mocks.systemMetadata.get.mockResolvedValue(systemConfigStub.machineLearningDisabled);
 
@@ -546,10 +544,8 @@ describe(PersonService.name, () => {
       mocks.job.getJobCounts.mockResolvedValue({
         active: 1,
         waiting: 1,
-        paused: 0,
-        completed: 0,
-        failed: 0,
         delayed: 0,
+        failed: 0,
       });
 
       await expect(sut.handleQueueRecognizeFaces({})).resolves.toBe(JobStatus.SKIPPED);
@@ -561,10 +557,8 @@ describe(PersonService.name, () => {
       mocks.job.getJobCounts.mockResolvedValue({
         active: 1,
         waiting: 0,
-        paused: 0,
-        completed: 0,
-        failed: 0,
         delayed: 0,
+        failed: 0,
       });
       mocks.person.getAllFaces.mockReturnValue(makeStream([faceStub.face1]));
       mocks.person.getAllWithoutFaces.mockResolvedValue([]);
@@ -590,10 +584,8 @@ describe(PersonService.name, () => {
       mocks.job.getJobCounts.mockResolvedValue({
         active: 1,
         waiting: 0,
-        paused: 0,
-        completed: 0,
-        failed: 0,
         delayed: 0,
+        failed: 0,
       });
       mocks.person.getAll.mockReturnValue(makeStream());
       mocks.person.getAllFaces.mockReturnValue(makeStream([faceStub.face1]));
@@ -619,10 +611,8 @@ describe(PersonService.name, () => {
       mocks.job.getJobCounts.mockResolvedValue({
         active: 1,
         waiting: 0,
-        paused: 0,
-        completed: 0,
-        failed: 0,
         delayed: 0,
+        failed: 0,
       });
       mocks.person.getAll.mockReturnValue(makeStream());
       mocks.person.getAllFaces.mockReturnValue(makeStream([faceStub.face1]));
@@ -666,10 +656,8 @@ describe(PersonService.name, () => {
       mocks.job.getJobCounts.mockResolvedValue({
         active: 1,
         waiting: 0,
-        paused: 0,
-        completed: 0,
-        failed: 0,
         delayed: 0,
+        failed: 0,
       });
       mocks.person.getAll.mockReturnValue(makeStream([faceStub.face1.person, personStub.randomPerson]));
       mocks.person.getAllFaces.mockReturnValue(makeStream([faceStub.face1]));

server/src/services/person.service.ts

@@ -392,7 +392,8 @@ export class PersonService extends BaseService {
       return JobStatus.SKIPPED;
     }
 
-    await this.jobRepository.waitForQueueCompletion(QueueName.THUMBNAIL_GENERATION, QueueName.FACE_DETECTION);
+    // todo
+    // await this.jobRepository.waitForQueueCompletion(QueueName.THUMBNAIL_GENERATION, QueueName.FACE_DETECTION);
 
     if (nightly) {
       const [state, latestFaceDate] = await Promise.all([

server/src/types.ts

@@ -256,16 +256,13 @@ export interface INotifyAlbumUpdateJob extends IEntityJob, IDelayedJob {
 
 export interface JobCounts {
   active: number;
-  completed: number;
-  failed: number;
-  delayed: number;
   waiting: number;
-  paused: number;
+  delayed: number;
+  failed: number;
 }
 
 export interface QueueStatus {
-  isActive: boolean;
-  isPaused: boolean;
+  paused: boolean;
 }
 
 export type JobItem =
@@ -450,6 +447,14 @@ export type MemoriesState = {
   lastOnThisDayDate: string;
 };
 
+export type QueueState = {
+  paused: boolean;
+};
+
+export type QueuesState = {
+  [key in QueueName]?: QueueState;
+};
+
 export interface SystemMetadata extends Record<SystemMetadataKey, Record<string, any>> {
   [SystemMetadataKey.ADMIN_ONBOARDING]: { isOnboarded: boolean };
   [SystemMetadataKey.FACIAL_RECOGNITION_STATE]: { lastRun?: string };
@@ -459,6 +464,7 @@ export interface SystemMetadata extends Record<SystemMetadataKey, Record<string,
   [SystemMetadataKey.SYSTEM_FLAGS]: DeepPartial<SystemFlags>;
   [SystemMetadataKey.VERSION_CHECK_STATE]: VersionCheckMetadata;
   [SystemMetadataKey.MEMORIES_STATE]: MemoriesState;
+  [SystemMetadataKey.QUEUES_STATE]: QueuesState;
 }
 
 export type UserMetadataItem<T extends keyof UserMetadata = UserMetadataKey> = {

server/src/utils/database.ts

@@ -32,7 +32,7 @@ export const asPostgresConnectionConfig = (params: DatabaseConnectionParams) =>
     return {
       host: params.host,
       port: params.port,
-      username: params.username,
+      user: params.username,
       password: params.password,
       database: params.database,
       ssl: undefined,
@@ -51,7 +51,7 @@ export const asPostgresConnectionConfig = (params: DatabaseConnectionParams) =>
   return {
     host: host ?? undefined,
     port: port ? Number(port) : undefined,
-    username: user,
+    user,
     password,
     database: database ?? undefined,
     ssl,
@@ -92,7 +92,7 @@ export const getKyselyConfig = (
         },
         host: config.host,
         port: config.port,
-        username: config.username,
+        user: config.user,
         password: config.password,
         database: config.database,
         ssl: config.ssl,