github-mirrors/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2026-05-18 03:10:24 +03:00
Code Issues Packages Projects Releases Wiki Activity

refactor: job vs queue naming (#23902)

Browse Source
This commit is contained in:
Jason Rasmussen
2025-11-14 14:42:00 -05:00
committed by GitHub
parent 1200bfad13
commit d784d431d0
36 changed files with 1356 additions and 1325 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/services/job.service.ts
+4 -248
View File
@@ -1,28 +1,12 @@
import { BadRequestException, Injectable } from '@nestjs/common';
import { ClassConstructor } from 'class-transformer';
import { SystemConfig } from 'src/config';
import { OnEvent } from 'src/decorators';
import { mapAsset } from 'src/dtos/asset-response.dto';
import { AllJobStatusResponseDto, JobCommandDto, JobCreateDto, JobStatusDto } from 'src/dtos/job.dto';
import {
AssetType,
AssetVisibility,
BootstrapEventPriority,
CronJob,
DatabaseLock,
ImmichWorker,
JobCommand,
JobName,
JobStatus,
ManualJobName,
QueueCleanType,
QueueName,
} from 'src/enum';
import { ArgOf, ArgsOf } from 'src/repositories/event.repository';
import { JobCreateDto } from 'src/dtos/job.dto';
import { AssetType, AssetVisibility, JobName, JobStatus, ManualJobName } from 'src/enum';
import { ArgsOf } from 'src/repositories/event.repository';
import { BaseService } from 'src/services/base.service';
import { ConcurrentQueueName, JobItem } from 'src/types';
import { JobItem } from 'src/types';
import { hexOrBufferToBase64 } from 'src/utils/bytes';
import { handlePromiseError } from 'src/utils/misc';
const asJobItem = (dto: JobCreateDto): JobItem => {
switch (dto.name) {
@@ -56,196 +40,12 @@ const asJobItem = (dto: JobCreateDto): JobItem => {
}
};
const asNightlyTasksCron = (config: SystemConfig) => {
const [hours, minutes] = config.nightlyTasks.startTime.split(':').map(Number);
return `${minutes} ${hours} * * *`;
};
@Injectable()
export class JobService extends BaseService {
private services: ClassConstructor<unknown>[] = [];
private nightlyJobsLock = false;
@OnEvent({ name: 'ConfigInit' })
async onConfigInit({ newConfig: config }: ArgOf<'ConfigInit'>) {
if (this.worker === ImmichWorker.Microservices) {
this.updateQueueConcurrency(config);
return;
}
this.nightlyJobsLock = await this.databaseRepository.tryLock(DatabaseLock.NightlyJobs);
if (this.nightlyJobsLock) {
const cronExpression = asNightlyTasksCron(config);
this.logger.debug(`Scheduling nightly jobs for ${cronExpression}`);
this.cronRepository.create({
name: CronJob.NightlyJobs,
expression: cronExpression,
start: true,
onTick: () => handlePromiseError(this.handleNightlyJobs(), this.logger),
});
}
}
@OnEvent({ name: 'ConfigUpdate', server: true })
onConfigUpdate({ newConfig: config }: ArgOf<'ConfigUpdate'>) {
if (this.worker === ImmichWorker.Microservices) {
this.updateQueueConcurrency(config);
return;
}
if (this.nightlyJobsLock) {
const cronExpression = asNightlyTasksCron(config);
this.logger.debug(`Scheduling nightly jobs for ${cronExpression}`);
this.cronRepository.update({ name: CronJob.NightlyJobs, expression: cronExpression, start: true });
}
}
@OnEvent({ name: 'AppBootstrap', priority: BootstrapEventPriority.JobService })
onBootstrap() {
this.jobRepository.setup(this.services);
if (this.worker === ImmichWorker.Microservices) {
this.jobRepository.startWorkers();
}
}
private updateQueueConcurrency(config: SystemConfig) {
this.logger.debug(`Updating queue concurrency settings`);
for (const queueName of Object.values(QueueName)) {
let concurrency = 1;
if (this.isConcurrentQueue(queueName)) {
concurrency = config.job[queueName].concurrency;
}
this.logger.debug(`Setting ${queueName} concurrency to ${concurrency}`);
this.jobRepository.setConcurrency(queueName, concurrency);
}
}
setServices(services: ClassConstructor<unknown>[]) {
this.services = services;
}
async create(dto: JobCreateDto): Promise<void> {
await this.jobRepository.queue(asJobItem(dto));
}
async handleCommand(queueName: QueueName, dto: JobCommandDto): Promise<JobStatusDto> {
this.logger.debug(`Handling command: queue=${queueName},command=${dto.command},force=${dto.force}`);
switch (dto.command) {
case JobCommand.Start: {
await this.start(queueName, dto);
break;
}
case JobCommand.Pause: {
await this.jobRepository.pause(queueName);
break;
}
case JobCommand.Resume: {
await this.jobRepository.resume(queueName);
break;
}
case JobCommand.Empty: {
await this.jobRepository.empty(queueName);
break;
}
case JobCommand.ClearFailed: {
const failedJobs = await this.jobRepository.clear(queueName, QueueCleanType.Failed);
this.logger.debug(`Cleared failed jobs: ${failedJobs}`);
break;
}
}
return this.getJobStatus(queueName);
}
async getJobStatus(queueName: QueueName): Promise<JobStatusDto> {
const [jobCounts, queueStatus] = await Promise.all([
this.jobRepository.getJobCounts(queueName),
this.jobRepository.getQueueStatus(queueName),
]);
return { jobCounts, queueStatus };
}
async getAllJobsStatus(): Promise<AllJobStatusResponseDto> {
const response = new AllJobStatusResponseDto();
for (const queueName of Object.values(QueueName)) {
response[queueName] = await this.getJobStatus(queueName);
}
return response;
}
private async start(name: QueueName, { force }: JobCommandDto): Promise<void> {
const { isActive } = await this.jobRepository.getQueueStatus(name);
if (isActive) {
throw new BadRequestException(`Job is already running`);
}
await this.eventRepository.emit('QueueStart', { name });
switch (name) {
case QueueName.VideoConversion: {
return this.jobRepository.queue({ name: JobName.AssetEncodeVideoQueueAll, data: { force } });
}
case QueueName.StorageTemplateMigration: {
return this.jobRepository.queue({ name: JobName.StorageTemplateMigration });
}
case QueueName.Migration: {
return this.jobRepository.queue({ name: JobName.FileMigrationQueueAll });
}
case QueueName.SmartSearch: {
return this.jobRepository.queue({ name: JobName.SmartSearchQueueAll, data: { force } });
}
case QueueName.DuplicateDetection: {
return this.jobRepository.queue({ name: JobName.AssetDetectDuplicatesQueueAll, data: { force } });
}
case QueueName.MetadataExtraction: {
return this.jobRepository.queue({ name: JobName.AssetExtractMetadataQueueAll, data: { force } });
}
case QueueName.Sidecar: {
return this.jobRepository.queue({ name: JobName.SidecarQueueAll, data: { force } });
}
case QueueName.ThumbnailGeneration: {
return this.jobRepository.queue({ name: JobName.AssetGenerateThumbnailsQueueAll, data: { force } });
}
case QueueName.FaceDetection: {
return this.jobRepository.queue({ name: JobName.AssetDetectFacesQueueAll, data: { force } });
}
case QueueName.FacialRecognition: {
return this.jobRepository.queue({ name: JobName.FacialRecognitionQueueAll, data: { force } });
}
case QueueName.Library: {
return this.jobRepository.queue({ name: JobName.LibraryScanQueueAll, data: { force } });
}
case QueueName.BackupDatabase: {
return this.jobRepository.queue({ name: JobName.DatabaseBackup, data: { force } });
}
case QueueName.Ocr: {
return this.jobRepository.queue({ name: JobName.OcrQueueAll, data: { force } });
}
default: {
throw new BadRequestException(`Invalid job name: ${name}`);
}
}
}
@OnEvent({ name: 'JobRun' })
async onJobRun(...[queueName, job]: ArgsOf<'JobRun'>) {
try {
@@ -262,50 +62,6 @@ export class JobService extends BaseService {
}
}
private isConcurrentQueue(name: QueueName): name is ConcurrentQueueName {
return ![
QueueName.FacialRecognition,
QueueName.StorageTemplateMigration,
QueueName.DuplicateDetection,
QueueName.BackupDatabase,
].includes(name);
}
async handleNightlyJobs() {
const config = await this.getConfig({ withCache: false });
const jobs: JobItem[] = [];
if (config.nightlyTasks.databaseCleanup) {
jobs.push(
{ name: JobName.AssetDeleteCheck },
{ name: JobName.UserDeleteCheck },
{ name: JobName.PersonCleanup },
{ name: JobName.MemoryCleanup },
{ name: JobName.SessionCleanup },
{ name: JobName.AuditTableCleanup },
{ name: JobName.AuditLogCleanup },
);
}
if (config.nightlyTasks.generateMemories) {
jobs.push({ name: JobName.MemoryGenerate });
}
if (config.nightlyTasks.syncQuotaUsage) {
jobs.push({ name: JobName.UserSyncUsage });
}
if (config.nightlyTasks.missingThumbnails) {
jobs.push({ name: JobName.AssetGenerateThumbnailsQueueAll, data: { force: false } });
}
if (config.nightlyTasks.clusterNewFaces) {
jobs.push({ name: JobName.FacialRecognitionQueueAll, data: { force: false, nightly: true } });
}
await this.jobRepository.queueAll(jobs);
}
/**
* Queue follow up jobs
*/