feat: ocr (#18836)

* feat: add OCR functionality and related configurations

* chore: update labeler configuration for machine learning files

* feat(i18n): enhance OCR model descriptions and add orientation classification and unwarping features

* chore: update Dockerfile to include ccache for improved build performance

* feat(ocr): enhance OCR model configuration with orientation classification and unwarping options, update PaddleOCR integration, and improve response structure

* refactor(ocr): remove OCR_CLEANUP job from enum and type definitions

* refactor(ocr): remove obsolete OCR entity and migration files, and update asset job status and schema to accommodate new OCR table structure

* refactor(ocr): update OCR schema and response structure to use individual coordinates instead of bounding box, and adjust related service and repository files

* feat: enhance OCR configuration and functionality

- Updated OCR settings to include minimum detection box score, minimum detection score, and minimum recognition score.
- Refactored PaddleOCRecognizer to utilize new scoring parameters.
- Introduced new database tables for asset OCR data and search functionality.
- Modified related services and repositories to support the new OCR features.
- Updated translations for improved clarity in settings UI.

* sql changes

* use rapidocr

* change dto

* update web

* update lock

* update api

* store positions as normalized floats

* match column order in db

* update admin ui settings descriptions

fix max resolution key

set min threshold to 0.1

fix bind

* apply config correctly, adjust defaults

* unnecessary model type

* unnecessary sources

* fix(ocr): switch RapidOCR lang type from LangDet to LangRec

* fix(ocr): expose lang_type (LangRec.CH) and font_path on OcrOptions for RapidOCR

* fix(ocr): make OCR text search case- and accent-insensitive using ILIKE + unaccent

* fix(ocr): add OCR search fields

* fix: Add OCR database migration and update ML prediction logic.

* trigrams are already case insensitive

* add tests

* format

* update migrations

* wrong uuid function

* linting

* maybe fix medium tests

* formatting

* fix weblate check

* openapi

* sql

* minor fixes

* maybe fix medium tests part 2

* passing medium tests

* format web

* readd sql

* format dart

* disabled in e2e

* chore: translation ordering

---------

Co-authored-by: mertalev <101130780+mertalev@users.noreply.github.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>

server/src/services/base.service.ts

@@ -32,6 +32,7 @@ import { MetadataRepository } from 'src/repositories/metadata.repository';
 import { MoveRepository } from 'src/repositories/move.repository';
 import { NotificationRepository } from 'src/repositories/notification.repository';
 import { OAuthRepository } from 'src/repositories/oauth.repository';
+import { OcrRepository } from 'src/repositories/ocr.repository';
 import { PartnerRepository } from 'src/repositories/partner.repository';
 import { PersonRepository } from 'src/repositories/person.repository';
 import { ProcessRepository } from 'src/repositories/process.repository';
@@ -84,6 +85,7 @@ export const BASE_SERVICE_DEPENDENCIES = [
   MoveRepository,
   NotificationRepository,
   OAuthRepository,
+  OcrRepository,
   PartnerRepository,
   PersonRepository,
   ProcessRepository,
@@ -137,6 +139,7 @@ export class BaseService {
     protected moveRepository: MoveRepository,
     protected notificationRepository: NotificationRepository,
     protected oauthRepository: OAuthRepository,
+    protected ocrRepository: OcrRepository,
     protected partnerRepository: PartnerRepository,
     protected personRepository: PersonRepository,
     protected processRepository: ProcessRepository,

server/src/services/index.ts

@@ -20,6 +20,7 @@ import { MemoryService } from 'src/services/memory.service';
 import { MetadataService } from 'src/services/metadata.service';
 import { NotificationAdminService } from 'src/services/notification-admin.service';
 import { NotificationService } from 'src/services/notification.service';
+import { OcrService } from 'src/services/ocr.service';
 import { PartnerService } from 'src/services/partner.service';
 import { PersonService } from 'src/services/person.service';
 import { SearchService } from 'src/services/search.service';
@@ -65,6 +66,7 @@ export const services = [
   MetadataService,
   NotificationService,
   NotificationAdminService,
+  OcrService,
   PartnerService,
   PersonService,
   SearchService,

server/src/services/job.service.spec.ts

@@ -24,7 +24,7 @@ describe(JobService.name, () => {
     it('should update concurrency', () => {
       sut.onConfigUpdate({ newConfig: defaults, oldConfig: {} as SystemConfig });
 
-      expect(mocks.job.setConcurrency).toHaveBeenCalledTimes(15);
+      expect(mocks.job.setConcurrency).toHaveBeenCalledTimes(16);
       expect(mocks.job.setConcurrency).toHaveBeenNthCalledWith(5, QueueName.FacialRecognition, 1);
       expect(mocks.job.setConcurrency).toHaveBeenNthCalledWith(7, QueueName.DuplicateDetection, 1);
       expect(mocks.job.setConcurrency).toHaveBeenNthCalledWith(8, QueueName.BackgroundTask, 5);
@@ -98,6 +98,7 @@ describe(JobService.name, () => {
         [QueueName.Library]: expectedJobStatus,
         [QueueName.Notification]: expectedJobStatus,
         [QueueName.BackupDatabase]: expectedJobStatus,
+        [QueueName.Ocr]: expectedJobStatus,
       });
     });
   });
@@ -268,12 +269,12 @@ describe(JobService.name, () => {
       },
       {
         item: { name: JobName.AssetGenerateThumbnails, data: { id: 'asset-1', source: 'upload' } },
-        jobs: [JobName.SmartSearch, JobName.AssetDetectFaces],
+        jobs: [JobName.SmartSearch, JobName.AssetDetectFaces, JobName.Ocr],
         stub: [assetStub.livePhotoStillAsset],
       },
       {
         item: { name: JobName.AssetGenerateThumbnails, data: { id: 'asset-1', source: 'upload' } },
-        jobs: [JobName.SmartSearch, JobName.AssetDetectFaces, JobName.AssetEncodeVideo],
+        jobs: [JobName.SmartSearch, JobName.AssetDetectFaces, JobName.Ocr, JobName.AssetEncodeVideo],
         stub: [assetStub.video],
       },
       {

server/src/services/job.service.ts

@@ -236,6 +236,10 @@ export class JobService extends BaseService {
         return this.jobRepository.queue({ name: JobName.DatabaseBackup, data: { force } });
       }
 
+      case QueueName.Ocr: {
+        return this.jobRepository.queue({ name: JobName.OcrQueueAll, data: { force } });
+      }
+
       default: {
         throw new BadRequestException(`Invalid job name: ${name}`);
       }
@@ -350,6 +354,7 @@ export class JobService extends BaseService {
         const jobs: JobItem[] = [
           { name: JobName.SmartSearch, data: item.data },
           { name: JobName.AssetDetectFaces, data: item.data },
+          { name: JobName.Ocr, data: item.data },
         ];
 
         if (asset.type === AssetType.Video) {

server/src/services/ocr.service.spec.ts

@@ -0,0 +1,177 @@
+import { AssetVisibility, ImmichWorker, JobName, JobStatus } from 'src/enum';
+import { OcrService } from 'src/services/ocr.service';
+import { assetStub } from 'test/fixtures/asset.stub';
+import { systemConfigStub } from 'test/fixtures/system-config.stub';
+import { makeStream, newTestService, ServiceMocks } from 'test/utils';
+
+describe(OcrService.name, () => {
+  let sut: OcrService;
+  let mocks: ServiceMocks;
+
+  beforeEach(() => {
+    ({ sut, mocks } = newTestService(OcrService));
+
+    mocks.config.getWorker.mockReturnValue(ImmichWorker.Microservices);
+  });
+
+  it('should work', () => {
+    expect(sut).toBeDefined();
+  });
+
+  describe('handleQueueOcr', () => {
+    it('should do nothing if machine learning is disabled', async () => {
+      mocks.systemMetadata.get.mockResolvedValue(systemConfigStub.machineLearningDisabled);
+
+      await sut.handleQueueOcr({ force: false });
+
+      expect(mocks.database.setDimensionSize).not.toHaveBeenCalled();
+    });
+
+    it('should queue the assets without ocr', async () => {
+      mocks.assetJob.streamForOcrJob.mockReturnValue(makeStream([assetStub.image]));
+
+      await sut.handleQueueOcr({ force: false });
+
+      expect(mocks.job.queueAll).toHaveBeenCalledWith([{ name: JobName.Ocr, data: { id: assetStub.image.id } }]);
+      expect(mocks.assetJob.streamForOcrJob).toHaveBeenCalledWith(false);
+    });
+
+    it('should queue all the assets', async () => {
+      mocks.assetJob.streamForOcrJob.mockReturnValue(makeStream([assetStub.image]));
+
+      await sut.handleQueueOcr({ force: true });
+
+      expect(mocks.job.queueAll).toHaveBeenCalledWith([{ name: JobName.Ocr, data: { id: assetStub.image.id } }]);
+      expect(mocks.assetJob.streamForOcrJob).toHaveBeenCalledWith(true);
+    });
+  });
+
+  describe('handleOcr', () => {
+    it('should do nothing if machine learning is disabled', async () => {
+      mocks.systemMetadata.get.mockResolvedValue(systemConfigStub.machineLearningDisabled);
+
+      expect(await sut.handleOcr({ id: '123' })).toEqual(JobStatus.Skipped);
+
+      expect(mocks.asset.getByIds).not.toHaveBeenCalled();
+      expect(mocks.machineLearning.encodeImage).not.toHaveBeenCalled();
+    });
+
+    it('should skip assets without a resize path', async () => {
+      mocks.assetJob.getForOcr.mockResolvedValue({ visibility: AssetVisibility.Timeline, previewFile: null });
+
+      expect(await sut.handleOcr({ id: assetStub.noResizePath.id })).toEqual(JobStatus.Failed);
+
+      expect(mocks.ocr.upsert).not.toHaveBeenCalled();
+      expect(mocks.machineLearning.ocr).not.toHaveBeenCalled();
+    });
+
+    it('should save the returned objects', async () => {
+      mocks.machineLearning.ocr.mockResolvedValue({
+        box: [10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130, 140, 150, 160],
+        boxScore: [0.9, 0.8],
+        text: ['One Two Three', 'Four Five'],
+        textScore: [0.95, 0.85],
+      });
+      mocks.assetJob.getForOcr.mockResolvedValue({
+        visibility: AssetVisibility.Timeline,
+        previewFile: assetStub.image.files[1].path,
+      });
+
+      expect(await sut.handleOcr({ id: assetStub.image.id })).toEqual(JobStatus.Success);
+
+      expect(mocks.machineLearning.ocr).toHaveBeenCalledWith(
+        '/uploads/user-id/thumbs/path.jpg',
+        expect.objectContaining({
+          modelName: 'PP-OCRv5_mobile',
+          minDetectionScore: 0.5,
+          minRecognitionScore: 0.8,
+          maxResolution: 736,
+        }),
+      );
+      expect(mocks.ocr.upsert).toHaveBeenCalledWith(assetStub.image.id, [
+        {
+          assetId: assetStub.image.id,
+          boxScore: 0.9,
+          text: 'One Two Three',
+          textScore: 0.95,
+          x1: 10,
+          y1: 20,
+          x2: 30,
+          y2: 40,
+          x3: 50,
+          y3: 60,
+          x4: 70,
+          y4: 80,
+        },
+        {
+          assetId: assetStub.image.id,
+          boxScore: 0.8,
+          text: 'Four Five',
+          textScore: 0.85,
+          x1: 90,
+          y1: 100,
+          x2: 110,
+          y2: 120,
+          x3: 130,
+          y3: 140,
+          x4: 150,
+          y4: 160,
+        },
+      ]);
+    });
+
+    it('should apply config settings', async () => {
+      mocks.systemMetadata.get.mockResolvedValue({
+        machineLearning: {
+          enabled: true,
+          ocr: {
+            modelName: 'PP-OCRv5_server',
+            enabled: true,
+            minDetectionScore: 0.8,
+            minRecognitionScore: 0.9,
+            maxResolution: 1500,
+          },
+        },
+      });
+      mocks.machineLearning.ocr.mockResolvedValue({ box: [], boxScore: [], text: [], textScore: [] });
+      mocks.assetJob.getForOcr.mockResolvedValue({
+        visibility: AssetVisibility.Timeline,
+        previewFile: assetStub.image.files[1].path,
+      });
+
+      expect(await sut.handleOcr({ id: assetStub.image.id })).toEqual(JobStatus.Success);
+
+      expect(mocks.machineLearning.ocr).toHaveBeenCalledWith(
+        '/uploads/user-id/thumbs/path.jpg',
+        expect.objectContaining({
+          modelName: 'PP-OCRv5_server',
+          minDetectionScore: 0.8,
+          minRecognitionScore: 0.9,
+          maxResolution: 1500,
+        }),
+      );
+      expect(mocks.ocr.upsert).toHaveBeenCalledWith(assetStub.image.id, []);
+    });
+
+    it('should skip invisible assets', async () => {
+      mocks.assetJob.getForOcr.mockResolvedValue({
+        visibility: AssetVisibility.Hidden,
+        previewFile: assetStub.image.files[1].path,
+      });
+
+      expect(await sut.handleOcr({ id: assetStub.livePhotoMotionAsset.id })).toEqual(JobStatus.Skipped);
+
+      expect(mocks.machineLearning.ocr).not.toHaveBeenCalled();
+      expect(mocks.ocr.upsert).not.toHaveBeenCalled();
+    });
+
+    it('should fail if asset could not be found', async () => {
+      mocks.assetJob.getForOcr.mockResolvedValue(void 0);
+
+      expect(await sut.handleOcr({ id: assetStub.image.id })).toEqual(JobStatus.Failed);
+
+      expect(mocks.machineLearning.ocr).not.toHaveBeenCalled();
+      expect(mocks.ocr.upsert).not.toHaveBeenCalled();
+    });
+  });
+});

server/src/services/ocr.service.ts

@@ -0,0 +1,86 @@
+import { Injectable } from '@nestjs/common';
+import { JOBS_ASSET_PAGINATION_SIZE } from 'src/constants';
+import { OnJob } from 'src/decorators';
+import { AssetVisibility, JobName, JobStatus, QueueName } from 'src/enum';
+import { OCR } from 'src/repositories/machine-learning.repository';
+import { BaseService } from 'src/services/base.service';
+import { JobItem, JobOf } from 'src/types';
+import { isOcrEnabled } from 'src/utils/misc';
+
+@Injectable()
+export class OcrService extends BaseService {
+  @OnJob({ name: JobName.OcrQueueAll, queue: QueueName.Ocr })
+  async handleQueueOcr({ force }: JobOf<JobName.OcrQueueAll>): Promise<JobStatus> {
+    const { machineLearning } = await this.getConfig({ withCache: false });
+    if (!isOcrEnabled(machineLearning)) {
+      return JobStatus.Skipped;
+    }
+
+    if (force) {
+      await this.ocrRepository.deleteAll();
+    }
+
+    let jobs: JobItem[] = [];
+    const assets = this.assetJobRepository.streamForOcrJob(force);
+
+    for await (const asset of assets) {
+      jobs.push({ name: JobName.Ocr, data: { id: asset.id } });
+
+      if (jobs.length >= JOBS_ASSET_PAGINATION_SIZE) {
+        await this.jobRepository.queueAll(jobs);
+        jobs = [];
+      }
+    }
+
+    await this.jobRepository.queueAll(jobs);
+    return JobStatus.Success;
+  }
+
+  @OnJob({ name: JobName.Ocr, queue: QueueName.Ocr })
+  async handleOcr({ id }: JobOf<JobName.Ocr>): Promise<JobStatus> {
+    const { machineLearning } = await this.getConfig({ withCache: true });
+    if (!isOcrEnabled(machineLearning)) {
+      return JobStatus.Skipped;
+    }
+
+    const asset = await this.assetJobRepository.getForOcr(id);
+    if (!asset || !asset.previewFile) {
+      return JobStatus.Failed;
+    }
+
+    if (asset.visibility === AssetVisibility.Hidden) {
+      return JobStatus.Skipped;
+    }
+
+    const ocrResults = await this.machineLearningRepository.ocr(asset.previewFile, machineLearning.ocr);
+
+    await this.ocrRepository.upsert(id, this.parseOcrResults(id, ocrResults));
+
+    await this.assetRepository.upsertJobStatus({ assetId: id, ocrAt: new Date() });
+
+    this.logger.debug(`Processed ${ocrResults.text.length} OCR result(s) for ${id}`);
+    return JobStatus.Success;
+  }
+
+  private parseOcrResults(id: string, { box, boxScore, text, textScore }: OCR) {
+    const ocrDataList = [];
+    for (let i = 0; i < text.length; i++) {
+      const boxOffset = i * 8;
+      ocrDataList.push({
+        assetId: id,
+        x1: box[boxOffset],
+        y1: box[boxOffset + 1],
+        x2: box[boxOffset + 2],
+        y2: box[boxOffset + 3],
+        x3: box[boxOffset + 4],
+        y3: box[boxOffset + 5],
+        x4: box[boxOffset + 6],
+        y4: box[boxOffset + 7],
+        boxScore: boxScore[i],
+        textScore: textScore[i],
+        text: text[i],
+      });
+    }
+    return ocrDataList;
+  }
+}

server/src/services/server.service.spec.ts

@@ -141,6 +141,7 @@ describe(ServerService.name, () => {
         reverseGeocoding: true,
         oauth: false,
         oauthAutoLaunch: false,
+        ocr: true,
         passwordLogin: true,
         search: true,
         sidecar: true,

server/src/services/server.service.ts

@@ -19,7 +19,12 @@ import { UserStatsQueryResponse } from 'src/repositories/user.repository';
 import { BaseService } from 'src/services/base.service';
 import { asHumanReadable } from 'src/utils/bytes';
 import { mimeTypes } from 'src/utils/mime-types';
-import { isDuplicateDetectionEnabled, isFacialRecognitionEnabled, isSmartSearchEnabled } from 'src/utils/misc';
+import {
+  isDuplicateDetectionEnabled,
+  isFacialRecognitionEnabled,
+  isOcrEnabled,
+  isSmartSearchEnabled,
+} from 'src/utils/misc';
 
 @Injectable()
 export class ServerService extends BaseService {
@@ -97,6 +102,7 @@ export class ServerService extends BaseService {
       trash: trash.enabled,
       oauth: oauth.enabled,
       oauthAutoLaunch: oauth.autoLaunch,
+      ocr: isOcrEnabled(machineLearning),
       passwordLogin: passwordLogin.enabled,
       configFile: !!configFile,
       email: notifications.smtp.enabled,

server/src/services/system-config.service.spec.ts

@@ -39,6 +39,7 @@ const updatedConfig = Object.freeze<SystemConfig>({
     [QueueName.ThumbnailGeneration]: { concurrency: 3 },
     [QueueName.VideoConversion]: { concurrency: 1 },
     [QueueName.Notification]: { concurrency: 5 },
+    [QueueName.Ocr]: { concurrency: 1 },
   },
   backup: {
     database: {
@@ -102,6 +103,13 @@ const updatedConfig = Object.freeze<SystemConfig>({
       maxDistance: 0.5,
       minFaces: 3,
     },
+    ocr: {
+      enabled: true,
+      modelName: 'PP-OCRv5_mobile',
+      minDetectionScore: 0.5,
+      minRecognitionScore: 0.8,
+      maxResolution: 736,
+    },
   },
   map: {
     enabled: true,