test: cont. integrity service tests

server/src/controllers/integrity.controller.ts

@@ -42,7 +42,7 @@ export class IntegrityController {
   })
   @Authenticated({ permission: Permission.Maintenance, admin: true })
   getIntegrityReport(@Body() dto: IntegrityGetReportDto): Promise<IntegrityReportResponseDto> {
-    return this.service.getIntegrityReport(dto);
+    return this.service.getIntegrityReport(dto.type);
   }
 
   @Delete('report/:id')

server/src/repositories/integrity.repository.ts

@@ -1,7 +1,6 @@
 import { Injectable } from '@nestjs/common';
 import { Insertable, Kysely, sql } from 'kysely';
 import { InjectKysely } from 'nestjs-kysely';
-import { Readable } from 'node:stream';
 import { DummyValue, GenerateSql } from 'src/decorators';
 import { IntegrityReportType } from 'src/enum';
 import { DB } from 'src/schema';
@@ -91,29 +90,6 @@ export class IntegrityRepository {
       .executeTakeFirstOrThrow();
   }
 
-  @GenerateSql({ params: [DummyValue.STRING], stream: true })
-  streamIntegrityReportsCSV(type: IntegrityReportType): Readable {
-    const items = this.db
-      .selectFrom('integrity_report')
-      .select(['id', 'type', 'path', 'assetId', 'fileAssetId'])
-      .where('type', '=', type)
-      .orderBy('createdAt', 'desc')
-      .stream();
-
-    // very rudimentary csv serialiser
-    async function* generator() {
-      yield 'id,type,assetId,fileAssetId,path\n';
-
-      for await (const item of items) {
-        // no expectation of particularly bad filenames
-        // but they could potentially have a newline or quote character
-        yield `${item.id},${item.type},${item.assetId},${item.fileAssetId},"${item.path.replaceAll('"', '""')}"\n`;
-      }
-    }
-
-    return Readable.from(generator());
-  }
-
   @GenerateSql({ params: [], stream: true })
   streamAllAssetPaths() {
     return this.db.selectFrom('asset').select(['originalPath', 'encodedVideoPath']).stream();
@@ -201,6 +177,16 @@ export class IntegrityRepository {
 
   @GenerateSql({ params: [DummyValue.STRING], stream: true })
   streamIntegrityReports(type: IntegrityReportType) {
+    return this.db
+      .selectFrom('integrity_report')
+      .select(['id', 'type', 'path', 'assetId', 'fileAssetId'])
+      .where('type', '=', type)
+      .orderBy('createdAt', 'desc')
+      .stream();
+  }
+
+  @GenerateSql({ params: [DummyValue.STRING], stream: true })
+  streamIntegrityReportsWithAssetChecksum(type: IntegrityReportType) {
     return this.db
       .selectFrom('integrity_report')
       .select(['integrity_report.id as reportId', 'integrity_report.path'])

server/src/services/integrity.service.spec.ts

@@ -1,10 +1,10 @@
+import { text } from 'node:stream/consumers';
+import { AssetStatus, IntegrityReportType, JobName } from 'src/enum';
 import { IntegrityService } from 'src/services/integrity.service';
 import { newTestService, ServiceMocks } from 'test/utils';
 
 describe(IntegrityService.name, () => {
   let sut: IntegrityService;
-  // impl. pending
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars
   let mocks: ServiceMocks;
 
   beforeEach(() => {
@@ -15,10 +15,211 @@ describe(IntegrityService.name, () => {
     expect(sut).toBeDefined();
   });
 
-  describe.skip('getIntegrityReportSummary'); // just calls repository
+  describe('getIntegrityReportSummary', () => {
-  describe.skip('getIntegrityReport'); // just calls repository
+    it('gets summary', async () => {
-  describe.skip('getIntegrityReportCsv'); // just calls repository
+      await sut.getIntegrityReportSummary();
+      expect(mocks.integrityReport.getIntegrityReportSummary).toHaveBeenCalled();
+    });
+  });
 
-  describe.todo('getIntegrityReportFile');
+  describe('getIntegrityReport', () => {
-  describe.todo('deleteIntegrityReport');
+    it('gets report', async () => {
+      await expect(sut.getIntegrityReport(IntegrityReportType.ChecksumFail)).resolves.toEqual(
+        expect.objectContaining({
+          items: undefined,
+        }),
+      );
+
+      expect(mocks.integrityReport.getIntegrityReports).toHaveBeenCalledWith(IntegrityReportType.ChecksumFail);
+    });
+  });
+
+  describe('getIntegrityReportCsv', () => {
+    it('gets report as csv', async () => {
+      mocks.integrityReport.streamIntegrityReports.mockReturnValue(
+        (function* () {
+          yield {
+            id: 'id',
+            createdAt: new Date(0),
+            path: '/path/to/file',
+            type: IntegrityReportType.ChecksumFail,
+            assetId: null,
+            fileAssetId: null,
+          };
+        })() as never,
+      );
+
+      await expect(text(sut.getIntegrityReportCsv(IntegrityReportType.ChecksumFail))).resolves.toMatchInlineSnapshot(`
+        "id,type,assetId,fileAssetId,path
+        id,checksum_mismatch,null,null,"/path/to/file"
+        "
+      `);
+
+      expect(mocks.integrityReport.streamIntegrityReports).toHaveBeenCalledWith(IntegrityReportType.ChecksumFail);
+    });
+  });
+
+  describe('getIntegrityReportFile', () => {
+    it('gets report file', async () => {
+      mocks.integrityReport.getById.mockResolvedValue({
+        id: 'id',
+        createdAt: new Date(0),
+        path: '/path/to/file',
+        type: IntegrityReportType.ChecksumFail,
+        assetId: null,
+        fileAssetId: null,
+      });
+
+      await expect(sut.getIntegrityReportFile('id')).resolves.toEqual({
+        path: '/path/to/file',
+        fileName: 'file',
+        contentType: 'application/octet-stream',
+        cacheControl: 'private_without_cache',
+      });
+
+      expect(mocks.integrityReport.getById).toHaveBeenCalledWith('id');
+    });
+  });
+
+  describe('deleteIntegrityReport', () => {
+    it('deletes asset if one is present', async () => {
+      mocks.integrityReport.getById.mockResolvedValue({
+        id: 'id',
+        createdAt: new Date(0),
+        path: '/path/to/file',
+        type: IntegrityReportType.ChecksumFail,
+        assetId: 'assetId',
+        fileAssetId: null,
+      });
+
+      await sut.deleteIntegrityReport(
+        {
+          user: {
+            id: 'userId',
+          },
+        } as never,
+        'id',
+      );
+
+      expect(mocks.asset.updateAll).toHaveBeenCalledWith(['assetId'], {
+        deletedAt: expect.any(Date),
+        status: AssetStatus.Trashed,
+      });
+
+      expect(mocks.event.emit).toHaveBeenCalledWith('AssetTrashAll', {
+        assetIds: ['assetId'],
+        userId: 'userId',
+      });
+
+      expect(mocks.integrityReport.deleteById).toHaveBeenCalledWith('id');
+    });
+
+    it('deletes file asset if one is present', async () => {
+      mocks.integrityReport.getById.mockResolvedValue({
+        id: 'id',
+        createdAt: new Date(0),
+        path: '/path/to/file',
+        type: IntegrityReportType.ChecksumFail,
+        assetId: null,
+        fileAssetId: 'fileAssetId',
+      });
+
+      await sut.deleteIntegrityReport(
+        {
+          user: {
+            id: 'userId',
+          },
+        } as never,
+        'id',
+      );
+
+      expect(mocks.asset.deleteFiles).toHaveBeenCalledWith([{ id: 'fileAssetId' }]);
+    });
+
+    it('deletes orphaned file', async () => {
+      mocks.integrityReport.getById.mockResolvedValue({
+        id: 'id',
+        createdAt: new Date(0),
+        path: '/path/to/file',
+        type: IntegrityReportType.ChecksumFail,
+        assetId: null,
+        fileAssetId: null,
+      });
+
+      await sut.deleteIntegrityReport(
+        {
+          user: {
+            id: 'userId',
+          },
+        } as never,
+        'id',
+      );
+
+      expect(mocks.storage.unlink).toHaveBeenCalledWith('/path/to/file');
+      expect(mocks.integrityReport.deleteById).toHaveBeenCalledWith('id');
+    });
+  });
+
+  describe('handleOrphanedFilesQueueAll', () => {
+    it('queues jobs for all detected files', async () => {
+      mocks.integrityReport.streamIntegrityReportsWithAssetChecksum.mockReturnValue((function* () {})() as never);
+      mocks.storage.walk.mockReturnValueOnce(
+        (function* () {
+          yield ['/path/to/file', '/path/to/file2'];
+          yield ['/path/to/batch2'];
+        })() as never,
+      );
+      mocks.storage.walk.mockReturnValueOnce(
+        (function* () {
+          yield ['/path/to/file3', '/path/to/file4'];
+          yield ['/path/to/batch4'];
+        })() as never,
+      );
+
+      await sut.handleOrphanedFilesQueueAll({ refreshOnly: false });
+
+      expect(mocks.job.queue).toBeCalledTimes(4);
+      expect(mocks.job.queue).toBeCalledWith({
+        name: JobName.IntegrityOrphanedFiles,
+        data: {
+          type: 'asset',
+          paths: expect.arrayContaining(['/path/to/file']),
+        },
+      });
+      expect(mocks.job.queue).toBeCalledWith({
+        name: JobName.IntegrityOrphanedFiles,
+        data: {
+          type: 'asset_file',
+          paths: expect.arrayContaining(['/path/to/file3']),
+        },
+      });
+    });
+
+    it('queues jobs to refresh reports', async () => {
+      mocks.integrityReport.streamIntegrityReportsWithAssetChecksum.mockReturnValue(
+        (function* () {
+          yield 'mockReport';
+        })() as never,
+      );
+
+      await sut.handleOrphanedFilesQueueAll({ refreshOnly: false });
+
+      expect(mocks.job.queue).toBeCalledTimes(1);
+      expect(mocks.job.queue).toBeCalledWith({
+        name: JobName.IntegrityOrphanedFilesRefresh,
+        data: {
+          items: expect.arrayContaining(['mockReport']),
+        },
+      });
+    });
+  });
+
+  describe.todo('handleOrphanedFiles');
+  describe.todo('handleOrphanedRefresh');
+  describe.todo('handleMissingFilesQueueAll');
+  describe.todo('handleMissingFiles');
+  describe.todo('handleMissingRefresh');
+  describe.todo('handleChecksumFiles');
+  describe.todo('handleChecksumRefresh');
+  describe.todo('handleDeleteIntegrityReport');
 });

server/src/services/integrity.service.ts

@@ -9,11 +9,7 @@ import { JOBS_LIBRARY_PAGINATION_SIZE } from 'src/constants';
 import { StorageCore } from 'src/cores/storage.core';
 import { OnEvent, OnJob } from 'src/decorators';
 import { AuthDto } from 'src/dtos/auth.dto';
-import {
+import { IntegrityReportResponseDto, IntegrityReportSummaryResponseDto } from 'src/dtos/integrity.dto';
-  IntegrityGetReportDto,
-  IntegrityReportResponseDto,
-  IntegrityReportSummaryResponseDto,
-} from 'src/dtos/integrity.dto';
 import {
   AssetStatus,
   CacheControl,
@@ -158,14 +154,27 @@ export class IntegrityService extends BaseService {
     return this.integrityRepository.getIntegrityReportSummary();
   }
 
-  async getIntegrityReport(dto: IntegrityGetReportDto): Promise<IntegrityReportResponseDto> {
+  async getIntegrityReport(type: IntegrityReportType): Promise<IntegrityReportResponseDto> {
     return {
-      items: await this.integrityRepository.getIntegrityReports(dto.type),
+      items: await this.integrityRepository.getIntegrityReports(type),
     };
   }
 
   getIntegrityReportCsv(type: IntegrityReportType): Readable {
-    return this.integrityRepository.streamIntegrityReportsCSV(type);
+    const items = this.integrityRepository.streamIntegrityReports(type);
+
+    // very rudimentary csv serialiser
+    async function* generator() {
+      yield 'id,type,assetId,fileAssetId,path\n';
+
+      for await (const item of items) {
+        // no expectation of particularly bad filenames
+        // but they could potentially have a newline or quote character
+        yield `${item.id},${item.type},${item.assetId},${item.fileAssetId},"${item.path.replaceAll('"', '""')}"\n`;
+      }
+    }
+
+    return Readable.from(generator());
   }
 
   async getIntegrityReportFile(id: string): Promise<ImmichFileResponse> {
@@ -206,7 +215,7 @@ export class IntegrityService extends BaseService {
   async handleOrphanedFilesQueueAll({ refreshOnly }: IIntegrityJob = {}): Promise<JobStatus> {
     this.logger.log(`Checking for out of date orphaned file reports...`);
 
-    const reports = this.integrityRepository.streamIntegrityReports(IntegrityReportType.OrphanFile);
+    const reports = this.integrityRepository.streamIntegrityReportsWithAssetChecksum(IntegrityReportType.OrphanFile);
 
     let total = 0;
     for await (const batchReports of chunk(reports, JOBS_LIBRARY_PAGINATION_SIZE)) {
@@ -332,7 +341,7 @@ export class IntegrityService extends BaseService {
     if (refreshOnly) {
       this.logger.log(`Checking for out of date missing file reports...`);
 
-      const reports = this.integrityRepository.streamIntegrityReports(IntegrityReportType.MissingFile);
+      const reports = this.integrityRepository.streamIntegrityReportsWithAssetChecksum(IntegrityReportType.MissingFile);
 
       let total = 0;
       for await (const batchReports of chunk(reports, JOBS_LIBRARY_PAGINATION_SIZE)) {
@@ -434,7 +443,9 @@ export class IntegrityService extends BaseService {
     if (refreshOnly) {
       this.logger.log(`Checking for out of date checksum file reports...`);
 
-      const reports = this.integrityRepository.streamIntegrityReports(IntegrityReportType.ChecksumFail);
+      const reports = this.integrityRepository.streamIntegrityReportsWithAssetChecksum(
+        IntegrityReportType.ChecksumFail,
+      );
 
       let total = 0;
       for await (const batchReports of chunk(reports, JOBS_LIBRARY_PAGINATION_SIZE)) {