import { createHash } from 'node:crypto'; import { Readable } from 'node:stream'; import { text } from 'node:stream/consumers'; import { AssetStatus, IntegrityReportType, JobName, JobStatus } from 'src/enum'; import { IntegrityService } from 'src/services/integrity.service'; import { newTestService, ServiceMocks } from 'test/utils'; describe(IntegrityService.name, () => { let sut: IntegrityService; let mocks: ServiceMocks; beforeEach(() => { ({ sut, mocks } = newTestService(IntegrityService)); }); it('should work', () => { expect(sut).toBeDefined(); }); describe('getIntegrityReportSummary', () => { it('gets summary', async () => { await sut.getIntegrityReportSummary(); expect(mocks.integrityReport.getIntegrityReportSummary).toHaveBeenCalled(); }); }); describe('getIntegrityReport', () => { it('gets report', async () => { mocks.integrityReport.getIntegrityReports.mockResolvedValue({ items: [], nextCursor: undefined, }); await expect(sut.getIntegrityReport({ type: IntegrityReportType.ChecksumFail })).resolves.toEqual({ items: [], nextCursor: undefined, }); expect(mocks.integrityReport.getIntegrityReports).toHaveBeenCalledWith( { cursor: undefined, limit: 100 }, IntegrityReportType.ChecksumFail, ); }); }); describe('getIntegrityReportCsv', () => { it('gets report as csv', async () => { mocks.integrityReport.streamIntegrityReports.mockReturnValue( (function* () { yield { id: 'id', createdAt: new Date(0), path: '/path/to/file', type: IntegrityReportType.ChecksumFail, assetId: null, fileAssetId: null, }; })() as never, ); await expect(text(sut.getIntegrityReportCsv(IntegrityReportType.ChecksumFail))).resolves.toMatchInlineSnapshot(` "id,type,assetId,fileAssetId,path id,checksum_mismatch,null,null,"/path/to/file" " `); expect(mocks.integrityReport.streamIntegrityReports).toHaveBeenCalledWith(IntegrityReportType.ChecksumFail); }); }); describe('getIntegrityReportFile', () => { it('gets report file', async () => { mocks.integrityReport.getById.mockResolvedValue({ id: 'id', createdAt: new Date(0), path: '/path/to/file', type: IntegrityReportType.ChecksumFail, assetId: null, fileAssetId: null, }); await expect(sut.getIntegrityReportFile('id')).resolves.toEqual({ path: '/path/to/file', fileName: 'file', contentType: 'application/octet-stream', cacheControl: 'private_without_cache', }); expect(mocks.integrityReport.getById).toHaveBeenCalledWith('id'); }); }); describe('deleteIntegrityReport', () => { it('deletes asset if one is present', async () => { mocks.integrityReport.getById.mockResolvedValue({ id: 'id', createdAt: new Date(0), path: '/path/to/file', type: IntegrityReportType.ChecksumFail, assetId: 'assetId', fileAssetId: null, }); await sut.deleteIntegrityReport( { user: { id: 'userId', }, } as never, 'id', ); expect(mocks.asset.updateAll).toHaveBeenCalledWith(['assetId'], { deletedAt: expect.any(Date), status: AssetStatus.Trashed, }); expect(mocks.event.emit).toHaveBeenCalledWith('AssetTrashAll', { assetIds: ['assetId'], userId: 'userId', }); expect(mocks.integrityReport.deleteById).toHaveBeenCalledWith('id'); }); it('deletes file asset if one is present', async () => { mocks.integrityReport.getById.mockResolvedValue({ id: 'id', createdAt: new Date(0), path: '/path/to/file', type: IntegrityReportType.ChecksumFail, assetId: null, fileAssetId: 'fileAssetId', }); await sut.deleteIntegrityReport( { user: { id: 'userId', }, } as never, 'id', ); expect(mocks.asset.deleteFiles).toHaveBeenCalledWith([{ id: 'fileAssetId' }]); }); it('deletes untracked file', async () => { mocks.integrityReport.getById.mockResolvedValue({ id: 'id', createdAt: new Date(0), path: '/path/to/file', type: IntegrityReportType.ChecksumFail, assetId: null, fileAssetId: null, }); await sut.deleteIntegrityReport( { user: { id: 'userId', }, } as never, 'id', ); expect(mocks.storage.unlink).toHaveBeenCalledWith('/path/to/file'); expect(mocks.integrityReport.deleteById).toHaveBeenCalledWith('id'); }); }); describe('handleUntrackedFilesQueueAll', () => { beforeEach(() => { mocks.integrityReport.streamIntegrityReportsWithAssetChecksum.mockReturnValue((function* () {})() as never); }); it('queues jobs for all detected files', async () => { mocks.storage.walk.mockReturnValueOnce( (function* () { yield ['/path/to/file', '/path/to/file2']; yield ['/path/to/batch2']; })() as never, ); mocks.storage.walk.mockReturnValueOnce( (function* () { yield ['/path/to/file3', '/path/to/file4']; yield ['/path/to/batch4']; })() as never, ); await sut.handleUntrackedFilesQueueAll({ refreshOnly: false }); expect(mocks.job.queue).toBeCalledTimes(4); expect(mocks.job.queue).toBeCalledWith({ name: JobName.IntegrityUntrackedFiles, data: { type: 'asset', paths: expect.arrayContaining(['/path/to/file']), }, }); expect(mocks.job.queue).toBeCalledWith({ name: JobName.IntegrityUntrackedFiles, data: { type: 'asset_file', paths: expect.arrayContaining(['/path/to/file3']), }, }); }); it('queues jobs to refresh reports', async () => { mocks.integrityReport.streamIntegrityReportsWithAssetChecksum.mockReturnValue( (function* () { yield 'mockReport'; })() as never, ); await sut.handleUntrackedFilesQueueAll({ refreshOnly: false }); expect(mocks.job.queue).toBeCalledTimes(1); expect(mocks.job.queue).toBeCalledWith({ name: JobName.IntegrityUntrackedFilesRefresh, data: { items: expect.arrayContaining(['mockReport']), }, }); }); it('should succeed', async () => { await expect(sut.handleUntrackedFilesQueueAll({ refreshOnly: false })).resolves.toBe(JobStatus.Success); }); }); describe('handleUntrackedFiles', () => { it('should detect untracked asset files', async () => { mocks.integrityReport.getAssetPathsByPaths.mockResolvedValue([ { originalPath: '/path/to/file1', encodedVideoPath: null }, ]); await sut.handleUntrackedFiles({ type: 'asset', paths: ['/path/to/file1', '/path/to/untracked'], }); expect(mocks.integrityReport.getAssetPathsByPaths).toHaveBeenCalledWith(['/path/to/file1', '/path/to/untracked']); expect(mocks.integrityReport.create).toHaveBeenCalledWith([ { type: IntegrityReportType.UntrackedFile, path: '/path/to/untracked' }, ]); }); it('should not create reports when no untracked files found for assets', async () => { mocks.integrityReport.getAssetPathsByPaths.mockResolvedValue([ { originalPath: '/path/to/file1', encodedVideoPath: '/path/to/encoded' }, ]); await sut.handleUntrackedFiles({ type: 'asset', paths: ['/path/to/file1', '/path/to/encoded'], }); expect(mocks.integrityReport.create).not.toHaveBeenCalled(); }); it('should detect untracked asset_file files', async () => { mocks.integrityReport.getAssetFilePathsByPaths.mockResolvedValue([{ path: '/path/to/thumb1' }]); await sut.handleUntrackedFiles({ type: 'asset_file', paths: ['/path/to/thumb1', '/path/to/untracked_thumb'], }); expect(mocks.integrityReport.create).toHaveBeenCalledWith([ { type: IntegrityReportType.UntrackedFile, path: '/path/to/untracked_thumb' }, ]); }); }); describe('handleUntrackedRefresh', () => { it('should delete reports for files that no longer exist', async () => { mocks.storage.stat .mockRejectedValueOnce(new Error('ENOENT')) .mockResolvedValueOnce({} as never) .mockRejectedValueOnce(new Error('ENOENT')); await sut.handleUntrackedRefresh({ items: [ { reportId: 'report1', path: '/path/to/missing1' }, { reportId: 'report2', path: '/path/to/existing' }, { reportId: 'report3', path: '/path/to/missing2' }, ], }); expect(mocks.integrityReport.deleteByIds).toHaveBeenCalledWith(['report1', 'report3']); }); it('should not delete reports for files that still exist', async () => { mocks.storage.stat.mockResolvedValue({} as never); await sut.handleUntrackedRefresh({ items: [{ reportId: 'report1', path: '/path/to/existing' }], }); expect(mocks.integrityReport.deleteByIds).not.toHaveBeenCalled(); }); it('should succeed', async () => { await expect(sut.handleUntrackedRefresh({ items: [] })).resolves.toBe(JobStatus.Success); }); }); describe('handleMissingFilesQueueAll', () => { beforeEach(() => { mocks.integrityReport.streamAssetPaths.mockReturnValue((function* () {})() as never); }); it('should queue jobs', async () => { mocks.integrityReport.streamAssetPaths.mockReturnValue( (function* () { yield { path: '/path/to/file1', assetId: 'asset1', fileAssetId: null }; yield { path: '/path/to/file2', assetId: 'asset2', fileAssetId: null }; })() as never, ); await sut.handleMissingFilesQueueAll({ refreshOnly: false }); expect(mocks.integrityReport.streamAssetPaths).toHaveBeenCalled(); expect(mocks.job.queue).toHaveBeenCalledWith({ name: JobName.IntegrityMissingFiles, data: { items: expect.arrayContaining([{ path: '/path/to/file1', assetId: 'asset1', fileAssetId: null }]), }, }); expect(mocks.job.queue).not.toHaveBeenCalledWith( expect.objectContaining({ name: JobName.IntegrityMissingFilesRefresh, }), ); }); it('should queue refresh jobs when refreshOnly is set', async () => { mocks.integrityReport.streamIntegrityReportsWithAssetChecksum.mockReturnValue( (function* () { yield { reportId: 'report1', path: '/path/to/file1' }; yield { reportId: 'report2', path: '/path/to/file2' }; })() as never, ); await sut.handleMissingFilesQueueAll({ refreshOnly: true }); expect(mocks.integrityReport.streamIntegrityReportsWithAssetChecksum).toHaveBeenCalledWith( IntegrityReportType.MissingFile, ); expect(mocks.job.queue).toHaveBeenCalledWith({ name: JobName.IntegrityMissingFilesRefresh, data: { items: expect.arrayContaining([{ reportId: 'report1', path: '/path/to/file1' }]), }, }); expect(mocks.job.queue).not.toHaveBeenCalledWith( expect.objectContaining({ name: JobName.IntegrityMissingFiles, }), ); }); it('should succeed', async () => { await expect(sut.handleMissingFilesQueueAll()).resolves.toBe(JobStatus.Success); }); }); describe('handleMissingFiles', () => { it('should detect missing files and remove outdated reports', async () => { mocks.storage.stat .mockResolvedValueOnce({} as never) .mockRejectedValueOnce(new Error('ENOENT')) .mockResolvedValueOnce({} as never); await sut.handleMissingFiles({ items: [ { path: '/path/to/existing', assetId: 'asset1', fileAssetId: null, reportId: null }, { path: '/path/to/missing', assetId: 'asset2', fileAssetId: null, reportId: null }, { path: '/path/to/restored', assetId: 'asset3', fileAssetId: null, reportId: 'report2' }, ], }); expect(mocks.integrityReport.deleteByIds).toHaveBeenCalledWith(['report2']); expect(mocks.integrityReport.create).toHaveBeenCalledWith([ { type: IntegrityReportType.MissingFile, path: '/path/to/missing', assetId: 'asset2', fileAssetId: null }, ]); }); it('should succeed', async () => { await expect(sut.handleMissingFiles({ items: [] })).resolves.toBe(JobStatus.Success); }); }); describe('handleMissingRefresh', () => { it('should remove outdated reports', async () => { mocks.storage.stat .mockResolvedValueOnce({} as never) .mockRejectedValueOnce(new Error('ENOENT')) .mockResolvedValueOnce({} as never); await sut.handleMissingRefresh({ items: [ { path: '/path/to/existing', reportId: null }, { path: '/path/to/missing', reportId: null }, { path: '/path/to/restored', reportId: 'report2' }, ], }); expect(mocks.integrityReport.deleteByIds).toHaveBeenCalledWith(['report2']); }); it('should succeed', async () => { await expect(sut.handleMissingFiles({ items: [] })).resolves.toBe(JobStatus.Success); }); }); describe('handleChecksumFiles', () => { beforeEach(() => { mocks.integrityReport.streamIntegrityReportsWithAssetChecksum.mockReturnValue((function* () {})() as never); mocks.integrityReport.streamAssetChecksums.mockReturnValue((function* () {})() as never); mocks.integrityReport.getAssetCount.mockResolvedValue({ count: 1000 }); mocks.systemMetadata.get.mockResolvedValue(null); }); it('should queue refresh jobs when refreshOnly', async () => { mocks.integrityReport.streamIntegrityReportsWithAssetChecksum.mockReturnValue( (function* () { yield { reportId: 'report1', path: '/path/to/file1', checksum: Buffer.from('abc123', 'hex') }; })() as never, ); await sut.handleChecksumFiles({ refreshOnly: true }); expect(mocks.integrityReport.streamIntegrityReportsWithAssetChecksum).toHaveBeenCalledWith( IntegrityReportType.ChecksumFail, ); expect(mocks.job.queue).toHaveBeenCalledWith({ name: JobName.IntegrityChecksumFilesRefresh, data: { items: [{ reportId: 'report1', path: '/path/to/file1', checksum: 'abc123' }], }, }); }); it('should create report for checksum mismatch and delete when fixed', async () => { const fileContent = Buffer.from('test content'); mocks.integrityReport.streamAssetChecksums.mockReturnValue( (function* () { yield { originalPath: '/path/to/mismatch', checksum: 'mismatched checksum', createdAt: new Date(), assetId: 'asset1', reportId: null, }; yield { originalPath: '/path/to/fixed', checksum: createHash('sha1').update(fileContent).digest(), createdAt: new Date(), assetId: 'asset2', reportId: 'report1', }; })() as never, ); mocks.storage.createPlainReadStream.mockImplementation(() => Readable.from(fileContent)); await sut.handleChecksumFiles({ refreshOnly: false }); expect(mocks.integrityReport.create).toHaveBeenCalledWith({ path: '/path/to/mismatch', type: IntegrityReportType.ChecksumFail, assetId: 'asset1', }); expect(mocks.integrityReport.deleteById).toHaveBeenCalledWith('report1'); }); it('should skip missing files', async () => { mocks.integrityReport.streamAssetChecksums.mockReturnValue( (function* () { yield { originalPath: '/path/to/missing', checksum: Buffer.from('abc', 'hex'), createdAt: new Date(), assetId: 'asset1', reportId: 'report1', }; })() as never, ); const error = new Error('ENOENT') as NodeJS.ErrnoException; error.code = 'ENOENT'; mocks.storage.createPlainReadStream.mockImplementation(() => { throw error; }); await sut.handleChecksumFiles({ refreshOnly: false }); expect(mocks.integrityReport.deleteById).toHaveBeenCalledWith('report1'); expect(mocks.integrityReport.create).not.toHaveBeenCalled(); }); it('should succeed', async () => { await expect(sut.handleChecksumFiles({ refreshOnly: false })).resolves.toBe(JobStatus.Success); }); }); describe('handleChecksumRefresh', () => { it('should delete reports when checksum now matches, file is missing, or asset is now missing', async () => { const fileContent = Buffer.from('test content'); const correctChecksum = createHash('sha1').update(fileContent).digest().toString('hex'); const error = new Error('ENOENT') as NodeJS.ErrnoException; error.code = 'ENOENT'; mocks.storage.createPlainReadStream .mockImplementationOnce(() => Readable.from(fileContent)) .mockImplementationOnce(() => { throw error; }) .mockImplementationOnce(() => Readable.from(fileContent)) .mockImplementationOnce(() => Readable.from(fileContent)); await sut.handleChecksumRefresh({ items: [ { reportId: 'report1', path: '/path/to/fixed', checksum: correctChecksum }, { reportId: 'report2', path: '/path/to/missing', checksum: 'abc123' }, { reportId: 'report3', path: '/path/to/bad', checksum: 'wrongchecksum' }, { reportId: 'report4', path: '/path/to/missing-asset', checksum: null }, ], }); expect(mocks.integrityReport.deleteByIds).toHaveBeenCalledWith(['report1', 'report2', 'report4']); }); it('should succeed', async () => { await expect(sut.handleChecksumRefresh({ items: [] })).resolves.toBe(JobStatus.Success); }); }); describe('handleDeleteAllIntegrityReports', () => { beforeEach(() => { mocks.integrityReport.streamIntegrityReportsByProperty.mockReturnValue((function* () {})() as never); }); it('should queue delete jobs for checksum fail reports', async () => { mocks.integrityReport.streamIntegrityReportsByProperty.mockReturnValue( (function* () { yield { id: 'report1', assetId: 'asset1', path: '/path/to/file1' }; })() as never, ); await sut.handleDeleteAllIntegrityReports({ type: IntegrityReportType.ChecksumFail }); expect(mocks.integrityReport.streamIntegrityReportsByProperty).toHaveBeenCalledWith( 'assetId', IntegrityReportType.ChecksumFail, ); expect(mocks.job.queue).toHaveBeenCalledWith({ name: JobName.IntegrityDeleteReports, data: { reports: [{ id: 'report1', assetId: 'asset1', path: '/path/to/file1' }], }, }); }); it('should queue delete jobs for missing file reports by assetId and fileAssetId', async () => { mocks.integrityReport.streamIntegrityReportsByProperty .mockReturnValueOnce( (function* () { yield { id: 'report1', assetId: 'asset1', path: '/path/to/file1' }; })() as never, ) .mockReturnValueOnce( (function* () { yield { id: 'report2', fileAssetId: 'fileAsset1', path: '/path/to/file2' }; })() as never, ); await sut.handleDeleteAllIntegrityReports({ type: IntegrityReportType.MissingFile }); expect(mocks.integrityReport.streamIntegrityReportsByProperty).toHaveBeenCalledWith( 'assetId', IntegrityReportType.MissingFile, ); expect(mocks.integrityReport.streamIntegrityReportsByProperty).toHaveBeenCalledWith( 'fileAssetId', IntegrityReportType.MissingFile, ); expect(mocks.job.queue).toHaveBeenCalledTimes(2); }); it('should queue delete jobs for untracked file reports', async () => { mocks.integrityReport.streamIntegrityReportsByProperty.mockReturnValue( (function* () { yield { id: 'report1', path: '/path/to/untracked' }; })() as never, ); await sut.handleDeleteAllIntegrityReports({ type: IntegrityReportType.UntrackedFile }); expect(mocks.integrityReport.streamIntegrityReportsByProperty).toHaveBeenCalledWith( undefined, IntegrityReportType.UntrackedFile, ); expect(mocks.job.queue).toHaveBeenCalledWith({ name: JobName.IntegrityDeleteReports, data: { reports: [{ id: 'report1', path: '/path/to/untracked' }], }, }); }); it('should query all property types when no type specified', async () => { await sut.handleDeleteAllIntegrityReports({}); expect(mocks.integrityReport.streamIntegrityReportsByProperty).toHaveBeenCalledWith(undefined, undefined); expect(mocks.integrityReport.streamIntegrityReportsByProperty).toHaveBeenCalledWith('assetId', undefined); expect(mocks.integrityReport.streamIntegrityReportsByProperty).toHaveBeenCalledWith('fileAssetId', undefined); }); it('should succeed', async () => { await expect(sut.handleDeleteAllIntegrityReports({})).resolves.toBe(JobStatus.Success); }); }); describe('handleDeleteIntegrityReports', () => { it('should handle all report types', async () => { mocks.storage.unlink.mockResolvedValue(void 0); await sut.handleDeleteIntegrityReports({ reports: [ { id: 'report1', assetId: 'asset1', fileAssetId: null, path: '/path/to/file1' }, { id: 'report2', assetId: 'asset2', fileAssetId: null, path: '/path/to/file2' }, { id: 'report3', assetId: null, fileAssetId: 'fileAsset1', path: '/path/to/file3' }, { id: 'report4', assetId: null, fileAssetId: null, path: '/path/to/untracked' }, ], }); expect(mocks.asset.updateAll).toHaveBeenCalledWith(['asset1', 'asset2'], { deletedAt: expect.any(Date), status: AssetStatus.Trashed, }); expect(mocks.event.emit).toHaveBeenCalledWith('AssetTrashAll', { assetIds: ['asset1', 'asset2'], userId: '', }); expect(mocks.integrityReport.deleteByIds).toHaveBeenCalledWith(['report1', 'report2']); expect(mocks.asset.deleteFiles).toHaveBeenCalledWith([{ id: 'fileAsset1' }]); expect(mocks.storage.unlink).toHaveBeenCalledWith('/path/to/untracked'); expect(mocks.integrityReport.deleteByIds).toHaveBeenCalledWith(['report4']); }); it('should succeed', async () => { await expect(sut.handleDeleteIntegrityReports({ reports: [] })).resolves.toBe(JobStatus.Success); }); }); });