diff --git a/server/src/controllers/integrity.controller.ts b/server/src/controllers/integrity.controller.ts index 84299899ab..8baf4d5fe8 100644 --- a/server/src/controllers/integrity.controller.ts +++ b/server/src/controllers/integrity.controller.ts @@ -42,7 +42,7 @@ export class IntegrityController { }) @Authenticated({ permission: Permission.Maintenance, admin: true }) getIntegrityReport(@Body() dto: IntegrityGetReportDto): Promise { - return this.service.getIntegrityReport(dto); + return this.service.getIntegrityReport(dto.type); } @Delete('report/:id') diff --git a/server/src/repositories/integrity.repository.ts b/server/src/repositories/integrity.repository.ts index 437095a98a..793c27798c 100644 --- a/server/src/repositories/integrity.repository.ts +++ b/server/src/repositories/integrity.repository.ts @@ -1,7 +1,6 @@ import { Injectable } from '@nestjs/common'; import { Insertable, Kysely, sql } from 'kysely'; import { InjectKysely } from 'nestjs-kysely'; -import { Readable } from 'node:stream'; import { DummyValue, GenerateSql } from 'src/decorators'; import { IntegrityReportType } from 'src/enum'; import { DB } from 'src/schema'; @@ -91,29 +90,6 @@ export class IntegrityRepository { .executeTakeFirstOrThrow(); } - @GenerateSql({ params: [DummyValue.STRING], stream: true }) - streamIntegrityReportsCSV(type: IntegrityReportType): Readable { - const items = this.db - .selectFrom('integrity_report') - .select(['id', 'type', 'path', 'assetId', 'fileAssetId']) - .where('type', '=', type) - .orderBy('createdAt', 'desc') - .stream(); - - // very rudimentary csv serialiser - async function* generator() { - yield 'id,type,assetId,fileAssetId,path\n'; - - for await (const item of items) { - // no expectation of particularly bad filenames - // but they could potentially have a newline or quote character - yield `${item.id},${item.type},${item.assetId},${item.fileAssetId},"${item.path.replaceAll('"', '""')}"\n`; - } - } - - return Readable.from(generator()); - } - @GenerateSql({ params: [], stream: true }) streamAllAssetPaths() { return this.db.selectFrom('asset').select(['originalPath', 'encodedVideoPath']).stream(); @@ -201,6 +177,16 @@ export class IntegrityRepository { @GenerateSql({ params: [DummyValue.STRING], stream: true }) streamIntegrityReports(type: IntegrityReportType) { + return this.db + .selectFrom('integrity_report') + .select(['id', 'type', 'path', 'assetId', 'fileAssetId']) + .where('type', '=', type) + .orderBy('createdAt', 'desc') + .stream(); + } + + @GenerateSql({ params: [DummyValue.STRING], stream: true }) + streamIntegrityReportsWithAssetChecksum(type: IntegrityReportType) { return this.db .selectFrom('integrity_report') .select(['integrity_report.id as reportId', 'integrity_report.path']) diff --git a/server/src/services/integrity.service.spec.ts b/server/src/services/integrity.service.spec.ts index 7266928565..e4aa9ae34d 100644 --- a/server/src/services/integrity.service.spec.ts +++ b/server/src/services/integrity.service.spec.ts @@ -1,10 +1,10 @@ +import { text } from 'node:stream/consumers'; +import { AssetStatus, IntegrityReportType, JobName } from 'src/enum'; import { IntegrityService } from 'src/services/integrity.service'; import { newTestService, ServiceMocks } from 'test/utils'; describe(IntegrityService.name, () => { let sut: IntegrityService; - // impl. pending - // eslint-disable-next-line @typescript-eslint/no-unused-vars let mocks: ServiceMocks; beforeEach(() => { @@ -15,10 +15,211 @@ describe(IntegrityService.name, () => { expect(sut).toBeDefined(); }); - describe.skip('getIntegrityReportSummary'); // just calls repository - describe.skip('getIntegrityReport'); // just calls repository - describe.skip('getIntegrityReportCsv'); // just calls repository + describe('getIntegrityReportSummary', () => { + it('gets summary', async () => { + await sut.getIntegrityReportSummary(); + expect(mocks.integrityReport.getIntegrityReportSummary).toHaveBeenCalled(); + }); + }); - describe.todo('getIntegrityReportFile'); - describe.todo('deleteIntegrityReport'); + describe('getIntegrityReport', () => { + it('gets report', async () => { + await expect(sut.getIntegrityReport(IntegrityReportType.ChecksumFail)).resolves.toEqual( + expect.objectContaining({ + items: undefined, + }), + ); + + expect(mocks.integrityReport.getIntegrityReports).toHaveBeenCalledWith(IntegrityReportType.ChecksumFail); + }); + }); + + describe('getIntegrityReportCsv', () => { + it('gets report as csv', async () => { + mocks.integrityReport.streamIntegrityReports.mockReturnValue( + (function* () { + yield { + id: 'id', + createdAt: new Date(0), + path: '/path/to/file', + type: IntegrityReportType.ChecksumFail, + assetId: null, + fileAssetId: null, + }; + })() as never, + ); + + await expect(text(sut.getIntegrityReportCsv(IntegrityReportType.ChecksumFail))).resolves.toMatchInlineSnapshot(` + "id,type,assetId,fileAssetId,path + id,checksum_mismatch,null,null,"/path/to/file" + " + `); + + expect(mocks.integrityReport.streamIntegrityReports).toHaveBeenCalledWith(IntegrityReportType.ChecksumFail); + }); + }); + + describe('getIntegrityReportFile', () => { + it('gets report file', async () => { + mocks.integrityReport.getById.mockResolvedValue({ + id: 'id', + createdAt: new Date(0), + path: '/path/to/file', + type: IntegrityReportType.ChecksumFail, + assetId: null, + fileAssetId: null, + }); + + await expect(sut.getIntegrityReportFile('id')).resolves.toEqual({ + path: '/path/to/file', + fileName: 'file', + contentType: 'application/octet-stream', + cacheControl: 'private_without_cache', + }); + + expect(mocks.integrityReport.getById).toHaveBeenCalledWith('id'); + }); + }); + + describe('deleteIntegrityReport', () => { + it('deletes asset if one is present', async () => { + mocks.integrityReport.getById.mockResolvedValue({ + id: 'id', + createdAt: new Date(0), + path: '/path/to/file', + type: IntegrityReportType.ChecksumFail, + assetId: 'assetId', + fileAssetId: null, + }); + + await sut.deleteIntegrityReport( + { + user: { + id: 'userId', + }, + } as never, + 'id', + ); + + expect(mocks.asset.updateAll).toHaveBeenCalledWith(['assetId'], { + deletedAt: expect.any(Date), + status: AssetStatus.Trashed, + }); + + expect(mocks.event.emit).toHaveBeenCalledWith('AssetTrashAll', { + assetIds: ['assetId'], + userId: 'userId', + }); + + expect(mocks.integrityReport.deleteById).toHaveBeenCalledWith('id'); + }); + + it('deletes file asset if one is present', async () => { + mocks.integrityReport.getById.mockResolvedValue({ + id: 'id', + createdAt: new Date(0), + path: '/path/to/file', + type: IntegrityReportType.ChecksumFail, + assetId: null, + fileAssetId: 'fileAssetId', + }); + + await sut.deleteIntegrityReport( + { + user: { + id: 'userId', + }, + } as never, + 'id', + ); + + expect(mocks.asset.deleteFiles).toHaveBeenCalledWith([{ id: 'fileAssetId' }]); + }); + + it('deletes orphaned file', async () => { + mocks.integrityReport.getById.mockResolvedValue({ + id: 'id', + createdAt: new Date(0), + path: '/path/to/file', + type: IntegrityReportType.ChecksumFail, + assetId: null, + fileAssetId: null, + }); + + await sut.deleteIntegrityReport( + { + user: { + id: 'userId', + }, + } as never, + 'id', + ); + + expect(mocks.storage.unlink).toHaveBeenCalledWith('/path/to/file'); + expect(mocks.integrityReport.deleteById).toHaveBeenCalledWith('id'); + }); + }); + + describe('handleOrphanedFilesQueueAll', () => { + it('queues jobs for all detected files', async () => { + mocks.integrityReport.streamIntegrityReportsWithAssetChecksum.mockReturnValue((function* () {})() as never); + mocks.storage.walk.mockReturnValueOnce( + (function* () { + yield ['/path/to/file', '/path/to/file2']; + yield ['/path/to/batch2']; + })() as never, + ); + mocks.storage.walk.mockReturnValueOnce( + (function* () { + yield ['/path/to/file3', '/path/to/file4']; + yield ['/path/to/batch4']; + })() as never, + ); + + await sut.handleOrphanedFilesQueueAll({ refreshOnly: false }); + + expect(mocks.job.queue).toBeCalledTimes(4); + expect(mocks.job.queue).toBeCalledWith({ + name: JobName.IntegrityOrphanedFiles, + data: { + type: 'asset', + paths: expect.arrayContaining(['/path/to/file']), + }, + }); + expect(mocks.job.queue).toBeCalledWith({ + name: JobName.IntegrityOrphanedFiles, + data: { + type: 'asset_file', + paths: expect.arrayContaining(['/path/to/file3']), + }, + }); + }); + + it('queues jobs to refresh reports', async () => { + mocks.integrityReport.streamIntegrityReportsWithAssetChecksum.mockReturnValue( + (function* () { + yield 'mockReport'; + })() as never, + ); + + await sut.handleOrphanedFilesQueueAll({ refreshOnly: false }); + + expect(mocks.job.queue).toBeCalledTimes(1); + expect(mocks.job.queue).toBeCalledWith({ + name: JobName.IntegrityOrphanedFilesRefresh, + data: { + items: expect.arrayContaining(['mockReport']), + }, + }); + }); + }); + + describe.todo('handleOrphanedFiles'); + describe.todo('handleOrphanedRefresh'); + describe.todo('handleMissingFilesQueueAll'); + describe.todo('handleMissingFiles'); + describe.todo('handleMissingRefresh'); + describe.todo('handleChecksumFiles'); + describe.todo('handleChecksumRefresh'); + describe.todo('handleDeleteIntegrityReport'); }); diff --git a/server/src/services/integrity.service.ts b/server/src/services/integrity.service.ts index 56d360ac2f..1b48d5cdfe 100644 --- a/server/src/services/integrity.service.ts +++ b/server/src/services/integrity.service.ts @@ -9,11 +9,7 @@ import { JOBS_LIBRARY_PAGINATION_SIZE } from 'src/constants'; import { StorageCore } from 'src/cores/storage.core'; import { OnEvent, OnJob } from 'src/decorators'; import { AuthDto } from 'src/dtos/auth.dto'; -import { - IntegrityGetReportDto, - IntegrityReportResponseDto, - IntegrityReportSummaryResponseDto, -} from 'src/dtos/integrity.dto'; +import { IntegrityReportResponseDto, IntegrityReportSummaryResponseDto } from 'src/dtos/integrity.dto'; import { AssetStatus, CacheControl, @@ -158,14 +154,27 @@ export class IntegrityService extends BaseService { return this.integrityRepository.getIntegrityReportSummary(); } - async getIntegrityReport(dto: IntegrityGetReportDto): Promise { + async getIntegrityReport(type: IntegrityReportType): Promise { return { - items: await this.integrityRepository.getIntegrityReports(dto.type), + items: await this.integrityRepository.getIntegrityReports(type), }; } getIntegrityReportCsv(type: IntegrityReportType): Readable { - return this.integrityRepository.streamIntegrityReportsCSV(type); + const items = this.integrityRepository.streamIntegrityReports(type); + + // very rudimentary csv serialiser + async function* generator() { + yield 'id,type,assetId,fileAssetId,path\n'; + + for await (const item of items) { + // no expectation of particularly bad filenames + // but they could potentially have a newline or quote character + yield `${item.id},${item.type},${item.assetId},${item.fileAssetId},"${item.path.replaceAll('"', '""')}"\n`; + } + } + + return Readable.from(generator()); } async getIntegrityReportFile(id: string): Promise { @@ -206,7 +215,7 @@ export class IntegrityService extends BaseService { async handleOrphanedFilesQueueAll({ refreshOnly }: IIntegrityJob = {}): Promise { this.logger.log(`Checking for out of date orphaned file reports...`); - const reports = this.integrityRepository.streamIntegrityReports(IntegrityReportType.OrphanFile); + const reports = this.integrityRepository.streamIntegrityReportsWithAssetChecksum(IntegrityReportType.OrphanFile); let total = 0; for await (const batchReports of chunk(reports, JOBS_LIBRARY_PAGINATION_SIZE)) { @@ -332,7 +341,7 @@ export class IntegrityService extends BaseService { if (refreshOnly) { this.logger.log(`Checking for out of date missing file reports...`); - const reports = this.integrityRepository.streamIntegrityReports(IntegrityReportType.MissingFile); + const reports = this.integrityRepository.streamIntegrityReportsWithAssetChecksum(IntegrityReportType.MissingFile); let total = 0; for await (const batchReports of chunk(reports, JOBS_LIBRARY_PAGINATION_SIZE)) { @@ -434,7 +443,9 @@ export class IntegrityService extends BaseService { if (refreshOnly) { this.logger.log(`Checking for out of date checksum file reports...`); - const reports = this.integrityRepository.streamIntegrityReports(IntegrityReportType.ChecksumFail); + const reports = this.integrityRepository.streamIntegrityReportsWithAssetChecksum( + IntegrityReportType.ChecksumFail, + ); let total = 0; for await (const batchReports of chunk(reports, JOBS_LIBRARY_PAGINATION_SIZE)) {