From 74878628c8603bb37600994be085e77b3af66a54 Mon Sep 17 00:00:00 2001 From: Paul Makles Date: Wed, 10 Jun 2026 21:02:27 +0200 Subject: [PATCH] feat: integrity check jobs (missing files, untracked files, checksums) (#24205) Co-authored-by: Daniel Dietzler Signed-off-by: izzy --- .../server/maintenance.e2e-spec.ts | 4 +- .../specs/server/api/integrity.e2e-spec.ts | 669 +++++++++++ e2e/src/specs/web/integrity.e2e-spec.ts | 41 + e2e/src/utils.ts | 49 +- i18n/en.json | 15 + mobile/openapi/README.md | 12 + mobile/openapi/lib/api.dart | 7 + .../lib/api/maintenance_admin_api.dart | 292 +++++ mobile/openapi/lib/api_client.dart | 14 + mobile/openapi/lib/api_helper.dart | 3 + .../openapi/lib/model/integrity_report.dart | 88 ++ .../model/integrity_report_response_dto.dart | 115 ++ ...grity_report_response_dto_items_inner.dart | 117 ++ ...integrity_report_summary_response_dto.dart | 121 ++ mobile/openapi/lib/model/job_name.dart | 30 + mobile/openapi/lib/model/manual_job_name.dart | 27 + mobile/openapi/lib/model/queue_name.dart | 3 + .../lib/model/queues_response_legacy_dto.dart | 10 +- .../openapi/lib/model/system_config_dto.dart | 10 +- .../model/system_config_integrity_checks.dart | 115 ++ .../system_config_integrity_checksum_job.dart | 133 +++ .../model/system_config_integrity_job.dart | 109 ++ .../lib/model/system_config_job_dto.dart | 10 +- open-api/immich-openapi-specs.json | 473 +++++++- packages/sdk/src/fetch-client.ts | 135 ++- server/src/config.ts | 33 + server/src/constants.ts | 1 + server/src/controllers/index.ts | 2 + .../controllers/integrity-admin.controller.ts | 91 ++ server/src/dtos/integrity.dto.ts | 41 + server/src/dtos/queue-legacy.dto.ts | 1 + server/src/dtos/system-config.dto.ts | 26 + server/src/enum.ts | 36 + server/src/queries/integrity.repository.sql | 179 +++ server/src/repositories/index.ts | 2 + .../src/repositories/integrity.repository.ts | 228 ++++ server/src/schema/index.ts | 4 + ...781089983296-CreateIntegrityReportTable.ts | 24 + server/src/schema/tables/asset.table.ts | 2 +- .../schema/tables/integrity-report.table.ts | 27 + server/src/services/base.service.ts | 4 + server/src/services/index.ts | 2 + server/src/services/integrity.service.spec.ts | 29 + server/src/services/integrity.service.ts | 724 ++++++++++++ server/src/services/job.service.ts | 38 +- server/src/services/queue.service.spec.ts | 3 +- .../services/system-config.service.spec.ts | 17 + server/src/types.ts | 51 + server/src/validation.ts | 11 + server/test/medium.factory.ts | 3 + .../specs/services/integrity.service.spec.ts | 1000 +++++++++++++++++ .../repositories/storage.repository.mock.ts | 2 +- server/test/utils.ts | 43 +- .../integrity/IntegrityReportTableItem.svelte | 41 + .../ServerStatisticsCard.svelte | 16 +- web/src/lib/managers/event-manager.svelte.ts | 7 + web/src/lib/modals/JobCreateModal.svelte | 24 + web/src/lib/route.ts | 8 +- web/src/lib/services/integrity.service.ts | 125 +++ web/src/lib/services/job.service.ts | 2 + web/src/lib/services/queue.service.ts | 5 + web/src/routes/admin/maintenance/+page.svelte | 152 ++- web/src/routes/admin/maintenance/+page.ts | 10 +- .../integrity-report/[type]/+page.svelte | 99 ++ .../integrity-report/[type]/+page.ts | 22 + .../admin/system-settings/JobSettings.svelte | 1 + 66 files changed, 5711 insertions(+), 27 deletions(-) create mode 100644 e2e/src/specs/server/api/integrity.e2e-spec.ts create mode 100644 e2e/src/specs/web/integrity.e2e-spec.ts create mode 100644 mobile/openapi/lib/model/integrity_report.dart create mode 100644 mobile/openapi/lib/model/integrity_report_response_dto.dart create mode 100644 mobile/openapi/lib/model/integrity_report_response_dto_items_inner.dart create mode 100644 mobile/openapi/lib/model/integrity_report_summary_response_dto.dart create mode 100644 mobile/openapi/lib/model/system_config_integrity_checks.dart create mode 100644 mobile/openapi/lib/model/system_config_integrity_checksum_job.dart create mode 100644 mobile/openapi/lib/model/system_config_integrity_job.dart create mode 100644 server/src/controllers/integrity-admin.controller.ts create mode 100644 server/src/dtos/integrity.dto.ts create mode 100644 server/src/queries/integrity.repository.sql create mode 100644 server/src/repositories/integrity.repository.ts create mode 100644 server/src/schema/migrations/1781089983296-CreateIntegrityReportTable.ts create mode 100644 server/src/schema/tables/integrity-report.table.ts create mode 100644 server/src/services/integrity.service.spec.ts create mode 100644 server/src/services/integrity.service.ts create mode 100644 server/test/medium/specs/services/integrity.service.spec.ts create mode 100644 web/src/lib/components/maintenance/integrity/IntegrityReportTableItem.svelte create mode 100644 web/src/lib/services/integrity.service.ts create mode 100644 web/src/routes/admin/maintenance/integrity-report/[type]/+page.svelte create mode 100644 web/src/routes/admin/maintenance/integrity-report/[type]/+page.ts diff --git a/e2e/src/specs/maintenance/server/maintenance.e2e-spec.ts b/e2e/src/specs/maintenance/server/maintenance.e2e-spec.ts index 8e4e154328..819a4e31f7 100644 --- a/e2e/src/specs/maintenance/server/maintenance.e2e-spec.ts +++ b/e2e/src/specs/maintenance/server/maintenance.e2e-spec.ts @@ -99,7 +99,7 @@ describe('/admin/maintenance', () => { }, { interval: 500, - timeout: 10_000, + timeout: 60_000, }, ) .toBeTruthy(); @@ -190,7 +190,7 @@ describe('/admin/maintenance', () => { }, { interval: 500, - timeout: 10_000, + timeout: 60_000, }, ) .toBeFalsy(); diff --git a/e2e/src/specs/server/api/integrity.e2e-spec.ts b/e2e/src/specs/server/api/integrity.e2e-spec.ts new file mode 100644 index 0000000000..8b2b4364ae --- /dev/null +++ b/e2e/src/specs/server/api/integrity.e2e-spec.ts @@ -0,0 +1,669 @@ +import { + AssetMediaResponseDto, + IntegrityReportResponseDto, + LoginResponseDto, + ManualJobName, + QueueCommand, + QueueName, +} from '@immich/sdk'; +import { readFile } from 'node:fs/promises'; +import { app, testAssetDir, utils } from 'src/utils'; +import request from 'supertest'; +import { afterEach, beforeAll, describe, expect, it } from 'vitest'; + +const assetFilepath = `${testAssetDir}/metadata/gps-position/thompson-springs.jpg`; +const asset1Filepath = `${testAssetDir}/albums/nature/el_torcal_rocks.jpg`; +const asset2Filepath = `${testAssetDir}/albums/nature/wood_anemones.jpg`; + +describe('/admin/integrity', () => { + let admin: LoginResponseDto; + let asset: AssetMediaResponseDto; + + let user1: LoginResponseDto; + let asset1: AssetMediaResponseDto; + + let user2: LoginResponseDto; + let asset2: AssetMediaResponseDto; + + beforeAll(async () => { + await utils.resetDatabase(); + admin = await utils.adminSetup(); + + user1 = await utils.userSetup(admin.accessToken, { + email: '1@example.com', + name: '1', + password: '1', + }); + + user2 = await utils.userSetup(admin.accessToken, { + email: '2@example.com', + name: '2', + password: '2', + }); + + for (const queue of Object.values(QueueName)) { + if (queue === QueueName.IntegrityCheck) { + continue; + } + + await utils.queueCommand(admin.accessToken, queue, { + command: QueueCommand.Pause, + }); + } + + asset = await utils.createAsset(admin.accessToken, { + assetData: { + filename: 'asset.jpg', + bytes: await readFile(assetFilepath), + }, + }); + + asset1 = await utils.createAsset(user1.accessToken, { + assetData: { + filename: 'asset.jpg', + bytes: await readFile(asset1Filepath), + }, + }); + + asset2 = await utils.createAsset(user2.accessToken, { + assetData: { + filename: 'asset.jpg', + bytes: await readFile(asset2Filepath), + }, + }); + + await utils.mkFolder('/data/bak'); + await utils.copyFolder(`/data/upload/${admin.userId}`, `/data/bak/${admin.userId}`); + + for (const queue of Object.values(QueueName)) { + if (queue === QueueName.IntegrityCheck) { + continue; + } + + await utils.queueCommand(admin.accessToken, queue, { + command: QueueCommand.Empty, + }); + + await utils.queueCommand(admin.accessToken, queue, { + command: QueueCommand.Resume, + }); + } + }); + + afterEach(async () => { + await utils.deleteFolder(`/data/upload/${admin.userId}`); + await utils.copyFolder(`/data/bak/${admin.userId}`, `/data/upload/${admin.userId}`); + }); + + describe('POST /summary (& jobs)', async () => { + it.sequential('reports no issues', async () => { + await utils.createJob(admin.accessToken, { + name: ManualJobName.IntegrityUntrackedFiles, + }); + + await utils.createJob(admin.accessToken, { + name: ManualJobName.IntegrityMissingFiles, + }); + + await utils.createJob(admin.accessToken, { + name: ManualJobName.IntegrityChecksumMismatch, + }); + + await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck); + + await utils.createJob(admin.accessToken, { + name: ManualJobName.IntegrityUntrackedFilesDeleteAll, + }); + + await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck); + + const { status, body } = await request(app) + .get('/admin/integrity/summary') + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + + expect(status).toBe(200); + expect(body).toEqual({ + missing_file: 0, + untracked_file: 0, + checksum_mismatch: 0, + }); + }); + + it.sequential('should detect an untracked file (job: check untracked files)', async () => { + await utils.putTextFile('untracked', `/data/upload/${admin.userId}/untracked1.png`); + + await utils.createJob(admin.accessToken, { + name: ManualJobName.IntegrityUntrackedFiles, + }); + + await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck); + + const { status, body } = await request(app) + .get('/admin/integrity/summary') + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + + expect(status).toBe(200); + expect(body).toEqual( + expect.objectContaining({ + untracked_file: 1, + }), + ); + }); + + it.sequential('should detect outdated untracked file reports (job: refresh untracked files)', async () => { + // these should not be detected: + await utils.putTextFile('untracked', `/data/upload/${admin.userId}/untracked2.png`); + await utils.putTextFile('untracked', `/data/upload/${admin.userId}/untracked3.png`); + + await utils.createJob(admin.accessToken, { + name: ManualJobName.IntegrityUntrackedFilesRefresh, + }); + + await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck); + + const { status, body } = await request(app) + .get('/admin/integrity/summary') + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + + expect(status).toBe(200); + expect(body).toEqual( + expect.objectContaining({ + untracked_file: 0, + }), + ); + }); + + it.sequential('should delete untracked files (job: delete all untracked file reports)', async () => { + await utils.putTextFile('untracked', `/data/upload/${admin.userId}/untracked1.png`); + + await utils.createJob(admin.accessToken, { + name: ManualJobName.IntegrityUntrackedFiles, + }); + + await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck); + + await utils.createJob(admin.accessToken, { + name: ManualJobName.IntegrityUntrackedFilesDeleteAll, + }); + + await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck); + + const { status, body } = await request(app) + .get('/admin/integrity/summary') + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + + expect(status).toBe(200); + expect(body).toEqual( + expect.objectContaining({ + untracked_file: 0, + }), + ); + }); + + it.sequential('should detect a missing file and not a checksum mismatch (job: check missing files)', async () => { + await utils.deleteFolder(`/data/upload/${admin.userId}`); + + await utils.createJob(admin.accessToken, { + name: ManualJobName.IntegrityMissingFiles, + }); + + await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck); + + const { status, body } = await request(app) + .get('/admin/integrity/summary') + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + + expect(status).toBe(200); + expect(body).toEqual( + expect.objectContaining({ + missing_file: 1, + checksum_mismatch: 0, + }), + ); + }); + + it.sequential('should detect outdated missing file reports (job: refresh missing files)', async () => { + await utils.createJob(admin.accessToken, { + name: ManualJobName.IntegrityMissingFilesRefresh, + }); + + await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck); + + const { status, body } = await request(app) + .get('/admin/integrity/summary') + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + + expect(status).toBe(200); + expect(body).toEqual( + expect.objectContaining({ + missing_file: 0, + checksum_mismatch: 0, + }), + ); + }); + + it.sequential('should delete assets with missing files (job: delete all missing file reports)', async () => { + await utils.deleteFolder(`/data/upload/${user1.userId}`); + + await utils.createJob(admin.accessToken, { + name: ManualJobName.IntegrityMissingFiles, + }); + + await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck); + + const { status: listStatus, body: listBody } = await request(app) + .get('/admin/integrity/summary') + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + + expect(listStatus).toBe(200); + expect(listBody).toEqual( + expect.objectContaining({ + missing_file: 1, + }), + ); + + await utils.createJob(admin.accessToken, { + name: ManualJobName.IntegrityMissingFilesDeleteAll, + }); + + await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck); + + const { status, body } = await request(app) + .get('/admin/integrity/summary') + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + + expect(status).toBe(200); + expect(body).toEqual( + expect.objectContaining({ + missing_file: 0, + }), + ); + + await expect(utils.getAssetInfo(user1.accessToken, asset1.id)).resolves.toEqual( + expect.objectContaining({ + isTrashed: true, + }), + ); + }); + + it.sequential('should detect a checksum mismatch (job: check file checksums)', async () => { + await utils.truncateFolder(`/data/upload/${admin.userId}`); + + await utils.createJob(admin.accessToken, { + name: ManualJobName.IntegrityChecksumMismatch, + }); + + await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck); + + const { status, body } = await request(app) + .get('/admin/integrity/summary') + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + + expect(status).toBe(200); + expect(body).toEqual( + expect.objectContaining({ + checksum_mismatch: 1, + }), + ); + }); + + it.sequential('should detect outdated checksum mismatch reports (job: refresh file checksums)', async () => { + await utils.createJob(admin.accessToken, { + name: ManualJobName.IntegrityChecksumMismatchRefresh, + }); + + await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck); + + const { status, body } = await request(app) + .get('/admin/integrity/summary') + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + + expect(status).toBe(200); + expect(body).toEqual( + expect.objectContaining({ + checksum_mismatch: 0, + }), + ); + }); + + it.sequential( + 'should delete assets with mismatched checksum (job: delete all checksum mismatch reports)', + async () => { + await utils.truncateFolder(`/data/upload/${user2.userId}`); + + await utils.createJob(admin.accessToken, { + name: ManualJobName.IntegrityChecksumMismatch, + }); + + await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck); + + const { status: listStatus, body: listBody } = await request(app) + .get('/admin/integrity/summary') + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + + expect(listStatus).toBe(200); + expect(listBody).toEqual( + expect.objectContaining({ + checksum_mismatch: 1, + }), + ); + + await utils.createJob(admin.accessToken, { + name: ManualJobName.IntegrityChecksumMismatchDeleteAll, + }); + + await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck); + + const { status, body } = await request(app) + .get('/admin/integrity/summary') + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + + expect(status).toBe(200); + expect(body).toEqual( + expect.objectContaining({ + checksum_mismatch: 0, + }), + ); + + await expect(utils.getAssetInfo(user2.accessToken, asset2.id)).resolves.toEqual( + expect.objectContaining({ + isTrashed: true, + }), + ); + }, + ); + }); + + describe('POST /report', async () => { + it.sequential('reports untracked files', async () => { + await utils.putTextFile('untracked', `/data/upload/${admin.userId}/untracked1.png`); + + await utils.createJob(admin.accessToken, { + name: ManualJobName.IntegrityUntrackedFiles, + }); + + await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck); + + const { status, body } = await request(app) + .get('/admin/integrity/report?type=untracked_file') + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + + expect(status).toBe(200); + expect(body).toEqual({ + nextCursor: undefined, + items: expect.arrayContaining([ + { + id: expect.any(String), + type: 'untracked_file', + path: `/data/upload/${admin.userId}/untracked1.png`, + assetId: null, + fileAssetId: null, + createdAt: expect.any(String), + }, + ]), + }); + }); + + it.sequential('reports missing files', async () => { + await utils.deleteFolder(`/data/upload/${admin.userId}`); + + await utils.createJob(admin.accessToken, { + name: ManualJobName.IntegrityMissingFiles, + }); + + await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck); + + const { status, body } = await request(app) + .get('/admin/integrity/report?type=missing_file') + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + + expect(status).toBe(200); + expect(body).toEqual({ + nextCursor: undefined, + items: expect.arrayContaining([ + { + id: expect.any(String), + type: 'missing_file', + path: expect.any(String), + assetId: asset.id, + fileAssetId: null, + createdAt: expect.any(String), + }, + ]), + }); + }); + + it.sequential('reports checksum mismatched files', async () => { + await utils.truncateFolder(`/data/upload/${admin.userId}`); + + await utils.createJob(admin.accessToken, { + name: ManualJobName.IntegrityChecksumMismatch, + }); + + await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck); + + const { status, body } = await request(app) + .get('/admin/integrity/report?type=checksum_mismatch') + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + + expect(status).toBe(200); + expect(body).toEqual({ + nextCursor: undefined, + items: expect.arrayContaining([ + { + id: expect.any(String), + type: 'checksum_mismatch', + path: expect.any(String), + assetId: asset.id, + fileAssetId: null, + createdAt: expect.any(String), + }, + ]), + }); + }); + }); + + describe('DELETE /report/:id', async () => { + it.sequential('delete untracked files', async () => { + await utils.putTextFile('untracked', `/data/upload/${admin.userId}/untracked1.png`); + + await utils.createJob(admin.accessToken, { + name: ManualJobName.IntegrityUntrackedFiles, + }); + + await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck); + + const { status: listStatus, body: listBody } = await request(app) + .get('/admin/integrity/report?type=untracked_file') + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + + expect(listStatus).toBe(200); + + const report = (listBody as IntegrityReportResponseDto).items.find( + (item) => item.path === `/data/upload/${admin.userId}/untracked1.png`, + )!; + + const { status } = await request(app) + .delete(`/admin/integrity/report/${report.id}`) + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + + expect(status).toBe(200); + + await utils.createJob(admin.accessToken, { + name: ManualJobName.IntegrityUntrackedFiles, + }); + + await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck); + + const { status: listStatus2, body: listBody2 } = await request(app) + .get('/admin/integrity/report?type=untracked_file') + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + + expect(listStatus2).toBe(200); + expect(listBody2).not.toBe( + expect.objectContaining({ + items: expect.arrayContaining([ + expect.objectContaining({ + id: report.id, + }), + ]), + }), + ); + }); + + it.sequential('delete assets missing files', async () => { + await utils.deleteFolder(`/data/upload/${admin.userId}`); + + await utils.createJob(admin.accessToken, { + name: ManualJobName.IntegrityMissingFiles, + }); + + await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck); + + const { status: listStatus, body: listBody } = await request(app) + .get('/admin/integrity/report?type=missing_file') + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + + expect(listStatus).toBe(200); + expect(listBody.items.length).toBe(1); + + const report = (listBody as IntegrityReportResponseDto).items[0]; + + const { status } = await request(app) + .delete(`/admin/integrity/report/${report.id}`) + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + + expect(status).toBe(200); + + await utils.createJob(admin.accessToken, { + name: ManualJobName.IntegrityMissingFiles, + }); + + await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck); + + const { status: listStatus2, body: listBody2 } = await request(app) + .get('/admin/integrity/report?type=missing_file') + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + + expect(listStatus2).toBe(200); + expect(listBody2.items.length).toBe(0); + }); + + it.sequential('delete assets with failing checksum', async () => { + await utils.truncateFolder(`/data/upload/${admin.userId}`); + + await utils.createJob(admin.accessToken, { + name: ManualJobName.IntegrityChecksumMismatch, + }); + + await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck); + + const { status: listStatus, body: listBody } = await request(app) + .get('/admin/integrity/report?type=checksum_mismatch') + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + + expect(listStatus).toBe(200); + expect(listBody.items.length).toBe(1); + + const report = (listBody as IntegrityReportResponseDto).items[0]; + + const { status } = await request(app) + .delete(`/admin/integrity/report/${report.id}`) + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + + expect(status).toBe(200); + + await utils.createJob(admin.accessToken, { + name: ManualJobName.IntegrityChecksumMismatch, + }); + + await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck); + + const { status: listStatus2, body: listBody2 } = await request(app) + .get('/admin/integrity/report?type=checksum_mismatch') + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + + expect(listStatus2).toBe(200); + expect(listBody2.items.length).toBe(0); + }); + }); + + describe('GET /report/:type/csv', () => { + it.sequential('exports untracked files as csv', async () => { + await utils.putTextFile('untracked', `/data/upload/${admin.userId}/untracked1.png`); + + await utils.createJob(admin.accessToken, { + name: ManualJobName.IntegrityUntrackedFiles, + }); + + await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck); + + const { status, headers, text } = await request(app) + .get('/admin/integrity/report/untracked_file/csv') + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + + expect(status).toBe(200); + expect(headers['content-type']).toContain('text/csv'); + expect(headers['content-disposition']).toContain('.csv'); + expect(text).toContain('id,type,assetId,fileAssetId,path'); + expect(text).toContain(`untracked_file`); + expect(text).toContain(`/data/upload/${admin.userId}/untracked1.png`); + }); + }); + + describe('GET /report/:id/file', () => { + it.sequential('downloads untracked file', async () => { + await utils.putTextFile('untracked-content', `/data/upload/${admin.userId}/untracked1.png`); + + await utils.createJob(admin.accessToken, { + name: ManualJobName.IntegrityUntrackedFiles, + }); + + await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck); + + const { body: listBody } = await request(app) + .get('/admin/integrity/report?type=untracked_file') + .set('Authorization', `Bearer ${admin.accessToken}`) + .send(); + + const report = (listBody as IntegrityReportResponseDto).items.find( + (item) => item.path === `/data/upload/${admin.userId}/untracked1.png`, + )!; + + const { status, headers, body } = await request(app) + .get(`/admin/integrity/report/${report.id}/file`) + .set('Authorization', `Bearer ${admin.accessToken}`) + .buffer(true) + .send(); + + expect(status).toBe(200); + expect(headers['content-type']).toContain('application/octet-stream'); + expect(body.toString()).toBe('untracked-content'); + }); + }); +}); diff --git a/e2e/src/specs/web/integrity.e2e-spec.ts b/e2e/src/specs/web/integrity.e2e-spec.ts new file mode 100644 index 0000000000..615b1dc538 --- /dev/null +++ b/e2e/src/specs/web/integrity.e2e-spec.ts @@ -0,0 +1,41 @@ +import { LoginResponseDto, ManualJobName, QueueName } from '@immich/sdk'; +import { expect, test } from '@playwright/test'; +import { utils } from 'src/utils'; + +test.describe.configure({ mode: 'serial' }); + +test.describe.skip('Integrity', () => { + let admin: LoginResponseDto; + + test.beforeAll(async () => { + utils.initSdk(); + await utils.resetDatabase(); + admin = await utils.adminSetup(); + }); + + test('run integrity jobs to update stats', async ({ context, page }) => { + await utils.setAuthCookies(context, admin.accessToken); + + await utils.createJob(admin.accessToken, { + name: ManualJobName.IntegrityUntrackedFiles, + }); + + await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck); + + await page.goto('/admin/maintenance'); + + const count = page.getByText('Untracked Files').locator('..').locator('..').locator('div').nth(1); + + const previousCount = Number.parseInt((await count.textContent()) ?? ''); + + await utils.mkFolder(`/data/upload/${admin.userId}`); + await utils.putTextFile('untracked', `/data/upload/${admin.userId}/untracked1.png`); + + const checkButton = page.getByText('Integrity Report').locator('..').getByRole('button', { name: 'Check All' }); + + await checkButton.click(); + await expect(checkButton).toBeEnabled(); + + await expect(count).toContainText((previousCount + 1).toString()); + }); +}); diff --git a/e2e/src/utils.ts b/e2e/src/utils.ts index 7e51b40f63..23b3b14306 100644 --- a/e2e/src/utils.ts +++ b/e2e/src/utils.ts @@ -192,6 +192,7 @@ export const utils = { 'user', 'system_metadata', 'tag', + 'integrity_report', ]; const truncateTables = tables.filter((table) => table !== 'system_metadata'); @@ -559,10 +560,54 @@ export const utils = { mkdirSync(`${testAssetDir}/temp`, { recursive: true }); }, + putFile(source: string, dest: string) { + return executeCommand('docker', ['cp', source, `immich-e2e-server:${dest}`]).promise; + }, + + async putTextFile(contents: string, dest: string) { + const dir = await mkdtemp(join(tmpdir(), 'test-')); + const fn = join(dir, 'file'); + await pipeline(Readable.from(contents), createWriteStream(fn)); + return executeCommand('docker', ['cp', fn, `immich-e2e-server:${dest}`]).promise; + }, + async move(source: string, dest: string) { return executeCommand('docker', ['exec', 'immich-e2e-server', 'mv', source, dest]).promise; }, + async copyFolder(source: string, dest: string) { + return executeCommand('docker', ['exec', 'immich-e2e-server', 'cp', '-r', source, dest]).promise; + }, + + async deleteFile(path: string) { + return executeCommand('docker', ['exec', 'immich-e2e-server', 'rm', path]).promise; + }, + + async deleteFolder(path: string) { + return executeCommand('docker', ['exec', 'immich-e2e-server', 'rm', '-r', path]).promise; + }, + + async truncateFolder(path: string) { + return executeCommand('docker', [ + 'exec', + 'immich-e2e-server', + 'find', + path, + '-type', + 'f', + '-exec', + 'truncate', + '-s', + '1', + '{}', + ';', + ]).promise; + }, + + async mkFolder(path: string) { + return executeCommand('docker', ['exec', 'immich-e2e-server', 'mkdir', '-p', path]).promise; + }, + createBackup: async (accessToken: string) => { await utils.createJob(accessToken, { name: ManualJobName.BackupDatabase, @@ -579,10 +624,8 @@ export const utils = { resetBackups: async (accessToken: string) => { const { backups } = await listDatabaseBackups({ headers: asBearerAuth(accessToken) }); - - const backupFiles = backups.map((b) => b.filename); await deleteDatabaseBackup( - { databaseBackupDeleteDto: { backups: backupFiles } }, + { databaseBackupDeleteDto: { backups: backups.map((dto) => dto.filename) } }, { headers: asBearerAuth(accessToken) }, ); }, diff --git a/i18n/en.json b/i18n/en.json index 455e1cad21..34e598d603 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -79,6 +79,7 @@ "cron_expression_description": "Set the scanning interval using the cron format. For more information please refer to e.g. Crontab Guru", "cron_expression_presets": "Cron expression presets", "disable_login": "Disable login", + "download_csv": "Download CSV", "duplicate_detection_job_description": "Run machine learning on assets to detect similar images. Relies on Smart Search", "exclusion_pattern_description": "Exclusion patterns lets you ignore files and folders when scanning your library. This is useful if you have folders that contain files you don't want to import, such as RAW files.", "export_config_as_json_description": "Download the current system config as a JSON file", @@ -191,6 +192,17 @@ "maintenance_delete_backup": "Delete Backup", "maintenance_delete_backup_description": "This file will be irrevocably deleted.", "maintenance_delete_error": "Failed to delete backup.", + "maintenance_integrity_check_all": "Check All", + "maintenance_integrity_checksum_mismatch": "Checksum Mismatch", + "maintenance_integrity_checksum_mismatch_job": "Check for checksum mismatches", + "maintenance_integrity_checksum_mismatch_refresh_job": "Refresh checksum mismatch reports", + "maintenance_integrity_missing_file": "Missing Files", + "maintenance_integrity_missing_file_job": "Check for missing files", + "maintenance_integrity_missing_file_refresh_job": "Refresh missing file reports", + "maintenance_integrity_report": "Integrity Report", + "maintenance_integrity_untracked_file": "Untracked Files", + "maintenance_integrity_untracked_file_job": "Check for untracked files", + "maintenance_integrity_untracked_file_refresh_job": "Refresh untracked file reports", "maintenance_restore_backup": "Restore Backup", "maintenance_restore_backup_description": "Immich will be wiped and restored from the chosen backup. A backup will be created before continuing.", "maintenance_restore_backup_different_version": "This backup was created with a different version of Immich!", @@ -1226,6 +1238,7 @@ "failed": "Failed", "failed_count": "Failed: {count}", "failed_to_authenticate": "Failed to authenticate", + "failed_to_delete_file": "Failed to delete file", "failed_to_load_assets": "Failed to load assets", "failed_to_load_folder": "Failed to load folder", "favorite": "Favorite", @@ -1356,6 +1369,7 @@ "individual_share": "Individual share", "individual_shares": "Individual shares", "info": "Info", + "integrity_checks": "Integrity Checks", "interval": { "day_at_onepm": "Every day at 1pm", "hours": "Every {hours, plural, one {hour} other {{hours, number} hours}}", @@ -1428,6 +1442,7 @@ "linked_oauth_account": "Linked OAuth account", "list": "List", "live": "Live", + "load_more": "Load More", "loading": "Loading", "loading_search_results_failed": "Loading search results failed", "local": "Local", diff --git a/mobile/openapi/README.md b/mobile/openapi/README.md index 08a9d0ec06..f220b6e97f 100644 --- a/mobile/openapi/README.md +++ b/mobile/openapi/README.md @@ -183,7 +183,12 @@ Class | Method | HTTP request | Description *LibrariesApi* | [**scanLibrary**](doc//LibrariesApi.md#scanlibrary) | **POST** /libraries/{id}/scan | Scan a library *LibrariesApi* | [**updateLibrary**](doc//LibrariesApi.md#updatelibrary) | **PUT** /libraries/{id} | Update a library *LibrariesApi* | [**validate**](doc//LibrariesApi.md#validate) | **POST** /libraries/{id}/validate | Validate library settings +*MaintenanceAdminApi* | [**deleteIntegrityReport**](doc//MaintenanceAdminApi.md#deleteintegrityreport) | **DELETE** /admin/integrity/report/{id} | Delete integrity report item *MaintenanceAdminApi* | [**detectPriorInstall**](doc//MaintenanceAdminApi.md#detectpriorinstall) | **GET** /admin/maintenance/detect-install | Detect existing install +*MaintenanceAdminApi* | [**getIntegrityReport**](doc//MaintenanceAdminApi.md#getintegrityreport) | **GET** /admin/integrity/report | Get integrity report by type +*MaintenanceAdminApi* | [**getIntegrityReportCsv**](doc//MaintenanceAdminApi.md#getintegrityreportcsv) | **GET** /admin/integrity/report/{type}/csv | Export integrity report by type as CSV +*MaintenanceAdminApi* | [**getIntegrityReportFile**](doc//MaintenanceAdminApi.md#getintegrityreportfile) | **GET** /admin/integrity/report/{id}/file | Download flagged file +*MaintenanceAdminApi* | [**getIntegrityReportSummary**](doc//MaintenanceAdminApi.md#getintegrityreportsummary) | **GET** /admin/integrity/summary | Get integrity report summary *MaintenanceAdminApi* | [**getMaintenanceStatus**](doc//MaintenanceAdminApi.md#getmaintenancestatus) | **GET** /admin/maintenance/status | Get maintenance mode status *MaintenanceAdminApi* | [**maintenanceLogin**](doc//MaintenanceAdminApi.md#maintenancelogin) | **POST** /admin/maintenance/login | Log into maintenance mode *MaintenanceAdminApi* | [**setMaintenanceMode**](doc//MaintenanceAdminApi.md#setmaintenancemode) | **POST** /admin/maintenance | Set maintenance mode @@ -448,6 +453,10 @@ Class | Method | HTTP request | Description - [FoldersResponse](doc//FoldersResponse.md) - [FoldersUpdate](doc//FoldersUpdate.md) - [ImageFormat](doc//ImageFormat.md) + - [IntegrityReport](doc//IntegrityReport.md) + - [IntegrityReportResponseDto](doc//IntegrityReportResponseDto.md) + - [IntegrityReportResponseDtoItemsInner](doc//IntegrityReportResponseDtoItemsInner.md) + - [IntegrityReportSummaryResponseDto](doc//IntegrityReportSummaryResponseDto.md) - [JobCreateDto](doc//JobCreateDto.md) - [JobName](doc//JobName.md) - [JobSettingsDto](doc//JobSettingsDto.md) @@ -630,6 +639,9 @@ Class | Method | HTTP request | Description - [SystemConfigGeneratedFullsizeImageDto](doc//SystemConfigGeneratedFullsizeImageDto.md) - [SystemConfigGeneratedImageDto](doc//SystemConfigGeneratedImageDto.md) - [SystemConfigImageDto](doc//SystemConfigImageDto.md) + - [SystemConfigIntegrityChecks](doc//SystemConfigIntegrityChecks.md) + - [SystemConfigIntegrityChecksumJob](doc//SystemConfigIntegrityChecksumJob.md) + - [SystemConfigIntegrityJob](doc//SystemConfigIntegrityJob.md) - [SystemConfigJobDto](doc//SystemConfigJobDto.md) - [SystemConfigLibraryDto](doc//SystemConfigLibraryDto.md) - [SystemConfigLibraryScanDto](doc//SystemConfigLibraryScanDto.md) diff --git a/mobile/openapi/lib/api.dart b/mobile/openapi/lib/api.dart index 891c6a1495..413a64040c 100644 --- a/mobile/openapi/lib/api.dart +++ b/mobile/openapi/lib/api.dart @@ -174,6 +174,10 @@ part 'model/facial_recognition_config.dart'; part 'model/folders_response.dart'; part 'model/folders_update.dart'; part 'model/image_format.dart'; +part 'model/integrity_report.dart'; +part 'model/integrity_report_response_dto.dart'; +part 'model/integrity_report_response_dto_items_inner.dart'; +part 'model/integrity_report_summary_response_dto.dart'; part 'model/job_create_dto.dart'; part 'model/job_name.dart'; part 'model/job_settings_dto.dart'; @@ -356,6 +360,9 @@ part 'model/system_config_faces_dto.dart'; part 'model/system_config_generated_fullsize_image_dto.dart'; part 'model/system_config_generated_image_dto.dart'; part 'model/system_config_image_dto.dart'; +part 'model/system_config_integrity_checks.dart'; +part 'model/system_config_integrity_checksum_job.dart'; +part 'model/system_config_integrity_job.dart'; part 'model/system_config_job_dto.dart'; part 'model/system_config_library_dto.dart'; part 'model/system_config_library_scan_dto.dart'; diff --git a/mobile/openapi/lib/api/maintenance_admin_api.dart b/mobile/openapi/lib/api/maintenance_admin_api.dart index 8bab193ddf..3e43f6d51a 100644 --- a/mobile/openapi/lib/api/maintenance_admin_api.dart +++ b/mobile/openapi/lib/api/maintenance_admin_api.dart @@ -16,6 +16,56 @@ class MaintenanceAdminApi { final ApiClient apiClient; + /// Delete integrity report item + /// + /// Delete a given report item and perform corresponding deletion (e.g. trash asset, delete file) + /// + /// Note: This method returns the HTTP [Response]. + /// + /// Parameters: + /// + /// * [String] id (required): + Future deleteIntegrityReportWithHttpInfo(String id, { Future? abortTrigger, }) async { + // ignore: prefer_const_declarations + final apiPath = r'/admin/integrity/report/{id}' + .replaceAll('{id}', id); + + // ignore: prefer_final_locals + Object? postBody; + + final queryParams = []; + final headerParams = {}; + final formParams = {}; + + const contentTypes = []; + + + return apiClient.invokeAPI( + apiPath, + 'DELETE', + queryParams, + postBody, + headerParams, + formParams, + contentTypes.isEmpty ? null : contentTypes.first, + abortTrigger: abortTrigger, + ); + } + + /// Delete integrity report item + /// + /// Delete a given report item and perform corresponding deletion (e.g. trash asset, delete file) + /// + /// Parameters: + /// + /// * [String] id (required): + Future deleteIntegrityReport(String id, { Future? abortTrigger, }) async { + final response = await deleteIntegrityReportWithHttpInfo(id, abortTrigger: abortTrigger,); + if (response.statusCode >= HttpStatus.badRequest) { + throw ApiException(response.statusCode, await _decodeBodyBytes(response)); + } + } + /// Detect existing install /// /// Collect integrity checks and other heuristics about local data. @@ -65,6 +115,248 @@ class MaintenanceAdminApi { return null; } + /// Get integrity report by type + /// + /// Get all flagged items by integrity report type + /// + /// Note: This method returns the HTTP [Response]. + /// + /// Parameters: + /// + /// * [IntegrityReport] type (required): + /// + /// * [String] cursor: + /// Cursor for pagination + /// + /// * [int] limit: + /// Number of items per page + Future getIntegrityReportWithHttpInfo(IntegrityReport type, { String? cursor, int? limit, Future? abortTrigger, }) async { + // ignore: prefer_const_declarations + final apiPath = r'/admin/integrity/report'; + + // ignore: prefer_final_locals + Object? postBody; + + final queryParams = []; + final headerParams = {}; + final formParams = {}; + + if (cursor != null) { + queryParams.addAll(_queryParams('', 'cursor', cursor)); + } + if (limit != null) { + queryParams.addAll(_queryParams('', 'limit', limit)); + } + queryParams.addAll(_queryParams('', 'type', type)); + + const contentTypes = []; + + + return apiClient.invokeAPI( + apiPath, + 'GET', + queryParams, + postBody, + headerParams, + formParams, + contentTypes.isEmpty ? null : contentTypes.first, + abortTrigger: abortTrigger, + ); + } + + /// Get integrity report by type + /// + /// Get all flagged items by integrity report type + /// + /// Parameters: + /// + /// * [IntegrityReport] type (required): + /// + /// * [String] cursor: + /// Cursor for pagination + /// + /// * [int] limit: + /// Number of items per page + Future getIntegrityReport(IntegrityReport type, { String? cursor, int? limit, Future? abortTrigger, }) async { + final response = await getIntegrityReportWithHttpInfo(type, cursor: cursor, limit: limit, abortTrigger: abortTrigger,); + if (response.statusCode >= HttpStatus.badRequest) { + throw ApiException(response.statusCode, await _decodeBodyBytes(response)); + } + // When a remote server returns no body with a status of 204, we shall not decode it. + // At the time of writing this, `dart:convert` will throw an "Unexpected end of input" + // FormatException when trying to decode an empty string. + if (response.body.isNotEmpty && response.statusCode != HttpStatus.noContent) { + return await apiClient.deserializeAsync(await _decodeBodyBytes(response), 'IntegrityReportResponseDto',) as IntegrityReportResponseDto; + + } + return null; + } + + /// Export integrity report by type as CSV + /// + /// Get all integrity report entries for a given type as a CSV + /// + /// Note: This method returns the HTTP [Response]. + /// + /// Parameters: + /// + /// * [IntegrityReport] type (required): + Future getIntegrityReportCsvWithHttpInfo(IntegrityReport type, { Future? abortTrigger, }) async { + // ignore: prefer_const_declarations + final apiPath = r'/admin/integrity/report/{type}/csv' + .replaceAll('{type}', type.toString()); + + // ignore: prefer_final_locals + Object? postBody; + + final queryParams = []; + final headerParams = {}; + final formParams = {}; + + const contentTypes = []; + + + return apiClient.invokeAPI( + apiPath, + 'GET', + queryParams, + postBody, + headerParams, + formParams, + contentTypes.isEmpty ? null : contentTypes.first, + abortTrigger: abortTrigger, + ); + } + + /// Export integrity report by type as CSV + /// + /// Get all integrity report entries for a given type as a CSV + /// + /// Parameters: + /// + /// * [IntegrityReport] type (required): + Future getIntegrityReportCsv(IntegrityReport type, { Future? abortTrigger, }) async { + final response = await getIntegrityReportCsvWithHttpInfo(type, abortTrigger: abortTrigger,); + if (response.statusCode >= HttpStatus.badRequest) { + throw ApiException(response.statusCode, await _decodeBodyBytes(response)); + } + // When a remote server returns no body with a status of 204, we shall not decode it. + // At the time of writing this, `dart:convert` will throw an "Unexpected end of input" + // FormatException when trying to decode an empty string. + if (response.body.isNotEmpty && response.statusCode != HttpStatus.noContent) { + return await apiClient.deserializeAsync(await _decodeBodyBytes(response), 'MultipartFile',) as MultipartFile; + + } + return null; + } + + /// Download flagged file + /// + /// Download the untracked/broken file if one exists + /// + /// Note: This method returns the HTTP [Response]. + /// + /// Parameters: + /// + /// * [String] id (required): + Future getIntegrityReportFileWithHttpInfo(String id, { Future? abortTrigger, }) async { + // ignore: prefer_const_declarations + final apiPath = r'/admin/integrity/report/{id}/file' + .replaceAll('{id}', id); + + // ignore: prefer_final_locals + Object? postBody; + + final queryParams = []; + final headerParams = {}; + final formParams = {}; + + const contentTypes = []; + + + return apiClient.invokeAPI( + apiPath, + 'GET', + queryParams, + postBody, + headerParams, + formParams, + contentTypes.isEmpty ? null : contentTypes.first, + abortTrigger: abortTrigger, + ); + } + + /// Download flagged file + /// + /// Download the untracked/broken file if one exists + /// + /// Parameters: + /// + /// * [String] id (required): + Future getIntegrityReportFile(String id, { Future? abortTrigger, }) async { + final response = await getIntegrityReportFileWithHttpInfo(id, abortTrigger: abortTrigger,); + if (response.statusCode >= HttpStatus.badRequest) { + throw ApiException(response.statusCode, await _decodeBodyBytes(response)); + } + // When a remote server returns no body with a status of 204, we shall not decode it. + // At the time of writing this, `dart:convert` will throw an "Unexpected end of input" + // FormatException when trying to decode an empty string. + if (response.body.isNotEmpty && response.statusCode != HttpStatus.noContent) { + return await apiClient.deserializeAsync(await _decodeBodyBytes(response), 'MultipartFile',) as MultipartFile; + + } + return null; + } + + /// Get integrity report summary + /// + /// Get a count of the items flagged in each integrity report + /// + /// Note: This method returns the HTTP [Response]. + Future getIntegrityReportSummaryWithHttpInfo({ Future? abortTrigger, }) async { + // ignore: prefer_const_declarations + final apiPath = r'/admin/integrity/summary'; + + // ignore: prefer_final_locals + Object? postBody; + + final queryParams = []; + final headerParams = {}; + final formParams = {}; + + const contentTypes = []; + + + return apiClient.invokeAPI( + apiPath, + 'GET', + queryParams, + postBody, + headerParams, + formParams, + contentTypes.isEmpty ? null : contentTypes.first, + abortTrigger: abortTrigger, + ); + } + + /// Get integrity report summary + /// + /// Get a count of the items flagged in each integrity report + Future getIntegrityReportSummary({ Future? abortTrigger, }) async { + final response = await getIntegrityReportSummaryWithHttpInfo(abortTrigger: abortTrigger,); + if (response.statusCode >= HttpStatus.badRequest) { + throw ApiException(response.statusCode, await _decodeBodyBytes(response)); + } + // When a remote server returns no body with a status of 204, we shall not decode it. + // At the time of writing this, `dart:convert` will throw an "Unexpected end of input" + // FormatException when trying to decode an empty string. + if (response.body.isNotEmpty && response.statusCode != HttpStatus.noContent) { + return await apiClient.deserializeAsync(await _decodeBodyBytes(response), 'IntegrityReportSummaryResponseDto',) as IntegrityReportSummaryResponseDto; + + } + return null; + } + /// Get maintenance mode status /// /// Fetch information about the currently running maintenance action. diff --git a/mobile/openapi/lib/api_client.dart b/mobile/openapi/lib/api_client.dart index baa5293222..90bff45431 100644 --- a/mobile/openapi/lib/api_client.dart +++ b/mobile/openapi/lib/api_client.dart @@ -393,6 +393,14 @@ class ApiClient { return FoldersUpdate.fromJson(value); case 'ImageFormat': return ImageFormatTypeTransformer().decode(value); + case 'IntegrityReport': + return IntegrityReportTypeTransformer().decode(value); + case 'IntegrityReportResponseDto': + return IntegrityReportResponseDto.fromJson(value); + case 'IntegrityReportResponseDtoItemsInner': + return IntegrityReportResponseDtoItemsInner.fromJson(value); + case 'IntegrityReportSummaryResponseDto': + return IntegrityReportSummaryResponseDto.fromJson(value); case 'JobCreateDto': return JobCreateDto.fromJson(value); case 'JobName': @@ -757,6 +765,12 @@ class ApiClient { return SystemConfigGeneratedImageDto.fromJson(value); case 'SystemConfigImageDto': return SystemConfigImageDto.fromJson(value); + case 'SystemConfigIntegrityChecks': + return SystemConfigIntegrityChecks.fromJson(value); + case 'SystemConfigIntegrityChecksumJob': + return SystemConfigIntegrityChecksumJob.fromJson(value); + case 'SystemConfigIntegrityJob': + return SystemConfigIntegrityJob.fromJson(value); case 'SystemConfigJobDto': return SystemConfigJobDto.fromJson(value); case 'SystemConfigLibraryDto': diff --git a/mobile/openapi/lib/api_helper.dart b/mobile/openapi/lib/api_helper.dart index e9040a6f7a..a985dece82 100644 --- a/mobile/openapi/lib/api_helper.dart +++ b/mobile/openapi/lib/api_helper.dart @@ -109,6 +109,9 @@ String parameterToString(dynamic value) { if (value is ImageFormat) { return ImageFormatTypeTransformer().encode(value).toString(); } + if (value is IntegrityReport) { + return IntegrityReportTypeTransformer().encode(value).toString(); + } if (value is JobName) { return JobNameTypeTransformer().encode(value).toString(); } diff --git a/mobile/openapi/lib/model/integrity_report.dart b/mobile/openapi/lib/model/integrity_report.dart new file mode 100644 index 0000000000..42a85bd0ce --- /dev/null +++ b/mobile/openapi/lib/model/integrity_report.dart @@ -0,0 +1,88 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +/// Integrity report type +class IntegrityReport { + /// Instantiate a new enum with the provided [value]. + const IntegrityReport._(this.value); + + /// The underlying value of this enum member. + final String value; + + @override + String toString() => value; + + String toJson() => value; + + static const untrackedFile = IntegrityReport._(r'untracked_file'); + static const missingFile = IntegrityReport._(r'missing_file'); + static const checksumMismatch = IntegrityReport._(r'checksum_mismatch'); + + /// List of all possible values in this [enum][IntegrityReport]. + static const values = [ + untrackedFile, + missingFile, + checksumMismatch, + ]; + + static IntegrityReport? fromJson(dynamic value) => IntegrityReportTypeTransformer().decode(value); + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = IntegrityReport.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } +} + +/// Transformation class that can [encode] an instance of [IntegrityReport] to String, +/// and [decode] dynamic data back to [IntegrityReport]. +class IntegrityReportTypeTransformer { + factory IntegrityReportTypeTransformer() => _instance ??= const IntegrityReportTypeTransformer._(); + + const IntegrityReportTypeTransformer._(); + + String encode(IntegrityReport data) => data.value; + + /// Decodes a [dynamic value][data] to a IntegrityReport. + /// + /// If [allowNull] is true and the [dynamic value][data] cannot be decoded successfully, + /// then null is returned. However, if [allowNull] is false and the [dynamic value][data] + /// cannot be decoded successfully, then an [UnimplementedError] is thrown. + /// + /// The [allowNull] is very handy when an API changes and a new enum value is added or removed, + /// and users are still using an old app with the old code. + IntegrityReport? decode(dynamic data, {bool allowNull = true}) { + if (data != null) { + switch (data) { + case r'untracked_file': return IntegrityReport.untrackedFile; + case r'missing_file': return IntegrityReport.missingFile; + case r'checksum_mismatch': return IntegrityReport.checksumMismatch; + default: + if (!allowNull) { + throw ArgumentError('Unknown enum value to decode: $data'); + } + } + } + return null; + } + + /// Singleton [IntegrityReportTypeTransformer] instance. + static IntegrityReportTypeTransformer? _instance; +} + diff --git a/mobile/openapi/lib/model/integrity_report_response_dto.dart b/mobile/openapi/lib/model/integrity_report_response_dto.dart new file mode 100644 index 0000000000..e9f8b91cca --- /dev/null +++ b/mobile/openapi/lib/model/integrity_report_response_dto.dart @@ -0,0 +1,115 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class IntegrityReportResponseDto { + /// Returns a new [IntegrityReportResponseDto] instance. + IntegrityReportResponseDto({ + this.items = const [], + this.nextCursor = const Optional.absent(), + }); + + List items; + + /// + /// Please note: This property should have been non-nullable! Since the specification file + /// does not include a default value (using the "default:" property), however, the generated + /// source code must fall back to having a nullable type. + /// Consider adding a "default:" property in the specification file to hide this note. + /// + Optional nextCursor; + + @override + bool operator ==(Object other) => identical(this, other) || other is IntegrityReportResponseDto && + _deepEquality.equals(other.items, items) && + other.nextCursor == nextCursor; + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (items.hashCode) + + (nextCursor == null ? 0 : nextCursor!.hashCode); + + @override + String toString() => 'IntegrityReportResponseDto[items=$items, nextCursor=$nextCursor]'; + + Map toJson() { + final json = {}; + json[r'items'] = this.items; + if (this.nextCursor.isPresent) { + final value = this.nextCursor.value; + json[r'nextCursor'] = value; + } + return json; + } + + /// Returns a new [IntegrityReportResponseDto] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static IntegrityReportResponseDto? fromJson(dynamic value) { + upgradeDto(value, "IntegrityReportResponseDto"); + if (value is Map) { + final json = value.cast(); + + return IntegrityReportResponseDto( + items: IntegrityReportResponseDtoItemsInner.listFromJson(json[r'items']), + nextCursor: json.containsKey(r'nextCursor') ? Optional.present(mapValueOfType(json, r'nextCursor')) : const Optional.absent(), + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = IntegrityReportResponseDto.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = IntegrityReportResponseDto.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of IntegrityReportResponseDto-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = IntegrityReportResponseDto.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'items', + }; +} + diff --git a/mobile/openapi/lib/model/integrity_report_response_dto_items_inner.dart b/mobile/openapi/lib/model/integrity_report_response_dto_items_inner.dart new file mode 100644 index 0000000000..db09f698f4 --- /dev/null +++ b/mobile/openapi/lib/model/integrity_report_response_dto_items_inner.dart @@ -0,0 +1,117 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class IntegrityReportResponseDtoItemsInner { + /// Returns a new [IntegrityReportResponseDtoItemsInner] instance. + IntegrityReportResponseDtoItemsInner({ + required this.id, + required this.path, + required this.type, + }); + + /// Integrity report item id + String id; + + /// Integrity report item path + String path; + + IntegrityReport type; + + @override + bool operator ==(Object other) => identical(this, other) || other is IntegrityReportResponseDtoItemsInner && + other.id == id && + other.path == path && + other.type == type; + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (id.hashCode) + + (path.hashCode) + + (type.hashCode); + + @override + String toString() => 'IntegrityReportResponseDtoItemsInner[id=$id, path=$path, type=$type]'; + + Map toJson() { + final json = {}; + json[r'id'] = this.id; + json[r'path'] = this.path; + json[r'type'] = this.type; + return json; + } + + /// Returns a new [IntegrityReportResponseDtoItemsInner] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static IntegrityReportResponseDtoItemsInner? fromJson(dynamic value) { + upgradeDto(value, "IntegrityReportResponseDtoItemsInner"); + if (value is Map) { + final json = value.cast(); + + return IntegrityReportResponseDtoItemsInner( + id: mapValueOfType(json, r'id')!, + path: mapValueOfType(json, r'path')!, + type: IntegrityReport.fromJson(json[r'type'])!, + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = IntegrityReportResponseDtoItemsInner.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = IntegrityReportResponseDtoItemsInner.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of IntegrityReportResponseDtoItemsInner-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = IntegrityReportResponseDtoItemsInner.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'id', + 'path', + 'type', + }; +} + diff --git a/mobile/openapi/lib/model/integrity_report_summary_response_dto.dart b/mobile/openapi/lib/model/integrity_report_summary_response_dto.dart new file mode 100644 index 0000000000..f95c036d14 --- /dev/null +++ b/mobile/openapi/lib/model/integrity_report_summary_response_dto.dart @@ -0,0 +1,121 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class IntegrityReportSummaryResponseDto { + /// Returns a new [IntegrityReportSummaryResponseDto] instance. + IntegrityReportSummaryResponseDto({ + required this.checksumMismatch, + required this.missingFile, + required this.untrackedFile, + }); + + /// Minimum value: 0 + /// Maximum value: 9007199254740991 + int checksumMismatch; + + /// Minimum value: 0 + /// Maximum value: 9007199254740991 + int missingFile; + + /// Minimum value: 0 + /// Maximum value: 9007199254740991 + int untrackedFile; + + @override + bool operator ==(Object other) => identical(this, other) || other is IntegrityReportSummaryResponseDto && + other.checksumMismatch == checksumMismatch && + other.missingFile == missingFile && + other.untrackedFile == untrackedFile; + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (checksumMismatch.hashCode) + + (missingFile.hashCode) + + (untrackedFile.hashCode); + + @override + String toString() => 'IntegrityReportSummaryResponseDto[checksumMismatch=$checksumMismatch, missingFile=$missingFile, untrackedFile=$untrackedFile]'; + + Map toJson() { + final json = {}; + json[r'checksum_mismatch'] = this.checksumMismatch; + json[r'missing_file'] = this.missingFile; + json[r'untracked_file'] = this.untrackedFile; + return json; + } + + /// Returns a new [IntegrityReportSummaryResponseDto] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static IntegrityReportSummaryResponseDto? fromJson(dynamic value) { + upgradeDto(value, "IntegrityReportSummaryResponseDto"); + if (value is Map) { + final json = value.cast(); + + return IntegrityReportSummaryResponseDto( + checksumMismatch: mapValueOfType(json, r'checksum_mismatch')!, + missingFile: mapValueOfType(json, r'missing_file')!, + untrackedFile: mapValueOfType(json, r'untracked_file')!, + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = IntegrityReportSummaryResponseDto.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = IntegrityReportSummaryResponseDto.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of IntegrityReportSummaryResponseDto-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = IntegrityReportSummaryResponseDto.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'checksum_mismatch', + 'missing_file', + 'untracked_file', + }; +} + diff --git a/mobile/openapi/lib/model/job_name.dart b/mobile/openapi/lib/model/job_name.dart index 435cffd623..048e5a60a4 100644 --- a/mobile/openapi/lib/model/job_name.dart +++ b/mobile/openapi/lib/model/job_name.dart @@ -79,6 +79,16 @@ class JobName { static const ocrQueueAll = JobName._(r'OcrQueueAll'); static const ocr = JobName._(r'Ocr'); static const workflowAssetTrigger = JobName._(r'WorkflowAssetTrigger'); + static const integrityUntrackedFilesQueueAll = JobName._(r'IntegrityUntrackedFilesQueueAll'); + static const integrityUntrackedFiles = JobName._(r'IntegrityUntrackedFiles'); + static const integrityUntrackedRefresh = JobName._(r'IntegrityUntrackedRefresh'); + static const integrityMissingFilesQueueAll = JobName._(r'IntegrityMissingFilesQueueAll'); + static const integrityMissingFiles = JobName._(r'IntegrityMissingFiles'); + static const integrityMissingFilesRefresh = JobName._(r'IntegrityMissingFilesRefresh'); + static const integrityChecksumFiles = JobName._(r'IntegrityChecksumFiles'); + static const integrityChecksumFilesRefresh = JobName._(r'IntegrityChecksumFilesRefresh'); + static const integrityDeleteReportType = JobName._(r'IntegrityDeleteReportType'); + static const integrityDeleteReports = JobName._(r'IntegrityDeleteReports'); /// List of all possible values in this [enum][JobName]. static const values = [ @@ -138,6 +148,16 @@ class JobName { ocrQueueAll, ocr, workflowAssetTrigger, + integrityUntrackedFilesQueueAll, + integrityUntrackedFiles, + integrityUntrackedRefresh, + integrityMissingFilesQueueAll, + integrityMissingFiles, + integrityMissingFilesRefresh, + integrityChecksumFiles, + integrityChecksumFilesRefresh, + integrityDeleteReportType, + integrityDeleteReports, ]; static JobName? fromJson(dynamic value) => JobNameTypeTransformer().decode(value); @@ -232,6 +252,16 @@ class JobNameTypeTransformer { case r'OcrQueueAll': return JobName.ocrQueueAll; case r'Ocr': return JobName.ocr; case r'WorkflowAssetTrigger': return JobName.workflowAssetTrigger; + case r'IntegrityUntrackedFilesQueueAll': return JobName.integrityUntrackedFilesQueueAll; + case r'IntegrityUntrackedFiles': return JobName.integrityUntrackedFiles; + case r'IntegrityUntrackedRefresh': return JobName.integrityUntrackedRefresh; + case r'IntegrityMissingFilesQueueAll': return JobName.integrityMissingFilesQueueAll; + case r'IntegrityMissingFiles': return JobName.integrityMissingFiles; + case r'IntegrityMissingFilesRefresh': return JobName.integrityMissingFilesRefresh; + case r'IntegrityChecksumFiles': return JobName.integrityChecksumFiles; + case r'IntegrityChecksumFilesRefresh': return JobName.integrityChecksumFilesRefresh; + case r'IntegrityDeleteReportType': return JobName.integrityDeleteReportType; + case r'IntegrityDeleteReports': return JobName.integrityDeleteReports; default: if (!allowNull) { throw ArgumentError('Unknown enum value to decode: $data'); diff --git a/mobile/openapi/lib/model/manual_job_name.dart b/mobile/openapi/lib/model/manual_job_name.dart index 27753eb9dc..44faaf4476 100644 --- a/mobile/openapi/lib/model/manual_job_name.dart +++ b/mobile/openapi/lib/model/manual_job_name.dart @@ -29,6 +29,15 @@ class ManualJobName { static const memoryCleanup = ManualJobName._(r'memory-cleanup'); static const memoryCreate = ManualJobName._(r'memory-create'); static const backupDatabase = ManualJobName._(r'backup-database'); + static const integrityMissingFiles = ManualJobName._(r'integrity-missing-files'); + static const integrityUntrackedFiles = ManualJobName._(r'integrity-untracked-files'); + static const integrityChecksumMismatch = ManualJobName._(r'integrity-checksum-mismatch'); + static const integrityMissingFilesRefresh = ManualJobName._(r'integrity-missing-files-refresh'); + static const integrityUntrackedFilesRefresh = ManualJobName._(r'integrity-untracked-files-refresh'); + static const integrityChecksumMismatchRefresh = ManualJobName._(r'integrity-checksum-mismatch-refresh'); + static const integrityMissingFilesDeleteAll = ManualJobName._(r'integrity-missing-files-delete-all'); + static const integrityUntrackedFilesDeleteAll = ManualJobName._(r'integrity-untracked-files-delete-all'); + static const integrityChecksumMismatchDeleteAll = ManualJobName._(r'integrity-checksum-mismatch-delete-all'); /// List of all possible values in this [enum][ManualJobName]. static const values = [ @@ -38,6 +47,15 @@ class ManualJobName { memoryCleanup, memoryCreate, backupDatabase, + integrityMissingFiles, + integrityUntrackedFiles, + integrityChecksumMismatch, + integrityMissingFilesRefresh, + integrityUntrackedFilesRefresh, + integrityChecksumMismatchRefresh, + integrityMissingFilesDeleteAll, + integrityUntrackedFilesDeleteAll, + integrityChecksumMismatchDeleteAll, ]; static ManualJobName? fromJson(dynamic value) => ManualJobNameTypeTransformer().decode(value); @@ -82,6 +100,15 @@ class ManualJobNameTypeTransformer { case r'memory-cleanup': return ManualJobName.memoryCleanup; case r'memory-create': return ManualJobName.memoryCreate; case r'backup-database': return ManualJobName.backupDatabase; + case r'integrity-missing-files': return ManualJobName.integrityMissingFiles; + case r'integrity-untracked-files': return ManualJobName.integrityUntrackedFiles; + case r'integrity-checksum-mismatch': return ManualJobName.integrityChecksumMismatch; + case r'integrity-missing-files-refresh': return ManualJobName.integrityMissingFilesRefresh; + case r'integrity-untracked-files-refresh': return ManualJobName.integrityUntrackedFilesRefresh; + case r'integrity-checksum-mismatch-refresh': return ManualJobName.integrityChecksumMismatchRefresh; + case r'integrity-missing-files-delete-all': return ManualJobName.integrityMissingFilesDeleteAll; + case r'integrity-untracked-files-delete-all': return ManualJobName.integrityUntrackedFilesDeleteAll; + case r'integrity-checksum-mismatch-delete-all': return ManualJobName.integrityChecksumMismatchDeleteAll; default: if (!allowNull) { throw ArgumentError('Unknown enum value to decode: $data'); diff --git a/mobile/openapi/lib/model/queue_name.dart b/mobile/openapi/lib/model/queue_name.dart index eb19d8957f..e9ecfcdd84 100644 --- a/mobile/openapi/lib/model/queue_name.dart +++ b/mobile/openapi/lib/model/queue_name.dart @@ -40,6 +40,7 @@ class QueueName { static const backupDatabase = QueueName._(r'backupDatabase'); static const ocr = QueueName._(r'ocr'); static const workflow = QueueName._(r'workflow'); + static const integrityCheck = QueueName._(r'integrityCheck'); static const editor = QueueName._(r'editor'); /// List of all possible values in this [enum][QueueName]. @@ -61,6 +62,7 @@ class QueueName { backupDatabase, ocr, workflow, + integrityCheck, editor, ]; @@ -117,6 +119,7 @@ class QueueNameTypeTransformer { case r'backupDatabase': return QueueName.backupDatabase; case r'ocr': return QueueName.ocr; case r'workflow': return QueueName.workflow; + case r'integrityCheck': return QueueName.integrityCheck; case r'editor': return QueueName.editor; default: if (!allowNull) { diff --git a/mobile/openapi/lib/model/queues_response_legacy_dto.dart b/mobile/openapi/lib/model/queues_response_legacy_dto.dart index c7bc23cb4d..80e74b9c41 100644 --- a/mobile/openapi/lib/model/queues_response_legacy_dto.dart +++ b/mobile/openapi/lib/model/queues_response_legacy_dto.dart @@ -19,6 +19,7 @@ class QueuesResponseLegacyDto { required this.editor, required this.faceDetection, required this.facialRecognition, + required this.integrityCheck, required this.library_, required this.metadataExtraction, required this.migration, @@ -45,6 +46,8 @@ class QueuesResponseLegacyDto { QueueResponseLegacyDto facialRecognition; + QueueResponseLegacyDto integrityCheck; + QueueResponseLegacyDto library_; QueueResponseLegacyDto metadataExtraction; @@ -77,6 +80,7 @@ class QueuesResponseLegacyDto { other.editor == editor && other.faceDetection == faceDetection && other.facialRecognition == facialRecognition && + other.integrityCheck == integrityCheck && other.library_ == library_ && other.metadataExtraction == metadataExtraction && other.migration == migration && @@ -99,6 +103,7 @@ class QueuesResponseLegacyDto { (editor.hashCode) + (faceDetection.hashCode) + (facialRecognition.hashCode) + + (integrityCheck.hashCode) + (library_.hashCode) + (metadataExtraction.hashCode) + (migration.hashCode) + @@ -113,7 +118,7 @@ class QueuesResponseLegacyDto { (workflow.hashCode); @override - String toString() => 'QueuesResponseLegacyDto[backgroundTask=$backgroundTask, backupDatabase=$backupDatabase, duplicateDetection=$duplicateDetection, editor=$editor, faceDetection=$faceDetection, facialRecognition=$facialRecognition, library_=$library_, metadataExtraction=$metadataExtraction, migration=$migration, notifications=$notifications, ocr=$ocr, search=$search, sidecar=$sidecar, smartSearch=$smartSearch, storageTemplateMigration=$storageTemplateMigration, thumbnailGeneration=$thumbnailGeneration, videoConversion=$videoConversion, workflow=$workflow]'; + String toString() => 'QueuesResponseLegacyDto[backgroundTask=$backgroundTask, backupDatabase=$backupDatabase, duplicateDetection=$duplicateDetection, editor=$editor, faceDetection=$faceDetection, facialRecognition=$facialRecognition, integrityCheck=$integrityCheck, library_=$library_, metadataExtraction=$metadataExtraction, migration=$migration, notifications=$notifications, ocr=$ocr, search=$search, sidecar=$sidecar, smartSearch=$smartSearch, storageTemplateMigration=$storageTemplateMigration, thumbnailGeneration=$thumbnailGeneration, videoConversion=$videoConversion, workflow=$workflow]'; Map toJson() { final json = {}; @@ -123,6 +128,7 @@ class QueuesResponseLegacyDto { json[r'editor'] = this.editor; json[r'faceDetection'] = this.faceDetection; json[r'facialRecognition'] = this.facialRecognition; + json[r'integrityCheck'] = this.integrityCheck; json[r'library'] = this.library_; json[r'metadataExtraction'] = this.metadataExtraction; json[r'migration'] = this.migration; @@ -153,6 +159,7 @@ class QueuesResponseLegacyDto { editor: QueueResponseLegacyDto.fromJson(json[r'editor'])!, faceDetection: QueueResponseLegacyDto.fromJson(json[r'faceDetection'])!, facialRecognition: QueueResponseLegacyDto.fromJson(json[r'facialRecognition'])!, + integrityCheck: QueueResponseLegacyDto.fromJson(json[r'integrityCheck'])!, library_: QueueResponseLegacyDto.fromJson(json[r'library'])!, metadataExtraction: QueueResponseLegacyDto.fromJson(json[r'metadataExtraction'])!, migration: QueueResponseLegacyDto.fromJson(json[r'migration'])!, @@ -218,6 +225,7 @@ class QueuesResponseLegacyDto { 'editor', 'faceDetection', 'facialRecognition', + 'integrityCheck', 'library', 'metadataExtraction', 'migration', diff --git a/mobile/openapi/lib/model/system_config_dto.dart b/mobile/openapi/lib/model/system_config_dto.dart index 38dbb30f0c..3a5e15b030 100644 --- a/mobile/openapi/lib/model/system_config_dto.dart +++ b/mobile/openapi/lib/model/system_config_dto.dart @@ -16,6 +16,7 @@ class SystemConfigDto { required this.backup, required this.ffmpeg, required this.image, + required this.integrityChecks, required this.job, required this.library_, required this.logging, @@ -42,6 +43,8 @@ class SystemConfigDto { SystemConfigImageDto image; + SystemConfigIntegrityChecks integrityChecks; + SystemConfigJobDto job; SystemConfigLibraryDto library_; @@ -83,6 +86,7 @@ class SystemConfigDto { other.backup == backup && other.ffmpeg == ffmpeg && other.image == image && + other.integrityChecks == integrityChecks && other.job == job && other.library_ == library_ && other.logging == logging && @@ -108,6 +112,7 @@ class SystemConfigDto { (backup.hashCode) + (ffmpeg.hashCode) + (image.hashCode) + + (integrityChecks.hashCode) + (job.hashCode) + (library_.hashCode) + (logging.hashCode) + @@ -128,13 +133,14 @@ class SystemConfigDto { (user.hashCode); @override - String toString() => 'SystemConfigDto[backup=$backup, ffmpeg=$ffmpeg, image=$image, job=$job, library_=$library_, logging=$logging, machineLearning=$machineLearning, map=$map, metadata=$metadata, newVersionCheck=$newVersionCheck, nightlyTasks=$nightlyTasks, notifications=$notifications, oauth=$oauth, passwordLogin=$passwordLogin, reverseGeocoding=$reverseGeocoding, server=$server, storageTemplate=$storageTemplate, templates=$templates, theme=$theme, trash=$trash, user=$user]'; + String toString() => 'SystemConfigDto[backup=$backup, ffmpeg=$ffmpeg, image=$image, integrityChecks=$integrityChecks, job=$job, library_=$library_, logging=$logging, machineLearning=$machineLearning, map=$map, metadata=$metadata, newVersionCheck=$newVersionCheck, nightlyTasks=$nightlyTasks, notifications=$notifications, oauth=$oauth, passwordLogin=$passwordLogin, reverseGeocoding=$reverseGeocoding, server=$server, storageTemplate=$storageTemplate, templates=$templates, theme=$theme, trash=$trash, user=$user]'; Map toJson() { final json = {}; json[r'backup'] = this.backup; json[r'ffmpeg'] = this.ffmpeg; json[r'image'] = this.image; + json[r'integrityChecks'] = this.integrityChecks; json[r'job'] = this.job; json[r'library'] = this.library_; json[r'logging'] = this.logging; @@ -168,6 +174,7 @@ class SystemConfigDto { backup: SystemConfigBackupsDto.fromJson(json[r'backup'])!, ffmpeg: SystemConfigFFmpegDto.fromJson(json[r'ffmpeg'])!, image: SystemConfigImageDto.fromJson(json[r'image'])!, + integrityChecks: SystemConfigIntegrityChecks.fromJson(json[r'integrityChecks'])!, job: SystemConfigJobDto.fromJson(json[r'job'])!, library_: SystemConfigLibraryDto.fromJson(json[r'library'])!, logging: SystemConfigLoggingDto.fromJson(json[r'logging'])!, @@ -236,6 +243,7 @@ class SystemConfigDto { 'backup', 'ffmpeg', 'image', + 'integrityChecks', 'job', 'library', 'logging', diff --git a/mobile/openapi/lib/model/system_config_integrity_checks.dart b/mobile/openapi/lib/model/system_config_integrity_checks.dart new file mode 100644 index 0000000000..ef047e156a --- /dev/null +++ b/mobile/openapi/lib/model/system_config_integrity_checks.dart @@ -0,0 +1,115 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class SystemConfigIntegrityChecks { + /// Returns a new [SystemConfigIntegrityChecks] instance. + SystemConfigIntegrityChecks({ + required this.checksumFiles, + required this.missingFiles, + required this.untrackedFiles, + }); + + SystemConfigIntegrityChecksumJob checksumFiles; + + SystemConfigIntegrityJob missingFiles; + + SystemConfigIntegrityJob untrackedFiles; + + @override + bool operator ==(Object other) => identical(this, other) || other is SystemConfigIntegrityChecks && + other.checksumFiles == checksumFiles && + other.missingFiles == missingFiles && + other.untrackedFiles == untrackedFiles; + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (checksumFiles.hashCode) + + (missingFiles.hashCode) + + (untrackedFiles.hashCode); + + @override + String toString() => 'SystemConfigIntegrityChecks[checksumFiles=$checksumFiles, missingFiles=$missingFiles, untrackedFiles=$untrackedFiles]'; + + Map toJson() { + final json = {}; + json[r'checksumFiles'] = this.checksumFiles; + json[r'missingFiles'] = this.missingFiles; + json[r'untrackedFiles'] = this.untrackedFiles; + return json; + } + + /// Returns a new [SystemConfigIntegrityChecks] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static SystemConfigIntegrityChecks? fromJson(dynamic value) { + upgradeDto(value, "SystemConfigIntegrityChecks"); + if (value is Map) { + final json = value.cast(); + + return SystemConfigIntegrityChecks( + checksumFiles: SystemConfigIntegrityChecksumJob.fromJson(json[r'checksumFiles'])!, + missingFiles: SystemConfigIntegrityJob.fromJson(json[r'missingFiles'])!, + untrackedFiles: SystemConfigIntegrityJob.fromJson(json[r'untrackedFiles'])!, + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = SystemConfigIntegrityChecks.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = SystemConfigIntegrityChecks.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of SystemConfigIntegrityChecks-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = SystemConfigIntegrityChecks.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'checksumFiles', + 'missingFiles', + 'untrackedFiles', + }; +} + diff --git a/mobile/openapi/lib/model/system_config_integrity_checksum_job.dart b/mobile/openapi/lib/model/system_config_integrity_checksum_job.dart new file mode 100644 index 0000000000..beed3c76be --- /dev/null +++ b/mobile/openapi/lib/model/system_config_integrity_checksum_job.dart @@ -0,0 +1,133 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class SystemConfigIntegrityChecksumJob { + /// Returns a new [SystemConfigIntegrityChecksumJob] instance. + SystemConfigIntegrityChecksumJob({ + required this.cronExpression, + required this.enabled, + required this.percentageLimit, + required this.timeLimit, + }); + + /// Cron expression for when the integrity check should run + String cronExpression; + + /// Enabled + bool enabled; + + /// Percentage limit of the integrity checksum job + /// + /// Minimum value: 0 + /// Maximum value: 9007199254740991 + int percentageLimit; + + /// How long the integrity checksum job may run for + /// + /// Minimum value: 0 + /// Maximum value: 9007199254740991 + int timeLimit; + + @override + bool operator ==(Object other) => identical(this, other) || other is SystemConfigIntegrityChecksumJob && + other.cronExpression == cronExpression && + other.enabled == enabled && + other.percentageLimit == percentageLimit && + other.timeLimit == timeLimit; + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (cronExpression.hashCode) + + (enabled.hashCode) + + (percentageLimit.hashCode) + + (timeLimit.hashCode); + + @override + String toString() => 'SystemConfigIntegrityChecksumJob[cronExpression=$cronExpression, enabled=$enabled, percentageLimit=$percentageLimit, timeLimit=$timeLimit]'; + + Map toJson() { + final json = {}; + json[r'cronExpression'] = this.cronExpression; + json[r'enabled'] = this.enabled; + json[r'percentageLimit'] = this.percentageLimit; + json[r'timeLimit'] = this.timeLimit; + return json; + } + + /// Returns a new [SystemConfigIntegrityChecksumJob] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static SystemConfigIntegrityChecksumJob? fromJson(dynamic value) { + upgradeDto(value, "SystemConfigIntegrityChecksumJob"); + if (value is Map) { + final json = value.cast(); + + return SystemConfigIntegrityChecksumJob( + cronExpression: mapValueOfType(json, r'cronExpression')!, + enabled: mapValueOfType(json, r'enabled')!, + percentageLimit: mapValueOfType(json, r'percentageLimit')!, + timeLimit: mapValueOfType(json, r'timeLimit')!, + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = SystemConfigIntegrityChecksumJob.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = SystemConfigIntegrityChecksumJob.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of SystemConfigIntegrityChecksumJob-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = SystemConfigIntegrityChecksumJob.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'cronExpression', + 'enabled', + 'percentageLimit', + 'timeLimit', + }; +} + diff --git a/mobile/openapi/lib/model/system_config_integrity_job.dart b/mobile/openapi/lib/model/system_config_integrity_job.dart new file mode 100644 index 0000000000..52afcfa187 --- /dev/null +++ b/mobile/openapi/lib/model/system_config_integrity_job.dart @@ -0,0 +1,109 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.18 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + +class SystemConfigIntegrityJob { + /// Returns a new [SystemConfigIntegrityJob] instance. + SystemConfigIntegrityJob({ + required this.cronExpression, + required this.enabled, + }); + + /// Cron expression for when the integrity check should run + String cronExpression; + + /// Enabled + bool enabled; + + @override + bool operator ==(Object other) => identical(this, other) || other is SystemConfigIntegrityJob && + other.cronExpression == cronExpression && + other.enabled == enabled; + + @override + int get hashCode => + // ignore: unnecessary_parenthesis + (cronExpression.hashCode) + + (enabled.hashCode); + + @override + String toString() => 'SystemConfigIntegrityJob[cronExpression=$cronExpression, enabled=$enabled]'; + + Map toJson() { + final json = {}; + json[r'cronExpression'] = this.cronExpression; + json[r'enabled'] = this.enabled; + return json; + } + + /// Returns a new [SystemConfigIntegrityJob] instance and imports its values from + /// [value] if it's a [Map], null otherwise. + // ignore: prefer_constructors_over_static_methods + static SystemConfigIntegrityJob? fromJson(dynamic value) { + upgradeDto(value, "SystemConfigIntegrityJob"); + if (value is Map) { + final json = value.cast(); + + return SystemConfigIntegrityJob( + cronExpression: mapValueOfType(json, r'cronExpression')!, + enabled: mapValueOfType(json, r'enabled')!, + ); + } + return null; + } + + static List listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = SystemConfigIntegrityJob.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } + + static Map mapFromJson(dynamic json) { + final map = {}; + if (json is Map && json.isNotEmpty) { + json = json.cast(); // ignore: parameter_assignments + for (final entry in json.entries) { + final value = SystemConfigIntegrityJob.fromJson(entry.value); + if (value != null) { + map[entry.key] = value; + } + } + } + return map; + } + + // maps a json object with a list of SystemConfigIntegrityJob-objects as value to a dart map + static Map> mapListFromJson(dynamic json, {bool growable = false,}) { + final map = >{}; + if (json is Map && json.isNotEmpty) { + // ignore: parameter_assignments + json = json.cast(); + for (final entry in json.entries) { + map[entry.key] = SystemConfigIntegrityJob.listFromJson(entry.value, growable: growable,); + } + } + return map; + } + + /// The list of required keys that must be present in a JSON. + static const requiredKeys = { + 'cronExpression', + 'enabled', + }; +} + diff --git a/mobile/openapi/lib/model/system_config_job_dto.dart b/mobile/openapi/lib/model/system_config_job_dto.dart index d54db6809f..08b07cc37c 100644 --- a/mobile/openapi/lib/model/system_config_job_dto.dart +++ b/mobile/openapi/lib/model/system_config_job_dto.dart @@ -16,6 +16,7 @@ class SystemConfigJobDto { required this.backgroundTask, required this.editor, required this.faceDetection, + required this.integrityCheck, required this.library_, required this.metadataExtraction, required this.migration, @@ -35,6 +36,8 @@ class SystemConfigJobDto { JobSettingsDto faceDetection; + JobSettingsDto integrityCheck; + JobSettingsDto library_; JobSettingsDto metadataExtraction; @@ -62,6 +65,7 @@ class SystemConfigJobDto { other.backgroundTask == backgroundTask && other.editor == editor && other.faceDetection == faceDetection && + other.integrityCheck == integrityCheck && other.library_ == library_ && other.metadataExtraction == metadataExtraction && other.migration == migration && @@ -80,6 +84,7 @@ class SystemConfigJobDto { (backgroundTask.hashCode) + (editor.hashCode) + (faceDetection.hashCode) + + (integrityCheck.hashCode) + (library_.hashCode) + (metadataExtraction.hashCode) + (migration.hashCode) + @@ -93,13 +98,14 @@ class SystemConfigJobDto { (workflow.hashCode); @override - String toString() => 'SystemConfigJobDto[backgroundTask=$backgroundTask, editor=$editor, faceDetection=$faceDetection, library_=$library_, metadataExtraction=$metadataExtraction, migration=$migration, notifications=$notifications, ocr=$ocr, search=$search, sidecar=$sidecar, smartSearch=$smartSearch, thumbnailGeneration=$thumbnailGeneration, videoConversion=$videoConversion, workflow=$workflow]'; + String toString() => 'SystemConfigJobDto[backgroundTask=$backgroundTask, editor=$editor, faceDetection=$faceDetection, integrityCheck=$integrityCheck, library_=$library_, metadataExtraction=$metadataExtraction, migration=$migration, notifications=$notifications, ocr=$ocr, search=$search, sidecar=$sidecar, smartSearch=$smartSearch, thumbnailGeneration=$thumbnailGeneration, videoConversion=$videoConversion, workflow=$workflow]'; Map toJson() { final json = {}; json[r'backgroundTask'] = this.backgroundTask; json[r'editor'] = this.editor; json[r'faceDetection'] = this.faceDetection; + json[r'integrityCheck'] = this.integrityCheck; json[r'library'] = this.library_; json[r'metadataExtraction'] = this.metadataExtraction; json[r'migration'] = this.migration; @@ -126,6 +132,7 @@ class SystemConfigJobDto { backgroundTask: JobSettingsDto.fromJson(json[r'backgroundTask'])!, editor: JobSettingsDto.fromJson(json[r'editor'])!, faceDetection: JobSettingsDto.fromJson(json[r'faceDetection'])!, + integrityCheck: JobSettingsDto.fromJson(json[r'integrityCheck'])!, library_: JobSettingsDto.fromJson(json[r'library'])!, metadataExtraction: JobSettingsDto.fromJson(json[r'metadataExtraction'])!, migration: JobSettingsDto.fromJson(json[r'migration'])!, @@ -187,6 +194,7 @@ class SystemConfigJobDto { 'backgroundTask', 'editor', 'faceDetection', + 'integrityCheck', 'library', 'metadataExtraction', 'migration', diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index 851c195c1d..41c494db5d 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -564,6 +564,298 @@ "x-immich-state": "Alpha" } }, + "/admin/integrity/report": { + "get": { + "description": "Get all flagged items by integrity report type", + "operationId": "getIntegrityReport", + "parameters": [ + { + "name": "cursor", + "required": false, + "in": "query", + "description": "Cursor for pagination", + "schema": { + "type": "string" + } + }, + { + "name": "limit", + "required": false, + "in": "query", + "description": "Number of items per page", + "schema": { + "maximum": 9007199254740991, + "exclusiveMinimum": true, + "default": 500, + "type": "integer", + "minimum": 0 + } + }, + { + "name": "type", + "required": true, + "in": "query", + "schema": { + "$ref": "#/components/schemas/IntegrityReport" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/IntegrityReportResponseDto" + } + } + }, + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "summary": "Get integrity report by type", + "tags": [ + "Maintenance (admin)" + ], + "x-immich-admin-only": true, + "x-immich-history": [ + { + "version": "v3.0.0", + "state": "Added" + }, + { + "version": "v3.0.0", + "state": "Alpha" + } + ], + "x-immich-permission": "maintenance", + "x-immich-state": "Alpha" + } + }, + "/admin/integrity/report/{id}": { + "delete": { + "description": "Delete a given report item and perform corresponding deletion (e.g. trash asset, delete file)", + "operationId": "deleteIntegrityReport", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-7[0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12})$", + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "summary": "Delete integrity report item", + "tags": [ + "Maintenance (admin)" + ], + "x-immich-admin-only": true, + "x-immich-history": [ + { + "version": "v3.0.0", + "state": "Added" + }, + { + "version": "v3.0.0", + "state": "Alpha" + } + ], + "x-immich-permission": "maintenance", + "x-immich-state": "Alpha" + } + }, + "/admin/integrity/report/{id}/file": { + "get": { + "description": "Download the untracked/broken file if one exists", + "operationId": "getIntegrityReportFile", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-7[0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12})$", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/octet-stream": { + "schema": { + "format": "binary", + "type": "string" + } + } + }, + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "summary": "Download flagged file", + "tags": [ + "Maintenance (admin)" + ], + "x-immich-admin-only": true, + "x-immich-history": [ + { + "version": "v3.0.0", + "state": "Added" + }, + { + "version": "v3.0.0", + "state": "Alpha" + } + ], + "x-immich-permission": "maintenance", + "x-immich-state": "Alpha" + } + }, + "/admin/integrity/report/{type}/csv": { + "get": { + "description": "Get all integrity report entries for a given type as a CSV", + "operationId": "getIntegrityReportCsv", + "parameters": [ + { + "name": "type", + "required": true, + "in": "path", + "schema": { + "$ref": "#/components/schemas/IntegrityReport" + } + } + ], + "responses": { + "200": { + "content": { + "application/octet-stream": { + "schema": { + "format": "binary", + "type": "string" + } + } + }, + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "summary": "Export integrity report by type as CSV", + "tags": [ + "Maintenance (admin)" + ], + "x-immich-admin-only": true, + "x-immich-history": [ + { + "version": "v3.0.0", + "state": "Added" + }, + { + "version": "v3.0.0", + "state": "Alpha" + } + ], + "x-immich-permission": "maintenance", + "x-immich-state": "Alpha" + } + }, + "/admin/integrity/summary": { + "get": { + "description": "Get a count of the items flagged in each integrity report", + "operationId": "getIntegrityReportSummary", + "parameters": [], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/IntegrityReportSummaryResponseDto" + } + } + }, + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "summary": "Get integrity report summary", + "tags": [ + "Maintenance (admin)" + ], + "x-immich-admin-only": true, + "x-immich-history": [ + { + "version": "v3.0.0", + "state": "Added" + }, + { + "version": "v3.0.0", + "state": "Alpha" + } + ], + "x-immich-permission": "maintenance", + "x-immich-state": "Alpha" + } + }, "/admin/maintenance": { "post": { "description": "Put Immich into or take it out of maintenance mode", @@ -15943,6 +16235,10 @@ "name": "Faces", "description": "A face is a detected human face within an asset, which can be associated with a person. Faces are normally detected via machine learning, but can also be created via manually." }, + { + "name": "Integrity (admin)", + "description": "Endpoints for viewing and managing integrity reports." + }, { "name": "Jobs", "description": "Queues and background jobs are used for processing tasks asynchronously. Queues can be paused and resumed as needed." @@ -18780,6 +19076,75 @@ ], "type": "string" }, + "IntegrityReport": { + "description": "Integrity report type", + "enum": [ + "untracked_file", + "missing_file", + "checksum_mismatch" + ], + "type": "string" + }, + "IntegrityReportResponseDto": { + "properties": { + "items": { + "items": { + "properties": { + "id": { + "description": "Integrity report item id", + "type": "string" + }, + "path": { + "description": "Integrity report item path", + "type": "string" + }, + "type": { + "$ref": "#/components/schemas/IntegrityReport" + } + }, + "required": [ + "id", + "type", + "path" + ], + "type": "object" + }, + "type": "array" + }, + "nextCursor": { + "type": "string" + } + }, + "required": [ + "items" + ], + "type": "object" + }, + "IntegrityReportSummaryResponseDto": { + "properties": { + "checksum_mismatch": { + "maximum": 9007199254740991, + "minimum": 0, + "type": "integer" + }, + "missing_file": { + "maximum": 9007199254740991, + "minimum": 0, + "type": "integer" + }, + "untracked_file": { + "maximum": 9007199254740991, + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "checksum_mismatch", + "missing_file", + "untracked_file" + ], + "type": "object" + }, "JobCreateDto": { "properties": { "name": { @@ -18849,7 +19214,17 @@ "VersionCheck", "OcrQueueAll", "Ocr", - "WorkflowAssetTrigger" + "WorkflowAssetTrigger", + "IntegrityUntrackedFilesQueueAll", + "IntegrityUntrackedFiles", + "IntegrityUntrackedRefresh", + "IntegrityMissingFilesQueueAll", + "IntegrityMissingFiles", + "IntegrityMissingFilesRefresh", + "IntegrityChecksumFiles", + "IntegrityChecksumFilesRefresh", + "IntegrityDeleteReportType", + "IntegrityDeleteReports" ], "type": "string" }, @@ -19223,7 +19598,16 @@ "user-cleanup", "memory-cleanup", "memory-create", - "backup-database" + "backup-database", + "integrity-missing-files", + "integrity-untracked-files", + "integrity-checksum-mismatch", + "integrity-missing-files-refresh", + "integrity-untracked-files-refresh", + "integrity-checksum-mismatch-refresh", + "integrity-missing-files-delete-all", + "integrity-untracked-files-delete-all", + "integrity-checksum-mismatch-delete-all" ], "type": "string" }, @@ -21092,6 +21476,7 @@ "backupDatabase", "ocr", "workflow", + "integrityCheck", "editor" ], "type": "string" @@ -21226,6 +21611,9 @@ "facialRecognition": { "$ref": "#/components/schemas/QueueResponseLegacyDto" }, + "integrityCheck": { + "$ref": "#/components/schemas/QueueResponseLegacyDto" + }, "library": { "$ref": "#/components/schemas/QueueResponseLegacyDto" }, @@ -21270,6 +21658,7 @@ "editor", "faceDetection", "facialRecognition", + "integrityCheck", "library", "metadataExtraction", "migration", @@ -24932,6 +25321,9 @@ "image": { "$ref": "#/components/schemas/SystemConfigImageDto" }, + "integrityChecks": { + "$ref": "#/components/schemas/SystemConfigIntegrityChecks" + }, "job": { "$ref": "#/components/schemas/SystemConfigJobDto" }, @@ -24991,6 +25383,7 @@ "backup", "ffmpeg", "image", + "integrityChecks", "job", "library", "logging", @@ -25249,6 +25642,78 @@ ], "type": "object" }, + "SystemConfigIntegrityChecks": { + "description": "Integrity checks config", + "properties": { + "checksumFiles": { + "$ref": "#/components/schemas/SystemConfigIntegrityChecksumJob" + }, + "missingFiles": { + "$ref": "#/components/schemas/SystemConfigIntegrityJob" + }, + "untrackedFiles": { + "$ref": "#/components/schemas/SystemConfigIntegrityJob" + } + }, + "required": [ + "checksumFiles", + "missingFiles", + "untrackedFiles" + ], + "type": "object" + }, + "SystemConfigIntegrityChecksumJob": { + "description": "Integrity checksum job config", + "properties": { + "cronExpression": { + "description": "Cron expression for when the integrity check should run", + "pattern": "(((\\d+,)+\\d+|(\\d+(\\/|-)\\d+)|\\d+|\\*) ?){5,7}", + "type": "string" + }, + "enabled": { + "description": "Enabled", + "type": "boolean" + }, + "percentageLimit": { + "description": "Percentage limit of the integrity checksum job", + "maximum": 9007199254740991, + "minimum": 0, + "type": "integer" + }, + "timeLimit": { + "description": "How long the integrity checksum job may run for", + "maximum": 9007199254740991, + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "cronExpression", + "enabled", + "percentageLimit", + "timeLimit" + ], + "type": "object" + }, + "SystemConfigIntegrityJob": { + "description": "Integrity job config", + "properties": { + "cronExpression": { + "description": "Cron expression for when the integrity check should run", + "pattern": "(((\\d+,)+\\d+|(\\d+(\\/|-)\\d+)|\\d+|\\*) ?){5,7}", + "type": "string" + }, + "enabled": { + "description": "Enabled", + "type": "boolean" + } + }, + "required": [ + "cronExpression", + "enabled" + ], + "type": "object" + }, "SystemConfigJobDto": { "properties": { "backgroundTask": { @@ -25260,6 +25725,9 @@ "faceDetection": { "$ref": "#/components/schemas/JobSettingsDto" }, + "integrityCheck": { + "$ref": "#/components/schemas/JobSettingsDto" + }, "library": { "$ref": "#/components/schemas/JobSettingsDto" }, @@ -25298,6 +25766,7 @@ "backgroundTask", "editor", "faceDetection", + "integrityCheck", "library", "metadataExtraction", "migration", diff --git a/packages/sdk/src/fetch-client.ts b/packages/sdk/src/fetch-client.ts index c982eca2c1..9bdc31fa6c 100644 --- a/packages/sdk/src/fetch-client.ts +++ b/packages/sdk/src/fetch-client.ts @@ -74,6 +74,21 @@ export type DatabaseBackupUploadDto = { /** Database backup file */ file?: Blob; }; +export type IntegrityReportResponseDto = { + items: { + /** Integrity report item id */ + id: string; + /** Integrity report item path */ + path: string; + "type": IntegrityReport; + }[]; + nextCursor?: string; +}; +export type IntegrityReportSummaryResponseDto = { + checksum_mismatch: number; + missing_file: number; + untracked_file: number; +}; export type SetMaintenanceModeDto = { action: MaintenanceAction; /** Restore backup filename */ @@ -1210,6 +1225,7 @@ export type QueuesResponseLegacyDto = { editor: QueueResponseLegacyDto; faceDetection: QueueResponseLegacyDto; facialRecognition: QueueResponseLegacyDto; + integrityCheck: QueueResponseLegacyDto; library: QueueResponseLegacyDto; metadataExtraction: QueueResponseLegacyDto; migration: QueueResponseLegacyDto; @@ -2342,6 +2358,27 @@ export type SystemConfigImageDto = { preview: SystemConfigGeneratedImageDto; thumbnail: SystemConfigGeneratedImageDto; }; +export type SystemConfigIntegrityChecksumJob = { + /** Cron expression for when the integrity check should run */ + cronExpression: string; + /** Enabled */ + enabled: boolean; + /** Percentage limit of the integrity checksum job */ + percentageLimit: number; + /** How long the integrity checksum job may run for */ + timeLimit: number; +}; +export type SystemConfigIntegrityJob = { + /** Cron expression for when the integrity check should run */ + cronExpression: string; + /** Enabled */ + enabled: boolean; +}; +export type SystemConfigIntegrityChecks = { + checksumFiles: SystemConfigIntegrityChecksumJob; + missingFiles: SystemConfigIntegrityJob; + untrackedFiles: SystemConfigIntegrityJob; +}; export type JobSettingsDto = { /** Concurrency */ concurrency: number; @@ -2350,6 +2387,7 @@ export type SystemConfigJobDto = { backgroundTask: JobSettingsDto; editor: JobSettingsDto; faceDetection: JobSettingsDto; + integrityCheck: JobSettingsDto; library: JobSettingsDto; metadataExtraction: JobSettingsDto; migration: JobSettingsDto; @@ -2567,6 +2605,7 @@ export type SystemConfigDto = { backup: SystemConfigBackupsDto; ffmpeg: SystemConfigFFmpegDto; image: SystemConfigImageDto; + integrityChecks: SystemConfigIntegrityChecks; job: SystemConfigJobDto; library: SystemConfigLibraryDto; logging: SystemConfigLoggingDto; @@ -3424,6 +3463,73 @@ export function downloadDatabaseBackup({ filename }: { ...opts })); } +/** + * Get integrity report by type + */ +export function getIntegrityReport({ cursor, limit, $type }: { + cursor?: string; + limit?: number; + $type: IntegrityReport; +}, opts?: Oazapfts.RequestOpts) { + return oazapfts.ok(oazapfts.fetchJson<{ + status: 200; + data: IntegrityReportResponseDto; + }>(`/admin/integrity/report${QS.query(QS.explode({ + cursor, + limit, + "type": $type + }))}`, { + ...opts + })); +} +/** + * Delete integrity report item + */ +export function deleteIntegrityReport({ id }: { + id: string; +}, opts?: Oazapfts.RequestOpts) { + return oazapfts.ok(oazapfts.fetchText(`/admin/integrity/report/${encodeURIComponent(id)}`, { + ...opts, + method: "DELETE" + })); +} +/** + * Download flagged file + */ +export function getIntegrityReportFile({ id }: { + id: string; +}, opts?: Oazapfts.RequestOpts) { + return oazapfts.ok(oazapfts.fetchBlob<{ + status: 200; + data: Blob; + }>(`/admin/integrity/report/${encodeURIComponent(id)}/file`, { + ...opts + })); +} +/** + * Export integrity report by type as CSV + */ +export function getIntegrityReportCsv({ $type }: { + $type: IntegrityReport; +}, opts?: Oazapfts.RequestOpts) { + return oazapfts.ok(oazapfts.fetchBlob<{ + status: 200; + data: Blob; + }>(`/admin/integrity/report/${encodeURIComponent($type)}/csv`, { + ...opts + })); +} +/** + * Get integrity report summary + */ +export function getIntegrityReportSummary(opts?: Oazapfts.RequestOpts) { + return oazapfts.ok(oazapfts.fetchJson<{ + status: 200; + data: IntegrityReportSummaryResponseDto; + }>("/admin/integrity/summary", { + ...opts + })); +} /** * Set maintenance mode */ @@ -6963,6 +7069,11 @@ export enum UserAvatarColor { Gray = "gray", Amber = "amber" } +export enum IntegrityReport { + UntrackedFile = "untracked_file", + MissingFile = "missing_file", + ChecksumMismatch = "checksum_mismatch" +} export enum MaintenanceAction { Start = "start", End = "end", @@ -7229,7 +7340,16 @@ export enum ManualJobName { UserCleanup = "user-cleanup", MemoryCleanup = "memory-cleanup", MemoryCreate = "memory-create", - BackupDatabase = "backup-database" + BackupDatabase = "backup-database", + IntegrityMissingFiles = "integrity-missing-files", + IntegrityUntrackedFiles = "integrity-untracked-files", + IntegrityChecksumMismatch = "integrity-checksum-mismatch", + IntegrityMissingFilesRefresh = "integrity-missing-files-refresh", + IntegrityUntrackedFilesRefresh = "integrity-untracked-files-refresh", + IntegrityChecksumMismatchRefresh = "integrity-checksum-mismatch-refresh", + IntegrityMissingFilesDeleteAll = "integrity-missing-files-delete-all", + IntegrityUntrackedFilesDeleteAll = "integrity-untracked-files-delete-all", + IntegrityChecksumMismatchDeleteAll = "integrity-checksum-mismatch-delete-all" } export enum QueueName { ThumbnailGeneration = "thumbnailGeneration", @@ -7249,6 +7369,7 @@ export enum QueueName { BackupDatabase = "backupDatabase", Ocr = "ocr", Workflow = "workflow", + IntegrityCheck = "integrityCheck", Editor = "editor" } export enum QueueCommand { @@ -7341,7 +7462,17 @@ export enum JobName { VersionCheck = "VersionCheck", OcrQueueAll = "OcrQueueAll", Ocr = "Ocr", - WorkflowAssetTrigger = "WorkflowAssetTrigger" + WorkflowAssetTrigger = "WorkflowAssetTrigger", + IntegrityUntrackedFilesQueueAll = "IntegrityUntrackedFilesQueueAll", + IntegrityUntrackedFiles = "IntegrityUntrackedFiles", + IntegrityUntrackedRefresh = "IntegrityUntrackedRefresh", + IntegrityMissingFilesQueueAll = "IntegrityMissingFilesQueueAll", + IntegrityMissingFiles = "IntegrityMissingFiles", + IntegrityMissingFilesRefresh = "IntegrityMissingFilesRefresh", + IntegrityChecksumFiles = "IntegrityChecksumFiles", + IntegrityChecksumFilesRefresh = "IntegrityChecksumFilesRefresh", + IntegrityDeleteReportType = "IntegrityDeleteReportType", + IntegrityDeleteReports = "IntegrityDeleteReports" } export enum SearchSuggestionType { Country = "country", diff --git a/server/src/config.ts b/server/src/config.ts index 730663d046..723e7e564e 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -50,6 +50,22 @@ export type SystemConfig = { enabled: boolean; }; }; + integrityChecks: { + missingFiles: { + enabled: boolean; + cronExpression: string; + }; + untrackedFiles: { + enabled: boolean; + cronExpression: string; + }; + checksumFiles: { + enabled: boolean; + cronExpression: string; + timeLimit: number; + percentageLimit: number; + }; + }; job: Record; logging: { enabled: boolean; @@ -233,6 +249,22 @@ export const defaults = Object.freeze({ enabled: false, }, }, + integrityChecks: { + missingFiles: { + enabled: true, + cronExpression: CronExpression.EVERY_DAY_AT_3AM, + }, + untrackedFiles: { + enabled: true, + cronExpression: CronExpression.EVERY_DAY_AT_3AM, + }, + checksumFiles: { + enabled: true, + cronExpression: CronExpression.EVERY_DAY_AT_3AM, + timeLimit: 60 * 60 * 1000, // 1 hour + percentageLimit: 1, // 100% of assets + }, + }, job: { [QueueName.BackgroundTask]: { concurrency: 5 }, [QueueName.SmartSearch]: { concurrency: 2 }, @@ -247,6 +279,7 @@ export const defaults = Object.freeze({ [QueueName.Notification]: { concurrency: 5 }, [QueueName.Ocr]: { concurrency: 1 }, [QueueName.Workflow]: { concurrency: 5 }, + [QueueName.IntegrityCheck]: { concurrency: 1 }, [QueueName.Editor]: { concurrency: 2 }, }, logging: { diff --git a/server/src/constants.ts b/server/src/constants.ts index 16b7731dce..8a0f979ff0 100644 --- a/server/src/constants.ts +++ b/server/src/constants.ts @@ -156,6 +156,7 @@ export const endpointTags: Record = { [ApiTag.Duplicates]: 'Endpoints for managing and identifying duplicate assets.', [ApiTag.Faces]: 'A face is a detected human face within an asset, which can be associated with a person. Faces are normally detected via machine learning, but can also be created via manually.', + [ApiTag.Integrity]: 'Endpoints for viewing and managing integrity reports.', [ApiTag.Jobs]: 'Queues and background jobs are used for processing tasks asynchronously. Queues can be paused and resumed as needed.', [ApiTag.Libraries]: diff --git a/server/src/controllers/index.ts b/server/src/controllers/index.ts index 336ea1cf91..e7a01643ab 100644 --- a/server/src/controllers/index.ts +++ b/server/src/controllers/index.ts @@ -10,6 +10,7 @@ import { DatabaseBackupController } from 'src/controllers/database-backup.contro import { DownloadController } from 'src/controllers/download.controller'; import { DuplicateController } from 'src/controllers/duplicate.controller'; import { FaceController } from 'src/controllers/face.controller'; +import { IntegrityAdminController } from 'src/controllers/integrity-admin.controller'; import { JobController } from 'src/controllers/job.controller'; import { LibraryController } from 'src/controllers/library.controller'; import { MaintenanceController } from 'src/controllers/maintenance.controller'; @@ -52,6 +53,7 @@ export const controllers = [ DownloadController, DuplicateController, FaceController, + IntegrityAdminController, JobController, LibraryController, MaintenanceController, diff --git a/server/src/controllers/integrity-admin.controller.ts b/server/src/controllers/integrity-admin.controller.ts new file mode 100644 index 0000000000..07a074ac58 --- /dev/null +++ b/server/src/controllers/integrity-admin.controller.ts @@ -0,0 +1,91 @@ +import { Controller, Delete, Get, HttpCode, HttpStatus, Next, Param, Query, Res } from '@nestjs/common'; +import { ApiTags } from '@nestjs/swagger'; +import { NextFunction, Response } from 'express'; +import { Endpoint, HistoryBuilder } from 'src/decorators'; +import { AuthDto } from 'src/dtos/auth.dto'; +import { + IntegrityGetReportDto, + IntegrityReportResponseDto, + IntegrityReportSummaryResponseDto, +} from 'src/dtos/integrity.dto'; +import { ApiTag, Permission } from 'src/enum'; +import { Auth, Authenticated, FileResponse } from 'src/middleware/auth.guard'; +import { LoggingRepository } from 'src/repositories/logging.repository'; +import { IntegrityService } from 'src/services/integrity.service'; +import { sendFile } from 'src/utils/file'; +import { IntegrityReportTypeParamDto, UUIDv7ParamDto } from 'src/validation'; + +@ApiTags(ApiTag.Maintenance) +@Controller('admin/integrity') +export class IntegrityAdminController { + constructor( + private logger: LoggingRepository, + private service: IntegrityService, + ) {} + + @Get('summary') + @Endpoint({ + summary: 'Get integrity report summary', + description: 'Get a count of the items flagged in each integrity report', + history: new HistoryBuilder().added('v3.0.0').alpha('v3.0.0'), + }) + @Authenticated({ permission: Permission.Maintenance, admin: true }) + getIntegrityReportSummary(): Promise { + return this.service.getIntegrityReportSummary(); + } + + @Get('report') + @HttpCode(HttpStatus.OK) + @Endpoint({ + summary: 'Get integrity report by type', + description: 'Get all flagged items by integrity report type', + history: new HistoryBuilder().added('v3.0.0').alpha('v3.0.0'), + }) + @Authenticated({ permission: Permission.Maintenance, admin: true }) + getIntegrityReport(@Query() dto: IntegrityGetReportDto): Promise { + return this.service.getIntegrityReport(dto); + } + + @Get('report/:id/file') + @Endpoint({ + summary: 'Download flagged file', + description: 'Download the untracked/broken file if one exists', + history: new HistoryBuilder().added('v3.0.0').alpha('v3.0.0'), + }) + @FileResponse() + @Authenticated({ permission: Permission.Maintenance, admin: true }) + async getIntegrityReportFile( + @Param() { id }: UUIDv7ParamDto, + @Res() res: Response, + @Next() next: NextFunction, + ): Promise { + await sendFile(res, next, () => this.service.getIntegrityReportFile(id), this.logger); + } + + @Delete('report/:id') + @Endpoint({ + summary: 'Delete integrity report item', + description: 'Delete a given report item and perform corresponding deletion (e.g. trash asset, delete file)', + history: new HistoryBuilder().added('v3.0.0').alpha('v3.0.0'), + }) + @Authenticated({ permission: Permission.Maintenance, admin: true }) + async deleteIntegrityReport(@Auth() auth: AuthDto, @Param() { id }: UUIDv7ParamDto): Promise { + await this.service.deleteIntegrityReport(auth.user.id, id); + } + + @Get('report/:type/csv') + @Endpoint({ + summary: 'Export integrity report by type as CSV', + description: 'Get all integrity report entries for a given type as a CSV', + history: new HistoryBuilder().added('v3.0.0').alpha('v3.0.0'), + }) + @FileResponse() + @Authenticated({ permission: Permission.Maintenance, admin: true }) + getIntegrityReportCsv(@Param() { type }: IntegrityReportTypeParamDto, @Res() res: Response): void { + res.setHeader('Content-Type', 'text/csv'); + res.setHeader('Cache-Control', 'private, no-cache, no-transform'); + res.setHeader('Content-Disposition', `attachment; filename="${encodeURIComponent(`${Date.now()}-${type}.csv`)}"`); + + this.service.getIntegrityReportCsv(type).pipe(res); + } +} diff --git a/server/src/dtos/integrity.dto.ts b/server/src/dtos/integrity.dto.ts new file mode 100644 index 0000000000..b4faf6005e --- /dev/null +++ b/server/src/dtos/integrity.dto.ts @@ -0,0 +1,41 @@ +import { createZodDto } from 'nestjs-zod'; +import { IntegrityReport, IntegrityReportSchema } from 'src/enum'; +import z from 'zod'; + +const IntegrityReportSummaryResponseSchema = z + .object({ + [IntegrityReport.ChecksumFail]: z.int().nonnegative(), + [IntegrityReport.MissingFile]: z.int().nonnegative(), + [IntegrityReport.UntrackedFile]: z.int().nonnegative(), + }) + .meta({ id: 'IntegrityReportSummaryResponseDto' }); + +export class IntegrityReportSummaryResponseDto extends createZodDto(IntegrityReportSummaryResponseSchema) {} + +const IntegrityGetReportSchema = z + .object({ + type: IntegrityReportSchema, + cursor: z.string().optional().describe('Cursor for pagination'), + limit: z.int().positive().default(500).optional().describe('Number of items per page'), + }) + .meta({ id: 'IntegrityGetReportDto' }); + +export class IntegrityGetReportDto extends createZodDto(IntegrityGetReportSchema) {} + +const IntegrityDeleteReportSchema = z.object({ type: IntegrityReport }).meta({ id: 'IntegrityDeleteReportDto' }); + +export class IntegrityDeleteReportDto extends createZodDto(IntegrityDeleteReportSchema) {} + +const IntegrityReportResponseItemSchema = z.object({ + id: z.string().describe('Integrity report item id'), + type: IntegrityReportSchema, + path: z.string().describe('Integrity report item path'), +}); +const IntegrityReportResponseSchema = z + .object({ + items: z.array(IntegrityReportResponseItemSchema), + nextCursor: z.string().optional(), + }) + .meta({ id: 'IntegrityReportResponseDto' }); + +export class IntegrityReportResponseDto extends createZodDto(IntegrityReportResponseSchema) {} diff --git a/server/src/dtos/queue-legacy.dto.ts b/server/src/dtos/queue-legacy.dto.ts index dbbcec2da5..44f988fd28 100644 --- a/server/src/dtos/queue-legacy.dto.ts +++ b/server/src/dtos/queue-legacy.dto.ts @@ -37,6 +37,7 @@ const QueuesResponseLegacySchema = z [QueueName.Ocr]: QueueResponseLegacySchema, [QueueName.Workflow]: QueueResponseLegacySchema, [QueueName.Editor]: QueueResponseLegacySchema, + [QueueName.IntegrityCheck]: QueueResponseLegacySchema, }) .meta({ id: 'QueuesResponseLegacyDto' }); diff --git a/server/src/dtos/system-config.dto.ts b/server/src/dtos/system-config.dto.ts index 3b31705918..1f6691c87b 100644 --- a/server/src/dtos/system-config.dto.ts +++ b/server/src/dtos/system-config.dto.ts @@ -54,6 +54,30 @@ const DatabaseBackupSchema = z }) .meta({ id: 'DatabaseBackupConfig' }); +const SystemConfigIntegrityJobSchema = z + .object({ + enabled: z.boolean().describe('Enabled'), + cronExpression: cronExpressionSchema.describe('Cron expression for when the integrity check should run'), + }) + .describe('Integrity job config') + .meta({ id: 'SystemConfigIntegrityJob' }); + +const SystemConfigIntegrityChecksumJobSchema = SystemConfigIntegrityJobSchema.extend({ + timeLimit: z.int().nonnegative().describe('How long the integrity checksum job may run for'), + percentageLimit: z.int().nonnegative().describe('Percentage limit of the integrity checksum job'), +}) + .describe('Integrity checksum job config') + .meta({ id: 'SystemConfigIntegrityChecksumJob' }); + +const SystemConfigIntegrityChecksSchema = z + .object({ + missingFiles: SystemConfigIntegrityJobSchema, + untrackedFiles: SystemConfigIntegrityJobSchema, + checksumFiles: SystemConfigIntegrityChecksumJobSchema, + }) + .describe('Integrity checks config') + .meta({ id: 'SystemConfigIntegrityChecks' }); + const SystemConfigBackupsSchema = z.object({ database: DatabaseBackupSchema }).meta({ id: 'SystemConfigBackupsDto' }); const SystemConfigFFmpegSchema = z @@ -103,6 +127,7 @@ const SystemConfigJobSchema = z ocr: JobSettingsSchema, workflow: JobSettingsSchema, editor: JobSettingsSchema, + integrityCheck: JobSettingsSchema, }) .meta({ id: 'SystemConfigJobDto' }); @@ -382,6 +407,7 @@ export const SystemConfigSchema = z templates: SystemConfigTemplatesSchema, server: SystemConfigServerSchema, user: SystemConfigUserSchema, + integrityChecks: SystemConfigIntegrityChecksSchema, }) .describe('System configuration') .meta({ id: 'SystemConfigDto' }); diff --git a/server/src/enum.ts b/server/src/enum.ts index c3f56b6850..d26d95ef21 100644 --- a/server/src/enum.ts +++ b/server/src/enum.ts @@ -341,6 +341,7 @@ export enum SystemMetadataKey { SystemFlags = 'system-flags', VersionCheckState = 'version-check-state', License = 'license', + IntegrityChecksumCheckpoint = 'integrity-checksum-checkpoint', } export enum UserMetadataKey { @@ -398,6 +399,17 @@ export enum SourceType { export const SourceTypeSchema = z.enum(SourceType).describe('Face detection source type').meta({ id: 'SourceType' }); +export enum IntegrityReport { + UntrackedFile = 'untracked_file', + MissingFile = 'missing_file', + ChecksumFail = 'checksum_mismatch', +} + +export const IntegrityReportSchema = z + .enum(IntegrityReport) + .describe('Integrity report type') + .meta({ id: 'IntegrityReport' }); + export enum ManualJobName { PersonCleanup = 'person-cleanup', TagCleanup = 'tag-cleanup', @@ -405,6 +417,15 @@ export enum ManualJobName { MemoryCleanup = 'memory-cleanup', MemoryCreate = 'memory-create', BackupDatabase = 'backup-database', + IntegrityMissingFiles = `integrity-missing-files`, + IntegrityUntrackedFiles = `integrity-untracked-files`, + IntegrityChecksumFiles = `integrity-checksum-mismatch`, + IntegrityMissingFilesRefresh = `integrity-missing-files-refresh`, + IntegrityUntrackedFilesRefresh = `integrity-untracked-files-refresh`, + IntegrityChecksumFilesRefresh = `integrity-checksum-mismatch-refresh`, + IntegrityMissingFilesDeleteAll = `integrity-missing-files-delete-all`, + IntegrityUntrackedFilesDeleteAll = `integrity-untracked-files-delete-all`, + IntegrityChecksumFilesDeleteAll = `integrity-checksum-mismatch-delete-all`, } export const ManualJobNameSchema = z.enum(ManualJobName).describe('Manual job name').meta({ id: 'ManualJobName' }); @@ -771,6 +792,7 @@ export enum QueueName { BackupDatabase = 'backupDatabase', Ocr = 'ocr', Workflow = 'workflow', + IntegrityCheck = 'integrityCheck', Editor = 'editor', } @@ -866,6 +888,18 @@ export enum JobName { // Workflow WorkflowAssetTrigger = 'WorkflowAssetTrigger', + + // Integrity + IntegrityUntrackedFilesQueueAll = 'IntegrityUntrackedFilesQueueAll', + IntegrityUntrackedFiles = 'IntegrityUntrackedFiles', + IntegrityUntrackedFilesRefresh = 'IntegrityUntrackedRefresh', + IntegrityMissingFilesQueueAll = 'IntegrityMissingFilesQueueAll', + IntegrityMissingFiles = 'IntegrityMissingFiles', + IntegrityMissingFilesRefresh = 'IntegrityMissingFilesRefresh', + IntegrityChecksumFiles = 'IntegrityChecksumFiles', + IntegrityChecksumFilesRefresh = 'IntegrityChecksumFilesRefresh', + IntegrityDeleteReportType = 'IntegrityDeleteReportType', + IntegrityDeleteReports = 'IntegrityDeleteReports', } export const JobNameSchema = z.enum(JobName).describe('Job name').meta({ id: 'JobName' }); @@ -917,6 +951,7 @@ export enum DatabaseLock { BackupDatabase = 42, MaintenanceOperation = 621, MemoryCreation = 777, + IntegrityCheck = 67, VersionCheck = 800, HlsSessionCleanup = 850, } @@ -1131,6 +1166,7 @@ export enum ApiTag { Download = 'Download', Duplicates = 'Duplicates', Faces = 'Faces', + Integrity = 'Integrity (admin)', Jobs = 'Jobs', Libraries = 'Libraries', Maintenance = 'Maintenance (admin)', diff --git a/server/src/queries/integrity.repository.sql b/server/src/queries/integrity.repository.sql new file mode 100644 index 0000000000..c6ca108d74 --- /dev/null +++ b/server/src/queries/integrity.repository.sql @@ -0,0 +1,179 @@ +-- NOTE: This file is auto generated by ./sql-generator + +-- IntegrityRepository.getById +select + "integrity_report".* +from + "integrity_report" +where + "id" = $1 + +-- IntegrityRepository.getIntegrityReportSummary +select + "type", + count(*) as "count" +from + "integrity_report" +group by + "type" + +-- IntegrityRepository.getIntegrityReport +select + "id", + "type", + "path", + "assetId", + "fileAssetId", + "createdAt" +from + "integrity_report" +where + "type" = $1 + and "id" <= $2 +order by + "id" desc +limit + $3 + +-- IntegrityRepository.getAssetPathsByPaths +select + "asset"."originalPath", + "asset_file"."path" as "encodedVideoPath" +from + "asset" + left join "asset_file" on "asset"."id" = "asset_file"."assetId" + and "asset_file"."type" = $1 +where + ( + "originalPath" in $2 + or "asset_file"."path" in $3 + ) + +-- IntegrityRepository.getAssetFilePathsByPaths +select + "path" +from + "asset_file" +where + "path" in $1 + +-- IntegrityRepository.getPersonThumbnailPathsByPaths +select + "person"."thumbnailPath" +from + "person" +where + "person"."thumbnailPath" in $1 + +-- IntegrityRepository.getAssetCount +select + count(*) as "count" +from + "asset" + +-- IntegrityRepository.streamAllAssetPaths +select + "originalPath", + "asset_file"."path" as "encodedVideoPath" +from + "asset" + left join "asset_file" on "asset"."id" = "asset_file"."assetId" + and "asset_file"."type" = $1 + +-- IntegrityRepository.streamAllAssetFilePaths +select + "path" +from + "asset_file" + +-- IntegrityRepository.streamAssetPaths +select + "allPaths"."path" as "path", + "allPaths"."assetId", + "allPaths"."fileAssetId", + "integrity_report"."id" as "reportId" +from + ( + select + "asset"."originalPath" as "path", + "asset"."id" as "assetId", + null::uuid as "fileAssetId" + from + "asset" + where + "asset"."deletedAt" is null + union all + select + "path", + null::uuid as "assetId", + "asset_file"."id" as "fileAssetId" + from + "asset_file" + ) as "allPaths" + left join "integrity_report" on "integrity_report"."type" = $1 + and ( + "integrity_report"."assetId" = "allPaths"."assetId" + or "integrity_report"."fileAssetId" = "allPaths"."fileAssetId" + ) + +-- IntegrityRepository.streamAssetChecksums +select + "asset"."originalPath", + "asset"."checksum", + "asset"."createdAt", + "asset"."id" as "assetId", + "integrity_report"."id" as "reportId" +from + "asset" + left join "integrity_report" on "integrity_report"."assetId" = "asset"."id" + and "integrity_report"."type" = $1 +where + "asset"."deletedAt" is null + and "createdAt" >= $2 + and "createdAt" <= $3 +order by + "createdAt" asc + +-- IntegrityRepository.streamIntegrityReports +select + "id", + "type", + "path", + "assetId", + "fileAssetId" +from + "integrity_report" +where + "type" = $1 +order by + "createdAt" desc + +-- IntegrityRepository.streamIntegrityReportsWithAssetChecksum +select + "integrity_report"."id" as "reportId", + "integrity_report"."path" +from + "integrity_report" +where + "integrity_report"."type" = $1 + +-- IntegrityRepository.streamIntegrityReportsByProperty +select + "id", + "path", + "assetId", + "fileAssetId" +from + "integrity_report" +where + "abcdefghi" is not null + +-- IntegrityRepository.deleteById +delete from "integrity_report" +where + "id" = $1 + +-- IntegrityRepository.deleteByIds +delete from "integrity_report" +where + "id" in $1 diff --git a/server/src/repositories/index.ts b/server/src/repositories/index.ts index 886f925ee8..534a71b6d6 100644 --- a/server/src/repositories/index.ts +++ b/server/src/repositories/index.ts @@ -15,6 +15,7 @@ import { DownloadRepository } from 'src/repositories/download.repository'; import { DuplicateRepository } from 'src/repositories/duplicate.repository'; import { EmailRepository } from 'src/repositories/email.repository'; import { EventRepository } from 'src/repositories/event.repository'; +import { IntegrityRepository } from 'src/repositories/integrity.repository'; import { JobRepository } from 'src/repositories/job.repository'; import { LibraryRepository } from 'src/repositories/library.repository'; import { LoggingRepository } from 'src/repositories/logging.repository'; @@ -69,6 +70,7 @@ export const repositories = [ DuplicateRepository, EmailRepository, EventRepository, + IntegrityRepository, JobRepository, LibraryRepository, LoggingRepository, diff --git a/server/src/repositories/integrity.repository.ts b/server/src/repositories/integrity.repository.ts new file mode 100644 index 0000000000..c77a0a6375 --- /dev/null +++ b/server/src/repositories/integrity.repository.ts @@ -0,0 +1,228 @@ +import { Injectable } from '@nestjs/common'; +import { Insertable, Kysely, sql } from 'kysely'; +import { InjectKysely } from 'nestjs-kysely'; +import { DummyValue, GenerateSql } from 'src/decorators'; +import { AssetFileType, IntegrityReport } from 'src/enum'; +import { DB } from 'src/schema'; +import { IntegrityReportTable } from 'src/schema/tables/integrity-report.table'; + +export type ReportPaginationOptions = { + cursor?: string; + limit: number; +}; + +@Injectable() +export class IntegrityRepository { + constructor(@InjectKysely() private db: Kysely) {} + + create(dto: Insertable | Insertable[]) { + return this.db + .insertInto('integrity_report') + .values(dto) + .onConflict((oc) => + oc.columns(['path', 'type']).doUpdateSet({ + assetId: (eb) => eb.ref('excluded.assetId'), + fileAssetId: (eb) => eb.ref('excluded.fileAssetId'), + }), + ) + .returningAll() + .executeTakeFirstOrThrow(); + } + + @GenerateSql({ params: [DummyValue.STRING] }) + getById(id: string) { + return this.db + .selectFrom('integrity_report') + .selectAll('integrity_report') + .where('id', '=', id) + .executeTakeFirstOrThrow(); + } + + @GenerateSql({ params: [] }) + async getIntegrityReportSummary() { + const counts = await this.db + .selectFrom('integrity_report') + .select(['type', this.db.fn.countAll().as('count')]) + .groupBy('type') + .execute(); + + return Object.fromEntries( + Object.values(IntegrityReport).map((type) => [type, counts.find((count) => count.type === type)?.count || 0]), + ) as Record; + } + + @GenerateSql({ params: [{ cursor: DummyValue.NUMBER, limit: 100 }, DummyValue.STRING] }) + async getIntegrityReport(pagination: ReportPaginationOptions, type: IntegrityReport) { + const items = await this.db + .selectFrom('integrity_report') + .select(['id', 'type', 'path', 'assetId', 'fileAssetId', 'createdAt']) + .where('type', '=', type) + .$if(pagination.cursor !== undefined, (eb) => eb.where('id', '<=', pagination.cursor!)) + .orderBy('id', 'desc') + .limit(pagination.limit + 1) + .execute(); + + return { + items: items.slice(0, pagination.limit), + nextCursor: items.at(pagination.limit)?.id, + }; + } + + @GenerateSql({ params: [DummyValue.STRING] }) + getAssetPathsByPaths(paths: string[]) { + return this.db + .selectFrom('asset') + .leftJoin('asset_file', (join) => + join.onRef('asset.id', '=', 'asset_file.assetId').on('asset_file.type', '=', AssetFileType.EncodedVideo), + ) + .select(['asset.originalPath', 'asset_file.path as encodedVideoPath']) + .where((eb) => eb.or([eb('originalPath', 'in', paths), eb('asset_file.path', 'in', paths)])) + .execute(); + } + + @GenerateSql({ params: [DummyValue.STRING] }) + getAssetFilePathsByPaths(paths: string[]) { + return this.db.selectFrom('asset_file').select('path').where('path', 'in', paths).execute(); + } + + @GenerateSql({ params: [DummyValue.STRING] }) + getPersonThumbnailPathsByPaths(paths: string[]) { + return this.db + .selectFrom('person') + .select('person.thumbnailPath') + .where('person.thumbnailPath', 'in', paths) + .execute(); + } + + @GenerateSql({ params: [] }) + getAssetCount() { + return this.db + .selectFrom('asset') + .select((eb) => eb.fn.countAll().as('count')) + .executeTakeFirstOrThrow(); + } + + @GenerateSql({ params: [], stream: true }) + streamAllAssetPaths() { + return this.db + .selectFrom('asset') + .leftJoin('asset_file', (join) => + join.onRef('asset.id', '=', 'asset_file.assetId').on('asset_file.type', '=', AssetFileType.EncodedVideo), + ) + .select(['originalPath', 'asset_file.path as encodedVideoPath']) + .stream(); + } + + @GenerateSql({ params: [], stream: true }) + streamAllAssetFilePaths() { + return this.db.selectFrom('asset_file').select(['path']).stream(); + } + + @GenerateSql({ params: [], stream: true }) + streamAssetPaths() { + return this.db + .selectFrom((eb) => + eb + .selectFrom('asset') + .where('asset.deletedAt', 'is', null) + .select('asset.originalPath as path') + .select((eb) => [ + eb.ref('asset.id').$castTo().as('assetId'), + sql`null::uuid`.as('fileAssetId'), + ]) + .unionAll( + eb + .selectFrom('asset_file') + .select(['path']) + .select((eb) => [ + sql`null::uuid`.as('assetId'), + eb.ref('asset_file.id').$castTo().as('fileAssetId'), + ]), + ) + .as('allPaths'), + ) + .leftJoin('integrity_report', (join) => + join + .on('integrity_report.type', '=', IntegrityReport.UntrackedFile) + .on((eb) => + eb.or([ + eb('integrity_report.assetId', '=', eb.ref('allPaths.assetId')), + eb('integrity_report.fileAssetId', '=', eb.ref('allPaths.fileAssetId')), + ]), + ), + ) + .select(['allPaths.path as path', 'allPaths.assetId', 'allPaths.fileAssetId', 'integrity_report.id as reportId']) + .stream() as AsyncIterableIterator< + { path: string; reportId: string | null } & ( + | { assetId: string; fileAssetId: null } + | { assetId: null; fileAssetId: string } + ) + >; + } + + @GenerateSql({ params: [DummyValue.DATE, DummyValue.DATE], stream: true }) + streamAssetChecksums(startMarker?: Date, endMarker?: Date) { + return this.db + .selectFrom('asset') + .where('asset.deletedAt', 'is', null) + .leftJoin('integrity_report', (join) => + join + .onRef('integrity_report.assetId', '=', 'asset.id') + .on('integrity_report.type', '=', IntegrityReport.ChecksumFail), + ) + .select([ + 'asset.originalPath', + 'asset.checksum', + 'asset.createdAt', + 'asset.id as assetId', + 'integrity_report.id as reportId', + ]) + .$if(startMarker !== undefined, (qb) => qb.where('createdAt', '>=', startMarker!)) + .$if(endMarker !== undefined, (qb) => qb.where('createdAt', '<=', endMarker!)) + .orderBy('createdAt', 'asc') + .stream(); + } + + @GenerateSql({ params: [DummyValue.STRING], stream: true }) + streamIntegrityReports(type: IntegrityReport) { + return this.db + .selectFrom('integrity_report') + .select(['id', 'type', 'path', 'assetId', 'fileAssetId']) + .where('type', '=', type) + .orderBy('createdAt', 'desc') + .stream(); + } + + @GenerateSql({ params: [DummyValue.STRING], stream: true }) + streamIntegrityReportsWithAssetChecksum(type: IntegrityReport) { + return this.db + .selectFrom('integrity_report') + .select(['integrity_report.id as reportId', 'integrity_report.path']) + .where('integrity_report.type', '=', type) + .$if(type === IntegrityReport.ChecksumFail, (eb) => + eb.leftJoin('asset', 'integrity_report.path', 'asset.originalPath').select('asset.checksum'), + ) + .stream(); + } + + @GenerateSql({ params: [DummyValue.STRING], stream: true }) + streamIntegrityReportsByProperty(property?: 'assetId' | 'fileAssetId', filterType?: IntegrityReport) { + return this.db + .selectFrom('integrity_report') + .select(['id', 'path', 'assetId', 'fileAssetId']) + .$if(filterType !== undefined, (eb) => eb.where('type', '=', filterType!)) + .$if(property === undefined, (eb) => eb.where('assetId', 'is', null).where('fileAssetId', 'is', null)) + .$if(property !== undefined, (eb) => eb.where(property!, 'is not', null)) + .stream(); + } + + @GenerateSql({ params: [DummyValue.STRING] }) + deleteById(id: string) { + return this.db.deleteFrom('integrity_report').where('id', '=', id).execute(); + } + + @GenerateSql({ params: [DummyValue.STRING] }) + deleteByIds(ids: string[]) { + return this.db.deleteFrom('integrity_report').where('id', 'in', ids).execute(); + } +} diff --git a/server/src/schema/index.ts b/server/src/schema/index.ts index 5bbe7cf4f4..4e55ff4cf6 100644 --- a/server/src/schema/index.ts +++ b/server/src/schema/index.ts @@ -49,6 +49,7 @@ import { AssetOcrTable } from 'src/schema/tables/asset-ocr.table'; import { AssetTable } from 'src/schema/tables/asset.table'; import { FaceSearchTable } from 'src/schema/tables/face-search.table'; import { GeodataPlacesTable } from 'src/schema/tables/geodata-places.table'; +import { IntegrityReportTable } from 'src/schema/tables/integrity-report.table'; import { LibraryTable } from 'src/schema/tables/library.table'; import { MemoryAssetAuditTable } from 'src/schema/tables/memory-asset-audit.table'; import { MemoryAssetTable } from 'src/schema/tables/memory-asset.table'; @@ -115,6 +116,7 @@ export class ImmichDatabase { AssetExifTable, FaceSearchTable, GeodataPlacesTable, + IntegrityReportTable, LibraryTable, MemoryTable, MemoryAuditTable, @@ -219,6 +221,8 @@ export interface DB { geodata_places: GeodataPlacesTable; + integrity_report: IntegrityReportTable; + library: LibraryTable; memory: MemoryTable; diff --git a/server/src/schema/migrations/1781089983296-CreateIntegrityReportTable.ts b/server/src/schema/migrations/1781089983296-CreateIntegrityReportTable.ts new file mode 100644 index 0000000000..69b0eae2df --- /dev/null +++ b/server/src/schema/migrations/1781089983296-CreateIntegrityReportTable.ts @@ -0,0 +1,24 @@ +import { Kysely, sql } from 'kysely'; + +export async function up(db: Kysely): Promise { + await sql`CREATE TABLE "integrity_report" ( + "id" uuid NOT NULL DEFAULT immich_uuid_v7(), + "type" character varying NOT NULL, + "path" character varying NOT NULL, + "createdAt" timestamp with time zone NOT NULL DEFAULT now(), + "assetId" uuid, + "fileAssetId" uuid, + CONSTRAINT "integrity_report_assetId_fkey" FOREIGN KEY ("assetId") REFERENCES "asset" ("id") ON UPDATE CASCADE ON DELETE CASCADE, + CONSTRAINT "integrity_report_fileAssetId_fkey" FOREIGN KEY ("fileAssetId") REFERENCES "asset_file" ("id") ON UPDATE CASCADE ON DELETE CASCADE, + CONSTRAINT "integrity_report_type_path_uq" UNIQUE ("type", "path"), + CONSTRAINT "integrity_report_pkey" PRIMARY KEY ("id") +);`.execute(db); + await sql`CREATE INDEX "integrity_report_assetId_idx" ON "integrity_report" ("assetId");`.execute(db); + await sql`CREATE INDEX "integrity_report_fileAssetId_idx" ON "integrity_report" ("fileAssetId");`.execute(db); + await sql`CREATE INDEX "asset_createdAt_idx" ON "asset" ("createdAt");`.execute(db); +} + +export async function down(db: Kysely): Promise { + await sql`DROP TABLE "integrity_report";`.execute(db); + await sql`DROP INDEX "asset_createdAt_idx";`.execute(db); +} diff --git a/server/src/schema/tables/asset.table.ts b/server/src/schema/tables/asset.table.ts index d4832648dd..80b25cd2aa 100644 --- a/server/src/schema/tables/asset.table.ts +++ b/server/src/schema/tables/asset.table.ts @@ -98,7 +98,7 @@ export class AssetTable { @UpdateDateColumn() updatedAt!: Generated; - @CreateDateColumn() + @CreateDateColumn({ index: true }) createdAt!: Generated; @Column({ index: true }) diff --git a/server/src/schema/tables/integrity-report.table.ts b/server/src/schema/tables/integrity-report.table.ts new file mode 100644 index 0000000000..0ef7967aac --- /dev/null +++ b/server/src/schema/tables/integrity-report.table.ts @@ -0,0 +1,27 @@ +import { Column, CreateDateColumn, ForeignKeyColumn, Generated, Table, Timestamp, Unique } from '@immich/sql-tools'; +import { PrimaryGeneratedUuidV7Column } from 'src/decorators'; +import { IntegrityReport } from 'src/enum'; +import { AssetFileTable } from 'src/schema/tables/asset-file.table'; +import { AssetTable } from 'src/schema/tables/asset.table'; + +@Table('integrity_report') +@Unique({ columns: ['type', 'path'] }) +export class IntegrityReportTable { + @PrimaryGeneratedUuidV7Column() + id!: Generated; + + @Column() + type!: IntegrityReport; + + @Column() + path!: string; + + @CreateDateColumn() + createdAt!: Generated; + + @ForeignKeyColumn(() => AssetTable, { onDelete: 'CASCADE', onUpdate: 'CASCADE', nullable: true }) + assetId!: string | null; + + @ForeignKeyColumn(() => AssetFileTable, { onDelete: 'CASCADE', onUpdate: 'CASCADE', nullable: true }) + fileAssetId!: string | null; +} diff --git a/server/src/services/base.service.ts b/server/src/services/base.service.ts index 33534f16de..62a0de8b56 100644 --- a/server/src/services/base.service.ts +++ b/server/src/services/base.service.ts @@ -22,6 +22,7 @@ import { DownloadRepository } from 'src/repositories/download.repository'; import { DuplicateRepository } from 'src/repositories/duplicate.repository'; import { EmailRepository } from 'src/repositories/email.repository'; import { EventRepository } from 'src/repositories/event.repository'; +import { IntegrityRepository } from 'src/repositories/integrity.repository'; import { JobRepository } from 'src/repositories/job.repository'; import { LibraryRepository } from 'src/repositories/library.repository'; import { LoggingRepository } from 'src/repositories/logging.repository'; @@ -81,6 +82,7 @@ export const BASE_SERVICE_DEPENDENCIES = [ DuplicateRepository, EmailRepository, EventRepository, + IntegrityRepository, JobRepository, LibraryRepository, MachineLearningRepository, @@ -140,6 +142,7 @@ export class BaseService { protected duplicateRepository: DuplicateRepository, protected emailRepository: EmailRepository, protected eventRepository: EventRepository, + protected integrityRepository: IntegrityRepository, protected jobRepository: JobRepository, protected libraryRepository: LibraryRepository, protected machineLearningRepository: MachineLearningRepository, @@ -208,6 +211,7 @@ export class BaseService { ctx.duplicateRepository, ctx.emailRepository, ctx.eventRepository, + ctx.integrityRepository, ctx.jobRepository, ctx.libraryRepository, ctx.machineLearningRepository, diff --git a/server/src/services/index.ts b/server/src/services/index.ts index 3c23e723bc..766b5979bc 100644 --- a/server/src/services/index.ts +++ b/server/src/services/index.ts @@ -12,6 +12,7 @@ import { DatabaseService } from 'src/services/database.service'; import { DownloadService } from 'src/services/download.service'; import { DuplicateService } from 'src/services/duplicate.service'; import { HlsService } from 'src/services/hls.service'; +import { IntegrityService } from 'src/services/integrity.service'; import { JobService } from 'src/services/job.service'; import { LibraryService } from 'src/services/library.service'; import { MaintenanceService } from 'src/services/maintenance.service'; @@ -63,6 +64,7 @@ export const services = [ DatabaseService, DownloadService, DuplicateService, + IntegrityService, HlsService, JobService, LibraryService, diff --git a/server/src/services/integrity.service.spec.ts b/server/src/services/integrity.service.spec.ts new file mode 100644 index 0000000000..997f337f2b --- /dev/null +++ b/server/src/services/integrity.service.spec.ts @@ -0,0 +1,29 @@ +import { IntegrityService } from 'src/services/integrity.service'; +import { newTestService, ServiceMocks } from 'test/utils'; + +describe(IntegrityService.name, () => { + let sut: IntegrityService; + let mocks: ServiceMocks; + + beforeEach(() => { + ({ sut, mocks } = newTestService(IntegrityService)); + }); + + it('should work', () => { + expect(sut).toBeDefined(); + }); + + describe('handleDeleteAllIntegrityReports', () => { + beforeEach(() => { + mocks.integrityReport.streamIntegrityReportsByProperty.mockReturnValue((function* () {})() as never); + }); + + it('should query all property types when no type specified', async () => { + await sut.handleDeleteAllIntegrityReports({}); + + expect(mocks.integrityReport.streamIntegrityReportsByProperty).toHaveBeenCalledWith(undefined, undefined); + expect(mocks.integrityReport.streamIntegrityReportsByProperty).toHaveBeenCalledWith('assetId', undefined); + expect(mocks.integrityReport.streamIntegrityReportsByProperty).toHaveBeenCalledWith('fileAssetId', undefined); + }); + }); +}); diff --git a/server/src/services/integrity.service.ts b/server/src/services/integrity.service.ts new file mode 100644 index 0000000000..a2c603fde1 --- /dev/null +++ b/server/src/services/integrity.service.ts @@ -0,0 +1,724 @@ +import { Injectable } from '@nestjs/common'; +import { createHash } from 'node:crypto'; +import { basename } from 'node:path'; +import { Readable, Writable } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; +import { JOBS_LIBRARY_PAGINATION_SIZE } from 'src/constants'; +import { StorageCore } from 'src/cores/storage.core'; +import { OnEvent, OnJob } from 'src/decorators'; +import { + IntegrityGetReportDto, + IntegrityReportResponseDto, + IntegrityReportSummaryResponseDto, +} from 'src/dtos/integrity.dto'; +import { + AssetStatus, + CacheControl, + DatabaseLock, + ImmichWorker, + IntegrityReport, + JobName, + JobStatus, + QueueName, + StorageFolder, + SystemMetadataKey, +} from 'src/enum'; +import { ArgOf } from 'src/repositories/event.repository'; +import { BaseService } from 'src/services/base.service'; +import { + IIntegrityDeleteReportsJob, + IIntegrityDeleteReportTypeJob, + IIntegrityJob, + IIntegrityMissingFilesJob, + IIntegrityPathWithChecksumJob, + IIntegrityPathWithReportJob, + IIntegrityUntrackedFilesJob, +} from 'src/types'; +import { ImmichFileResponse } from 'src/utils/file'; +import { handlePromiseError } from 'src/utils/misc'; + +/** + * Untracked Files: + * Files are detected in /data/encoded-video, /data/library, /data/upload + * Checked against the asset table + * Files are detected in /data/thumbs + * Checked against the asset_file table + * + * * Can perform download or delete of files + * + * Missing Files: + * Paths are queried from asset(originalPath, encodedVideoPath), asset_file(path) + * Check whether files exist on disk + * + * * Reports must include origin (asset or asset_file) & ID for further action + * * Can perform trash (asset) or delete (asset_file) + * + * Checksum Mismatch: + * Paths & checksums are queried from asset(originalPath, checksum) + * Check whether files match checksum, missing files ignored + * + * * Reports must include origin (as above) for further action + * * Can perform download or trash (asset) + */ + +@Injectable() +export class IntegrityService extends BaseService { + private integrityLock = false; + + @OnEvent({ name: 'ConfigInit', workers: [ImmichWorker.Microservices] }) + async onConfigInit({ + newConfig: { + integrityChecks: { untrackedFiles, missingFiles, checksumFiles }, + }, + }: ArgOf<'ConfigInit'>) { + this.integrityLock = await this.databaseRepository.tryLock(DatabaseLock.IntegrityCheck); + if (!this.integrityLock) { + return; + } + + this.cronRepository.create({ + name: 'integrityUntrackedFiles', + expression: untrackedFiles.cronExpression, + onTick: () => + handlePromiseError( + this.jobRepository.queue({ name: JobName.IntegrityUntrackedFilesQueueAll, data: {} }), + this.logger, + ), + start: untrackedFiles.enabled, + }); + + this.cronRepository.create({ + name: 'integrityMissingFiles', + expression: missingFiles.cronExpression, + onTick: () => + handlePromiseError( + this.jobRepository.queue({ name: JobName.IntegrityMissingFilesQueueAll, data: {} }), + this.logger, + ), + start: missingFiles.enabled, + }); + + this.cronRepository.create({ + name: 'integrityChecksumFiles', + expression: checksumFiles.cronExpression, + onTick: () => + handlePromiseError(this.jobRepository.queue({ name: JobName.IntegrityChecksumFiles, data: {} }), this.logger), + start: checksumFiles.enabled, + }); + } + + @OnEvent({ name: 'ConfigUpdate', server: true }) + onConfigUpdate({ + newConfig: { + integrityChecks: { untrackedFiles, missingFiles, checksumFiles }, + }, + }: ArgOf<'ConfigUpdate'>) { + if (!this.integrityLock) { + return; + } + + this.cronRepository.update({ + name: 'integrityUntrackedFiles', + expression: untrackedFiles.cronExpression, + start: untrackedFiles.enabled, + }); + + this.cronRepository.update({ + name: 'integrityMissingFiles', + expression: missingFiles.cronExpression, + start: missingFiles.enabled, + }); + + this.cronRepository.update({ + name: 'integrityChecksumFiles', + expression: checksumFiles.cronExpression, + start: checksumFiles.enabled, + }); + } + + getIntegrityReportSummary(): Promise { + return this.integrityRepository.getIntegrityReportSummary(); + } + + getIntegrityReport(dto: IntegrityGetReportDto): Promise { + return this.integrityRepository.getIntegrityReport({ cursor: dto.cursor, limit: dto.limit ?? 100 }, dto.type); + } + + getIntegrityReportCsv(type: IntegrityReport): Readable { + const items = this.integrityRepository.streamIntegrityReports(type); + + // very rudimentary csv serialiser + async function* generator() { + yield 'id,type,assetId,fileAssetId,path\n'; + + for await (const item of items) { + // no expectation of particularly bad filenames + // but they could potentially have a newline or quote character + yield `${item.id},${item.type},${item.assetId},${item.fileAssetId},"${item.path.replaceAll('"', '""')}"\n`; + } + } + + return Readable.from(generator()); + } + + async getIntegrityReportFile(id: string): Promise { + const { path } = await this.integrityRepository.getById(id); + + return new ImmichFileResponse({ + path, + fileName: basename(path), + contentType: 'application/octet-stream', + cacheControl: CacheControl.PrivateWithoutCache, + }); + } + + async deleteIntegrityReport(userId: string, id: string): Promise { + const { path, assetId, fileAssetId } = await this.integrityRepository.getById(id); + + if (assetId) { + await this.assetRepository.updateAll([assetId], { + deletedAt: new Date(), + status: AssetStatus.Trashed, + }); + + await this.eventRepository.emit('AssetTrashAll', { + assetIds: [assetId], + userId, + }); + + await this.integrityRepository.deleteById(id); + } else if (fileAssetId) { + await this.assetRepository.deleteFiles([{ id: fileAssetId }]); + } else { + await this.storageRepository.unlink(path); + await this.integrityRepository.deleteById(id); + } + } + + private async queueRefreshAllUntrackedFiles() { + this.logger.log(`Checking for out of date untracked file reports...`); + + const reports = this.integrityRepository.streamIntegrityReportsWithAssetChecksum(IntegrityReport.UntrackedFile); + + let total = 0; + for await (const batchReports of chunk(reports, JOBS_LIBRARY_PAGINATION_SIZE)) { + await this.jobRepository.queue({ + name: JobName.IntegrityUntrackedFilesRefresh, + data: { + items: batchReports, + }, + }); + + total += batchReports.length; + this.logger.log(`Queued report check of ${batchReports.length} report(s) (${total} so far)`); + } + } + + @OnJob({ name: JobName.IntegrityUntrackedFilesQueueAll, queue: QueueName.IntegrityCheck }) + async handleUntrackedFilesQueueAll({ refreshOnly }: IIntegrityJob = {}): Promise { + await this.queueRefreshAllUntrackedFiles(); + + if (refreshOnly) { + this.logger.log('Refresh complete.'); + return JobStatus.Success; + } + + this.logger.log(`Scanning for untracked files...`); + + const assetPaths = this.storageRepository.walk({ + pathsToCrawl: [StorageFolder.EncodedVideo, StorageFolder.Library, StorageFolder.Upload].map((folder) => + StorageCore.getBaseFolder(folder), + ), + includeHidden: false, + take: JOBS_LIBRARY_PAGINATION_SIZE, + }); + + const assetFilePaths = this.storageRepository.walk({ + pathsToCrawl: [StorageCore.getBaseFolder(StorageFolder.Thumbnails)], + includeHidden: false, + take: JOBS_LIBRARY_PAGINATION_SIZE, + }); + + async function* paths() { + for await (const batch of assetPaths) { + yield ['asset', batch] as const; + } + + for await (const batch of assetFilePaths) { + yield ['asset_file', batch] as const; + } + } + + let total = 0; + for await (const [batchType, batchPaths] of paths()) { + await this.jobRepository.queue({ + name: JobName.IntegrityUntrackedFiles, + data: { + type: batchType, + paths: batchPaths, + }, + }); + + const count = batchPaths.length; + total += count; + + this.logger.log(`Queued untracked check of ${count} file(s) (${total} so far)`); + } + + return JobStatus.Success; + } + + @OnJob({ name: JobName.IntegrityUntrackedFiles, queue: QueueName.IntegrityCheck }) + async handleUntrackedFiles({ type, paths }: IIntegrityUntrackedFilesJob): Promise { + this.logger.log(`Processing batch of ${paths.length} files to check if they are untracked.`); + + const untrackedFiles = new Set(paths); + if (type === 'asset') { + const assets = await this.integrityRepository.getAssetPathsByPaths(paths); + for (const { originalPath, encodedVideoPath } of assets) { + untrackedFiles.delete(originalPath); + + if (encodedVideoPath) { + untrackedFiles.delete(encodedVideoPath); + } + } + } else { + const assets = await this.integrityRepository.getAssetFilePathsByPaths(paths); + for (const { path } of assets) { + untrackedFiles.delete(path); + } + } + + const personThumbnailPaths = await this.integrityRepository.getPersonThumbnailPathsByPaths(paths); + for (const { thumbnailPath } of personThumbnailPaths) { + untrackedFiles.delete(thumbnailPath); + } + + if (untrackedFiles.size > 0) { + await this.integrityRepository.create( + [...untrackedFiles].map((path) => ({ + type: IntegrityReport.UntrackedFile, + path, + })), + ); + } + + this.logger.log(`Processed ${paths.length} and found ${untrackedFiles.size} untracked file(s).`); + return JobStatus.Success; + } + + @OnJob({ name: JobName.IntegrityUntrackedFilesRefresh, queue: QueueName.IntegrityCheck }) + async handleUntrackedRefresh({ items }: IIntegrityPathWithReportJob): Promise { + this.logger.log(`Processing batch of ${items.length} reports to check if they are out of date.`); + + const results = await Promise.all( + items.map(({ reportId, path }) => + this.storageRepository + .stat(path) + .then(() => void 0) + .catch(() => reportId), + ), + ); + + const reportIds = results.filter(Boolean) as string[]; + + if (reportIds.length > 0) { + await this.integrityRepository.deleteByIds(reportIds); + } + + this.logger.log(`Processed ${items.length} paths and found ${reportIds.length} report(s) out of date.`); + return JobStatus.Success; + } + + private async queueRefreshAllMissingFiles() { + this.logger.log(`Checking for out of date missing file reports...`); + + const reports = this.integrityRepository.streamIntegrityReportsWithAssetChecksum(IntegrityReport.MissingFile); + + let total = 0; + for await (const batchReports of chunk(reports, JOBS_LIBRARY_PAGINATION_SIZE)) { + await this.jobRepository.queue({ + name: JobName.IntegrityMissingFilesRefresh, + data: { + items: batchReports, + }, + }); + + total += batchReports.length; + this.logger.log(`Queued report check of ${batchReports.length} report(s) (${total} so far)`); + } + + this.logger.log('Refresh complete.'); + } + + @OnJob({ name: JobName.IntegrityMissingFilesQueueAll, queue: QueueName.IntegrityCheck }) + async handleMissingFilesQueueAll({ refreshOnly }: IIntegrityJob = {}): Promise { + if (refreshOnly) { + await this.queueRefreshAllMissingFiles(); + return JobStatus.Success; + } + + this.logger.log(`Scanning for missing files...`); + + const assetPaths = this.integrityRepository.streamAssetPaths(); + + let total = 0; + for await (const batchPaths of chunk(assetPaths, JOBS_LIBRARY_PAGINATION_SIZE)) { + await this.jobRepository.queue({ + name: JobName.IntegrityMissingFiles, + data: { + items: batchPaths, + }, + }); + + total += batchPaths.length; + this.logger.log(`Queued missing check of ${batchPaths.length} file(s) (${total} so far)`); + } + + return JobStatus.Success; + } + + @OnJob({ name: JobName.IntegrityMissingFiles, queue: QueueName.IntegrityCheck }) + async handleMissingFiles({ items }: IIntegrityMissingFilesJob): Promise { + this.logger.log(`Processing batch of ${items.length} files to check if they are missing.`); + + const results = await Promise.all( + items.map((item) => + this.storageRepository + .stat(item.path) + .then(() => ({ ...item, exists: true })) + .catch(() => ({ ...item, exists: false })), + ), + ); + + const outdatedReports = results + .filter(({ exists, reportId }) => exists && reportId) + .map(({ reportId }) => reportId!); + + if (outdatedReports.length > 0) { + await this.integrityRepository.deleteByIds(outdatedReports); + } + + const missingFiles = results.filter(({ exists }) => !exists); + if (missingFiles.length > 0) { + await this.integrityRepository.create( + missingFiles.map(({ path, assetId, fileAssetId }) => ({ + type: IntegrityReport.MissingFile, + path, + assetId, + fileAssetId, + })), + ); + } + + this.logger.log(`Processed ${items.length} and found ${missingFiles.length} missing file(s).`); + return JobStatus.Success; + } + + @OnJob({ name: JobName.IntegrityMissingFilesRefresh, queue: QueueName.IntegrityCheck }) + async handleMissingRefresh({ items: paths }: IIntegrityPathWithReportJob): Promise { + this.logger.log(`Processing batch of ${paths.length} reports to check if they are out of date.`); + + const results = await Promise.all( + paths.map(({ reportId, path }) => + this.storageRepository + .stat(path) + .then(() => reportId) + .catch(() => void 0), + ), + ); + + const reportIds = results.filter(Boolean) as string[]; + + if (reportIds.length > 0) { + await this.integrityRepository.deleteByIds(reportIds); + } + + this.logger.log(`Processed ${paths.length} paths and found ${reportIds.length} report(s) out of date.`); + return JobStatus.Success; + } + + private async queueRefreshAllChecksumFiles() { + this.logger.log(`Checking for out of date checksum file reports...`); + + const reports = this.integrityRepository.streamIntegrityReportsWithAssetChecksum(IntegrityReport.ChecksumFail); + + let total = 0; + for await (const batchReports of chunk(reports, JOBS_LIBRARY_PAGINATION_SIZE)) { + await this.jobRepository.queue({ + name: JobName.IntegrityChecksumFilesRefresh, + data: { + items: batchReports.map(({ path, reportId, checksum }) => ({ + path, + reportId, + checksum: checksum?.toString('hex'), + })), + }, + }); + + total += batchReports.length; + this.logger.log(`Queued report check of ${batchReports.length} report(s) (${total} so far)`); + } + + this.logger.log('Refresh complete.'); + } + + private async checkAssetChecksum( + originalPath: string, + checksum: Buffer, + assetId: string, + reportId: string | null, + ) { + const hash = createHash('sha1'); + + try { + await pipeline([ + this.storageRepository.createPlainReadStream(originalPath), + new Writable({ + write(chunk, _encoding, callback) { + hash.update(chunk); + callback(); + }, + }), + ]); + + if (checksum.equals(hash.digest())) { + if (reportId) { + await this.integrityRepository.deleteById(reportId); + } + } else { + throw new Error('File failed checksum'); + } + } catch (error) { + if ((error as { code?: string }).code === 'ENOENT') { + if (reportId) { + await this.integrityRepository.deleteById(reportId); + } + + // missing file; handled by the missing files job + return; + } + + this.logger.warn('Failed to process a file: ' + error); + await this.integrityRepository.create({ + path: originalPath, + type: IntegrityReport.ChecksumFail, + assetId, + }); + } + } + + @OnJob({ name: JobName.IntegrityChecksumFiles, queue: QueueName.IntegrityCheck }) + async handleChecksumFiles({ refreshOnly }: IIntegrityJob = {}): Promise { + if (refreshOnly) { + await this.queueRefreshAllChecksumFiles(); + return JobStatus.Success; + } + + const { + integrityChecks: { + checksumFiles: { timeLimit, percentageLimit }, + }, + } = await this.getConfig({ + withCache: true, + }); + + this.logger.log( + `Checking file checksums... (will run for up to ${(timeLimit / (60 * 60 * 1000)).toFixed(2)} hours or until ${(percentageLimit * 100).toFixed(2)}% of assets are processed)`, + ); + + let processed = 0; + const startedAt = Date.now(); + const { count } = await this.integrityRepository.getAssetCount(); + const checkpoint = await this.systemMetadataRepository.get(SystemMetadataKey.IntegrityChecksumCheckpoint); + + let startMarker: Date | undefined = checkpoint?.date ? new Date(checkpoint.date) : undefined; + let endMarker: Date | undefined; + + const printStats = () => { + const averageTime = ((Date.now() - startedAt) / processed).toFixed(2); + const completionProgress = ((processed / count) * 100).toFixed(2); + + this.logger.log( + `Processed ${processed} files so far... (avg. ${averageTime} ms/asset, ${completionProgress}% of all assets)`, + ); + }; + + let lastCreatedAt: Date | undefined; + + finishEarly: do { + this.logger.log( + `Processing assets in range [${startMarker?.toISOString() ?? 'beginning'}, ${endMarker?.toISOString() ?? 'end'}]`, + ); + + const assets = this.integrityRepository.streamAssetChecksums(startMarker, endMarker); + endMarker = startMarker; + startMarker = undefined; + + for await (const { originalPath, checksum, createdAt, assetId, reportId } of assets) { + await this.checkAssetChecksum(originalPath, checksum, assetId, reportId); + + processed++; + + if (processed % 100 === 0) { + printStats(); + } + + if (Date.now() > startedAt + timeLimit || processed > count * percentageLimit) { + this.logger.log('Reached stop criteria.'); + lastCreatedAt = createdAt; + break finishEarly; + } + } + } while (endMarker); + + await this.systemMetadataRepository.set(SystemMetadataKey.IntegrityChecksumCheckpoint, { + date: lastCreatedAt?.toISOString(), + }); + + printStats(); + + if (lastCreatedAt) { + this.logger.log(`Finished checksum job, will continue from ${lastCreatedAt.toISOString()}.`); + } else { + this.logger.log(`Finished checksum job, covered all assets.`); + } + + return JobStatus.Success; + } + + @OnJob({ name: JobName.IntegrityChecksumFilesRefresh, queue: QueueName.IntegrityCheck }) + async handleChecksumRefresh({ items: paths }: IIntegrityPathWithChecksumJob): Promise { + this.logger.log(`Processing batch of ${paths.length} reports to check if they are out of date.`); + + const results = await Promise.all( + paths.map(async ({ reportId, path, checksum }) => { + if (!checksum) { + return reportId; + } + + const hash = createHash('sha1'); + + try { + await pipeline([ + this.storageRepository.createPlainReadStream(path), + new Writable({ + write(chunk, _encoding, callback) { + hash.update(chunk); + callback(); + }, + }), + ]); + } catch (error) { + if ((error as { code?: string }).code === 'ENOENT') { + return reportId; + } + } + + if (Buffer.from(checksum, 'hex').equals(hash.digest())) { + return reportId; + } + }), + ); + + const reportIds = results.filter(Boolean) as string[]; + + if (reportIds.length > 0) { + await this.integrityRepository.deleteByIds(reportIds); + } + + this.logger.log(`Processed ${paths.length} paths and found ${reportIds.length} report(s) out of date.`); + return JobStatus.Success; + } + + @OnJob({ name: JobName.IntegrityDeleteReportType, queue: QueueName.IntegrityCheck }) + async handleDeleteAllIntegrityReports({ type }: IIntegrityDeleteReportTypeJob): Promise { + this.logger.log(`Deleting all entries for ${type ?? 'all types of'} integrity report`); + + let properties; + switch (type) { + case IntegrityReport.ChecksumFail: { + properties = ['assetId'] as const; + break; + } + case IntegrityReport.MissingFile: { + properties = ['assetId', 'fileAssetId'] as const; + break; + } + case IntegrityReport.UntrackedFile: { + properties = [void 0] as const; + break; + } + default: { + properties = [void 0, 'assetId', 'fileAssetId'] as const; + break; + } + } + + for (const property of properties) { + const reports = this.integrityRepository.streamIntegrityReportsByProperty(property, type); + for await (const batch of chunk(reports, JOBS_LIBRARY_PAGINATION_SIZE)) { + await this.jobRepository.queue({ + name: JobName.IntegrityDeleteReports, + data: { + reports: batch, + }, + }); + + this.logger.log(`Queued ${batch.length} reports to delete.`); + } + } + + return JobStatus.Success; + } + + @OnJob({ name: JobName.IntegrityDeleteReports, queue: QueueName.IntegrityCheck }) + async handleDeleteIntegrityReports({ reports }: IIntegrityDeleteReportsJob): Promise { + const byAsset = reports.filter((report) => report.assetId); + const byFileAsset = reports.filter((report) => report.fileAssetId); + const byPath = reports.filter((report) => !report.assetId && !report.fileAssetId); + + if (byAsset.length > 0) { + const ids = byAsset.map(({ assetId }) => assetId!); + await this.assetRepository.updateAll(ids, { + deletedAt: new Date(), + status: AssetStatus.Trashed, + }); + + await this.eventRepository.emit('AssetTrashAll', { + assetIds: ids, + userId: '', // we don't notify any users currently + }); + + await this.integrityRepository.deleteByIds(byAsset.map(({ id }) => id)); + } + + if (byFileAsset.length > 0) { + await this.assetRepository.deleteFiles(byFileAsset.map(({ fileAssetId }) => ({ id: fileAssetId! }))); + } + + if (byPath.length > 0) { + await Promise.all(byPath.map(({ path }) => this.storageRepository.unlink(path).catch(() => void 0))); + await this.integrityRepository.deleteByIds(byPath.map(({ id }) => id)); + } + + this.logger.log(`Deleted ${reports.length} reports.`); + return JobStatus.Success; + } +} + +async function* chunk(generator: AsyncIterableIterator, n: number) { + let chunk: T[] = []; + for await (const item of generator) { + chunk.push(item); + + if (chunk.length === n) { + yield chunk; + chunk = []; + } + } + + if (chunk.length > 0) { + yield chunk; + } +} diff --git a/server/src/services/job.service.ts b/server/src/services/job.service.ts index a8721a5fde..00105103e8 100644 --- a/server/src/services/job.service.ts +++ b/server/src/services/job.service.ts @@ -2,7 +2,7 @@ import { BadRequestException, Injectable } from '@nestjs/common'; import { OnEvent } from 'src/decorators'; import { mapAsset } from 'src/dtos/asset-response.dto'; import { JobCreateDto } from 'src/dtos/job.dto'; -import { AssetType, AssetVisibility, JobName, JobStatus, ManualJobName } from 'src/enum'; +import { AssetType, AssetVisibility, IntegrityReport, JobName, JobStatus, ManualJobName } from 'src/enum'; import { ArgsOf } from 'src/repositories/event.repository'; import { BaseService } from 'src/services/base.service'; import { JobItem } from 'src/types'; @@ -34,6 +34,42 @@ const asJobItem = (dto: JobCreateDto): JobItem => { return { name: JobName.DatabaseBackup }; } + case ManualJobName.IntegrityMissingFiles: { + return { name: JobName.IntegrityMissingFilesQueueAll }; + } + + case ManualJobName.IntegrityUntrackedFiles: { + return { name: JobName.IntegrityUntrackedFilesQueueAll }; + } + + case ManualJobName.IntegrityChecksumFiles: { + return { name: JobName.IntegrityChecksumFiles }; + } + + case ManualJobName.IntegrityMissingFilesRefresh: { + return { name: JobName.IntegrityMissingFilesQueueAll, data: { refreshOnly: true } }; + } + + case ManualJobName.IntegrityUntrackedFilesRefresh: { + return { name: JobName.IntegrityUntrackedFilesQueueAll, data: { refreshOnly: true } }; + } + + case ManualJobName.IntegrityChecksumFilesRefresh: { + return { name: JobName.IntegrityChecksumFiles, data: { refreshOnly: true } }; + } + + case ManualJobName.IntegrityMissingFilesDeleteAll: { + return { name: JobName.IntegrityDeleteReportType, data: { type: IntegrityReport.MissingFile } }; + } + + case ManualJobName.IntegrityUntrackedFilesDeleteAll: { + return { name: JobName.IntegrityDeleteReportType, data: { type: IntegrityReport.UntrackedFile } }; + } + + case ManualJobName.IntegrityChecksumFilesDeleteAll: { + return { name: JobName.IntegrityDeleteReportType, data: { type: IntegrityReport.ChecksumFail } }; + } + default: { throw new BadRequestException('Invalid job name'); } diff --git a/server/src/services/queue.service.spec.ts b/server/src/services/queue.service.spec.ts index 48c61c0951..5643c5eced 100644 --- a/server/src/services/queue.service.spec.ts +++ b/server/src/services/queue.service.spec.ts @@ -23,7 +23,7 @@ describe(QueueService.name, () => { it('should update concurrency', () => { sut.onConfigUpdate({ newConfig: defaults, oldConfig: {} as SystemConfig }); - expect(mocks.job.setConcurrency).toHaveBeenCalledTimes(18); + expect(mocks.job.setConcurrency).toHaveBeenCalledTimes(19); expect(mocks.job.setConcurrency).toHaveBeenNthCalledWith(5, QueueName.FacialRecognition, 1); expect(mocks.job.setConcurrency).toHaveBeenNthCalledWith(7, QueueName.DuplicateDetection, 1); expect(mocks.job.setConcurrency).toHaveBeenNthCalledWith(8, QueueName.BackgroundTask, 5); @@ -77,6 +77,7 @@ describe(QueueService.name, () => { [QueueName.BackupDatabase]: expected, [QueueName.Ocr]: expected, [QueueName.Workflow]: expected, + [QueueName.IntegrityCheck]: expected, [QueueName.Editor]: expected, }); }); diff --git a/server/src/services/system-config.service.spec.ts b/server/src/services/system-config.service.spec.ts index 2d0850ac58..7b14832155 100644 --- a/server/src/services/system-config.service.spec.ts +++ b/server/src/services/system-config.service.spec.ts @@ -42,6 +42,7 @@ const updatedConfig = Object.freeze({ [QueueName.Notification]: { concurrency: 5 }, [QueueName.Ocr]: { concurrency: 1 }, [QueueName.Workflow]: { concurrency: 5 }, + [QueueName.IntegrityCheck]: { concurrency: 1 }, [QueueName.Editor]: { concurrency: 2 }, }, backup: { @@ -77,6 +78,22 @@ const updatedConfig = Object.freeze({ enabled: false, }, }, + integrityChecks: { + untrackedFiles: { + enabled: true, + cronExpression: '0 03 * * *', + }, + missingFiles: { + enabled: true, + cronExpression: '0 03 * * *', + }, + checksumFiles: { + enabled: true, + cronExpression: '0 03 * * *', + timeLimit: 60 * 60 * 1000, + percentageLimit: 1, + }, + }, logging: { enabled: true, level: LogLevel.Log, diff --git a/server/src/types.ts b/server/src/types.ts index 4e5a383cca..598d900e70 100644 --- a/server/src/types.ts +++ b/server/src/types.ts @@ -21,6 +21,7 @@ import { H264Profile, HevcProfile, ImageFormat, + IntegrityReport, JobName, MemoryType, QueueName, @@ -311,6 +312,43 @@ export type IWorkflowJob = { type: T; }; +export interface IIntegrityJob { + refreshOnly?: boolean; +} + +export interface IIntegrityDeleteReportTypeJob { + type?: IntegrityReport; +} + +export interface IIntegrityDeleteReportsJob { + reports: { + id: string; + assetId: string | null; + fileAssetId: string | null; + path: string; + }[]; +} + +export interface IIntegrityUntrackedFilesJob { + type: 'asset' | 'asset_file'; + paths: string[]; +} + +export interface IIntegrityMissingFilesJob { + items: ({ path: string; reportId: string | null } & ( + | { assetId: string; fileAssetId: null } + | { assetId: null; fileAssetId: string } + ))[]; +} + +export interface IIntegrityPathWithReportJob { + items: { path: string; reportId: string | null }[]; +} + +export interface IIntegrityPathWithChecksumJob { + items: { path: string; reportId: string | null; checksum?: string | null }[]; +} + export interface JobCounts { active: number; completed: number; @@ -422,6 +460,18 @@ export type JobItem = // Workflow | { name: JobName.WorkflowAssetTrigger; data: { workflowId: string; assetId: string } } + // Integrity + | { name: JobName.IntegrityUntrackedFilesQueueAll; data?: IIntegrityJob } + | { name: JobName.IntegrityUntrackedFiles; data: IIntegrityUntrackedFilesJob } + | { name: JobName.IntegrityUntrackedFilesRefresh; data: IIntegrityPathWithReportJob } + | { name: JobName.IntegrityMissingFilesQueueAll; data?: IIntegrityJob } + | { name: JobName.IntegrityMissingFiles; data: IIntegrityPathWithReportJob } + | { name: JobName.IntegrityMissingFilesRefresh; data: IIntegrityPathWithReportJob } + | { name: JobName.IntegrityChecksumFiles; data?: IIntegrityJob } + | { name: JobName.IntegrityChecksumFilesRefresh; data?: IIntegrityPathWithChecksumJob } + | { name: JobName.IntegrityDeleteReportType; data: IIntegrityDeleteReportTypeJob } + | { name: JobName.IntegrityDeleteReports; data: IIntegrityDeleteReportsJob } + // Editor | { name: JobName.AssetEditThumbnailGeneration; data: IEntityJob }; @@ -522,6 +572,7 @@ export interface SystemMetadata extends Record; [SystemMetadataKey.VersionCheckState]: VersionCheckMetadata; [SystemMetadataKey.MemoriesState]: MemoriesState; + [SystemMetadataKey.IntegrityChecksumCheckpoint]: { date?: string }; } export type UserPreferences = { diff --git a/server/src/validation.ts b/server/src/validation.ts index 95bfe003a4..f94ad4f2ae 100644 --- a/server/src/validation.ts +++ b/server/src/validation.ts @@ -1,6 +1,7 @@ import { ArgumentMetadata, FileValidator, Injectable, ParseUUIDPipe } from '@nestjs/common'; import { createZodDto } from 'nestjs-zod'; import sanitize from 'sanitize-filename'; +import { IntegrityReportSchema } from 'src/enum'; import { isIP, isIPRange } from 'validator'; import z from 'zod'; @@ -110,6 +111,12 @@ const UUIDParamSchema = z.object({ export class UUIDParamDto extends createZodDto(UUIDParamSchema) {} +const UUIDv7ParamSchema = z.object({ + id: z.uuidv7(), +}); + +export class UUIDv7ParamDto extends createZodDto(UUIDv7ParamSchema) {} + const UUIDAssetIDParamSchema = z.object({ id: z.uuidv4(), assetId: z.uuidv4(), @@ -125,6 +132,10 @@ const FilenameParamSchema = z.object({ export class FilenameParamDto extends createZodDto(FilenameParamSchema) {} +const IntegrityReportParamSchema = z.object({ type: IntegrityReportSchema }).meta({ id: 'IntegrityReportDto' }); + +export class IntegrityReportTypeParamDto extends createZodDto(IntegrityReportParamSchema) {} + /** * Unified email validation * Converts email strings to lowercase and validates against HTML5 email regex diff --git a/server/test/medium.factory.ts b/server/test/medium.factory.ts index e7915d3f1c..599cfe7cc2 100644 --- a/server/test/medium.factory.ts +++ b/server/test/medium.factory.ts @@ -30,6 +30,7 @@ import { CryptoRepository } from 'src/repositories/crypto.repository'; import { DatabaseRepository } from 'src/repositories/database.repository'; import { EmailRepository } from 'src/repositories/email.repository'; import { EventRepository } from 'src/repositories/event.repository'; +import { IntegrityRepository } from 'src/repositories/integrity.repository'; import { JobRepository } from 'src/repositories/job.repository'; import { LoggingRepository } from 'src/repositories/logging.repository'; import { MachineLearningRepository } from 'src/repositories/machine-learning.repository'; @@ -415,6 +416,7 @@ const newRealRepository = (key: ClassConstructor, db: Kysely): T => { case AssetRepository: case AssetEditRepository: case AssetJobRepository: + case IntegrityRepository: case MemoryRepository: case NotificationRepository: case OcrRepository: @@ -483,6 +485,7 @@ const newMockRepository = (key: ClassConstructor) => { case ConfigRepository: case CryptoRepository: case MemoryRepository: + case IntegrityRepository: case NotificationRepository: case OcrRepository: case PartnerRepository: diff --git a/server/test/medium/specs/services/integrity.service.spec.ts b/server/test/medium/specs/services/integrity.service.spec.ts new file mode 100644 index 0000000000..758fb5ab00 --- /dev/null +++ b/server/test/medium/specs/services/integrity.service.spec.ts @@ -0,0 +1,1000 @@ +import { Kysely } from 'kysely'; +import { createHash, randomUUID } from 'node:crypto'; +import { Readable } from 'node:stream'; +import { text } from 'node:stream/consumers'; +import { StorageCore } from 'src/cores/storage.core'; +import { AssetFileType, IntegrityReport, JobName, JobStatus } from 'src/enum'; +import { AssetRepository } from 'src/repositories/asset.repository'; +import { ConfigRepository } from 'src/repositories/config.repository'; +import { EventRepository } from 'src/repositories/event.repository'; +import { IntegrityRepository } from 'src/repositories/integrity.repository'; +import { JobRepository } from 'src/repositories/job.repository'; +import { LoggingRepository } from 'src/repositories/logging.repository'; +import { StorageRepository } from 'src/repositories/storage.repository'; +import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository'; +import { DB } from 'src/schema'; +import { IntegrityService } from 'src/services/integrity.service'; +import { newMediumService } from 'test/medium.factory'; +import { getKyselyDB, makeStream } from 'test/utils'; + +let defaultDatabase: Kysely; + +const setup = (db?: Kysely) => { + return newMediumService(IntegrityService, { + database: db || defaultDatabase, + real: [IntegrityRepository, AssetRepository, ConfigRepository, SystemMetadataRepository], + mock: [LoggingRepository, EventRepository, StorageRepository, JobRepository], + }); +}; + +beforeAll(async () => { + defaultDatabase = await getKyselyDB(); + StorageCore.setMediaLocation('/path/to/file'); +}); + +afterAll(() => { + StorageCore.reset(); +}); + +describe(IntegrityService.name, () => { + beforeEach(async () => { + await defaultDatabase.deleteFrom('asset_file').execute(); + await defaultDatabase.deleteFrom('asset').execute(); + await defaultDatabase.deleteFrom('integrity_report').execute(); + }); + + it('should work', () => { + const { sut } = setup(); + expect(sut).toBeDefined(); + }); + + describe('getIntegrityReportSummary', () => { + it('gets summary', async () => { + const { sut } = setup(); + + await expect(sut.getIntegrityReportSummary()).resolves.toEqual({ + checksum_mismatch: 0, + missing_file: 0, + untracked_file: 0, + }); + }); + }); + + describe('getIntegrityReport', () => { + it('gets report', async () => { + const { sut } = setup(); + + await expect(sut.getIntegrityReport({ type: IntegrityReport.ChecksumFail })).resolves.toEqual({ + items: [], + nextCursor: undefined, + }); + }); + }); + + describe('getIntegrityReportCsv', () => { + it('gets report as csv', async () => { + const { sut, ctx } = setup(); + + const { id } = await ctx.get(IntegrityRepository).create({ + type: IntegrityReport.ChecksumFail, + path: '/path/to/file', + }); + + await expect(text(sut.getIntegrityReportCsv(IntegrityReport.ChecksumFail))).resolves.toMatchInlineSnapshot(` + "id,type,assetId,fileAssetId,path + ${id},checksum_mismatch,null,null,"/path/to/file" + " + `); + }); + }); + + describe('getIntegrityReportFile', () => { + it('gets report file', async () => { + const { sut, ctx } = setup(); + + const { id } = await ctx.get(IntegrityRepository).create({ + type: IntegrityReport.ChecksumFail, + path: '/path/to/file', + }); + + await expect(sut.getIntegrityReportFile(id)).resolves.toEqual({ + path: '/path/to/file', + fileName: 'file', + contentType: 'application/octet-stream', + cacheControl: 'private_without_cache', + }); + }); + }); + + describe('deleteIntegrityReport', () => { + it('deletes asset if one is present', async () => { + const { sut, ctx } = setup(); + const events = ctx.getMock(EventRepository); + events.emit.mockResolvedValue(void 0); + + const { + result: { id: ownerId }, + } = await ctx.newUser(); + + const { + result: { id: assetId }, + } = await ctx.newAsset({ ownerId }); + + const { id } = await ctx.get(IntegrityRepository).create({ + type: IntegrityReport.ChecksumFail, + path: '/path/to/file', + assetId, + }); + + await sut.deleteIntegrityReport(ownerId, id); + + await expect(ctx.get(AssetRepository).getById(assetId)).resolves.toEqual( + expect.objectContaining({ + status: 'trashed', + }), + ); + + expect(events.emit).toHaveBeenCalledWith('AssetTrashAll', { + assetIds: [assetId], + userId: ownerId, + }); + + await expect(sut.getIntegrityReport({ type: IntegrityReport.ChecksumFail })).resolves.toEqual({ + items: [], + nextCursor: undefined, + }); + }); + + it('deletes file asset if one is present', async () => { + const { sut, ctx } = setup(); + + const { + result: { id: ownerId }, + } = await ctx.newUser(); + + const { + result: { id: assetId }, + } = await ctx.newAsset({ ownerId }); + + const fileAssetId = randomUUID(); + await ctx.newAssetFile({ id: fileAssetId, assetId, type: AssetFileType.Thumbnail, path: '/path/to/file' }); + + const { id } = await ctx.get(IntegrityRepository).create({ + type: IntegrityReport.ChecksumFail, + path: '/path/to/file', + fileAssetId, + }); + + await sut.deleteIntegrityReport('userId', id); + + await expect(ctx.get(AssetRepository).getForThumbnail(assetId, AssetFileType.Thumbnail, false)).resolves.toEqual( + expect.objectContaining({ + path: null, + }), + ); + }); + + it('deletes untracked file', async () => { + const { sut, ctx } = setup(); + const storage = ctx.getMock(StorageRepository); + storage.unlink.mockResolvedValue(void 0); + + const { + result: { id: userId }, + } = await ctx.newUser(); + + const { id } = await ctx.get(IntegrityRepository).create({ + type: IntegrityReport.ChecksumFail, + path: '/path/to/file', + }); + + await sut.deleteIntegrityReport(userId, id); + + expect(storage.unlink).toHaveBeenCalledWith('/path/to/file'); + }); + }); + + describe('handleUntrackedFilesQueueAll', () => { + it('should succeed', async () => { + const { sut, ctx } = setup(); + const storage = ctx.getMock(StorageRepository); + const job = ctx.getMock(JobRepository); + + job.queue.mockResolvedValue(void 0); + storage.walk.mockImplementation(() => makeStream([['/path/to/file', '/path/to/file2'], ['/path/to/batch2']])); + + await expect(sut.handleUntrackedFilesQueueAll({ refreshOnly: false })).resolves.toBe(JobStatus.Success); + }); + + it('queues jobs for all detected files', async () => { + const { sut, ctx } = setup(); + const storage = ctx.getMock(StorageRepository); + const job = ctx.getMock(JobRepository); + + job.queue.mockResolvedValue(void 0); + + storage.walk.mockReturnValueOnce(makeStream([['/path/to/file', '/path/to/file2'], ['/path/to/batch2']])); + storage.walk.mockReturnValueOnce(makeStream([['/path/to/file3', '/path/to/file4'], ['/path/to/batch4']])); + + await sut.handleUntrackedFilesQueueAll({ refreshOnly: false }); + + expect(job.queue).toBeCalledTimes(4); + expect(job.queue).toBeCalledWith({ + name: JobName.IntegrityUntrackedFiles, + data: { + type: 'asset', + paths: expect.arrayContaining(['/path/to/file']), + }, + }); + + expect(job.queue).toBeCalledWith({ + name: JobName.IntegrityUntrackedFiles, + data: { + type: 'asset_file', + paths: expect.arrayContaining(['/path/to/file3']), + }, + }); + }); + + it('queues jobs to refresh reports', async () => { + const { sut, ctx } = setup(); + const storage = ctx.getMock(StorageRepository); + const job = ctx.getMock(JobRepository); + + job.queue.mockResolvedValue(void 0); + storage.walk.mockImplementation(() => makeStream([['/path/to/file', '/path/to/file2'], ['/path/to/batch2']])); + + const { id } = await ctx.get(IntegrityRepository).create({ + type: IntegrityReport.UntrackedFile, + path: '/path/to/file', + }); + + await sut.handleUntrackedFilesQueueAll({ refreshOnly: true }); + + expect(job.queue).toBeCalledTimes(1); + expect(job.queue).toBeCalledWith({ + name: JobName.IntegrityUntrackedFilesRefresh, + data: { + items: expect.arrayContaining([ + { + path: '/path/to/file', + reportId: id, + }, + ]), + }, + }); + }); + }); + + describe('handleUntrackedFiles', () => { + it('should detect untracked asset files', async () => { + const { sut, ctx } = setup(); + + const { + result: { id: ownerId }, + } = await ctx.newUser(); + + const { asset } = await ctx.newAsset({ ownerId, originalPath: '/path/to/file1' }); + await ctx.newAssetFile({ assetId: asset.id, type: AssetFileType.EncodedVideo, path: '/path/to/file2' }); + + await sut.handleUntrackedFiles({ + type: 'asset', + paths: ['/path/to/file1', '/path/to/file2', '/path/to/untracked'], + }); + + await expect( + ctx.get(IntegrityRepository).getIntegrityReport( + { + limit: 100, + }, + IntegrityReport.UntrackedFile, + ), + ).resolves.toEqual({ + items: [ + expect.objectContaining({ + path: '/path/to/untracked', + }), + ], + nextCursor: undefined, + }); + }); + + it('should detect untracked asset_file files', async () => { + const { sut, ctx } = setup(); + + const { + result: { id: ownerId }, + } = await ctx.newUser(); + + const { + result: { id: assetId }, + } = await ctx.newAsset({ ownerId }); + + await ctx.newAssetFile({ assetId, type: AssetFileType.Thumbnail, path: '/path/to/file1' }); + + await sut.handleUntrackedFiles({ + type: 'asset_file', + paths: ['/path/to/file1', '/path/to/untracked'], + }); + + await expect( + ctx.get(IntegrityRepository).getIntegrityReport( + { + limit: 100, + }, + IntegrityReport.UntrackedFile, + ), + ).resolves.toEqual({ + items: [ + expect.objectContaining({ + path: '/path/to/untracked', + }), + ], + nextCursor: undefined, + }); + }); + }); + + describe('handleUntrackedRefresh', () => { + it('should succeed', async () => { + const { sut } = setup(); + await expect(sut.handleUntrackedRefresh({ items: [] })).resolves.toBe(JobStatus.Success); + }); + + it('should delete reports for files that no longer exist', async () => { + const { sut, ctx } = setup(); + const integrity = ctx.get(IntegrityRepository); + const storage = ctx.getMock(StorageRepository); + + const report1 = await integrity.create({ + type: IntegrityReport.UntrackedFile, + path: '/path/to/missing1', + }); + + const report2 = await integrity.create({ + type: IntegrityReport.UntrackedFile, + path: '/path/to/existing', + }); + + storage.stat.mockRejectedValueOnce(new Error('ENOENT')).mockResolvedValueOnce({} as never); + + await sut.handleUntrackedRefresh({ + items: [ + { reportId: report1.id, path: report1.path }, + { reportId: report2.id, path: report2.path }, + ], + }); + + await expect( + ctx.get(IntegrityRepository).getIntegrityReport( + { + limit: 100, + }, + IntegrityReport.UntrackedFile, + ), + ).resolves.toEqual({ + items: [ + expect.objectContaining({ + path: '/path/to/existing', + }), + ], + nextCursor: undefined, + }); + }); + }); + + describe('handleMissingFilesQueueAll', () => { + it('should succeed', async () => { + const { sut } = setup(); + await expect(sut.handleMissingFilesQueueAll()).resolves.toBe(JobStatus.Success); + }); + + it('should queue jobs', async () => { + const { sut, ctx } = setup(); + const job = ctx.getMock(JobRepository); + job.queue.mockResolvedValue(void 0); + + const { + result: { id: ownerId }, + } = await ctx.newUser(); + + const { + result: { id: assetId }, + } = await ctx.newAsset({ ownerId, originalPath: '/path/to/file1' }); + + const { + result: { id: assetId2 }, + } = await ctx.newAsset({ ownerId, originalPath: '/path/to/file2' }); + + const { id: reportId } = await ctx.get(IntegrityRepository).create({ + type: IntegrityReport.UntrackedFile, + path: '/path/to/file2', + assetId: assetId2, + }); + + await sut.handleMissingFilesQueueAll({ refreshOnly: false }); + + expect(job.queue).toHaveBeenCalledWith({ + name: JobName.IntegrityMissingFiles, + data: { + items: expect.arrayContaining([ + { path: '/path/to/file1', assetId, fileAssetId: null, reportId: null }, + { path: '/path/to/file2', assetId: assetId2, fileAssetId: null, reportId }, + ]), + }, + }); + + expect(job.queue).not.toHaveBeenCalledWith( + expect.objectContaining({ + name: JobName.IntegrityMissingFilesRefresh, + }), + ); + }); + + it('should queue refresh jobs when refreshOnly is set', async () => { + const { sut, ctx } = setup(); + const job = ctx.getMock(JobRepository); + job.queue.mockResolvedValue(void 0); + + const { id: reportId } = await ctx.get(IntegrityRepository).create({ + type: IntegrityReport.MissingFile, + path: '/path/to/file1', + }); + + await sut.handleMissingFilesQueueAll({ refreshOnly: true }); + + expect(job.queue).toHaveBeenCalledWith({ + name: JobName.IntegrityMissingFilesRefresh, + data: { + items: expect.arrayContaining([{ reportId, path: '/path/to/file1' }]), + }, + }); + + expect(job.queue).not.toHaveBeenCalledWith( + expect.objectContaining({ + name: JobName.IntegrityMissingFiles, + }), + ); + }); + }); + + describe('handleMissingFiles', () => { + it('should succeed', async () => { + const { sut } = setup(); + await expect(sut.handleMissingFiles({ items: [] })).resolves.toBe(JobStatus.Success); + }); + + it('should detect missing files and remove outdated reports', async () => { + const { sut, ctx } = setup(); + const integrity = ctx.get(IntegrityRepository); + const storage = ctx.getMock(StorageRepository); + + const { + result: { id: ownerId }, + } = await ctx.newUser(); + + const { + result: { id: assetId }, + } = await ctx.newAsset({ ownerId, originalPath: '/path/to/file1' }); + + const { id: restoredId } = await integrity.create({ + type: IntegrityReport.MissingFile, + path: '/path/to/restored', + assetId, + }); + + storage.stat + .mockResolvedValueOnce({} as never) + .mockRejectedValueOnce(new Error('ENOENT')) + .mockResolvedValueOnce({} as never); + + await sut.handleMissingFiles({ + items: [ + { path: '/path/to/existing', assetId, fileAssetId: null, reportId: null }, + { path: '/path/to/missing', assetId, fileAssetId: null, reportId: null }, + { path: '/path/to/restored', assetId, fileAssetId: null, reportId: restoredId }, + ], + }); + + await expect( + ctx.get(IntegrityRepository).getIntegrityReport( + { + limit: 100, + }, + IntegrityReport.MissingFile, + ), + ).resolves.toEqual({ + items: [ + expect.objectContaining({ + path: '/path/to/missing', + }), + ], + nextCursor: undefined, + }); + }); + }); + + describe('handleMissingRefresh', () => { + it('should succeed', async () => { + const { sut } = setup(); + await expect(sut.handleMissingRefresh({ items: [] })).resolves.toBe(JobStatus.Success); + }); + + it('should remove outdated reports', async () => { + const { sut, ctx } = setup(); + const integrity = ctx.get(IntegrityRepository); + const storage = ctx.getMock(StorageRepository); + + const { id: restoredId } = await integrity.create({ + type: IntegrityReport.MissingFile, + path: '/path/to/restored', + }); + + storage.stat + .mockResolvedValueOnce({} as never) + .mockRejectedValueOnce(new Error('ENOENT')) + .mockResolvedValueOnce({} as never); + + await sut.handleMissingRefresh({ + items: [ + { path: '/path/to/existing', reportId: null }, + { path: '/path/to/missing', reportId: null }, + { path: '/path/to/restored', reportId: restoredId }, + ], + }); + + await expect( + ctx.get(IntegrityRepository).getIntegrityReport( + { + limit: 100, + }, + IntegrityReport.MissingFile, + ), + ).resolves.toEqual({ + items: [], + nextCursor: undefined, + }); + }); + }); + + describe('handleChecksumFiles', () => { + it('should succeed', async () => { + const { sut } = setup(); + await expect(sut.handleChecksumFiles({ refreshOnly: false })).resolves.toBe(JobStatus.Success); + }); + + it('should queue refresh jobs when refreshOnly', async () => { + const { sut, ctx } = setup(); + const job = ctx.getMock(JobRepository); + job.queue.mockResolvedValue(void 0); + + const { + result: { id: ownerId }, + } = await ctx.newUser(); + + const { + result: { id: assetId }, + } = await ctx.newAsset({ ownerId, originalPath: '/path/to/file1', checksum: Buffer.from('a') }); + + const { id: reportId } = await ctx.get(IntegrityRepository).create({ + type: IntegrityReport.ChecksumFail, + path: '/path/to/file1', + assetId, + }); + + await sut.handleChecksumFiles({ refreshOnly: true }); + + expect(job.queue).toHaveBeenCalledWith({ + name: JobName.IntegrityChecksumFilesRefresh, + data: { + items: [{ reportId, path: '/path/to/file1', checksum: '61' }], + }, + }); + }); + + it('should create report for checksum mismatch and delete when fixed', async () => { + const { sut, ctx } = setup(); + const storage = ctx.getMock(StorageRepository); + const job = ctx.getMock(JobRepository); + job.queue.mockResolvedValue(void 0); + + const { + result: { id: ownerId }, + } = await ctx.newUser(); + + const { + result: { id: assetId }, + } = await ctx.newAsset({ ownerId, originalPath: '/path/to/file1', checksum: Buffer.from('mismatched') }); + + const fileContent1 = Buffer.from('test content'); + await ctx.newAsset({ + ownerId, + originalPath: '/path/to/file2', + checksum: createHash('sha1').update(fileContent1).digest(), + }); + + const fileContent2 = Buffer.from('test content 2'); + const { + result: { id: assetId3 }, + } = await ctx.newAsset({ + ownerId, + originalPath: '/path/to/file3', + checksum: createHash('sha1').update(fileContent2).digest(), + }); + + await ctx.get(IntegrityRepository).create({ + type: IntegrityReport.ChecksumFail, + path: '/path/to/file3', + assetId: assetId3, + }); + + storage.createPlainReadStream.mockImplementation((path) => + Readable.from( + path === '/path/to/file2' ? fileContent1 : path === '/path/to/file3' ? fileContent2 : 'garbage data', + ), + ); + + await sut.handleChecksumFiles({ refreshOnly: false }); + + await expect( + ctx.get(IntegrityRepository).getIntegrityReport( + { + limit: 100, + }, + IntegrityReport.ChecksumFail, + ), + ).resolves.toEqual({ + items: [ + expect.objectContaining({ + assetId, + path: '/path/to/file1', + }), + ], + nextCursor: undefined, + }); + }); + + it('should skip missing files', async () => { + const { sut, ctx } = setup(); + const storage = ctx.getMock(StorageRepository); + const job = ctx.getMock(JobRepository); + job.queue.mockResolvedValue(void 0); + + const { + result: { id: ownerId }, + } = await ctx.newUser(); + + await ctx.newAsset({ ownerId, originalPath: '/path/to/file1', checksum: Buffer.from('a') }); + + const error = new Error('ENOENT') as NodeJS.ErrnoException; + error.code = 'ENOENT'; + storage.createPlainReadStream.mockImplementation(() => { + throw error; + }); + + await sut.handleChecksumFiles({ refreshOnly: false }); + + await expect( + ctx.get(IntegrityRepository).getIntegrityReport( + { + limit: 100, + }, + IntegrityReport.ChecksumFail, + ), + ).resolves.toEqual({ + items: [], + nextCursor: undefined, + }); + }); + }); + + describe('handleChecksumRefresh', () => { + it('should succeed', async () => { + const { sut } = setup(); + await expect(sut.handleChecksumRefresh({ items: [] })).resolves.toBe(JobStatus.Success); + }); + + it('should delete reports when checksum now matches, file is missing, or asset is now missing', async () => { + const { sut, ctx } = setup(); + const integrity = ctx.get(IntegrityRepository); + const storage = ctx.getMock(StorageRepository); + const job = ctx.getMock(JobRepository); + job.queue.mockResolvedValue(void 0); + + const fileContent = Buffer.from('test content'); + const correctChecksum = createHash('sha1').update(fileContent).digest(); + + const { + result: { id: ownerId }, + } = await ctx.newUser(); + + const { + result: { id: fixedAssetId }, + } = await ctx.newAsset({ ownerId, originalPath: '/path/to/fixed', checksum: correctChecksum }); + + const { id: fixedReportId } = await integrity.create({ + type: IntegrityReport.ChecksumFail, + path: '/path/to/fixed', + assetId: fixedAssetId, + }); + + const { + result: { id: missingAssetId }, + } = await ctx.newAsset({ ownerId, originalPath: '/path/to/missing', checksum: Buffer.from('1') }); + + const { id: missingReportId } = await integrity.create({ + type: IntegrityReport.ChecksumFail, + path: '/path/to/missing', + assetId: missingAssetId, + }); + + const { + result: { id: badAssetId }, + } = await ctx.newAsset({ ownerId, originalPath: '/path/to/missing', checksum: Buffer.from('2') }); + + const { id: badReportId } = await integrity.create({ + type: IntegrityReport.ChecksumFail, + path: '/path/to/bad', + assetId: badAssetId, + }); + + const { id: missingAssetReportId } = await integrity.create({ + type: IntegrityReport.ChecksumFail, + path: '/path/to/missing-asset', + }); + + const error = new Error('ENOENT') as NodeJS.ErrnoException; + error.code = 'ENOENT'; + + storage.createPlainReadStream + .mockImplementationOnce(() => Readable.from(fileContent)) + .mockImplementationOnce(() => { + throw error; + }) + .mockImplementationOnce(() => Readable.from(fileContent)) + .mockImplementationOnce(() => Readable.from(fileContent)); + + await sut.handleChecksumRefresh({ + items: [ + { reportId: fixedReportId, path: '/path/to/fixed', checksum: correctChecksum.toString('hex') }, + { reportId: missingReportId, path: '/path/to/missing', checksum: 'abc123' }, + { reportId: badReportId, path: '/path/to/bad', checksum: 'wrongchecksum' }, + { reportId: missingAssetReportId, path: '/path/to/missing-asset', checksum: null }, + ], + }); + + await expect( + ctx.get(IntegrityRepository).getIntegrityReport( + { + limit: 100, + }, + IntegrityReport.ChecksumFail, + ), + ).resolves.toEqual({ + items: [ + expect.objectContaining({ + id: badReportId, + assetId: badAssetId, + path: '/path/to/bad', + }), + ], + nextCursor: undefined, + }); + }); + }); + + describe('handleDeleteAllIntegrityReports', () => { + it('should succeed', async () => { + const { sut } = setup(); + await expect(sut.handleDeleteAllIntegrityReports({})).resolves.toBe(JobStatus.Success); + }); + + it('should queue delete jobs for checksum fail reports', async () => { + const { sut, ctx } = setup(); + const integrity = ctx.get(IntegrityRepository); + const job = ctx.getMock(JobRepository); + job.queue.mockResolvedValue(void 0); + + const { + result: { id: ownerId }, + } = await ctx.newUser(); + + const { + result: { id: assetId }, + } = await ctx.newAsset({ ownerId, originalPath: '/path/to/file1' }); + + const { id: reportId } = await integrity.create({ + type: IntegrityReport.ChecksumFail, + path: '/path/to/file1', + assetId, + }); + + await sut.handleDeleteAllIntegrityReports({ type: IntegrityReport.ChecksumFail }); + + expect(job.queue).toHaveBeenCalledWith({ + name: JobName.IntegrityDeleteReports, + data: { + reports: [{ id: reportId, assetId, path: '/path/to/file1', fileAssetId: null }], + }, + }); + }); + + it('should queue delete jobs for missing file reports by assetId and fileAssetId', async () => { + const { sut, ctx } = setup(); + const integrity = ctx.get(IntegrityRepository); + const job = ctx.getMock(JobRepository); + job.queue.mockResolvedValue(void 0); + + const { + result: { id: ownerId }, + } = await ctx.newUser(); + + const { + result: { id: assetId }, + } = await ctx.newAsset({ ownerId, originalPath: '/path/to/file1' }); + + const { id: assetReportId } = await integrity.create({ + type: IntegrityReport.MissingFile, + path: '/path/to/file1', + assetId, + }); + + const { + result: { id: assetId2 }, + } = await ctx.newAsset({ ownerId, originalPath: '/path/to/file2' }); + + const fileAssetId = randomUUID(); + await ctx.newAssetFile({ + id: fileAssetId, + assetId: assetId2, + path: '/path/to/file3', + type: AssetFileType.Thumbnail, + }); + + const { id: fileAssetReportId } = await integrity.create({ + type: IntegrityReport.MissingFile, + path: '/path/to/file3', + fileAssetId, + }); + + await sut.handleDeleteAllIntegrityReports({ type: IntegrityReport.MissingFile }); + + expect(job.queue).toHaveBeenCalledTimes(2); + + expect(job.queue).toHaveBeenCalledWith({ + name: JobName.IntegrityDeleteReports, + data: { + reports: [{ id: assetReportId, assetId, path: '/path/to/file1', fileAssetId: null }], + }, + }); + + expect(job.queue).toHaveBeenCalledWith({ + name: JobName.IntegrityDeleteReports, + data: { + reports: [{ id: fileAssetReportId, assetId: null, path: '/path/to/file3', fileAssetId }], + }, + }); + }); + + it('should queue delete jobs for untracked file reports', async () => { + const { sut, ctx } = setup(); + const integrity = ctx.get(IntegrityRepository); + const job = ctx.getMock(JobRepository); + job.queue.mockResolvedValue(void 0); + + const { id: reportId } = await integrity.create({ + type: IntegrityReport.UntrackedFile, + path: '/path/to/untracked', + }); + + await sut.handleDeleteAllIntegrityReports({ type: IntegrityReport.UntrackedFile }); + + expect(job.queue).toHaveBeenCalledWith({ + name: JobName.IntegrityDeleteReports, + data: { + reports: [{ id: reportId, path: '/path/to/untracked', assetId: null, fileAssetId: null }], + }, + }); + }); + }); + + describe('handleDeleteIntegrityReports', () => { + it('should succeed', async () => { + const { sut } = setup(); + await expect(sut.handleDeleteIntegrityReports({ reports: [] })).resolves.toBe(JobStatus.Success); + }); + + it('should handle all report types', async () => { + const { sut, ctx } = setup(); + const integrity = ctx.get(IntegrityRepository); + const storage = ctx.getMock(StorageRepository); + const events = ctx.getMock(EventRepository); + + storage.unlink.mockResolvedValue(void 0); + events.emit.mockResolvedValue(void 0); + + const { + result: { id: ownerId }, + } = await ctx.newUser(); + + const { + result: { id: assetId1 }, + } = await ctx.newAsset({ ownerId, originalPath: '/path/to/file1' }); + + const { id: reportId1 } = await integrity.create({ + path: '/path/to/file1', + type: IntegrityReport.ChecksumFail, + assetId: assetId1, + }); + + const { + result: { id: assetId2 }, + } = await ctx.newAsset({ ownerId, originalPath: '/path/to/file2' }); + + const { id: reportId2 } = await integrity.create({ + path: '/path/to/file2', + type: IntegrityReport.MissingFile, + assetId: assetId2, + }); + + const { + result: { id: assetId3 }, + } = await ctx.newAsset({ ownerId, originalPath: '/path/to/file3' }); + + const fileAssetId = randomUUID(); + await ctx.newAssetFile({ + id: fileAssetId, + assetId: assetId3, + path: '/path/to/file4', + type: AssetFileType.Thumbnail, + }); + + const { id: reportId3 } = await integrity.create({ + path: '/path/to/file4', + type: IntegrityReport.MissingFile, + fileAssetId, + }); + + const { id: reportId4 } = await integrity.create({ + path: '/path/to/untracked', + type: IntegrityReport.UntrackedFile, + }); + + await sut.handleDeleteIntegrityReports({ + reports: [ + { id: reportId1, assetId: assetId1, fileAssetId: null, path: '/path/to/file1' }, + { id: reportId2, assetId: assetId2, fileAssetId: null, path: '/path/to/file2' }, + { id: reportId3, assetId: null, fileAssetId, path: '/path/to/file4' }, + { id: reportId4, assetId: null, fileAssetId: null, path: '/path/to/untracked' }, + ], + }); + + expect(events.emit).toHaveBeenCalledWith('AssetTrashAll', { + assetIds: [assetId1, assetId2], + userId: '', + }); + + expect(storage.unlink).toHaveBeenCalledWith('/path/to/untracked'); + + await expect(ctx.get(AssetRepository).getByIds([assetId1, assetId2, assetId3])).resolves.toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: assetId1, + status: 'trashed', + }), + expect.objectContaining({ + id: assetId2, + status: 'trashed', + }), + expect.objectContaining({ + id: assetId3, + status: 'active', + }), + ]), + ); + + await expect(defaultDatabase.selectFrom('asset_file').execute()).resolves.toEqual([]); + await expect(defaultDatabase.selectFrom('integrity_report').execute()).resolves.toEqual([]); + }); + }); +}); diff --git a/server/test/repositories/storage.repository.mock.ts b/server/test/repositories/storage.repository.mock.ts index c1fb7ceaa6..f05826a424 100644 --- a/server/test/repositories/storage.repository.mock.ts +++ b/server/test/repositories/storage.repository.mock.ts @@ -48,8 +48,8 @@ export const newStorageRepositoryMock = (): Mocked { email: automock(EmailRepository, { args: [loggerMock] }), // eslint-disable-next-line no-sparse-arrays event: automock(EventRepository, { args: [, , loggerMock], strict: false }), + integrityReport: automock(IntegrityRepository, { strict: false }), job: newJobRepositoryMock(), apiKey: automock(ApiKeyRepository), library: automock(LibraryRepository, { strict: false }), @@ -385,6 +388,7 @@ export const newTestService = ( overrides.duplicateRepository || (mocks.duplicateRepository as As), overrides.email || (mocks.email as As), overrides.event || (mocks.event as As), + overrides.integrityReport || (mocks.integrityReport as As), overrides.job || (mocks.job as As), overrides.library || (mocks.library as As), overrides.machineLearning || (mocks.machineLearning as As), @@ -540,7 +544,44 @@ export const mockDuplex = return duplex; }; -export async function* makeStream(items: T[] = []): AsyncIterableIterator { +export const mockFork = vitest.fn((exitCode: number, stdout: string, stderr: string, error?: unknown) => { + const stdoutStream = new Readable({ + read() { + this.push(stdout); // write mock data to stdout + this.push(null); // end stream + }, + }); + + return { + stdout: stdoutStream, + stderr: new Readable({ + read() { + this.push(stderr); // write mock data to stderr + this.push(null); // end stream + }, + }), + stdin: new Writable({ + write(chunk, encoding, callback) { + callback(); + }, + }), + exitCode, + on: vitest.fn((event, callback: any) => { + if (event === 'close') { + stdoutStream.once('end', () => callback(0)); + } + if (event === 'error' && error) { + stdoutStream.once('end', () => callback(error)); + } + if (event === 'exit') { + stdoutStream.once('end', () => callback(exitCode)); + } + }), + kill: vitest.fn(), + } as unknown as ChildProcessWithoutNullStreams; +}); + +export async function* makeStream(items: T[] = []): AsyncGenerator { for (const item of items) { await Promise.resolve(); yield item; diff --git a/web/src/lib/components/maintenance/integrity/IntegrityReportTableItem.svelte b/web/src/lib/components/maintenance/integrity/IntegrityReportTableItem.svelte new file mode 100644 index 0000000000..2b4285b803 --- /dev/null +++ b/web/src/lib/components/maintenance/integrity/IntegrityReportTableItem.svelte @@ -0,0 +1,41 @@ + + + + + + {path} + + + + diff --git a/web/src/lib/components/server-statistics/ServerStatisticsCard.svelte b/web/src/lib/components/server-statistics/ServerStatisticsCard.svelte index b9498b08c4..b8ece40b49 100644 --- a/web/src/lib/components/server-statistics/ServerStatisticsCard.svelte +++ b/web/src/lib/components/server-statistics/ServerStatisticsCard.svelte @@ -1,6 +1,7 @@ + + +
+
+ + {$t('admin.maintenance_integrity_report')} + + + +
+ {#each reportTypes as reportType (reportType)} + + {#snippet footer()} + + + + + + {/snippet} + + {/each} +
+
+
+
+ {$t('admin.maintenance_settings')} + { await authenticate(url, { admin: true }); - const { backups } = await listDatabaseBackups(); - const $t = await getFormatter(); + + const integrityReport = await getIntegrityReportSummary(); const { major, minor, patch } = await getServerVersion(); + const { backups } = await listDatabaseBackups(); + + const $t = await getFormatter(); return { backups, + integrityReport, expectedVersion: `${major}.${minor}.${patch}`, meta: { title: $t('admin.maintenance_settings'), diff --git a/web/src/routes/admin/maintenance/integrity-report/[type]/+page.svelte b/web/src/routes/admin/maintenance/integrity-report/[type]/+page.svelte new file mode 100644 index 0000000000..581194fe36 --- /dev/null +++ b/web/src/routes/admin/maintenance/integrity-report/[type]/+page.svelte @@ -0,0 +1,99 @@ + + + + + +
+
+ + + {$t('filename')} + + + + + {#each integrityReport.items as { id, path } (id)} + + {/each} + + + {#if integrityReport.nextCursor} + + + + {/if} +
+
+
+ diff --git a/web/src/routes/admin/maintenance/integrity-report/[type]/+page.ts b/web/src/routes/admin/maintenance/integrity-report/[type]/+page.ts new file mode 100644 index 0000000000..45542385a9 --- /dev/null +++ b/web/src/routes/admin/maintenance/integrity-report/[type]/+page.ts @@ -0,0 +1,22 @@ +import { getIntegrityReport, IntegrityReport } from '@immich/sdk'; +import { authenticate } from '$lib/utils/auth'; +import { getFormatter } from '$lib/utils/i18n'; +import type { PageLoad } from './$types'; + +export const load = (async ({ params, url }) => { + const type = params.type as IntegrityReport; + + await authenticate(url, { admin: true }); + const integrityReport = await getIntegrityReport({ + $type: type, + }); + const $t = await getFormatter(); + + return { + type, + integrityReport, + meta: { + title: $t(`admin.maintenance_integrity_${type}`), + }, + }; +}) satisfies PageLoad; diff --git a/web/src/routes/admin/system-settings/JobSettings.svelte b/web/src/routes/admin/system-settings/JobSettings.svelte index b18d134d81..a2d51e0a4a 100644 --- a/web/src/routes/admin/system-settings/JobSettings.svelte +++ b/web/src/routes/admin/system-settings/JobSettings.svelte @@ -49,6 +49,7 @@ [QueueName.Ocr]: $t('admin.machine_learning_ocr'), [QueueName.Workflow]: $t('workflows'), [QueueName.Editor]: $t('editor'), + [QueueName.IntegrityCheck]: $t('integrity_checks'), });