feat: manually trigger integrity jobs

feat: update summary after job runs

i18n/en.json

@@ -183,8 +183,14 @@
     "maintenance_start_error": "Failed to start maintenance mode.",
     "maintenance_integrity_report": "Integrity Report",
     "maintenance_integrity_orphan_file": "Orphan Files",
+    "maintenance_integrity_orphan_file_job": "Check for orphaned files",
+    "maintenance_integrity_orphan_file_refresh_job": "Refresh orphan file reports",
     "maintenance_integrity_missing_file": "Missing Files",
+    "maintenance_integrity_missing_file_job": "Check for missing files",
+    "maintenance_integrity_missing_file_refresh_job": "Refresh missing file reports",
     "maintenance_integrity_checksum_mismatch": "Checksum Mismatch",
+    "maintenance_integrity_checksum_mismatch_job": "Check for checksum mismatches",
+    "maintenance_integrity_checksum_mismatch_refresh_job": "Refresh checksum mismatch reports",
     "manage_concurrency": "Manage Concurrency",
     "manage_log_settings": "Manage log settings",
     "map_dark_style": "Dark style",

mobile/openapi/lib/model/manual_job_name.dart

@@ -29,6 +29,12 @@ class ManualJobName {
   static const memoryCleanup = ManualJobName._(r'memory-cleanup');
   static const memoryCreate = ManualJobName._(r'memory-create');
   static const backupDatabase = ManualJobName._(r'backup-database');
+  static const integrityMissingFiles = ManualJobName._(r'integrity-missing-files');
+  static const integrityOrphanFiles = ManualJobName._(r'integrity-orphan-files');
+  static const integrityChecksumMismatch = ManualJobName._(r'integrity-checksum-mismatch');
+  static const integrityMissingFilesRefresh = ManualJobName._(r'integrity-missing-files-refresh');
+  static const integrityOrphanFilesRefresh = ManualJobName._(r'integrity-orphan-files-refresh');
+  static const integrityChecksumMismatchRefresh = ManualJobName._(r'integrity-checksum-mismatch-refresh');
 
   /// List of all possible values in this [enum][ManualJobName].
   static const values = <ManualJobName>[
@@ -38,6 +44,12 @@ class ManualJobName {
     memoryCleanup,
     memoryCreate,
     backupDatabase,
+    integrityMissingFiles,
+    integrityOrphanFiles,
+    integrityChecksumMismatch,
+    integrityMissingFilesRefresh,
+    integrityOrphanFilesRefresh,
+    integrityChecksumMismatchRefresh,
   ];
 
   static ManualJobName? fromJson(dynamic value) => ManualJobNameTypeTransformer().decode(value);
@@ -82,6 +94,12 @@ class ManualJobNameTypeTransformer {
         case r'memory-cleanup': return ManualJobName.memoryCleanup;
         case r'memory-create': return ManualJobName.memoryCreate;
         case r'backup-database': return ManualJobName.backupDatabase;
+        case r'integrity-missing-files': return ManualJobName.integrityMissingFiles;
+        case r'integrity-orphan-files': return ManualJobName.integrityOrphanFiles;
+        case r'integrity-checksum-mismatch': return ManualJobName.integrityChecksumMismatch;
+        case r'integrity-missing-files-refresh': return ManualJobName.integrityMissingFilesRefresh;
+        case r'integrity-orphan-files-refresh': return ManualJobName.integrityOrphanFilesRefresh;
+        case r'integrity-checksum-mismatch-refresh': return ManualJobName.integrityChecksumMismatchRefresh;
         default:
           if (!allowNull) {
             throw ArgumentError('Unknown enum value to decode: $data');

open-api/immich-openapi-specs.json

@@ -17118,7 +17118,13 @@
           "user-cleanup",
           "memory-cleanup",
           "memory-create",
-          "backup-database"
+          "backup-database",
+          "integrity-missing-files",
+          "integrity-orphan-files",
+          "integrity-checksum-mismatch",
+          "integrity-missing-files-refresh",
+          "integrity-orphan-files-refresh",
+          "integrity-checksum-mismatch-refresh"
         ],
         "type": "string"
       },

open-api/typescript-sdk/src/fetch-client.ts

@@ -5441,7 +5441,13 @@ export enum ManualJobName {
     UserCleanup = "user-cleanup",
     MemoryCleanup = "memory-cleanup",
     MemoryCreate = "memory-create",
-    BackupDatabase = "backup-database"
+    BackupDatabase = "backup-database",
+    IntegrityMissingFiles = "integrity-missing-files",
+    IntegrityOrphanFiles = "integrity-orphan-files",
+    IntegrityChecksumMismatch = "integrity-checksum-mismatch",
+    IntegrityMissingFilesRefresh = "integrity-missing-files-refresh",
+    IntegrityOrphanFilesRefresh = "integrity-orphan-files-refresh",
+    IntegrityChecksumMismatchRefresh = "integrity-checksum-mismatch-refresh"
 }
 export enum QueueName {
     ThumbnailGeneration = "thumbnailGeneration",

server/src/enum.ts

@@ -345,6 +345,12 @@ export enum SourceType {
   Manual = 'manual',
 }
 
+export enum IntegrityReportType {
+  OrphanFile = 'orphan_file',
+  MissingFile = 'missing_file',
+  ChecksumFail = 'checksum_mismatch',
+}
+
 export enum ManualJobName {
   PersonCleanup = 'person-cleanup',
   TagCleanup = 'tag-cleanup',
@@ -352,6 +358,12 @@ export enum ManualJobName {
   MemoryCleanup = 'memory-cleanup',
   MemoryCreate = 'memory-create',
   BackupDatabase = 'backup-database',
+  IntegrityMissingFiles = `integrity-missing-files`,
+  IntegrityOrphanFiles = `integrity-orphan-files`,
+  IntegrityChecksumFiles = `integrity-checksum-mismatch`,
+  IntegrityMissingFilesRefresh = `integrity-missing-files-refresh`,
+  IntegrityOrphanFilesRefresh = `integrity-orphan-files-refresh`,
+  IntegrityChecksumFilesRefresh = `integrity-checksum-mismatch-refresh`,
 }
 
 export enum AssetPathType {
@@ -482,12 +494,6 @@ export enum CacheControl {
   None = 'none',
 }
 
-export enum IntegrityReportType {
-  OrphanFile = 'orphan_file',
-  MissingFile = 'missing_file',
-  ChecksumFail = 'checksum_mismatch',
-}
-
 export enum ImmichEnvironment {
   Development = 'development',
   Testing = 'testing',

server/src/services/integrity.service.ts

@@ -19,7 +19,7 @@ import {
 } from 'src/enum';
 import { ArgOf } from 'src/repositories/event.repository';
 import { BaseService } from 'src/services/base.service';
-import { IIntegrityOrphanedFilesJob, IIntegrityPathWithReportJob } from 'src/types';
+import { IIntegrityJob, IIntegrityOrphanedFilesJob, IIntegrityPathWithReportJob } from 'src/types';
 import { handlePromiseError } from 'src/utils/misc';
 
 async function* chunk<T>(generator: AsyncIterableIterator<T>, n: number) {
@@ -130,7 +130,7 @@ export class IntegrityService extends BaseService {
   }
 
   @OnJob({ name: JobName.IntegrityOrphanedFilesQueueAll, queue: QueueName.BackgroundTask })
-  async handleOrphanedFilesQueueAll(): Promise<JobStatus> {
+  async handleOrphanedFilesQueueAll({ refreshOnly }: IIntegrityJob = {}): Promise<JobStatus> {
     this.logger.log(`Checking for out of date orphaned file reports...`);
 
     const reports = this.assetJobRepository.streamIntegrityReports(IntegrityReportType.OrphanFile);
@@ -148,6 +148,11 @@ export class IntegrityService extends BaseService {
       this.logger.log(`Queued report check of ${batchReports.length} report(s) (${total} so far)`);
     }
 
+    if (refreshOnly) {
+      this.logger.log('Refresh complete.');
+      return JobStatus.Success;
+    }
+
     this.logger.log(`Scanning for orphaned files...`);
 
     const assetPaths = this.storageRepository.walk({
@@ -232,8 +237,8 @@ export class IntegrityService extends BaseService {
     const results = await Promise.all(
       paths.map(({ reportId, path }) =>
         stat(path)
-          .then(() => reportId)
+          .then(() => void 0)
-          .catch(() => void 0),
+          .catch(() => reportId),
       ),
     );
 
@@ -243,12 +248,18 @@ export class IntegrityService extends BaseService {
       await this.integrityReportRepository.deleteByIds(reportIds);
     }
 
-    this.logger.log(`Processed ${paths.length} and found ${reportIds.length} orphaned file(s).`);
+    this.logger.log(`Processed ${paths.length} paths and found ${reportIds.length} report(s) out of date.`);
     return JobStatus.Success;
   }
 
   @OnJob({ name: JobName.IntegrityMissingFilesQueueAll, queue: QueueName.BackgroundTask })
-  async handleMissingFilesQueueAll(): Promise<JobStatus> {
+  async handleMissingFilesQueueAll({ refreshOnly }: IIntegrityJob = {}): Promise<JobStatus> {
+    if (refreshOnly) {
+      // TODO
+      this.logger.log('Refresh complete.');
+      return JobStatus.Success;
+    }
+
     this.logger.log(`Scanning for missing files...`);
 
     const assetPaths = this.assetJobRepository.streamAssetPaths();
@@ -304,7 +315,13 @@ export class IntegrityService extends BaseService {
   }
 
   @OnJob({ name: JobName.IntegrityChecksumFiles, queue: QueueName.BackgroundTask })
-  async handleChecksumFiles(): Promise<JobStatus> {
+  async handleChecksumFiles({ refreshOnly }: IIntegrityJob = {}): Promise<JobStatus> {
+    if (refreshOnly) {
+      // TODO
+      this.logger.log('Refresh complete.');
+      return JobStatus.Success;
+    }
+
     const timeLimit = 60 * 60 * 1000; // 1000;
     const percentageLimit = 1; // 0.25;
 

server/src/services/job.service.ts

@@ -34,6 +34,30 @@ const asJobItem = (dto: JobCreateDto): JobItem => {
       return { name: JobName.DatabaseBackup };
     }
 
+    case ManualJobName.IntegrityMissingFiles: {
+      return { name: JobName.IntegrityMissingFilesQueueAll };
+    }
+
+    case ManualJobName.IntegrityOrphanFiles: {
+      return { name: JobName.IntegrityOrphanedFilesQueueAll };
+    }
+
+    case ManualJobName.IntegrityChecksumFiles: {
+      return { name: JobName.IntegrityChecksumFiles };
+    }
+
+    case ManualJobName.IntegrityMissingFilesRefresh: {
+      return { name: JobName.IntegrityMissingFilesQueueAll, data: { refreshOnly: true } };
+    }
+
+    case ManualJobName.IntegrityOrphanFilesRefresh: {
+      return { name: JobName.IntegrityOrphanedFilesQueueAll, data: { refreshOnly: true } };
+    }
+
+    case ManualJobName.IntegrityChecksumFilesRefresh: {
+      return { name: JobName.IntegrityChecksumFiles, data: { refreshOnly: true } };
+    }
+
     default: {
       throw new BadRequestException('Invalid job name');
     }

server/src/types.ts

@@ -282,6 +282,10 @@ export interface IWorkflowJob<T extends PluginTriggerType = PluginTriggerType> {
   event: WorkflowData[T];
 }
 
+export interface IIntegrityJob {
+  refreshOnly?: boolean;
+}
+
 export interface IIntegrityOrphanedFilesJob {
   type: 'asset' | 'asset_file';
   paths: string[];
@@ -403,12 +407,12 @@ export type JobItem =
   | { name: JobName.WorkflowRun; data: IWorkflowJob }
 
   // Integrity
-  | { name: JobName.IntegrityOrphanedFilesQueueAll; data: IBaseJob }
+  | { name: JobName.IntegrityOrphanedFilesQueueAll; data?: IIntegrityJob }
   | { name: JobName.IntegrityOrphanedFiles; data: IIntegrityOrphanedFilesJob }
   | { name: JobName.IntegrityOrphanedCheckReports; data: IIntegrityPathWithReportJob }
-  | { name: JobName.IntegrityMissingFilesQueueAll; data: IBaseJob }
+  | { name: JobName.IntegrityMissingFilesQueueAll; data?: IIntegrityJob }
   | { name: JobName.IntegrityMissingFiles; data: IIntegrityPathWithReportJob }
-  | { name: JobName.IntegrityChecksumFiles; data: IBaseJob };
+  | { name: JobName.IntegrityChecksumFiles; data?: IIntegrityJob };
 
 export type VectorExtension = (typeof VECTOR_EXTENSIONS)[number];
 

web/src/lib/modals/JobCreateModal.svelte

@@ -16,6 +16,30 @@
     { title: $t('admin.memory_cleanup_job'), value: ManualJobName.MemoryCleanup },
     { title: $t('admin.memory_generate_job'), value: ManualJobName.MemoryCreate },
     { title: $t('admin.backup_database'), value: ManualJobName.BackupDatabase },
+    {
+      title: $t('admin.maintenance_integrity_missing_file_job'),
+      value: ManualJobName.IntegrityMissingFiles,
+    },
+    {
+      title: $t('admin.maintenance_integrity_orphan_file_job'),
+      value: ManualJobName.IntegrityOrphanFiles,
+    },
+    {
+      title: $t('admin.maintenance_integrity_checksum_mismatch_job'),
+      value: ManualJobName.IntegrityChecksumMismatch,
+    },
+    {
+      title: $t('admin.maintenance_integrity_missing_file_refresh_job'),
+      value: ManualJobName.IntegrityMissingFilesRefresh,
+    },
+    {
+      title: $t('admin.maintenance_integrity_orphan_file_refresh_job'),
+      value: ManualJobName.IntegrityOrphanFilesRefresh,
+    },
+    {
+      title: $t('admin.maintenance_integrity_checksum_mismatch_refresh_job'),
+      value: ManualJobName.IntegrityChecksumMismatchRefresh,
+    },
   ].map(({ value, title }) => ({ id: value, label: title, value }));
 
   let selectedJob: ComboBoxOption | undefined = $state(undefined);

web/src/routes/admin/maintenance/+page.svelte

@@ -2,10 +2,22 @@
   import AdminPageLayout from '$lib/components/layouts/AdminPageLayout.svelte';
   import ServerStatisticsCard from '$lib/components/server-statistics/ServerStatisticsCard.svelte';
   import { AppRoute } from '$lib/constants';
+  import { asyncTimeout } from '$lib/utils';
   import { handleError } from '$lib/utils/handle-error';
-  import { MaintenanceAction, setMaintenanceMode } from '@immich/sdk';
+  import {
-  import { Button, HStack, Text } from '@immich/ui';
+    createJob,
+    getIntegrityReportSummary,
+    getQueuesLegacy,
+    IntegrityReportType,
+    MaintenanceAction,
+    ManualJobName,
+    setMaintenanceMode,
+    type MaintenanceIntegrityReportSummaryResponseDto,
+    type QueuesResponseLegacyDto,
+  } from '@immich/sdk';
+  import { Button, HStack, Text, toastManager } from '@immich/ui';
   import { mdiProgressWrench } from '@mdi/js';
+  import { onDestroy, onMount } from 'svelte';
   import { t } from 'svelte-i18n';
   import type { PageData } from './$types';
 
@@ -15,6 +27,14 @@
 
   let { data }: Props = $props();
 
+  let integrityReport: MaintenanceIntegrityReportSummaryResponseDto | undefined = $state(data.integrityReport);
+
+  const TYPES: IntegrityReportType[] = [
+    IntegrityReportType.OrphanFile,
+    IntegrityReportType.MissingFile,
+    IntegrityReportType.ChecksumMismatch,
+  ];
+
   async function switchToMaintenance() {
     try {
       await setMaintenanceMode({
@@ -26,6 +46,56 @@
       handleError(error, $t('admin.maintenance_start_error'));
     }
   }
+
+  let jobs: QueuesResponseLegacyDto | undefined = $state();
+  let expectingUpdate: boolean = $state(false);
+
+  async function runJob(reportType: IntegrityReportType, refreshOnly?: boolean) {
+    let name: ManualJobName;
+    switch (reportType) {
+      case IntegrityReportType.OrphanFile: {
+        name = refreshOnly ? ManualJobName.IntegrityOrphanFilesRefresh : ManualJobName.IntegrityOrphanFiles;
+        break;
+      }
+      case IntegrityReportType.MissingFile: {
+        name = refreshOnly ? ManualJobName.IntegrityMissingFilesRefresh : ManualJobName.IntegrityMissingFiles;
+        break;
+      }
+      case IntegrityReportType.ChecksumMismatch: {
+        name = refreshOnly ? ManualJobName.IntegrityChecksumMismatchRefresh : ManualJobName.IntegrityChecksumMismatch;
+        break;
+      }
+    }
+
+    try {
+      await createJob({ jobCreateDto: { name } });
+      if (jobs) {
+        expectingUpdate = true;
+        jobs.backgroundTask.queueStatus.isActive = true;
+      }
+      toastManager.success($t('admin.job_created'));
+    } catch (error) {
+      handleError(error, $t('errors.unable_to_submit_job'));
+    }
+  }
+
+  let running = true;
+
+  onMount(async () => {
+    while (running) {
+      jobs = await getQueuesLegacy();
+      if (jobs.backgroundTask.queueStatus.isActive) {
+        expectingUpdate = true;
+      } else if (expectingUpdate) {
+        integrityReport = await getIntegrityReportSummary();
+      }
+      await asyncTimeout(5000);
+    }
+  });
+
+  onDestroy(() => {
+    running = false;
+  });
 </script>
 
 <AdminPageLayout breadcrumbs={[{ title: data.meta.title }]}>
@@ -48,17 +118,33 @@
       <p class="text-sm dark:text-immich-dark-fg uppercase">{$t('admin.maintenance_integrity_report')}</p>
 
       <div class="mt-5 hidden justify-between lg:flex gap-4">
-        {#each ['orphan_file', 'missing_file', 'checksum_mismatch'] as const as reportType (reportType)}
+        {#each TYPES as reportType (reportType)}
           <ServerStatisticsCard
             title={$t(`admin.maintenance_integrity_${reportType}`)}
-            value={data.integrityReport[reportType]}
+            value={integrityReport[reportType]}
           >
             {#snippet footer()}
-              <Button
+              <HStack gap={1} class="justify-end">
-                href={`${AppRoute.ADMIN_MAINTENANCE_INTEGRITY_REPORT + reportType}`}
+                <Button
-                size="tiny"
+                  onclick={() => runJob(reportType)}
-                class="self-end mt-1">View Report</Button
+                  size="tiny"
-              >
+                  variant="ghost"
+                  class="self-end mt-1"
+                  disabled={jobs?.backgroundTask.queueStatus.isActive}>Check All</Button
+                >
+                <Button
+                  onclick={() => runJob(reportType, true)}
+                  size="tiny"
+                  variant="ghost"
+                  class="self-end mt-1"
+                  disabled={jobs?.backgroundTask.queueStatus.isActive}>Refresh</Button
+                >
+                <Button
+                  href={`${AppRoute.ADMIN_MAINTENANCE_INTEGRITY_REPORT + reportType}`}
+                  size="tiny"
+                  class="self-end mt-1">View</Button
+                >
+              </HStack>
             {/snippet}
           </ServerStatisticsCard>
         {/each}