refactor: use separate queue for integrity checks

e2e/src/api/specs/maintenance.e2e-spec.ts

@@ -69,7 +69,7 @@ describe('/admin/maintenance', () => {
         name: ManualJobName.IntegrityChecksumMismatch,
       });
 
-      await utils.waitForQueueFinish(admin.accessToken, QueueName.BackgroundTask);
+      await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck);
 
       const { status, body } = await request(app)
         .get('/admin/maintenance/integrity/summary')
@@ -93,7 +93,7 @@ describe('/admin/maintenance', () => {
         name: ManualJobName.IntegrityOrphanFiles,
       });
 
-      await utils.waitForQueueFinish(admin.accessToken, QueueName.BackgroundTask);
+      await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck);
 
       const { status, body } = await request(app)
         .get('/admin/maintenance/integrity/summary')
@@ -117,7 +117,7 @@ describe('/admin/maintenance', () => {
         name: ManualJobName.IntegrityOrphanFilesRefresh,
       });
 
-      await utils.waitForQueueFinish(admin.accessToken, QueueName.BackgroundTask);
+      await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck);
 
       const { status, body } = await request(app)
         .get('/admin/maintenance/integrity/summary')
@@ -139,7 +139,7 @@ describe('/admin/maintenance', () => {
         name: ManualJobName.IntegrityMissingFiles,
       });
 
-      await utils.waitForQueueFinish(admin.accessToken, QueueName.BackgroundTask);
+      await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck);
 
       const { status, body } = await request(app)
         .get('/admin/maintenance/integrity/summary')
@@ -160,7 +160,7 @@ describe('/admin/maintenance', () => {
         name: ManualJobName.IntegrityMissingFilesRefresh,
       });
 
-      await utils.waitForQueueFinish(admin.accessToken, QueueName.BackgroundTask);
+      await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck);
 
       const { status, body } = await request(app)
         .get('/admin/maintenance/integrity/summary')
@@ -183,7 +183,7 @@ describe('/admin/maintenance', () => {
         name: ManualJobName.IntegrityChecksumMismatch,
       });
 
-      await utils.waitForQueueFinish(admin.accessToken, QueueName.BackgroundTask);
+      await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck);
 
       const { status, body } = await request(app)
         .get('/admin/maintenance/integrity/summary')
@@ -203,7 +203,7 @@ describe('/admin/maintenance', () => {
         name: ManualJobName.IntegrityChecksumMismatchRefresh,
       });
 
-      await utils.waitForQueueFinish(admin.accessToken, QueueName.BackgroundTask);
+      await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck);
 
       const { status, body } = await request(app)
         .get('/admin/maintenance/integrity/summary')

mobile/openapi/lib/model/queue_name.dart

@@ -40,6 +40,7 @@ class QueueName {
   static const backupDatabase = QueueName._(r'backupDatabase');
   static const ocr = QueueName._(r'ocr');
   static const workflow = QueueName._(r'workflow');
+  static const integrityCheck = QueueName._(r'integrityCheck');
 
   /// List of all possible values in this [enum][QueueName].
   static const values = <QueueName>[
@@ -60,6 +61,7 @@ class QueueName {
     backupDatabase,
     ocr,
     workflow,
+    integrityCheck,
   ];
 
   static QueueName? fromJson(dynamic value) => QueueNameTypeTransformer().decode(value);
@@ -115,6 +117,7 @@ class QueueNameTypeTransformer {
         case r'backupDatabase': return QueueName.backupDatabase;
         case r'ocr': return QueueName.ocr;
         case r'workflow': return QueueName.workflow;
+        case r'integrityCheck': return QueueName.integrityCheck;
         default:
           if (!allowNull) {
             throw ArgumentError('Unknown enum value to decode: $data');

mobile/openapi/lib/model/queues_response_legacy_dto.dart

@@ -18,6 +18,7 @@ class QueuesResponseLegacyDto {
     required this.duplicateDetection,
     required this.faceDetection,
     required this.facialRecognition,
+    required this.integrityCheck,
     required this.library_,
     required this.metadataExtraction,
     required this.migration,
@@ -42,6 +43,8 @@ class QueuesResponseLegacyDto {
 
   QueueResponseLegacyDto facialRecognition;
 
+  QueueResponseLegacyDto integrityCheck;
+
   QueueResponseLegacyDto library_;
 
   QueueResponseLegacyDto metadataExtraction;
@@ -73,6 +76,7 @@ class QueuesResponseLegacyDto {
     other.duplicateDetection == duplicateDetection &&
     other.faceDetection == faceDetection &&
     other.facialRecognition == facialRecognition &&
+    other.integrityCheck == integrityCheck &&
     other.library_ == library_ &&
     other.metadataExtraction == metadataExtraction &&
     other.migration == migration &&
@@ -94,6 +98,7 @@ class QueuesResponseLegacyDto {
     (duplicateDetection.hashCode) +
     (faceDetection.hashCode) +
     (facialRecognition.hashCode) +
+    (integrityCheck.hashCode) +
     (library_.hashCode) +
     (metadataExtraction.hashCode) +
     (migration.hashCode) +
@@ -108,7 +113,7 @@ class QueuesResponseLegacyDto {
     (workflow.hashCode);
 
   @override
-  String toString() => 'QueuesResponseLegacyDto[backgroundTask=$backgroundTask, backupDatabase=$backupDatabase, duplicateDetection=$duplicateDetection, faceDetection=$faceDetection, facialRecognition=$facialRecognition, library_=$library_, metadataExtraction=$metadataExtraction, migration=$migration, notifications=$notifications, ocr=$ocr, search=$search, sidecar=$sidecar, smartSearch=$smartSearch, storageTemplateMigration=$storageTemplateMigration, thumbnailGeneration=$thumbnailGeneration, videoConversion=$videoConversion, workflow=$workflow]';
+  String toString() => 'QueuesResponseLegacyDto[backgroundTask=$backgroundTask, backupDatabase=$backupDatabase, duplicateDetection=$duplicateDetection, faceDetection=$faceDetection, facialRecognition=$facialRecognition, integrityCheck=$integrityCheck, library_=$library_, metadataExtraction=$metadataExtraction, migration=$migration, notifications=$notifications, ocr=$ocr, search=$search, sidecar=$sidecar, smartSearch=$smartSearch, storageTemplateMigration=$storageTemplateMigration, thumbnailGeneration=$thumbnailGeneration, videoConversion=$videoConversion, workflow=$workflow]';
 
   Map<String, dynamic> toJson() {
     final json = <String, dynamic>{};
@@ -117,6 +122,7 @@ class QueuesResponseLegacyDto {
       json[r'duplicateDetection'] = this.duplicateDetection;
       json[r'faceDetection'] = this.faceDetection;
       json[r'facialRecognition'] = this.facialRecognition;
+      json[r'integrityCheck'] = this.integrityCheck;
       json[r'library'] = this.library_;
       json[r'metadataExtraction'] = this.metadataExtraction;
       json[r'migration'] = this.migration;
@@ -146,6 +152,7 @@ class QueuesResponseLegacyDto {
         duplicateDetection: QueueResponseLegacyDto.fromJson(json[r'duplicateDetection'])!,
         faceDetection: QueueResponseLegacyDto.fromJson(json[r'faceDetection'])!,
         facialRecognition: QueueResponseLegacyDto.fromJson(json[r'facialRecognition'])!,
+        integrityCheck: QueueResponseLegacyDto.fromJson(json[r'integrityCheck'])!,
         library_: QueueResponseLegacyDto.fromJson(json[r'library'])!,
         metadataExtraction: QueueResponseLegacyDto.fromJson(json[r'metadataExtraction'])!,
         migration: QueueResponseLegacyDto.fromJson(json[r'migration'])!,
@@ -210,6 +217,7 @@ class QueuesResponseLegacyDto {
     'duplicateDetection',
     'faceDetection',
     'facialRecognition',
+    'integrityCheck',
     'library',
     'metadataExtraction',
     'migration',

mobile/openapi/lib/model/system_config_job_dto.dart

@@ -15,6 +15,7 @@ class SystemConfigJobDto {
   SystemConfigJobDto({
     required this.backgroundTask,
     required this.faceDetection,
+    required this.integrityCheck,
     required this.library_,
     required this.metadataExtraction,
     required this.migration,
@@ -32,6 +33,8 @@ class SystemConfigJobDto {
 
   JobSettingsDto faceDetection;
 
+  JobSettingsDto integrityCheck;
+
   JobSettingsDto library_;
 
   JobSettingsDto metadataExtraction;
@@ -58,6 +61,7 @@ class SystemConfigJobDto {
   bool operator ==(Object other) => identical(this, other) || other is SystemConfigJobDto &&
     other.backgroundTask == backgroundTask &&
     other.faceDetection == faceDetection &&
+    other.integrityCheck == integrityCheck &&
     other.library_ == library_ &&
     other.metadataExtraction == metadataExtraction &&
     other.migration == migration &&
@@ -75,6 +79,7 @@ class SystemConfigJobDto {
     // ignore: unnecessary_parenthesis
     (backgroundTask.hashCode) +
     (faceDetection.hashCode) +
+    (integrityCheck.hashCode) +
     (library_.hashCode) +
     (metadataExtraction.hashCode) +
     (migration.hashCode) +
@@ -88,12 +93,13 @@ class SystemConfigJobDto {
     (workflow.hashCode);
 
   @override
-  String toString() => 'SystemConfigJobDto[backgroundTask=$backgroundTask, faceDetection=$faceDetection, library_=$library_, metadataExtraction=$metadataExtraction, migration=$migration, notifications=$notifications, ocr=$ocr, search=$search, sidecar=$sidecar, smartSearch=$smartSearch, thumbnailGeneration=$thumbnailGeneration, videoConversion=$videoConversion, workflow=$workflow]';
+  String toString() => 'SystemConfigJobDto[backgroundTask=$backgroundTask, faceDetection=$faceDetection, integrityCheck=$integrityCheck, library_=$library_, metadataExtraction=$metadataExtraction, migration=$migration, notifications=$notifications, ocr=$ocr, search=$search, sidecar=$sidecar, smartSearch=$smartSearch, thumbnailGeneration=$thumbnailGeneration, videoConversion=$videoConversion, workflow=$workflow]';
 
   Map<String, dynamic> toJson() {
     final json = <String, dynamic>{};
       json[r'backgroundTask'] = this.backgroundTask;
       json[r'faceDetection'] = this.faceDetection;
+      json[r'integrityCheck'] = this.integrityCheck;
       json[r'library'] = this.library_;
       json[r'metadataExtraction'] = this.metadataExtraction;
       json[r'migration'] = this.migration;
@@ -119,6 +125,7 @@ class SystemConfigJobDto {
       return SystemConfigJobDto(
         backgroundTask: JobSettingsDto.fromJson(json[r'backgroundTask'])!,
         faceDetection: JobSettingsDto.fromJson(json[r'faceDetection'])!,
+        integrityCheck: JobSettingsDto.fromJson(json[r'integrityCheck'])!,
         library_: JobSettingsDto.fromJson(json[r'library'])!,
         metadataExtraction: JobSettingsDto.fromJson(json[r'metadataExtraction'])!,
         migration: JobSettingsDto.fromJson(json[r'migration'])!,
@@ -179,6 +186,7 @@ class SystemConfigJobDto {
   static const requiredKeys = <String>{
     'backgroundTask',
     'faceDetection',
+    'integrityCheck',
     'library',
     'metadataExtraction',
     'migration',

open-api/immich-openapi-specs.json

@@ -18903,7 +18903,8 @@
           "notifications",
           "backupDatabase",
           "ocr",
-          "workflow"
+          "workflow",
+          "integrityCheck"
         ],
         "type": "string"
       },
@@ -19016,6 +19017,9 @@
           "facialRecognition": {
             "$ref": "#/components/schemas/QueueResponseLegacyDto"
           },
+          "integrityCheck": {
+            "$ref": "#/components/schemas/QueueResponseLegacyDto"
+          },
           "library": {
             "$ref": "#/components/schemas/QueueResponseLegacyDto"
           },
@@ -19059,6 +19063,7 @@
           "duplicateDetection",
           "faceDetection",
           "facialRecognition",
+          "integrityCheck",
           "library",
           "metadataExtraction",
           "migration",
@@ -21971,6 +21976,9 @@
           "faceDetection": {
             "$ref": "#/components/schemas/JobSettingsDto"
           },
+          "integrityCheck": {
+            "$ref": "#/components/schemas/JobSettingsDto"
+          },
           "library": {
             "$ref": "#/components/schemas/JobSettingsDto"
           },
@@ -22008,6 +22016,7 @@
         "required": [
           "backgroundTask",
           "faceDetection",
+          "integrityCheck",
           "library",
           "metadataExtraction",
           "migration",

open-api/typescript-sdk/src/fetch-client.ts

@@ -746,6 +746,7 @@ export type QueuesResponseLegacyDto = {
     duplicateDetection: QueueResponseLegacyDto;
     faceDetection: QueueResponseLegacyDto;
     facialRecognition: QueueResponseLegacyDto;
+    integrityCheck: QueueResponseLegacyDto;
     library: QueueResponseLegacyDto;
     metadataExtraction: QueueResponseLegacyDto;
     migration: QueueResponseLegacyDto;
@@ -1491,6 +1492,7 @@ export type JobSettingsDto = {
 export type SystemConfigJobDto = {
     backgroundTask: JobSettingsDto;
     faceDetection: JobSettingsDto;
+    integrityCheck: JobSettingsDto;
     library: JobSettingsDto;
     metadataExtraction: JobSettingsDto;
     migration: JobSettingsDto;
@@ -5506,7 +5508,8 @@ export enum QueueName {
     Notifications = "notifications",
     BackupDatabase = "backupDatabase",
     Ocr = "ocr",
-    Workflow = "workflow"
+    Workflow = "workflow",
+    IntegrityCheck = "integrityCheck"
 }
 export enum QueueCommand {
     Start = "start",

server/src/config.ts

@@ -268,6 +268,7 @@ export const defaults = Object.freeze<SystemConfig>({
     [QueueName.Notification]: { concurrency: 5 },
     [QueueName.Ocr]: { concurrency: 1 },
     [QueueName.Workflow]: { concurrency: 5 },
+    [QueueName.IntegrityCheck]: { concurrency: 1 },
   },
   logging: {
     enabled: true,

server/src/dtos/queue-legacy.dto.ts

@@ -66,6 +66,9 @@ export class QueuesResponseLegacyDto implements Record<QueueName, QueueResponseL
 
   @ApiProperty({ type: QueueResponseLegacyDto })
   [QueueName.Workflow]!: QueueResponseLegacyDto;
+
+  @ApiProperty({ type: QueueResponseLegacyDto })
+  [QueueName.IntegrityCheck]!: QueueResponseLegacyDto;
 }
 
 export const mapQueueLegacy = (response: QueueResponseDto): QueueResponseLegacyDto => {

server/src/dtos/system-config.dto.ts

@@ -267,6 +267,12 @@ class SystemConfigJobDto implements Record<ConcurrentQueueName, JobSettingsDto>
   @IsObject()
   @Type(() => JobSettingsDto)
   [QueueName.Workflow]!: JobSettingsDto;
+
+  @ApiProperty({ type: JobSettingsDto })
+  @ValidateNested()
+  @IsObject()
+  @Type(() => JobSettingsDto)
+  [QueueName.IntegrityCheck]!: JobSettingsDto;
 }
 
 class SystemConfigLibraryScanDto {

server/src/enum.ts

@@ -565,6 +565,7 @@ export enum QueueName {
   BackupDatabase = 'backupDatabase',
   Ocr = 'ocr',
   Workflow = 'workflow',
+  IntegrityCheck = 'integrityCheck',
 }
 
 export enum QueueJobStatus {

server/src/services/integrity.service.ts

@@ -145,7 +145,7 @@ export class IntegrityService extends BaseService {
     });
   }
 
-  @OnJob({ name: JobName.IntegrityOrphanedFilesQueueAll, queue: QueueName.BackgroundTask })
+  @OnJob({ name: JobName.IntegrityOrphanedFilesQueueAll, queue: QueueName.IntegrityCheck })
   async handleOrphanedFilesQueueAll({ refreshOnly }: IIntegrityJob = {}): Promise<JobStatus> {
     this.logger.log(`Checking for out of date orphaned file reports...`);
 
@@ -214,7 +214,7 @@ export class IntegrityService extends BaseService {
     return JobStatus.Success;
   }
 
-  @OnJob({ name: JobName.IntegrityOrphanedFiles, queue: QueueName.BackgroundTask })
+  @OnJob({ name: JobName.IntegrityOrphanedFiles, queue: QueueName.IntegrityCheck })
   async handleOrphanedFiles({ type, paths }: IIntegrityOrphanedFilesJob): Promise<JobStatus> {
     this.logger.log(`Processing batch of ${paths.length} files to check if they are orphaned.`);
 
@@ -248,7 +248,7 @@ export class IntegrityService extends BaseService {
     return JobStatus.Success;
   }
 
-  @OnJob({ name: JobName.IntegrityOrphanedFilesRefresh, queue: QueueName.BackgroundTask })
+  @OnJob({ name: JobName.IntegrityOrphanedFilesRefresh, queue: QueueName.IntegrityCheck })
   async handleOrphanedRefresh({ items }: IIntegrityPathWithReportJob): Promise<JobStatus> {
     this.logger.log(`Processing batch of ${items.length} reports to check if they are out of date.`);
 
@@ -270,7 +270,7 @@ export class IntegrityService extends BaseService {
     return JobStatus.Success;
   }
 
-  @OnJob({ name: JobName.IntegrityMissingFilesQueueAll, queue: QueueName.BackgroundTask })
+  @OnJob({ name: JobName.IntegrityMissingFilesQueueAll, queue: QueueName.IntegrityCheck })
   async handleMissingFilesQueueAll({ refreshOnly }: IIntegrityJob = {}): Promise<JobStatus> {
     if (refreshOnly) {
       this.logger.log(`Checking for out of date missing file reports...`);
@@ -314,7 +314,7 @@ export class IntegrityService extends BaseService {
     return JobStatus.Success;
   }
 
-  @OnJob({ name: JobName.IntegrityMissingFiles, queue: QueueName.BackgroundTask })
+  @OnJob({ name: JobName.IntegrityMissingFiles, queue: QueueName.IntegrityCheck })
   async handleMissingFiles({ items }: IIntegrityMissingFilesJob): Promise<JobStatus> {
     this.logger.log(`Processing batch of ${items.length} files to check if they are missing.`);
 
@@ -350,7 +350,7 @@ export class IntegrityService extends BaseService {
     return JobStatus.Success;
   }
 
-  @OnJob({ name: JobName.IntegrityMissingFilesRefresh, queue: QueueName.BackgroundTask })
+  @OnJob({ name: JobName.IntegrityMissingFilesRefresh, queue: QueueName.IntegrityCheck })
   async handleMissingRefresh({ items: paths }: IIntegrityPathWithReportJob): Promise<JobStatus> {
     this.logger.log(`Processing batch of ${paths.length} reports to check if they are out of date.`);
 
@@ -372,7 +372,7 @@ export class IntegrityService extends BaseService {
     return JobStatus.Success;
   }
 
-  @OnJob({ name: JobName.IntegrityChecksumFiles, queue: QueueName.BackgroundTask })
+  @OnJob({ name: JobName.IntegrityChecksumFiles, queue: QueueName.IntegrityCheck })
   async handleChecksumFiles({ refreshOnly }: IIntegrityJob = {}): Promise<JobStatus> {
     if (refreshOnly) {
       this.logger.log(`Checking for out of date checksum file reports...`);
@@ -507,7 +507,7 @@ export class IntegrityService extends BaseService {
     return JobStatus.Success;
   }
 
-  @OnJob({ name: JobName.IntegrityChecksumFilesRefresh, queue: QueueName.BackgroundTask })
+  @OnJob({ name: JobName.IntegrityChecksumFilesRefresh, queue: QueueName.IntegrityCheck })
   async handleChecksumRefresh({ items: paths }: IIntegrityPathWithChecksumJob): Promise<JobStatus> {
     this.logger.log(`Processing batch of ${paths.length} reports to check if they are out of date.`);
 
@@ -551,7 +551,7 @@ export class IntegrityService extends BaseService {
     return JobStatus.Success;
   }
 
-  @OnJob({ name: JobName.IntegrityReportDelete, queue: QueueName.BackgroundTask })
+  @OnJob({ name: JobName.IntegrityReportDelete, queue: QueueName.IntegrityCheck })
   async handleDeleteIntegrityReport({ type }: IIntegrityDeleteReportJob): Promise<JobStatus> {
     this.logger.log(`Deleting all entries for ${type ?? 'all types of'} integrity report`);