immich-app/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2025-12-21 09:15:44 +03:00
Code Issues Packages Projects Releases 18 Wiki Activity

refactor: use separate queue for integrity checks

Browse Source
This commit is contained in:
izzy
2025-12-17 14:37:43 +00:00
parent 6e7854b5bb
commit 8fdec465c5
11 changed files with 62 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
e2e/src/api/specs/maintenance.e2e-spec.ts
View File

@@ -69,7 +69,7 @@ describe('/admin/maintenance', () => {
name: ManualJobName.IntegrityChecksumMismatch,
});
await utils.waitForQueueFinish(admin.accessToken, QueueName.BackgroundTask);
await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck);
const { status, body } = await request(app)
.get('/admin/maintenance/integrity/summary')
@@ -93,7 +93,7 @@ describe('/admin/maintenance', () => {
name: ManualJobName.IntegrityOrphanFiles,
});
await utils.waitForQueueFinish(admin.accessToken, QueueName.BackgroundTask);
await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck);
const { status, body } = await request(app)
.get('/admin/maintenance/integrity/summary')
@@ -117,7 +117,7 @@ describe('/admin/maintenance', () => {
name: ManualJobName.IntegrityOrphanFilesRefresh,
});
await utils.waitForQueueFinish(admin.accessToken, QueueName.BackgroundTask);
await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck);
const { status, body } = await request(app)
.get('/admin/maintenance/integrity/summary')
@@ -139,7 +139,7 @@ describe('/admin/maintenance', () => {
name: ManualJobName.IntegrityMissingFiles,
});
await utils.waitForQueueFinish(admin.accessToken, QueueName.BackgroundTask);
await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck);
const { status, body } = await request(app)
.get('/admin/maintenance/integrity/summary')
@@ -160,7 +160,7 @@ describe('/admin/maintenance', () => {
name: ManualJobName.IntegrityMissingFilesRefresh,
});
await utils.waitForQueueFinish(admin.accessToken, QueueName.BackgroundTask);
await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck);
const { status, body } = await request(app)
.get('/admin/maintenance/integrity/summary')
@@ -183,7 +183,7 @@ describe('/admin/maintenance', () => {
name: ManualJobName.IntegrityChecksumMismatch,
});
await utils.waitForQueueFinish(admin.accessToken, QueueName.BackgroundTask);
await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck);
const { status, body } = await request(app)
.get('/admin/maintenance/integrity/summary')
@@ -203,7 +203,7 @@ describe('/admin/maintenance', () => {
name: ManualJobName.IntegrityChecksumMismatchRefresh,
});
await utils.waitForQueueFinish(admin.accessToken, QueueName.BackgroundTask);
await utils.waitForQueueFinish(admin.accessToken, QueueName.IntegrityCheck);
const { status, body } = await request(app)
.get('/admin/maintenance/integrity/summary')

3
mobile/openapi/lib/model/queue_name.dart generated
View File

@@ -40,6 +40,7 @@ class QueueName {
static const backupDatabase = QueueName._(r'backupDatabase');
static const ocr = QueueName._(r'ocr');
static const workflow = QueueName._(r'workflow');
static const integrityCheck = QueueName._(r'integrityCheck');
/// List of all possible values in this [enum][QueueName].
static const values = <QueueName>[
@@ -60,6 +61,7 @@ class QueueName {
backupDatabase,
ocr,
workflow,
integrityCheck,
];
static QueueName? fromJson(dynamic value) => QueueNameTypeTransformer().decode(value);
@@ -115,6 +117,7 @@ class QueueNameTypeTransformer {
case r'backupDatabase': return QueueName.backupDatabase;
case r'ocr': return QueueName.ocr;
case r'workflow': return QueueName.workflow;
case r'integrityCheck': return QueueName.integrityCheck;
default:
if (!allowNull) {
throw ArgumentError('Unknown enum value to decode: $data');

10
mobile/openapi/lib/model/queues_response_legacy_dto.dart generated
View File

@@ -18,6 +18,7 @@ class QueuesResponseLegacyDto {
required this.duplicateDetection,
required this.faceDetection,
required this.facialRecognition,
required this.integrityCheck,
required this.library_,
required this.metadataExtraction,
required this.migration,
@@ -42,6 +43,8 @@ class QueuesResponseLegacyDto {
QueueResponseLegacyDto facialRecognition;
QueueResponseLegacyDto integrityCheck;
QueueResponseLegacyDto library_;
QueueResponseLegacyDto metadataExtraction;
@@ -73,6 +76,7 @@ class QueuesResponseLegacyDto {
other.duplicateDetection == duplicateDetection &&
other.faceDetection == faceDetection &&
other.facialRecognition == facialRecognition &&
other.integrityCheck == integrityCheck &&
other.library_ == library_ &&
other.metadataExtraction == metadataExtraction &&
other.migration == migration &&
@@ -94,6 +98,7 @@ class QueuesResponseLegacyDto {
(duplicateDetection.hashCode) +
(faceDetection.hashCode) +
(facialRecognition.hashCode) +
(integrityCheck.hashCode) +
(library_.hashCode) +
(metadataExtraction.hashCode) +
(migration.hashCode) +
@@ -108,7 +113,7 @@ class QueuesResponseLegacyDto {
(workflow.hashCode);
@override
String toString() => 'QueuesResponseLegacyDto[backgroundTask=$backgroundTask, backupDatabase=$backupDatabase, duplicateDetection=$duplicateDetection, faceDetection=$faceDetection, facialRecognition=$facialRecognition, library_=$library_, metadataExtraction=$metadataExtraction, migration=$migration, notifications=$notifications, ocr=$ocr, search=$search, sidecar=$sidecar, smartSearch=$smartSearch, storageTemplateMigration=$storageTemplateMigration, thumbnailGeneration=$thumbnailGeneration, videoConversion=$videoConversion, workflow=$workflow]';
String toString() => 'QueuesResponseLegacyDto[backgroundTask=$backgroundTask, backupDatabase=$backupDatabase, duplicateDetection=$duplicateDetection, faceDetection=$faceDetection, facialRecognition=$facialRecognition, integrityCheck=$integrityCheck, library_=$library_, metadataExtraction=$metadataExtraction, migration=$migration, notifications=$notifications, ocr=$ocr, search=$search, sidecar=$sidecar, smartSearch=$smartSearch, storageTemplateMigration=$storageTemplateMigration, thumbnailGeneration=$thumbnailGeneration, videoConversion=$videoConversion, workflow=$workflow]';
Map<String, dynamic> toJson() {
final json = <String, dynamic>{};
@@ -117,6 +122,7 @@ class QueuesResponseLegacyDto {
json[r'duplicateDetection'] = this.duplicateDetection;
json[r'faceDetection'] = this.faceDetection;
json[r'facialRecognition'] = this.facialRecognition;
json[r'integrityCheck'] = this.integrityCheck;
json[r'library'] = this.library_;
json[r'metadataExtraction'] = this.metadataExtraction;
json[r'migration'] = this.migration;
@@ -146,6 +152,7 @@ class QueuesResponseLegacyDto {
duplicateDetection: QueueResponseLegacyDto.fromJson(json[r'duplicateDetection'])!,
faceDetection: QueueResponseLegacyDto.fromJson(json[r'faceDetection'])!,
facialRecognition: QueueResponseLegacyDto.fromJson(json[r'facialRecognition'])!,
integrityCheck: QueueResponseLegacyDto.fromJson(json[r'integrityCheck'])!,
library_: QueueResponseLegacyDto.fromJson(json[r'library'])!,
metadataExtraction: QueueResponseLegacyDto.fromJson(json[r'metadataExtraction'])!,
migration: QueueResponseLegacyDto.fromJson(json[r'migration'])!,
@@ -210,6 +217,7 @@ class QueuesResponseLegacyDto {
'duplicateDetection',
'faceDetection',
'facialRecognition',
'integrityCheck',
'library',
'metadataExtraction',
'migration',

10
mobile/openapi/lib/model/system_config_job_dto.dart generated
View File

@@ -15,6 +15,7 @@ class SystemConfigJobDto {
SystemConfigJobDto({
required this.backgroundTask,
required this.faceDetection,
required this.integrityCheck,
required this.library_,
required this.metadataExtraction,
required this.migration,
@@ -32,6 +33,8 @@ class SystemConfigJobDto {
JobSettingsDto faceDetection;
JobSettingsDto integrityCheck;
JobSettingsDto library_;
JobSettingsDto metadataExtraction;
@@ -58,6 +61,7 @@ class SystemConfigJobDto {
bool operator ==(Object other) => identical(this, other) || other is SystemConfigJobDto &&
other.backgroundTask == backgroundTask &&
other.faceDetection == faceDetection &&
other.integrityCheck == integrityCheck &&
other.library_ == library_ &&
other.metadataExtraction == metadataExtraction &&
other.migration == migration &&
@@ -75,6 +79,7 @@ class SystemConfigJobDto {
// ignore: unnecessary_parenthesis
(backgroundTask.hashCode) +
(faceDetection.hashCode) +
(integrityCheck.hashCode) +
(library_.hashCode) +
(metadataExtraction.hashCode) +
(migration.hashCode) +
@@ -88,12 +93,13 @@ class SystemConfigJobDto {
(workflow.hashCode);
@override
String toString() => 'SystemConfigJobDto[backgroundTask=$backgroundTask, faceDetection=$faceDetection, library_=$library_, metadataExtraction=$metadataExtraction, migration=$migration, notifications=$notifications, ocr=$ocr, search=$search, sidecar=$sidecar, smartSearch=$smartSearch, thumbnailGeneration=$thumbnailGeneration, videoConversion=$videoConversion, workflow=$workflow]';
String toString() => 'SystemConfigJobDto[backgroundTask=$backgroundTask, faceDetection=$faceDetection, integrityCheck=$integrityCheck, library_=$library_, metadataExtraction=$metadataExtraction, migration=$migration, notifications=$notifications, ocr=$ocr, search=$search, sidecar=$sidecar, smartSearch=$smartSearch, thumbnailGeneration=$thumbnailGeneration, videoConversion=$videoConversion, workflow=$workflow]';
Map<String, dynamic> toJson() {
final json = <String, dynamic>{};
json[r'backgroundTask'] = this.backgroundTask;
json[r'faceDetection'] = this.faceDetection;
json[r'integrityCheck'] = this.integrityCheck;
json[r'library'] = this.library_;
json[r'metadataExtraction'] = this.metadataExtraction;
json[r'migration'] = this.migration;
@@ -119,6 +125,7 @@ class SystemConfigJobDto {
return SystemConfigJobDto(
backgroundTask: JobSettingsDto.fromJson(json[r'backgroundTask'])!,
faceDetection: JobSettingsDto.fromJson(json[r'faceDetection'])!,
integrityCheck: JobSettingsDto.fromJson(json[r'integrityCheck'])!,
library_: JobSettingsDto.fromJson(json[r'library'])!,
metadataExtraction: JobSettingsDto.fromJson(json[r'metadataExtraction'])!,
migration: JobSettingsDto.fromJson(json[r'migration'])!,
@@ -179,6 +186,7 @@ class SystemConfigJobDto {
static const requiredKeys = <String>{
'backgroundTask',
'faceDetection',
'integrityCheck',
'library',
'metadataExtraction',
'migration',

11
open-api/immich-openapi-specs.json
View File

@@ -18903,7 +18903,8 @@
"notifications",
"backupDatabase",
"ocr",
"workflow"
"workflow",
"integrityCheck"
],
"type": "string"
},
@@ -19016,6 +19017,9 @@
"facialRecognition": {
"$ref": "#/components/schemas/QueueResponseLegacyDto"
},
"integrityCheck": {
"$ref": "#/components/schemas/QueueResponseLegacyDto"
},
"library": {
"$ref": "#/components/schemas/QueueResponseLegacyDto"
},
@@ -19059,6 +19063,7 @@
"duplicateDetection",
"faceDetection",
"facialRecognition",
"integrityCheck",
"library",
"metadataExtraction",
"migration",
@@ -21971,6 +21976,9 @@
"faceDetection": {
"$ref": "#/components/schemas/JobSettingsDto"
},
"integrityCheck": {
"$ref": "#/components/schemas/JobSettingsDto"
},
"library": {
"$ref": "#/components/schemas/JobSettingsDto"
},
@@ -22008,6 +22016,7 @@
"required": [
"backgroundTask",
"faceDetection",
"integrityCheck",
"library",
"metadataExtraction",
"migration",

5
open-api/typescript-sdk/src/fetch-client.ts
View File

@@ -746,6 +746,7 @@ export type QueuesResponseLegacyDto = {
duplicateDetection: QueueResponseLegacyDto;
faceDetection: QueueResponseLegacyDto;
facialRecognition: QueueResponseLegacyDto;
integrityCheck: QueueResponseLegacyDto;
library: QueueResponseLegacyDto;
metadataExtraction: QueueResponseLegacyDto;
migration: QueueResponseLegacyDto;
@@ -1491,6 +1492,7 @@ export type JobSettingsDto = {
export type SystemConfigJobDto = {
backgroundTask: JobSettingsDto;
faceDetection: JobSettingsDto;
integrityCheck: JobSettingsDto;
library: JobSettingsDto;
metadataExtraction: JobSettingsDto;
migration: JobSettingsDto;
@@ -5506,7 +5508,8 @@ export enum QueueName {
Notifications = "notifications",
BackupDatabase = "backupDatabase",
Ocr = "ocr",
Workflow = "workflow"
Workflow = "workflow",
IntegrityCheck = "integrityCheck"
}
export enum QueueCommand {
Start = "start",

1
server/src/config.ts
View File

@@ -268,6 +268,7 @@ export const defaults = Object.freeze<SystemConfig>({
[QueueName.Notification]: { concurrency: 5 },
[QueueName.Ocr]: { concurrency: 1 },
[QueueName.Workflow]: { concurrency: 5 },
[QueueName.IntegrityCheck]: { concurrency: 1 },
},
logging: {
enabled: true,

3
server/src/dtos/queue-legacy.dto.ts
View File

@@ -66,6 +66,9 @@ export class QueuesResponseLegacyDto implements Record<QueueName, QueueResponseL
@ApiProperty({ type: QueueResponseLegacyDto })
[QueueName.Workflow]!: QueueResponseLegacyDto;
@ApiProperty({ type: QueueResponseLegacyDto })
[QueueName.IntegrityCheck]!: QueueResponseLegacyDto;
}
export const mapQueueLegacy = (response: QueueResponseDto): QueueResponseLegacyDto => {

6
server/src/dtos/system-config.dto.ts
View File

@@ -267,6 +267,12 @@ class SystemConfigJobDto implements Record<ConcurrentQueueName, JobSettingsDto>
@IsObject()
@Type(() => JobSettingsDto)
[QueueName.Workflow]!: JobSettingsDto;
@ApiProperty({ type: JobSettingsDto })
@ValidateNested()
@IsObject()
@Type(() => JobSettingsDto)
[QueueName.IntegrityCheck]!: JobSettingsDto;
}
class SystemConfigLibraryScanDto {

1
server/src/enum.ts
View File

@@ -565,6 +565,7 @@ export enum QueueName {
BackupDatabase = 'backupDatabase',
Ocr = 'ocr',
Workflow = 'workflow',
IntegrityCheck = 'integrityCheck',
}
export enum QueueJobStatus {

18
server/src/services/integrity.service.ts
View File

@@ -145,7 +145,7 @@ export class IntegrityService extends BaseService {
});
}
@OnJob({ name: JobName.IntegrityOrphanedFilesQueueAll, queue: QueueName.BackgroundTask })
@OnJob({ name: JobName.IntegrityOrphanedFilesQueueAll, queue: QueueName.IntegrityCheck })
async handleOrphanedFilesQueueAll({ refreshOnly }: IIntegrityJob = {}): Promise<JobStatus> {
this.logger.log(`Checking for out of date orphaned file reports...`);
@@ -214,7 +214,7 @@ export class IntegrityService extends BaseService {
return JobStatus.Success;
}
@OnJob({ name: JobName.IntegrityOrphanedFiles, queue: QueueName.BackgroundTask })
@OnJob({ name: JobName.IntegrityOrphanedFiles, queue: QueueName.IntegrityCheck })
async handleOrphanedFiles({ type, paths }: IIntegrityOrphanedFilesJob): Promise<JobStatus> {
this.logger.log(`Processing batch of ${paths.length} files to check if they are orphaned.`);
@@ -248,7 +248,7 @@ export class IntegrityService extends BaseService {
return JobStatus.Success;
}
@OnJob({ name: JobName.IntegrityOrphanedFilesRefresh, queue: QueueName.BackgroundTask })
@OnJob({ name: JobName.IntegrityOrphanedFilesRefresh, queue: QueueName.IntegrityCheck })
async handleOrphanedRefresh({ items }: IIntegrityPathWithReportJob): Promise<JobStatus> {
this.logger.log(`Processing batch of ${items.length} reports to check if they are out of date.`);
@@ -270,7 +270,7 @@ export class IntegrityService extends BaseService {
return JobStatus.Success;
}
@OnJob({ name: JobName.IntegrityMissingFilesQueueAll, queue: QueueName.BackgroundTask })
@OnJob({ name: JobName.IntegrityMissingFilesQueueAll, queue: QueueName.IntegrityCheck })
async handleMissingFilesQueueAll({ refreshOnly }: IIntegrityJob = {}): Promise<JobStatus> {
if (refreshOnly) {
this.logger.log(`Checking for out of date missing file reports...`);
@@ -314,7 +314,7 @@ export class IntegrityService extends BaseService {
return JobStatus.Success;
}
@OnJob({ name: JobName.IntegrityMissingFiles, queue: QueueName.BackgroundTask })
@OnJob({ name: JobName.IntegrityMissingFiles, queue: QueueName.IntegrityCheck })
async handleMissingFiles({ items }: IIntegrityMissingFilesJob): Promise<JobStatus> {
this.logger.log(`Processing batch of ${items.length} files to check if they are missing.`);
@@ -350,7 +350,7 @@ export class IntegrityService extends BaseService {
return JobStatus.Success;
}
@OnJob({ name: JobName.IntegrityMissingFilesRefresh, queue: QueueName.BackgroundTask })
@OnJob({ name: JobName.IntegrityMissingFilesRefresh, queue: QueueName.IntegrityCheck })
async handleMissingRefresh({ items: paths }: IIntegrityPathWithReportJob): Promise<JobStatus> {
this.logger.log(`Processing batch of ${paths.length} reports to check if they are out of date.`);
@@ -372,7 +372,7 @@ export class IntegrityService extends BaseService {
return JobStatus.Success;
}
@OnJob({ name: JobName.IntegrityChecksumFiles, queue: QueueName.BackgroundTask })
@OnJob({ name: JobName.IntegrityChecksumFiles, queue: QueueName.IntegrityCheck })
async handleChecksumFiles({ refreshOnly }: IIntegrityJob = {}): Promise<JobStatus> {
if (refreshOnly) {
this.logger.log(`Checking for out of date checksum file reports...`);
@@ -507,7 +507,7 @@ export class IntegrityService extends BaseService {
return JobStatus.Success;
}
@OnJob({ name: JobName.IntegrityChecksumFilesRefresh, queue: QueueName.BackgroundTask })
@OnJob({ name: JobName.IntegrityChecksumFilesRefresh, queue: QueueName.IntegrityCheck })
async handleChecksumRefresh({ items: paths }: IIntegrityPathWithChecksumJob): Promise<JobStatus> {
this.logger.log(`Processing batch of ${paths.length} reports to check if they are out of date.`);
@@ -551,7 +551,7 @@ export class IntegrityService extends BaseService {
return JobStatus.Success;
}
@OnJob({ name: JobName.IntegrityReportDelete, queue: QueueName.BackgroundTask })
@OnJob({ name: JobName.IntegrityReportDelete, queue: QueueName.IntegrityCheck })
async handleDeleteIntegrityReport({ type }: IIntegrityDeleteReportJob): Promise<JobStatus> {
this.logger.log(`Deleting all entries for ${type ?? 'all types of'} integrity report`);