immich-app/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2025-12-22 17:24:56 +03:00
Code Issues Packages Projects Releases 18 Wiki Activity

feat: manually trigger integrity jobs

Browse Source 
feat: update summary after job runs
This commit is contained in:
izzy
2025-11-28 15:27:12 +00:00
parent 13e9cf0ed9
commit 2779fce7d0
10 changed files with 224 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
i18n/en.json
View File

@@ -183,8 +183,14 @@
"maintenance_start_error": "Failed to start maintenance mode.",
"maintenance_integrity_report": "Integrity Report",
"maintenance_integrity_orphan_file": "Orphan Files",
"maintenance_integrity_orphan_file_job": "Check for orphaned files",
"maintenance_integrity_orphan_file_refresh_job": "Refresh orphan file reports",
"maintenance_integrity_missing_file": "Missing Files",
"maintenance_integrity_missing_file_job": "Check for missing files",
"maintenance_integrity_missing_file_refresh_job": "Refresh missing file reports",
"maintenance_integrity_checksum_mismatch": "Checksum Mismatch",
"maintenance_integrity_checksum_mismatch_job": "Check for checksum mismatches",
"maintenance_integrity_checksum_mismatch_refresh_job": "Refresh checksum mismatch reports",
"manage_concurrency": "Manage Concurrency",
"manage_log_settings": "Manage log settings",
"map_dark_style": "Dark style",

18
mobile/openapi/lib/model/manual_job_name.dart generated
View File

@@ -29,6 +29,12 @@ class ManualJobName {
static const memoryCleanup = ManualJobName._(r'memory-cleanup');
static const memoryCreate = ManualJobName._(r'memory-create');
static const backupDatabase = ManualJobName._(r'backup-database');
static const integrityMissingFiles = ManualJobName._(r'integrity-missing-files');
static const integrityOrphanFiles = ManualJobName._(r'integrity-orphan-files');
static const integrityChecksumMismatch = ManualJobName._(r'integrity-checksum-mismatch');
static const integrityMissingFilesRefresh = ManualJobName._(r'integrity-missing-files-refresh');
static const integrityOrphanFilesRefresh = ManualJobName._(r'integrity-orphan-files-refresh');
static const integrityChecksumMismatchRefresh = ManualJobName._(r'integrity-checksum-mismatch-refresh');
/// List of all possible values in this [enum][ManualJobName].
static const values = <ManualJobName>[
@@ -38,6 +44,12 @@ class ManualJobName {
memoryCleanup,
memoryCreate,
backupDatabase,
integrityMissingFiles,
integrityOrphanFiles,
integrityChecksumMismatch,
integrityMissingFilesRefresh,
integrityOrphanFilesRefresh,
integrityChecksumMismatchRefresh,
];
static ManualJobName? fromJson(dynamic value) => ManualJobNameTypeTransformer().decode(value);
@@ -82,6 +94,12 @@ class ManualJobNameTypeTransformer {
case r'memory-cleanup': return ManualJobName.memoryCleanup;
case r'memory-create': return ManualJobName.memoryCreate;
case r'backup-database': return ManualJobName.backupDatabase;
case r'integrity-missing-files': return ManualJobName.integrityMissingFiles;
case r'integrity-orphan-files': return ManualJobName.integrityOrphanFiles;
case r'integrity-checksum-mismatch': return ManualJobName.integrityChecksumMismatch;
case r'integrity-missing-files-refresh': return ManualJobName.integrityMissingFilesRefresh;
case r'integrity-orphan-files-refresh': return ManualJobName.integrityOrphanFilesRefresh;
case r'integrity-checksum-mismatch-refresh': return ManualJobName.integrityChecksumMismatchRefresh;
default:
if (!allowNull) {
throw ArgumentError('Unknown enum value to decode: $data');

8
open-api/immich-openapi-specs.json
View File

@@ -17118,7 +17118,13 @@
"user-cleanup",
"memory-cleanup",
"memory-create",
"backup-database"
"backup-database",
"integrity-missing-files",
"integrity-orphan-files",
"integrity-checksum-mismatch",
"integrity-missing-files-refresh",
"integrity-orphan-files-refresh",
"integrity-checksum-mismatch-refresh"
],
"type": "string"
},

8
open-api/typescript-sdk/src/fetch-client.ts
View File

@@ -5441,7 +5441,13 @@ export enum ManualJobName {
UserCleanup = "user-cleanup",
MemoryCleanup = "memory-cleanup",
MemoryCreate = "memory-create",
BackupDatabase = "backup-database"
BackupDatabase = "backup-database",
IntegrityMissingFiles = "integrity-missing-files",
IntegrityOrphanFiles = "integrity-orphan-files",
IntegrityChecksumMismatch = "integrity-checksum-mismatch",
IntegrityMissingFilesRefresh = "integrity-missing-files-refresh",
IntegrityOrphanFilesRefresh = "integrity-orphan-files-refresh",
IntegrityChecksumMismatchRefresh = "integrity-checksum-mismatch-refresh"
}
export enum QueueName {
ThumbnailGeneration = "thumbnailGeneration",

18
server/src/enum.ts
View File

@@ -345,6 +345,12 @@ export enum SourceType {
Manual = 'manual',
}
export enum IntegrityReportType {
OrphanFile = 'orphan_file',
MissingFile = 'missing_file',
ChecksumFail = 'checksum_mismatch',
}
export enum ManualJobName {
PersonCleanup = 'person-cleanup',
TagCleanup = 'tag-cleanup',
@@ -352,6 +358,12 @@ export enum ManualJobName {
MemoryCleanup = 'memory-cleanup',
MemoryCreate = 'memory-create',
BackupDatabase = 'backup-database',
IntegrityMissingFiles = `integrity-missing-files`,
IntegrityOrphanFiles = `integrity-orphan-files`,
IntegrityChecksumFiles = `integrity-checksum-mismatch`,
IntegrityMissingFilesRefresh = `integrity-missing-files-refresh`,
IntegrityOrphanFilesRefresh = `integrity-orphan-files-refresh`,
IntegrityChecksumFilesRefresh = `integrity-checksum-mismatch-refresh`,
}
export enum AssetPathType {
@@ -482,12 +494,6 @@ export enum CacheControl {
None = 'none',
}
export enum IntegrityReportType {
OrphanFile = 'orphan_file',
MissingFile = 'missing_file',
ChecksumFail = 'checksum_mismatch',
}
export enum ImmichEnvironment {
Development = 'development',
Testing = 'testing',

31
server/src/services/integrity.service.ts
View File

@@ -19,7 +19,7 @@ import {
} from 'src/enum';
import { ArgOf } from 'src/repositories/event.repository';
import { BaseService } from 'src/services/base.service';
import { IIntegrityOrphanedFilesJob, IIntegrityPathWithReportJob } from 'src/types';
import { IIntegrityJob, IIntegrityOrphanedFilesJob, IIntegrityPathWithReportJob } from 'src/types';
import { handlePromiseError } from 'src/utils/misc';
async function* chunk<T>(generator: AsyncIterableIterator<T>, n: number) {
@@ -130,7 +130,7 @@ export class IntegrityService extends BaseService {
}
@OnJob({ name: JobName.IntegrityOrphanedFilesQueueAll, queue: QueueName.BackgroundTask })
async handleOrphanedFilesQueueAll(): Promise<JobStatus> {
async handleOrphanedFilesQueueAll({ refreshOnly }: IIntegrityJob = {}): Promise<JobStatus> {
this.logger.log(`Checking for out of date orphaned file reports...`);
const reports = this.assetJobRepository.streamIntegrityReports(IntegrityReportType.OrphanFile);
@@ -148,6 +148,11 @@ export class IntegrityService extends BaseService {
this.logger.log(`Queued report check of ${batchReports.length} report(s) (${total} so far)`);
}
if (refreshOnly) {
this.logger.log('Refresh complete.');
return JobStatus.Success;
}
this.logger.log(`Scanning for orphaned files...`);
const assetPaths = this.storageRepository.walk({
@@ -232,8 +237,8 @@ export class IntegrityService extends BaseService {
const results = await Promise.all(
paths.map(({ reportId, path }) =>
stat(path)
.then(() => reportId)
.catch(() => void 0),
.then(() => void 0)
.catch(() => reportId),
),
);
@@ -243,12 +248,18 @@ export class IntegrityService extends BaseService {
await this.integrityReportRepository.deleteByIds(reportIds);
}
this.logger.log(`Processed ${paths.length} and found ${reportIds.length} orphaned file(s).`);
this.logger.log(`Processed ${paths.length} paths and found ${reportIds.length} report(s) out of date.`);
return JobStatus.Success;
}
@OnJob({ name: JobName.IntegrityMissingFilesQueueAll, queue: QueueName.BackgroundTask })
async handleMissingFilesQueueAll(): Promise<JobStatus> {
async handleMissingFilesQueueAll({ refreshOnly }: IIntegrityJob = {}): Promise<JobStatus> {
if (refreshOnly) {
// TODO
this.logger.log('Refresh complete.');
return JobStatus.Success;
}
this.logger.log(`Scanning for missing files...`);
const assetPaths = this.assetJobRepository.streamAssetPaths();
@@ -304,7 +315,13 @@ export class IntegrityService extends BaseService {
}
@OnJob({ name: JobName.IntegrityChecksumFiles, queue: QueueName.BackgroundTask })
async handleChecksumFiles(): Promise<JobStatus> {
async handleChecksumFiles({ refreshOnly }: IIntegrityJob = {}): Promise<JobStatus> {
if (refreshOnly) {
// TODO
this.logger.log('Refresh complete.');
return JobStatus.Success;
}
const timeLimit = 60 * 60 * 1000; // 1000;
const percentageLimit = 1; // 0.25;

24
server/src/services/job.service.ts
View File

@@ -34,6 +34,30 @@ const asJobItem = (dto: JobCreateDto): JobItem => {
return { name: JobName.DatabaseBackup };
}
case ManualJobName.IntegrityMissingFiles: {
return { name: JobName.IntegrityMissingFilesQueueAll };
}
case ManualJobName.IntegrityOrphanFiles: {
return { name: JobName.IntegrityOrphanedFilesQueueAll };
}
case ManualJobName.IntegrityChecksumFiles: {
return { name: JobName.IntegrityChecksumFiles };
}
case ManualJobName.IntegrityMissingFilesRefresh: {
return { name: JobName.IntegrityMissingFilesQueueAll, data: { refreshOnly: true } };
}
case ManualJobName.IntegrityOrphanFilesRefresh: {
return { name: JobName.IntegrityOrphanedFilesQueueAll, data: { refreshOnly: true } };
}
case ManualJobName.IntegrityChecksumFilesRefresh: {
return { name: JobName.IntegrityChecksumFiles, data: { refreshOnly: true } };
}
default: {
throw new BadRequestException('Invalid job name');
}

10
server/src/types.ts
View File

@@ -282,6 +282,10 @@ export interface IWorkflowJob<T extends PluginTriggerType = PluginTriggerType> {
event: WorkflowData[T];
}
export interface IIntegrityJob {
refreshOnly?: boolean;
}
export interface IIntegrityOrphanedFilesJob {
type: 'asset' | 'asset_file';
paths: string[];
@@ -403,12 +407,12 @@ export type JobItem =
| { name: JobName.WorkflowRun; data: IWorkflowJob }
// Integrity
| { name: JobName.IntegrityOrphanedFilesQueueAll; data: IBaseJob }
| { name: JobName.IntegrityOrphanedFilesQueueAll; data?: IIntegrityJob }
| { name: JobName.IntegrityOrphanedFiles; data: IIntegrityOrphanedFilesJob }
| { name: JobName.IntegrityOrphanedCheckReports; data: IIntegrityPathWithReportJob }
| { name: JobName.IntegrityMissingFilesQueueAll; data: IBaseJob }
| { name: JobName.IntegrityMissingFilesQueueAll; data?: IIntegrityJob }
| { name: JobName.IntegrityMissingFiles; data: IIntegrityPathWithReportJob }
| { name: JobName.IntegrityChecksumFiles; data: IBaseJob };
| { name: JobName.IntegrityChecksumFiles; data?: IIntegrityJob };
export type VectorExtension = (typeof VECTOR_EXTENSIONS)[number];

24
web/src/lib/modals/JobCreateModal.svelte
View File

@@ -16,6 +16,30 @@
{ title: $t('admin.memory_cleanup_job'), value: ManualJobName.MemoryCleanup },
{ title: $t('admin.memory_generate_job'), value: ManualJobName.MemoryCreate },
{ title: $t('admin.backup_database'), value: ManualJobName.BackupDatabase },
{
title: $t('admin.maintenance_integrity_missing_file_job'),
value: ManualJobName.IntegrityMissingFiles,
},
{
title: $t('admin.maintenance_integrity_orphan_file_job'),
value: ManualJobName.IntegrityOrphanFiles,
},
{
title: $t('admin.maintenance_integrity_checksum_mismatch_job'),
value: ManualJobName.IntegrityChecksumMismatch,
},
{
title: $t('admin.maintenance_integrity_missing_file_refresh_job'),
value: ManualJobName.IntegrityMissingFilesRefresh,
},
{
title: $t('admin.maintenance_integrity_orphan_file_refresh_job'),
value: ManualJobName.IntegrityOrphanFilesRefresh,
},
{
title: $t('admin.maintenance_integrity_checksum_mismatch_refresh_job'),
value: ManualJobName.IntegrityChecksumMismatchRefresh,
},
].map(({ value, title }) => ({ id: value, label: title, value }));
let selectedJob: ComboBoxOption | undefined = $state(undefined);

104
web/src/routes/admin/maintenance/+page.svelte
View File

@@ -2,10 +2,22 @@
import AdminPageLayout from '$lib/components/layouts/AdminPageLayout.svelte';
import ServerStatisticsCard from '$lib/components/server-statistics/ServerStatisticsCard.svelte';
import { AppRoute } from '$lib/constants';
import { asyncTimeout } from '$lib/utils';
import { handleError } from '$lib/utils/handle-error';
import { MaintenanceAction, setMaintenanceMode } from '@immich/sdk';
import { Button, HStack, Text } from '@immich/ui';
import {
createJob,
getIntegrityReportSummary,
getQueuesLegacy,
IntegrityReportType,
MaintenanceAction,
ManualJobName,
setMaintenanceMode,
type MaintenanceIntegrityReportSummaryResponseDto,
type QueuesResponseLegacyDto,
} from '@immich/sdk';
import { Button, HStack, Text, toastManager } from '@immich/ui';
import { mdiProgressWrench } from '@mdi/js';
import { onDestroy, onMount } from 'svelte';
import { t } from 'svelte-i18n';
import type { PageData } from './$types';
@@ -15,6 +27,14 @@
let { data }: Props = $props();
let integrityReport: MaintenanceIntegrityReportSummaryResponseDto | undefined = $state(data.integrityReport);
const TYPES: IntegrityReportType[] = [
IntegrityReportType.OrphanFile,
IntegrityReportType.MissingFile,
IntegrityReportType.ChecksumMismatch,
];
async function switchToMaintenance() {
try {
await setMaintenanceMode({
@@ -26,6 +46,56 @@
handleError(error, $t('admin.maintenance_start_error'));
}
}
let jobs: QueuesResponseLegacyDto | undefined = $state();
let expectingUpdate: boolean = $state(false);
async function runJob(reportType: IntegrityReportType, refreshOnly?: boolean) {
let name: ManualJobName;
switch (reportType) {
case IntegrityReportType.OrphanFile: {
name = refreshOnly ? ManualJobName.IntegrityOrphanFilesRefresh : ManualJobName.IntegrityOrphanFiles;
break;
}
case IntegrityReportType.MissingFile: {
name = refreshOnly ? ManualJobName.IntegrityMissingFilesRefresh : ManualJobName.IntegrityMissingFiles;
break;
}
case IntegrityReportType.ChecksumMismatch: {
name = refreshOnly ? ManualJobName.IntegrityChecksumMismatchRefresh : ManualJobName.IntegrityChecksumMismatch;
break;
}
}
try {
await createJob({ jobCreateDto: { name } });
if (jobs) {
expectingUpdate = true;
jobs.backgroundTask.queueStatus.isActive = true;
}
toastManager.success($t('admin.job_created'));
} catch (error) {
handleError(error, $t('errors.unable_to_submit_job'));
}
}
let running = true;
onMount(async () => {
while (running) {
jobs = await getQueuesLegacy();
if (jobs.backgroundTask.queueStatus.isActive) {
expectingUpdate = true;
} else if (expectingUpdate) {
integrityReport = await getIntegrityReportSummary();
}
await asyncTimeout(5000);
}
});
onDestroy(() => {
running = false;
});
</script>
<AdminPageLayout breadcrumbs={[{ title: data.meta.title }]}>
@@ -48,17 +118,33 @@
<p class="text-sm dark:text-immich-dark-fg uppercase">{$t('admin.maintenance_integrity_report')}</p>
<div class="mt-5 hidden justify-between lg:flex gap-4">
{#each ['orphan_file', 'missing_file', 'checksum_mismatch'] as const as reportType (reportType)}
{#each TYPES as reportType (reportType)}
<ServerStatisticsCard
title={$t(`admin.maintenance_integrity_${reportType}`)}
value={data.integrityReport[reportType]}
value={integrityReport[reportType]}
>
{#snippet footer()}
<Button
href={`${AppRoute.ADMIN_MAINTENANCE_INTEGRITY_REPORT + reportType}`}
size="tiny"
class="self-end mt-1">View Report</Button
>
<HStack gap={1} class="justify-end">
<Button
onclick={() => runJob(reportType)}
size="tiny"
variant="ghost"
class="self-end mt-1"
disabled={jobs?.backgroundTask.queueStatus.isActive}>Check All</Button
>
<Button
onclick={() => runJob(reportType, true)}
size="tiny"
variant="ghost"
class="self-end mt-1"
disabled={jobs?.backgroundTask.queueStatus.isActive}>Refresh</Button
>
<Button
href={`${AppRoute.ADMIN_MAINTENANCE_INTEGRITY_REPORT + reportType}`}
size="tiny"
class="self-end mt-1">View</Button
>
</HStack>
{/snippet}
</ServerStatisticsCard>
{/each}