feat: nightly tasks (#19879)

server/src/services/api.service.ts

@@ -1,5 +1,5 @@
 import { Injectable } from '@nestjs/common';
-import { Cron, CronExpression, Interval } from '@nestjs/schedule';
+import { Interval } from '@nestjs/schedule';
 import { NextFunction, Request, Response } from 'express';
 import { readFileSync } from 'node:fs';
 import sanitizeHtml from 'sanitize-html';
@@ -54,11 +54,6 @@ export class ApiService {
     await this.versionService.handleQueueVersionCheck();
   }
 
-  @Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)
-  async onNightlyJob() {
-    await this.jobService.handleNightlyJobs();
-  }
-
   ssr(excludePaths: string[]) {
     const { resourcePaths } = this.configRepository.getEnv();
 

server/src/services/job.service.spec.ts

@@ -41,12 +41,12 @@ describe(JobService.name, () => {
         { name: JobName.USER_DELETE_CHECK },
         { name: JobName.PERSON_CLEANUP },
         { name: JobName.MEMORIES_CLEANUP },
-        { name: JobName.MEMORIES_CREATE },
-        { name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } },
-        { name: JobName.CLEAN_OLD_AUDIT_LOGS },
-        { name: JobName.USER_SYNC_USAGE },
-        { name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false, nightly: true } },
         { name: JobName.CLEAN_OLD_SESSION_TOKENS },
+        { name: JobName.CLEAN_OLD_AUDIT_LOGS },
+        { name: JobName.MEMORIES_CREATE },
+        { name: JobName.USER_SYNC_USAGE },
+        { name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } },
+        { name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false, nightly: true } },
       ]);
     });
   });

server/src/services/job.service.ts

@@ -1,6 +1,7 @@
 import { BadRequestException, Injectable } from '@nestjs/common';
 import { ClassConstructor } from 'class-transformer';
 import { snakeCase } from 'lodash';
+import { SystemConfig } from 'src/config';
 import { OnEvent } from 'src/decorators';
 import { mapAsset } from 'src/dtos/asset-response.dto';
 import { AllJobStatusResponseDto, JobCommandDto, JobCreateDto, JobStatusDto } from 'src/dtos/job.dto';
@@ -8,6 +9,8 @@ import {
   AssetType,
   AssetVisibility,
   BootstrapEventPriority,
+  CronJob,
+  DatabaseLock,
   ImmichWorker,
   JobCommand,
   JobName,
@@ -20,6 +23,7 @@ import { ArgOf, ArgsOf } from 'src/repositories/event.repository';
 import { BaseService } from 'src/services/base.service';
 import { ConcurrentQueueName, JobItem } from 'src/types';
 import { hexOrBufferToBase64 } from 'src/utils/bytes';
+import { handlePromiseError } from 'src/utils/misc';
 
 const asJobItem = (dto: JobCreateDto): JobItem => {
   switch (dto.name) {
@@ -53,12 +57,59 @@ const asJobItem = (dto: JobCreateDto): JobItem => {
   }
 };
 
+const asNightlyTasksCron = (config: SystemConfig) => {
+  const [hours, minutes] = config.nightlyTasks.startTime.split(':').map(Number);
+  return `${minutes} ${hours} * * *`;
+};
+
 @Injectable()
 export class JobService extends BaseService {
   private services: ClassConstructor<unknown>[] = [];
+  private nightlyJobsLock = false;
 
-  @OnEvent({ name: 'config.init', workers: [ImmichWorker.MICROSERVICES] })
-  onConfigInit({ newConfig: config }: ArgOf<'config.init'>) {
+  @OnEvent({ name: 'config.init' })
+  async onConfigInit({ newConfig: config }: ArgOf<'config.init'>) {
+    if (this.worker === ImmichWorker.MICROSERVICES) {
+      this.updateQueueConcurrency(config);
+      return;
+    }
+
+    this.nightlyJobsLock = await this.databaseRepository.tryLock(DatabaseLock.NightlyJobs);
+    if (this.nightlyJobsLock) {
+      const cronExpression = asNightlyTasksCron(config);
+      this.logger.debug(`Scheduling nightly jobs for ${cronExpression}`);
+      this.cronRepository.create({
+        name: CronJob.NightlyJobs,
+        expression: cronExpression,
+        start: true,
+        onTick: () => handlePromiseError(this.handleNightlyJobs(), this.logger),
+      });
+    }
+  }
+
+  @OnEvent({ name: 'config.update', server: true })
+  onConfigUpdate({ newConfig: config }: ArgOf<'config.update'>) {
+    if (this.worker === ImmichWorker.MICROSERVICES) {
+      this.updateQueueConcurrency(config);
+      return;
+    }
+
+    if (this.nightlyJobsLock) {
+      const cronExpression = asNightlyTasksCron(config);
+      this.logger.debug(`Scheduling nightly jobs for ${cronExpression}`);
+      this.cronRepository.update({ name: CronJob.NightlyJobs, expression: cronExpression, start: true });
+    }
+  }
+
+  @OnEvent({ name: 'app.bootstrap', priority: BootstrapEventPriority.JobService })
+  onBootstrap() {
+    this.jobRepository.setup(this.services);
+    if (this.worker === ImmichWorker.MICROSERVICES) {
+      this.jobRepository.startWorkers();
+    }
+  }
+
+  private updateQueueConcurrency(config: SystemConfig) {
     this.logger.debug(`Updating queue concurrency settings`);
     for (const queueName of Object.values(QueueName)) {
       let concurrency = 1;
@@ -70,19 +121,6 @@ export class JobService extends BaseService {
     }
   }
 
-  @OnEvent({ name: 'config.update', server: true, workers: [ImmichWorker.MICROSERVICES] })
-  onConfigUpdate({ newConfig: config }: ArgOf<'config.update'>) {
-    this.onConfigInit({ newConfig: config });
-  }
-
-  @OnEvent({ name: 'app.bootstrap', priority: BootstrapEventPriority.JobService })
-  onBootstrap() {
-    this.jobRepository.setup(this.services);
-    if (this.worker === ImmichWorker.MICROSERVICES) {
-      this.jobRepository.startWorkers();
-    }
-  }
-
   setServices(services: ClassConstructor<unknown>[]) {
     this.services = services;
   }
@@ -233,18 +271,37 @@ export class JobService extends BaseService {
   }
 
   async handleNightlyJobs() {
-    await this.jobRepository.queueAll([
-      { name: JobName.ASSET_DELETION_CHECK },
-      { name: JobName.USER_DELETE_CHECK },
-      { name: JobName.PERSON_CLEANUP },
-      { name: JobName.MEMORIES_CLEANUP },
-      { name: JobName.MEMORIES_CREATE },
-      { name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } },
-      { name: JobName.CLEAN_OLD_AUDIT_LOGS },
-      { name: JobName.USER_SYNC_USAGE },
-      { name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false, nightly: true } },
-      { name: JobName.CLEAN_OLD_SESSION_TOKENS },
-    ]);
+    const config = await this.getConfig({ withCache: false });
+    const jobs: JobItem[] = [];
+
+    if (config.nightlyTasks.databaseCleanup) {
+      jobs.push(
+        { name: JobName.ASSET_DELETION_CHECK },
+        { name: JobName.USER_DELETE_CHECK },
+        { name: JobName.PERSON_CLEANUP },
+        { name: JobName.MEMORIES_CLEANUP },
+        { name: JobName.CLEAN_OLD_SESSION_TOKENS },
+        { name: JobName.CLEAN_OLD_AUDIT_LOGS },
+      );
+    }
+
+    if (config.nightlyTasks.generateMemories) {
+      jobs.push({ name: JobName.MEMORIES_CREATE });
+    }
+
+    if (config.nightlyTasks.syncQuotaUsage) {
+      jobs.push({ name: JobName.USER_SYNC_USAGE });
+    }
+
+    if (config.nightlyTasks.missingThumbnails) {
+      jobs.push({ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } });
+    }
+
+    if (config.nightlyTasks.clusterNewFaces) {
+      jobs.push({ name: JobName.QUEUE_FACIAL_RECOGNITION, data: { force: false, nightly: true } });
+    }
+
+    await this.jobRepository.queueAll(jobs);
   }
 
   /**

server/src/services/library.service.spec.ts

@@ -3,7 +3,7 @@ import { Stats } from 'node:fs';
 import { defaults, SystemConfig } from 'src/config';
 import { JOBS_LIBRARY_PAGINATION_SIZE } from 'src/constants';
 import { mapLibrary } from 'src/dtos/library.dto';
-import { AssetType, ImmichWorker, JobName, JobStatus } from 'src/enum';
+import { AssetType, CronJob, ImmichWorker, JobName, JobStatus } from 'src/enum';
 import { LibraryService } from 'src/services/library.service';
 import { ILibraryBulkIdsJob, ILibraryFileJob } from 'src/types';
 import { assetStub } from 'test/fixtures/asset.stub';
@@ -56,7 +56,11 @@ describe(LibraryService.name, () => {
         } as SystemConfig,
       });
 
-      expect(mocks.cron.update).toHaveBeenCalledWith({ name: 'libraryScan', expression: '0 1 * * *', start: true });
+      expect(mocks.cron.update).toHaveBeenCalledWith({
+        name: CronJob.LibraryScan,
+        expression: '0 1 * * *',
+        start: true,
+      });
     });
 
     it('should initialize watcher for all external libraries', async () => {
@@ -128,7 +132,7 @@ describe(LibraryService.name, () => {
       });
 
       expect(mocks.cron.update).toHaveBeenCalledWith({
-        name: 'libraryScan',
+        name: CronJob.LibraryScan,
         expression: systemConfigStub.libraryScan.library.scan.cronExpression,
         start: systemConfigStub.libraryScan.library.scan.enabled,
       });
@@ -149,7 +153,7 @@ describe(LibraryService.name, () => {
       });
 
       expect(mocks.cron.update).toHaveBeenCalledWith({
-        name: 'libraryScan',
+        name: CronJob.LibraryScan,
         expression: systemConfigStub.libraryScan.library.scan.cronExpression,
         start: systemConfigStub.libraryScan.library.scan.enabled,
       });

server/src/services/library.service.ts

@@ -17,7 +17,7 @@ import {
   ValidateLibraryImportPathResponseDto,
   ValidateLibraryResponseDto,
 } from 'src/dtos/library.dto';
-import { AssetStatus, AssetType, DatabaseLock, ImmichWorker, JobName, JobStatus, QueueName } from 'src/enum';
+import { AssetStatus, AssetType, CronJob, DatabaseLock, ImmichWorker, JobName, JobStatus, QueueName } from 'src/enum';
 import { ArgOf } from 'src/repositories/event.repository';
 import { AssetSyncResult } from 'src/repositories/library.repository';
 import { AssetTable } from 'src/schema/tables/asset.table';
@@ -45,7 +45,7 @@ export class LibraryService extends BaseService {
 
     if (this.lock) {
       this.cronRepository.create({
-        name: 'libraryScan',
+        name: CronJob.LibraryScan,
         expression: scan.cronExpression,
         onTick: () =>
           handlePromiseError(this.jobRepository.queue({ name: JobName.LIBRARY_QUEUE_SCAN_ALL }), this.logger),
@@ -65,7 +65,7 @@ export class LibraryService extends BaseService {
     }
 
     this.cronRepository.update({
-      name: 'libraryScan',
+      name: CronJob.LibraryScan,
       expression: library.scan.cronExpression,
       start: library.scan.enabled,
     });

server/src/services/system-config.service.spec.ts

@@ -103,6 +103,14 @@ const updatedConfig = Object.freeze<SystemConfig>({
     lightStyle: 'https://tiles.immich.cloud/v1/style/light.json',
     darkStyle: 'https://tiles.immich.cloud/v1/style/dark.json',
   },
+  nightlyTasks: {
+    startTime: '00:00',
+    databaseCleanup: true,
+    clusterNewFaces: true,
+    missingThumbnails: true,
+    generateMemories: true,
+    syncQuotaUsage: true,
+  },
   reverseGeocoding: {
     enabled: true,
   },