feat(server): custom library scanning interval (#4390)

* add automatic library scan config options

* add validation

* open api

* use CronJob instead of cron-validator

* fix tests

* catch potential error of the library scan initialization

* better description for input field

* move library scan job initialization to server app service

* fix tests

* add comments to all parameters of cronjob contructor

* make scan a child of a more general library object

* open api

* chore: cleanup

* move cronjob handling to job repoistory

* web: select for common cron expressions

* fix open api

* fix tests

* put scanning settings in nested accordion

* fix system config validation

* refactor, tests

---------

Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>

server/src/domain/domain.util.ts

@@ -1,6 +1,7 @@
 import { applyDecorators } from '@nestjs/common';
 import { ApiProperty } from '@nestjs/swagger';
 import { IsArray, IsNotEmpty, IsOptional, IsString, IsUUID, ValidateIf, ValidationOptions } from 'class-validator';
+import { CronJob } from 'cron';
 import { basename, extname } from 'node:path';
 import sanitize from 'sanitize-filename';
 
@@ -18,6 +19,16 @@ export function ValidateUUID({ optional, each }: Options = { optional: false, ea
   );
 }
 
+export function validateCronExpression(expression: string) {
+  try {
+    new CronJob(expression, () => {});
+  } catch (error) {
+    return false;
+  }
+
+  return true;
+}
+
 interface IValue {
   value?: string;
 }

server/src/domain/job/job.service.spec.ts

@@ -61,7 +61,6 @@ describe(JobService.name, () => {
         [{ name: JobName.PERSON_CLEANUP }],
         [{ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } }],
         [{ name: JobName.CLEAN_OLD_AUDIT_LOGS }],
-        [{ name: JobName.LIBRARY_QUEUE_SCAN_ALL, data: { force: false } }],
       ]);
     });
   });

server/src/domain/job/job.service.ts

@@ -153,7 +153,6 @@ export class JobService {
     await this.jobRepository.queue({ name: JobName.PERSON_CLEANUP });
     await this.jobRepository.queue({ name: JobName.QUEUE_GENERATE_THUMBNAILS, data: { force: false } });
     await this.jobRepository.queue({ name: JobName.CLEAN_OLD_AUDIT_LOGS });
-    await this.jobRepository.queue({ name: JobName.LIBRARY_QUEUE_SCAN_ALL, data: { force: false } });
   }
 
   /**

server/src/domain/library/library.service.spec.ts

@@ -1,4 +1,4 @@
-import { AssetType, LibraryType, UserEntity } from '@app/infra/entities';
+import { AssetType, LibraryType, SystemConfig, SystemConfigKey, UserEntity } from '@app/infra/entities';
 import { BadRequestException } from '@nestjs/common';
 
 import {
@@ -12,6 +12,7 @@ import {
   newJobRepositoryMock,
   newLibraryRepositoryMock,
   newStorageRepositoryMock,
+  newSystemConfigRepositoryMock,
   newUserRepositoryMock,
   userStub,
 } from '@test';
@@ -23,8 +24,10 @@ import {
   IJobRepository,
   ILibraryRepository,
   IStorageRepository,
+  ISystemConfigRepository,
   IUserRepository,
 } from '../repositories';
+import { SystemConfigCore } from '../system-config/system-config.core';
 import { LibraryService } from './library.service';
 
 describe(LibraryService.name, () => {
@@ -32,6 +35,7 @@ describe(LibraryService.name, () => {
 
   let accessMock: IAccessRepositoryMock;
   let assetMock: jest.Mocked<IAssetRepository>;
+  let configMock: jest.Mocked<ISystemConfigRepository>;
   let cryptoMock: jest.Mocked<ICryptoRepository>;
   let userMock: jest.Mocked<IUserRepository>;
   let jobMock: jest.Mocked<IJobRepository>;
@@ -40,6 +44,7 @@ describe(LibraryService.name, () => {
 
   beforeEach(() => {
     accessMock = newAccessRepositoryMock();
+    configMock = newSystemConfigRepositoryMock();
     libraryMock = newLibraryRepositoryMock();
     userMock = newUserRepositoryMock();
     assetMock = newAssetRepositoryMock();
@@ -55,13 +60,46 @@ describe(LibraryService.name, () => {
 
     accessMock.library.hasOwnerAccess.mockResolvedValue(true);
 
-    sut = new LibraryService(accessMock, assetMock, cryptoMock, jobMock, libraryMock, storageMock, userMock);
+    sut = new LibraryService(
+      accessMock,
+      assetMock,
+      configMock,
+      cryptoMock,
+      jobMock,
+      libraryMock,
+      storageMock,
+      userMock,
+    );
   });
 
   it('should work', () => {
     expect(sut).toBeDefined();
   });
 
+  describe('init', () => {
+    it('should init cron job and subscribe to config changes', async () => {
+      configMock.load.mockResolvedValue([
+        { key: SystemConfigKey.LIBRARY_SCAN_ENABLED, value: true },
+        { key: SystemConfigKey.LIBRARY_SCAN_CRON_EXPRESSION, value: '0 0 * * *' },
+      ]);
+
+      await sut.init();
+      expect(configMock.load).toHaveBeenCalled();
+      expect(jobMock.addCronJob).toHaveBeenCalled();
+
+      SystemConfigCore.create(newSystemConfigRepositoryMock(false)).config$.next({
+        library: {
+          scan: {
+            enabled: true,
+            cronExpression: '0 1 * * *',
+          },
+        },
+      } as SystemConfig);
+
+      expect(jobMock.updateCronJob).toHaveBeenCalledWith('libraryScan', '0 1 * * *', true);
+    });
+  });
+
   describe('handleQueueAssetRefresh', () => {
     it("should not queue assets outside of user's external path", async () => {
       const mockLibraryJob: ILibraryRefreshJob = {

server/src/domain/library/library.service.ts

@@ -7,7 +7,7 @@ import { basename, parse } from 'path';
 import { AccessCore, Permission } from '../access';
 import { AuthUserDto } from '../auth';
 import { mimeTypes } from '../domain.constant';
-import { usePagination } from '../domain.util';
+import { usePagination, validateCronExpression } from '../domain.util';
 import { IBaseJob, IEntityJob, ILibraryFileJob, ILibraryRefreshJob, JOBS_ASSET_PAGINATION_SIZE, JobName } from '../job';
 
 import {
@@ -17,9 +17,11 @@ import {
   IJobRepository,
   ILibraryRepository,
   IStorageRepository,
+  ISystemConfigRepository,
   IUserRepository,
   WithProperty,
 } from '../repositories';
+import { SystemConfigCore } from '../system-config';
 import {
   CreateLibraryDto,
   LibraryResponseDto,
@@ -33,10 +35,12 @@ import {
 export class LibraryService {
   readonly logger = new Logger(LibraryService.name);
   private access: AccessCore;
+  private configCore: SystemConfigCore;
 
   constructor(
     @Inject(IAccessRepository) accessRepository: IAccessRepository,
     @Inject(IAssetRepository) private assetRepository: IAssetRepository,
+    @Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
     @Inject(ICryptoRepository) private cryptoRepository: ICryptoRepository,
     @Inject(IJobRepository) private jobRepository: IJobRepository,
     @Inject(ILibraryRepository) private repository: ILibraryRepository,
@@ -44,6 +48,26 @@ export class LibraryService {
     @Inject(IUserRepository) private userRepository: IUserRepository,
   ) {
     this.access = AccessCore.create(accessRepository);
+    this.configCore = SystemConfigCore.create(configRepository);
+    this.configCore.addValidator((config) => {
+      if (!validateCronExpression(config.library.scan.cronExpression)) {
+        throw new Error(`Invalid cron expression ${config.library.scan.cronExpression}`);
+      }
+    });
+  }
+
+  async init() {
+    const config = await this.configCore.getConfig();
+    this.jobRepository.addCronJob(
+      'libraryScan',
+      config.library.scan.cronExpression,
+      () => this.jobRepository.queue({ name: JobName.LIBRARY_QUEUE_SCAN_ALL, data: { force: false } }),
+      config.library.scan.enabled,
+    );
+
+    this.configCore.config$.subscribe((config) => {
+      this.jobRepository.updateCronJob('libraryScan', config.library.scan.cronExpression, config.library.scan.enabled);
+    });
   }
 
   async getStatistics(authUser: AuthUserDto, id: string): Promise<LibraryStatsResponseDto> {

server/src/domain/repositories/job.repository.ts

@@ -111,6 +111,9 @@ export const IJobRepository = 'IJobRepository';
 
 export interface IJobRepository {
   addHandler(queueName: QueueName, concurrency: number, handler: JobItemHandler): void;
+  addCronJob(name: string, expression: string, onTick: () => void, start?: boolean): void;
+  updateCronJob(name: string, expression?: string, start?: boolean): void;
+  deleteCronJob(name: string): void;
   setConcurrency(queueName: QueueName, concurrency: number): void;
   queue(item: JobItem): Promise<void>;
   pause(name: QueueName): Promise<void>;

server/src/domain/system-config/dto/index.ts

@@ -1,4 +1,5 @@
 export * from './system-config-ffmpeg.dto';
+export * from './system-config-library.dto';
 export * from './system-config-oauth.dto';
 export * from './system-config-password-login.dto';
 export * from './system-config-storage-template.dto';

server/src/domain/system-config/dto/system-config-library.dto.ts

@@ -0,0 +1,40 @@
+import { validateCronExpression } from '@app/domain';
+import { Type } from 'class-transformer';
+import {
+  IsBoolean,
+  IsNotEmpty,
+  IsObject,
+  IsString,
+  Validate,
+  ValidateIf,
+  ValidateNested,
+  ValidatorConstraint,
+  ValidatorConstraintInterface,
+} from 'class-validator';
+
+const isEnabled = (config: SystemConfigLibraryScanDto) => config.enabled;
+
+@ValidatorConstraint({ name: 'cronValidator' })
+class CronValidator implements ValidatorConstraintInterface {
+  validate(expression: string): boolean {
+    return validateCronExpression(expression);
+  }
+}
+
+export class SystemConfigLibraryScanDto {
+  @IsBoolean()
+  enabled!: boolean;
+
+  @ValidateIf(isEnabled)
+  @IsNotEmpty()
+  @Validate(CronValidator, { message: 'Invalid cron expression' })
+  @IsString()
+  cronExpression!: string;
+}
+
+export class SystemConfigLibraryDto {
+  @Type(() => SystemConfigLibraryScanDto)
+  @ValidateNested()
+  @IsObject()
+  scan!: SystemConfigLibraryScanDto;
+}

server/src/domain/system-config/dto/system-config.dto.ts

@@ -3,6 +3,7 @@ import { Type } from 'class-transformer';
 import { IsObject, ValidateNested } from 'class-validator';
 import { SystemConfigFFmpegDto } from './system-config-ffmpeg.dto';
 import { SystemConfigJobDto } from './system-config-job.dto';
+import { SystemConfigLibraryDto } from './system-config-library.dto';
 import { SystemConfigMachineLearningDto } from './system-config-machine-learning.dto';
 import { SystemConfigMapDto } from './system-config-map.dto';
 import { SystemConfigNewVersionCheckDto } from './system-config-new-version-check.dto';
@@ -74,6 +75,11 @@ export class SystemConfigDto implements SystemConfig {
   @ValidateNested()
   @IsObject()
   theme!: SystemConfigThemeDto;
+
+  @Type(() => SystemConfigLibraryDto)
+  @ValidateNested()
+  @IsObject()
+  library!: SystemConfigLibraryDto;
 }
 
 export function mapConfig(config: SystemConfig): SystemConfigDto {

server/src/domain/system-config/system-config.core.ts

@@ -13,6 +13,7 @@ import {
   VideoCodec,
 } from '@app/infra/entities';
 import { BadRequestException, ForbiddenException, Injectable, Logger } from '@nestjs/common';
+import { CronExpression } from '@nestjs/schedule';
 import { plainToInstance } from 'class-transformer';
 import { validate } from 'class-validator';
 import * as _ from 'lodash';
@@ -120,6 +121,12 @@ export const defaults = Object.freeze<SystemConfig>({
   theme: {
     customCss: '',
   },
+  library: {
+    scan: {
+      enabled: true,
+      cronExpression: CronExpression.EVERY_DAY_AT_MIDNIGHT,
+    },
+  },
 });
 
 export enum FeatureFlag {

server/src/domain/system-config/system-config.service.spec.ts

@@ -121,6 +121,12 @@ const updatedConfig = Object.freeze<SystemConfig>({
   theme: {
     customCss: '',
   },
+  library: {
+    scan: {
+      enabled: true,
+      cronExpression: '0 0 * * *',
+    },
+  },
 });
 
 describe(SystemConfigService.name, () => {

server/src/immich/app.service.ts

@@ -1,4 +1,4 @@
-import { JobService, ONE_HOUR, SearchService, ServerInfoService, StorageService } from '@app/domain';
+import { JobService, LibraryService, ONE_HOUR, SearchService, ServerInfoService, StorageService } from '@app/domain';
 import { Injectable, Logger } from '@nestjs/common';
 import { Cron, CronExpression, Interval } from '@nestjs/schedule';
 
@@ -8,6 +8,7 @@ export class AppService {
 
   constructor(
     private jobService: JobService,
+    private libraryService: LibraryService,
     private searchService: SearchService,
     private storageService: StorageService,
     private serverService: ServerInfoService,
@@ -28,6 +29,7 @@ export class AppService {
     await this.searchService.init();
     await this.serverService.handleVersionCheck();
     this.logger.log(`Feature Flags: ${JSON.stringify(await this.serverService.getFeatures(), null, 2)}`);
+    await this.libraryService.init();
   }
 
   async destroy() {

server/src/infra/entities/system-config.entity.ts

@@ -94,6 +94,9 @@ export enum SystemConfigKey {
   TRASH_DAYS = 'trash.days',
 
   THEME_CUSTOM_CSS = 'theme.customCss',
+
+  LIBRARY_SCAN_ENABLED = 'library.scan.enabled',
+  LIBRARY_SCAN_CRON_EXPRESSION = 'library.scan.cronExpression',
 }
 
 export enum TranscodePolicy {
@@ -232,4 +235,10 @@ export interface SystemConfig {
   theme: {
     customCss: string;
   };
+  library: {
+    scan: {
+      enabled: boolean;
+      cronExpression: string;
+    };
+  };
 }

server/src/infra/repositories/job.repository.ts

@@ -2,7 +2,9 @@ import { IJobRepository, JobCounts, JobItem, JobName, JOBS_TO_QUEUE, QueueName,
 import { getQueueToken } from '@nestjs/bullmq';
 import { Injectable, Logger } from '@nestjs/common';
 import { ModuleRef } from '@nestjs/core';
+import { SchedulerRegistry } from '@nestjs/schedule';
 import { Job, JobsOptions, Processor, Queue, Worker, WorkerOptions } from 'bullmq';
+import { CronJob, CronTime } from 'cron';
 import { bullConfig } from '../infra.config';
 
 @Injectable()
@@ -10,7 +12,10 @@ export class JobRepository implements IJobRepository {
   private workers: Partial<Record<QueueName, Worker>> = {};
   private logger = new Logger(JobRepository.name);
 
-  constructor(private moduleRef: ModuleRef) {}
+  constructor(
+    private moduleRef: ModuleRef,
+    private schedulerReqistry: SchedulerRegistry,
+  ) {}
 
   addHandler(queueName: QueueName, concurrency: number, handler: (item: JobItem) => Promise<void>) {
     const workerHandler: Processor = async (job: Job) => handler(job as JobItem);
@@ -18,6 +23,43 @@ export class JobRepository implements IJobRepository {
     this.workers[queueName] = new Worker(queueName, workerHandler, workerOptions);
   }
 
+  addCronJob(name: string, expression: string, onTick: () => void, start = true): void {
+    const job = new CronJob(
+      expression,
+      onTick,
+      // function to run onComplete
+      undefined,
+      // whether it should start directly
+      start,
+      // timezone
+      undefined,
+      // context
+      undefined,
+      // runOnInit
+      undefined,
+      // utcOffset
+      undefined,
+      // prevents memory leaking by automatically stopping when the node process finishes
+      true,
+    );
+
+    this.schedulerReqistry.addCronJob(name, job);
+  }
+
+  updateCronJob(name: string, expression?: string, start?: boolean): void {
+    const job = this.schedulerReqistry.getCronJob(name);
+    if (expression) {
+      job.setTime(new CronTime(expression));
+    }
+    if (start !== undefined) {
+      start ? job.start() : job.stop();
+    }
+  }
+
+  deleteCronJob(name: string): void {
+    this.schedulerReqistry.deleteCronJob(name);
+  }
+
   setConcurrency(queueName: QueueName, concurrency: number) {
     const worker = this.workers[queueName];
     if (!worker) {