feat: maintenance mode (#23431)

* feat: add a `maintenance.enabled` config flag

* feat: implement graceful restart
feat: restart when maintenance config is toggled

* feat: boot a stripped down maintenance api if enabled

* feat: cli command to toggle maintenance mode

* chore: fallback IMMICH_SERVER_URL environment variable in process

* chore: add additional routes to maintenance controller

* fix: don't wait for nest application to close to finish request response

* chore: add a failsafe on restart to prevent other exit codes from preventing restart

* feat: redirect into/from maintenance page

* refactor: use system metadata for maintenance status

* refactor: wait on WebSocket connection to refresh

* feat: broadcast websocket event on server restart
refactor: listen to WS instead of polling

* refactor: bubble up maintenance information instead of hijacking in fetch function
feat: show modal when server is restarting

* chore: increase timeout for ungraceful restart

* refactor: deduplicate code between api/maintenance workers

* fix: skip config check if database is not initialised

* fix: add `maintenanceMode` field to system config test

* refactor: move maintenance resolution code to static method in service

* chore: clean up linter issues

* chore: generate dart openapi

* refactor: use try{} block for maintenance mode check

* fix: logic error in server redirect

* chore: include `maintenanceMode` key in e2e test

* chore: add i18n entries for maintenance screens

* chore: remove negated condition from hook

* fix: should set default value not override in service

* fix: minor error in page

* feat: initial draft of maintenance module, repo., worker controller, worker service

* refactor: move broadcast code into notification service

* chore: connect websocket on client if in maintenance

* chore: set maintenance module app name

* refactor: rename repository to include worker
chore: configure websocket adapter

* feat: reimplement maintenance mode exit with new module

* refactor: add a constant enum for ExitCode

* refactor: remove redundant route for maintenance

* refactor: only spin up kysely on boot (rather than a Nest app)

* refactor(web): move redirect logic into +layout file where modal is setup

* feat: add Maintenance permission

* refactor: merge common code between api/maintenance

* fix: propagate changes from the CLI to servers

* feat: maintenance authentication guard

* refactor: unify maintenance code into repository
feat: add a step to generate maintenance mode token

* feat: jwt auth for maintenance

* refactor: switch from nest jwt to just jsonwebtokens

* feat: log into maintenance mode from CLI command

* refactor: use `secret` instead of `token` in jwt terminology
chore: log maintenance mode login URL on boot
chore: don't make CLI actions reload if already in target state

* docs: initial draft for maintenance mode page

* refactor: always validate the maintenance auth on the server

* feat: add a link to maintenance mode documentation

* feat: redirect users back to the last page they were on when exiting maintenance

* refactor: provide closeFn in both maintenance repos.

* refactor: ensure the user is also redirected by the server

* chore: swap jsonwebtoken for jose

* refactor: introduce AppRestartEvent w/o secret passing

* refactor: use navigation goto

* refactor: use `continue` instead of `next`

* chore: lint fixes for server

* chore: lint fixes for web

* test: add mock for maintenance repository

* test: add base service dependency to maintenance

* chore: remove @types/jsonwebtoken

* refactor: close database connection after startup check

* refactor: use `request#auth` key

* refactor: use service instead of repository
chore: read token from cookie if possible
chore: rename client event to AppRestartV1

* refactor: more concise redirect logic on web

* refactor: move redirect check into utils
refactor: update translation strings to be more sensible

* refactor: always validate login (i.e. check cookie)

* refactor: lint, open-api, remove old dto

* refactor: encode at point of usage

* refactor: remove business logic from repositories

* chore: fix server/web lints

* refactor: remove repository mock

* chore: fix formatting

* test: write service mocks for maintenance mode

* test: write cli service tests

* fix: catch errors when closing app

* fix: always report no maintenance when usual API is available

* test: api e2e maintenance spec

* chore: add response builder

* chore: add helper to set maint. auth cookie

* feat: add SSR to maintenance API

* test(e2e): write web spec for maintenance

* chore: clean up lint issues

* chore: format files

* feat: perform 302 redirect at server level during maintenance

* fix: keep trying to stop immich until it succeeds (CLI issue)

* chore: lint/format

* refactor: annotate references to other services in worker service

* chore: lint

* refactor: remove unnecessary await

Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>

* refactor: move static methods into util

* refactor: assert secret exists in maintenance worker

* refactor: remove assertion which isn't necessary anymore

* refactor: remove assertion

* refactor: remove outer try {} catch block from loadMaintenanceAuth

* refactor: undo earlier change to vite.config.ts

* chore: update tests due to refactors

* revert: vite.config.ts

* test: expect string jwt

* chore: move blanket exceptions into controllers

* test: update tests according with last change

* refactor: use respondWithCookie
refactor: merge start/end into one route
refactor: rename MaintenanceRepository to AppRepository
chore: use new ApiTag/Endpoint
refactor: apply other requested changes

* chore: regenerate openapi

* chore: lint/format

* chore: remove secureOnly for maint. cookie

* refactor: move maintenance worker code into src/maintenance\nfix: various test fixes

* refactor: use `action` property for setting maint. mode

* refactor: remove Websocket#restartApp in favour of individual methods

* chore: incomplete commit

* chore: remove stray log

* fix: call exitApp from maintenance worker on exit

* fix: add app repository mock

* fix: ensure maintenance cookies are secure

* fix: run playwright tests over secure context (localhost)

* test: update other references to 127.0.0.1

* refactor: use serverSideEmitWithAck

* chore: correct the logic in tryTerminate

* test: juggle cookies ourselves

* chore: fix lint error for e2e spec

* chore: format e2e test

* fix: set cookie secure/non-secure depending on context

* chore: format files

---------

Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>

server/src/app.module.ts

@@ -9,15 +9,21 @@ import { commandsAndQuestions } from 'src/commands';
 import { IWorker } from 'src/constants';
 import { controllers } from 'src/controllers';
 import { ImmichWorker } from 'src/enum';
+import { MaintenanceAuthGuard } from 'src/maintenance/maintenance-auth.guard';
+import { MaintenanceWebsocketRepository } from 'src/maintenance/maintenance-websocket.repository';
+import { MaintenanceWorkerController } from 'src/maintenance/maintenance-worker.controller';
+import { MaintenanceWorkerService } from 'src/maintenance/maintenance-worker.service';
 import { AuthGuard } from 'src/middleware/auth.guard';
 import { ErrorInterceptor } from 'src/middleware/error.interceptor';
 import { FileUploadInterceptor } from 'src/middleware/file-upload.interceptor';
 import { GlobalExceptionFilter } from 'src/middleware/global-exception.filter';
 import { LoggingInterceptor } from 'src/middleware/logging.interceptor';
 import { repositories } from 'src/repositories';
+import { AppRepository } from 'src/repositories/app.repository';
 import { ConfigRepository } from 'src/repositories/config.repository';
 import { EventRepository } from 'src/repositories/event.repository';
 import { LoggingRepository } from 'src/repositories/logging.repository';
+import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository';
 import { teardownTelemetry, TelemetryRepository } from 'src/repositories/telemetry.repository';
 import { WebsocketRepository } from 'src/repositories/websocket.repository';
 import { services } from 'src/services';
@@ -28,27 +34,27 @@ import { getKyselyConfig } from 'src/utils/database';
 
 const common = [...repositories, ...services, GlobalExceptionFilter];
 
-export const middleware = [
-  FileUploadInterceptor,
+const commonMiddleware = [
   { provide: APP_FILTER, useClass: GlobalExceptionFilter },
   { provide: APP_PIPE, useValue: new ValidationPipe({ transform: true, whitelist: true }) },
   { provide: APP_INTERCEPTOR, useClass: LoggingInterceptor },
   { provide: APP_INTERCEPTOR, useClass: ErrorInterceptor },
-  { provide: APP_GUARD, useClass: AuthGuard },
 ];
 
+const apiMiddleware = [FileUploadInterceptor, ...commonMiddleware, { provide: APP_GUARD, useClass: AuthGuard }];
+
 const configRepository = new ConfigRepository();
 const { bull, cls, database, otel } = configRepository.getEnv();
 
-const imports = [
-  BullModule.forRoot(bull.config),
-  BullModule.registerQueue(...bull.queues),
+const commonImports = [
   ClsModule.forRoot(cls.config),
-  OpenTelemetryModule.forRoot(otel),
   KyselyModule.forRoot(getKyselyConfig(database.config)),
+  OpenTelemetryModule.forRoot(otel),
 ];
 
-class BaseModule implements OnModuleInit, OnModuleDestroy {
+const bullImports = [BullModule.forRoot(bull.config), BullModule.registerQueue(...bull.queues)];
+
+export class BaseModule implements OnModuleInit, OnModuleDestroy {
   constructor(
     @Inject(IWorker) private worker: ImmichWorker,
     logger: LoggingRepository,
@@ -85,20 +91,44 @@ class BaseModule implements OnModuleInit, OnModuleDestroy {
 }
 
 @Module({
-  imports: [...imports, ScheduleModule.forRoot()],
+  imports: [...bullImports, ...commonImports, ScheduleModule.forRoot()],
   controllers: [...controllers],
-  providers: [...common, ...middleware, { provide: IWorker, useValue: ImmichWorker.Api }],
+  providers: [...common, ...apiMiddleware, { provide: IWorker, useValue: ImmichWorker.Api }],
 })
 export class ApiModule extends BaseModule {}
 
 @Module({
-  imports: [...imports],
+  imports: [...commonImports],
+  controllers: [MaintenanceWorkerController],
+  providers: [
+    ConfigRepository,
+    LoggingRepository,
+    SystemMetadataRepository,
+    AppRepository,
+    MaintenanceWebsocketRepository,
+    MaintenanceWorkerService,
+    ...commonMiddleware,
+    { provide: APP_GUARD, useClass: MaintenanceAuthGuard },
+    { provide: IWorker, useValue: ImmichWorker.Maintenance },
+  ],
+})
+export class MaintenanceModule {
+  constructor(
+    @Inject(IWorker) private worker: ImmichWorker,
+    logger: LoggingRepository,
+  ) {
+    logger.setAppName(this.worker);
+  }
+}
+
+@Module({
+  imports: [...bullImports, ...commonImports],
   providers: [...common, { provide: IWorker, useValue: ImmichWorker.Microservices }, SchedulerRegistry],
 })
 export class MicroservicesModule extends BaseModule {}
 
 @Module({
-  imports: [...imports],
+  imports: [...bullImports, ...commonImports],
   providers: [...common, ...commandsAndQuestions, SchedulerRegistry],
 })
 export class ImmichAdminModule implements OnModuleDestroy {