feat: wait on maintenance operation lock on boot

server/src/enum.ts

@@ -657,12 +657,15 @@ export enum DatabaseLock {
   MediaLocation = 700,
   GetSystemConfig = 69,
   BackupDatabase = 42,
+  MaintenanceOperation = 621,
   MemoryCreation = 777,
 }
 
 export enum MaintenanceAction {
   Start = 'start',
   End = 'end',
+  RestoreFlow = 'restore_flow',
+  RestoreDatabase = 'restore_database',
 }
 
 export enum ExitCode {

server/src/main.ts

@@ -1,11 +1,11 @@
-import { Kysely } from 'kysely';
+import { Kysely, sql } from 'kysely';
 import { CommandFactory } from 'nest-commander';
 import { ChildProcess, fork } from 'node:child_process';
 import { dirname, join } from 'node:path';
 import { Worker } from 'node:worker_threads';
 import { PostgresError } from 'postgres';
 import { ImmichAdminModule } from 'src/app.module';
-import { ExitCode, ImmichWorker, LogLevel, SystemMetadataKey } from 'src/enum';
+import { DatabaseLock, ExitCode, ImmichWorker, LogLevel, SystemMetadataKey } from 'src/enum';
 import { ConfigRepository } from 'src/repositories/config.repository';
 import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository';
 import { type DB } from 'src/schema';
@@ -35,16 +35,14 @@ class Workers {
     if (isMaintenanceMode) {
       this.startWorker(ImmichWorker.Maintenance);
     } else {
+      this.waitForFreeLock();
+
       for (const worker of workers) {
         this.startWorker(worker);
       }
     }
   }
 
-  /**
-   * Initialise a short-lived Nest application to build configuration
-   * @returns System configuration
-   */
   private async isMaintenanceMode(): Promise<boolean> {
     const { database } = new ConfigRepository().getEnv();
     const kysely = new Kysely<DB>(getKyselyConfig(database.config));
@@ -65,6 +63,30 @@ class Workers {
     }
   }
 
+  private async waitForFreeLock() {
+    const { database } = new ConfigRepository().getEnv();
+    const kysely = new Kysely<DB>(getKyselyConfig(database.config));
+
+    let locked = false;
+    while (!locked) {
+      locked = await kysely.connection().execute(async (conn) => {
+        const { rows } = await sql<{
+          pg_try_advisory_lock: boolean;
+        }>`SELECT pg_try_advisory_lock(${DatabaseLock.MaintenanceOperation})`.execute(conn);
+
+        const isLocked = rows[0].pg_try_advisory_lock;
+
+        if (isLocked) {
+          await sql`SELECT pg_advisory_unlock(${DatabaseLock.MaintenanceOperation})`.execute(conn);
+        }
+
+        return isLocked;
+      });
+    }
+
+    await kysely.destroy();
+  }
+
   /**
    * Start an individual worker
    * @param name Worker