feat(server): better mount checks (#13092)

server/src/interfaces/config.interface.ts

@@ -7,6 +7,9 @@ export interface EnvData {
     skipMigrations: boolean;
     vectorExtension: VectorExtension;
   };
+  storage: {
+    ignoreMountCheckErrors: boolean;
+  };
 }
 
 export interface IConfigRepository {

server/src/repositories/config.repository.ts

@@ -10,6 +10,9 @@ export class ConfigRepository implements IConfigRepository {
         skipMigrations: process.env.DB_SKIP_MIGRATIONS === 'true',
         vectorExtension: getVectorExtension(),
       },
+      storage: {
+        ignoreMountCheckErrors: process.env.IMMICH_IGNORE_MOUNT_CHECK_ERRORS === 'true',
+      },
     };
   }
 }

server/src/services/database.service.spec.ts

@@ -7,7 +7,7 @@ import {
 } from 'src/interfaces/database.interface';
 import { ILoggerRepository } from 'src/interfaces/logger.interface';
 import { DatabaseService } from 'src/services/database.service';
-import { newConfigRepositoryMock } from 'test/repositories/config.repository.mock';
+import { mockEnvData, newConfigRepositoryMock } from 'test/repositories/config.repository.mock';
 import { newDatabaseRepositoryMock } from 'test/repositories/database.repository.mock';
 import { newLoggerRepositoryMock } from 'test/repositories/logger.repository.mock';
 import { Mocked } from 'vitest';
@@ -60,7 +60,9 @@ describe(DatabaseService.name, () => {
     { extension: DatabaseExtension.VECTORS, extensionName: EXTENSION_NAMES[DatabaseExtension.VECTORS] },
   ])('should work with $extensionName', ({ extension, extensionName }) => {
     beforeEach(() => {
-      configMock.getEnv.mockReturnValue({ database: { skipMigrations: false, vectorExtension: extension } });
+      configMock.getEnv.mockReturnValue(
+        mockEnvData({ database: { skipMigrations: false, vectorExtension: extension } }),
+      );
     });
 
     it(`should start up successfully with ${extension}`, async () => {
@@ -244,12 +246,14 @@ describe(DatabaseService.name, () => {
   });
 
   it('should skip migrations if DB_SKIP_MIGRATIONS=true', async () => {
-    configMock.getEnv.mockReturnValue({
-      database: {
-        skipMigrations: true,
-        vectorExtension: DatabaseExtension.VECTORS,
-      },
-    });
+    configMock.getEnv.mockReturnValue(
+      mockEnvData({
+        database: {
+          skipMigrations: true,
+          vectorExtension: DatabaseExtension.VECTORS,
+        },
+      }),
+    );
 
     await expect(sut.onBootstrap()).resolves.toBeUndefined();
 
@@ -257,12 +261,14 @@ describe(DatabaseService.name, () => {
   });
 
   it(`should throw error if pgvector extension could not be created`, async () => {
-    configMock.getEnv.mockReturnValue({
-      database: {
-        skipMigrations: true,
-        vectorExtension: DatabaseExtension.VECTOR,
-      },
-    });
+    configMock.getEnv.mockReturnValue(
+      mockEnvData({
+        database: {
+          skipMigrations: true,
+          vectorExtension: DatabaseExtension.VECTOR,
+        },
+      }),
+    );
     databaseMock.getExtensionVersion.mockResolvedValue({
       installedVersion: null,
       availableVersion: minVersionInRange,

server/src/services/storage.service.spec.ts

@@ -1,9 +1,11 @@
 import { SystemMetadataKey } from 'src/enum';
+import { IConfigRepository } from 'src/interfaces/config.interface';
 import { IDatabaseRepository } from 'src/interfaces/database.interface';
 import { ILoggerRepository } from 'src/interfaces/logger.interface';
 import { IStorageRepository } from 'src/interfaces/storage.interface';
 import { ISystemMetadataRepository } from 'src/interfaces/system-metadata.interface';
 import { StorageService } from 'src/services/storage.service';
+import { mockEnvData, newConfigRepositoryMock } from 'test/repositories/config.repository.mock';
 import { newDatabaseRepositoryMock } from 'test/repositories/database.repository.mock';
 import { newLoggerRepositoryMock } from 'test/repositories/logger.repository.mock';
 import { newStorageRepositoryMock } from 'test/repositories/storage.repository.mock';
@@ -12,18 +14,20 @@ import { Mocked } from 'vitest';
 
 describe(StorageService.name, () => {
   let sut: StorageService;
+  let configMock: Mocked<IConfigRepository>;
   let databaseMock: Mocked<IDatabaseRepository>;
   let storageMock: Mocked<IStorageRepository>;
   let loggerMock: Mocked<ILoggerRepository>;
   let systemMock: Mocked<ISystemMetadataRepository>;
 
   beforeEach(() => {
+    configMock = newConfigRepositoryMock();
     databaseMock = newDatabaseRepositoryMock();
     storageMock = newStorageRepositoryMock();
     loggerMock = newLoggerRepositoryMock();
     systemMock = newSystemMetadataRepositoryMock();
 
-    sut = new StorageService(databaseMock, storageMock, loggerMock, systemMock);
+    sut = new StorageService(configMock, databaseMock, storageMock, loggerMock, systemMock);
   });
 
   it('should work', () => {
@@ -52,7 +56,7 @@ describe(StorageService.name, () => {
       systemMock.get.mockResolvedValue({ mountFiles: true });
       storageMock.readFile.mockRejectedValue(new Error("ENOENT: no such file or directory, open '/app/.immich'"));
 
-      await expect(sut.onBootstrap()).rejects.toThrow('Failed to validate folder mount');
+      await expect(sut.onBootstrap()).rejects.toThrow('Failed to read');
 
       expect(storageMock.createOrOverwriteFile).not.toHaveBeenCalled();
       expect(systemMock.set).not.toHaveBeenCalled();
@@ -62,7 +66,21 @@ describe(StorageService.name, () => {
       systemMock.get.mockResolvedValue({ mountFiles: true });
       storageMock.overwriteFile.mockRejectedValue(new Error("ENOENT: no such file or directory, open '/app/.immich'"));
 
-      await expect(sut.onBootstrap()).rejects.toThrow('Failed to validate folder mount');
+      await expect(sut.onBootstrap()).rejects.toThrow('Failed to write');
+
+      expect(systemMock.set).not.toHaveBeenCalled();
+    });
+
+    it('should startup if checks are disabled', async () => {
+      systemMock.get.mockResolvedValue({ mountFiles: true });
+      configMock.getEnv.mockReturnValue(
+        mockEnvData({
+          storage: { ignoreMountCheckErrors: true },
+        }),
+      );
+      storageMock.overwriteFile.mockRejectedValue(new Error("ENOENT: no such file or directory, open '/app/.immich'"));
+
+      await expect(sut.onBootstrap()).resolves.toBeUndefined();
 
       expect(systemMock.set).not.toHaveBeenCalled();
     });

server/src/services/storage.service.ts

@@ -3,6 +3,7 @@ import { join } from 'node:path';
 import { StorageCore } from 'src/cores/storage.core';
 import { OnEvent } from 'src/decorators';
 import { StorageFolder, SystemMetadataKey } from 'src/enum';
+import { IConfigRepository } from 'src/interfaces/config.interface';
 import { DatabaseLock, IDatabaseRepository } from 'src/interfaces/database.interface';
 import { IDeleteFilesJob, JobStatus } from 'src/interfaces/job.interface';
 import { ILoggerRepository } from 'src/interfaces/logger.interface';
@@ -10,9 +11,12 @@ import { IStorageRepository } from 'src/interfaces/storage.interface';
 import { ISystemMetadataRepository } from 'src/interfaces/system-metadata.interface';
 import { ImmichStartupError } from 'src/utils/events';
 
+const docsMessage = `Please see https://immich.app/docs/administration/system-integrity#folder-checks for more information.`;
+
 @Injectable()
 export class StorageService {
   constructor(
+    @Inject(IConfigRepository) private configRepository: IConfigRepository,
     @Inject(IDatabaseRepository) private databaseRepository: IDatabaseRepository,
     @Inject(IStorageRepository) private storageRepository: IStorageRepository,
     @Inject(ILoggerRepository) private logger: ILoggerRepository,
@@ -23,30 +27,41 @@ export class StorageService {
 
   @OnEvent({ name: 'app.bootstrap' })
   async onBootstrap() {
+    const envData = this.configRepository.getEnv();
+
     await this.databaseRepository.withLock(DatabaseLock.SystemFileMounts, async () => {
       const flags = (await this.systemMetadata.get(SystemMetadataKey.SYSTEM_FLAGS)) || { mountFiles: false };
       const enabled = flags.mountFiles ?? false;
 
       this.logger.log(`Verifying system mount folder checks (enabled=${enabled})`);
 
-      // check each folder exists and is writable
-      for (const folder of Object.values(StorageFolder)) {
-        if (!enabled) {
-          this.logger.log(`Writing initial mount file for the ${folder} folder`);
-          await this.createMountFile(folder);
+      try {
+        // check each folder exists and is writable
+        for (const folder of Object.values(StorageFolder)) {
+          if (!enabled) {
+            this.logger.log(`Writing initial mount file for the ${folder} folder`);
+            await this.createMountFile(folder);
+          }
+
+          await this.verifyReadAccess(folder);
+          await this.verifyWriteAccess(folder);
         }
 
-        await this.verifyReadAccess(folder);
-        await this.verifyWriteAccess(folder);
-      }
+        if (!flags.mountFiles) {
+          flags.mountFiles = true;
+          await this.systemMetadata.set(SystemMetadataKey.SYSTEM_FLAGS, flags);
+          this.logger.log('Successfully enabled system mount folders checks');
+        }
 
-      if (!flags.mountFiles) {
-        flags.mountFiles = true;
-        await this.systemMetadata.set(SystemMetadataKey.SYSTEM_FLAGS, flags);
-        this.logger.log('Successfully enabled system mount folders checks');
+        this.logger.log('Successfully verified system mount folder checks');
+      } catch (error) {
+        if (envData.storage.ignoreMountCheckErrors) {
+          this.logger.error(error);
+          this.logger.warn('Ignoring mount folder errors');
+        } else {
+          throw error;
+        }
       }
-
-      this.logger.log('Successfully verified system mount folder checks');
     });
   }
 
@@ -70,49 +85,45 @@ export class StorageService {
   }
 
   private async verifyReadAccess(folder: StorageFolder) {
-    const { filePath } = this.getMountFilePaths(folder);
+    const { internalPath, externalPath } = this.getMountFilePaths(folder);
     try {
-      await this.storageRepository.readFile(filePath);
+      await this.storageRepository.readFile(internalPath);
     } catch (error) {
-      this.logger.error(`Failed to read ${filePath}: ${error}`);
-      this.logger.error(
-        `The "${folder}" folder appears to be offline/missing, please make sure the volume is mounted with the correct permissions`,
-      );
-      throw new ImmichStartupError(`Failed to validate folder mount (read from "<MEDIA_LOCATION>/${folder}")`);
+      this.logger.error(`Failed to read ${internalPath}: ${error}`);
+      throw new ImmichStartupError(`Failed to read "${externalPath} - ${docsMessage}"`);
     }
   }
 
   private async createMountFile(folder: StorageFolder) {
-    const { folderPath, filePath } = this.getMountFilePaths(folder);
+    const { folderPath, internalPath, externalPath } = this.getMountFilePaths(folder);
     try {
       this.storageRepository.mkdirSync(folderPath);
-      await this.storageRepository.createFile(filePath, Buffer.from(`${Date.now()}`));
+      await this.storageRepository.createFile(internalPath, Buffer.from(`${Date.now()}`));
     } catch (error) {
-      this.logger.error(`Failed to create ${filePath}: ${error}`);
-      this.logger.error(
-        `The "${folder}" folder cannot be written to, please make sure the volume is mounted with the correct permissions`,
-      );
-      throw new ImmichStartupError(`Failed to validate folder mount (write to "<UPLOAD_LOCATION>/${folder}")`);
+      if ((error as NodeJS.ErrnoException).code === 'EEXIST') {
+        this.logger.warn('Found existing mount file, skipping creation');
+        return;
+      }
+      this.logger.error(`Failed to create ${internalPath}: ${error}`);
+      throw new ImmichStartupError(`Failed to create "${externalPath} - ${docsMessage}"`);
     }
   }
 
   private async verifyWriteAccess(folder: StorageFolder) {
-    const { filePath } = this.getMountFilePaths(folder);
+    const { internalPath, externalPath } = this.getMountFilePaths(folder);
     try {
-      await this.storageRepository.overwriteFile(filePath, Buffer.from(`${Date.now()}`));
+      await this.storageRepository.overwriteFile(internalPath, Buffer.from(`${Date.now()}`));
     } catch (error) {
-      this.logger.error(`Failed to write ${filePath}: ${error}`);
-      this.logger.error(
-        `The "${folder}" folder cannot be written to, please make sure the volume is mounted with the correct permissions`,
-      );
-      throw new ImmichStartupError(`Failed to validate folder mount (write to "<UPLOAD_LOCATION>/${folder}")`);
+      this.logger.error(`Failed to write ${internalPath}: ${error}`);
+      throw new ImmichStartupError(`Failed to write "${externalPath} - ${docsMessage}"`);
     }
   }
 
   private getMountFilePaths(folder: StorageFolder) {
     const folderPath = StorageCore.getBaseFolder(folder);
-    const filePath = join(folderPath, '.immich');
+    const internalPath = join(folderPath, '.immich');
+    const externalPath = `<UPLOAD_LOCATION>/${folder}/.immich`;
 
-    return { folderPath, filePath };
+    return { folderPath, internalPath, externalPath };
   }
 }