feat(web,server): offline/untracked files admin tool (#4447)

* feat: admin repair orphans tool

* chore: open api

* fix: include upload folder

* fix: bugs

* feat: empty placeholder

* fix: checks

* feat: move buttons to top of page

* feat: styling and clipboard

* styling

* better clicking hitbox

* fix: show title on hover

* feat: download report

* restrict file access to immich related files

* Add description

---------

Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
Co-authored-by: Daniel Dietzler <mail@ddietzler.dev>

server/src/domain/audit/audit.dto.ts

@@ -1,8 +1,10 @@
-import { EntityType } from '@app/infra/entities';
+import { AssetPathType, EntityType, PathType, PersonPathType, UserPathType } from '@app/infra/entities';
 import { ApiProperty } from '@nestjs/swagger';
 import { Type } from 'class-transformer';
-import { IsDate, IsEnum, IsUUID } from 'class-validator';
-import { Optional } from '../domain.util';
+import { IsArray, IsDate, IsEnum, IsString, IsUUID, ValidateNested } from 'class-validator';
+import { Optional, ValidateUUID } from '../domain.util';
+
+const PathEnum = Object.values({ ...AssetPathType, ...PersonPathType, ...UserPathType });
 
 export class AuditDeletesDto {
   @IsDate()
@@ -19,7 +21,54 @@ export class AuditDeletesDto {
   userId?: string;
 }
 
+export enum PathEntityType {
+  ASSET = 'asset',
+  PERSON = 'person',
+  USER = 'user',
+}
+
 export class AuditDeletesResponseDto {
   needsFullSync!: boolean;
   ids!: string[];
 }
+
+export class FileReportDto {
+  orphans!: FileReportItemDto[];
+  extras!: string[];
+}
+
+export class FileChecksumDto {
+  @IsString({ each: true })
+  filenames!: string[];
+}
+
+export class FileChecksumResponseDto {
+  filename!: string;
+  checksum!: string;
+}
+
+export class FileReportFixDto {
+  @IsArray()
+  @ValidateNested({ each: true })
+  @Type(() => FileReportItemDto)
+  items!: FileReportItemDto[];
+}
+
+// used both as request and response dto
+export class FileReportItemDto {
+  @ValidateUUID()
+  entityId!: string;
+
+  @ApiProperty({ enumName: 'PathEntityType', enum: PathEntityType })
+  @IsEnum(PathEntityType)
+  entityType!: PathEntityType;
+
+  @ApiProperty({ enumName: 'PathType', enum: PathEnum })
+  @IsEnum(PathEnum)
+  pathType!: PathType;
+
+  @IsString()
+  pathValue!: string;
+
+  checksum?: string;
+}

server/src/domain/audit/audit.service.spec.ts

@@ -1,17 +1,45 @@
 import { DatabaseAction, EntityType } from '@app/infra/entities';
-import { IAccessRepositoryMock, auditStub, authStub, newAccessRepositoryMock, newAuditRepositoryMock } from '@test';
-import { IAuditRepository } from '../repositories';
+import {
+  IAccessRepositoryMock,
+  auditStub,
+  authStub,
+  newAccessRepositoryMock,
+  newAssetRepositoryMock,
+  newAuditRepositoryMock,
+  newCryptoRepositoryMock,
+  newPersonRepositoryMock,
+  newStorageRepositoryMock,
+  newUserRepositoryMock,
+} from '@test';
+import {
+  IAssetRepository,
+  IAuditRepository,
+  ICryptoRepository,
+  IPersonRepository,
+  IStorageRepository,
+  IUserRepository,
+} from '../repositories';
 import { AuditService } from './audit.service';
 
 describe(AuditService.name, () => {
   let sut: AuditService;
   let accessMock: IAccessRepositoryMock;
+  let assetMock: jest.Mocked<IAssetRepository>;
   let auditMock: jest.Mocked<IAuditRepository>;
+  let cryptoMock: jest.Mocked<ICryptoRepository>;
+  let personMock: jest.Mocked<IPersonRepository>;
+  let storageMock: jest.Mocked<IStorageRepository>;
+  let userMock: jest.Mocked<IUserRepository>;
 
   beforeEach(async () => {
     accessMock = newAccessRepositoryMock();
+    assetMock = newAssetRepositoryMock();
+    cryptoMock = newCryptoRepositoryMock();
     auditMock = newAuditRepositoryMock();
-    sut = new AuditService(accessMock, auditMock);
+    personMock = newPersonRepositoryMock();
+    storageMock = newStorageRepositoryMock();
+    userMock = newUserRepositoryMock();
+    sut = new AuditService(accessMock, assetMock, cryptoMock, personMock, auditMock, storageMock, userMock);
   });
 
   it('should work', () => {

server/src/domain/audit/audit.service.ts

@@ -1,19 +1,44 @@
-import { DatabaseAction } from '@app/infra/entities';
-import { Inject, Injectable } from '@nestjs/common';
+import { AssetPathType, DatabaseAction, PersonPathType, UserPathType } from '@app/infra/entities';
+import { BadRequestException, Inject, Injectable, Logger } from '@nestjs/common';
 import { DateTime } from 'luxon';
+import { resolve } from 'node:path';
 import { AccessCore, Permission } from '../access';
 import { AuthUserDto } from '../auth';
 import { AUDIT_LOG_MAX_DURATION } from '../domain.constant';
-import { IAccessRepository, IAuditRepository } from '../repositories';
-import { AuditDeletesDto, AuditDeletesResponseDto } from './audit.dto';
+import { usePagination } from '../domain.util';
+import { JOBS_ASSET_PAGINATION_SIZE } from '../job';
+import {
+  IAccessRepository,
+  IAssetRepository,
+  IAuditRepository,
+  ICryptoRepository,
+  IPersonRepository,
+  IStorageRepository,
+  IUserRepository,
+} from '../repositories';
+import { StorageCore, StorageFolder } from '../storage';
+import {
+  AuditDeletesDto,
+  AuditDeletesResponseDto,
+  FileChecksumDto,
+  FileChecksumResponseDto,
+  FileReportItemDto,
+  PathEntityType,
+} from './audit.dto';
 
 @Injectable()
 export class AuditService {
   private access: AccessCore;
+  private logger = new Logger(AuditService.name);
 
   constructor(
     @Inject(IAccessRepository) accessRepository: IAccessRepository,
+    @Inject(IAssetRepository) private assetRepository: IAssetRepository,
+    @Inject(ICryptoRepository) private cryptoRepository: ICryptoRepository,
+    @Inject(IPersonRepository) private personRepository: IPersonRepository,
     @Inject(IAuditRepository) private repository: IAuditRepository,
+    @Inject(IStorageRepository) private storageRepository: IStorageRepository,
+    @Inject(IUserRepository) private userRepository: IUserRepository,
   ) {
     this.access = new AccessCore(accessRepository);
   }
@@ -40,4 +65,160 @@ export class AuditService {
       ids: audits.map(({ entityId }) => entityId),
     };
   }
+
+  async getChecksums(dto: FileChecksumDto) {
+    const results: FileChecksumResponseDto[] = [];
+    for (const filename of dto.filenames) {
+      if (!StorageCore.isImmichPath(filename)) {
+        throw new BadRequestException(
+          `Could not get the checksum of ${filename} because the file isn't accessible by Immich`,
+        );
+      }
+
+      const checksum = await this.cryptoRepository.hashFile(filename);
+      results.push({ filename, checksum: checksum.toString('base64') });
+    }
+    return results;
+  }
+
+  async fixItems(items: FileReportItemDto[]) {
+    for (const { entityId: id, pathType, pathValue } of items) {
+      if (!StorageCore.isImmichPath(pathValue)) {
+        throw new BadRequestException(
+          `Could not fix item ${id} with path ${pathValue} because the file isn't accessible by Immich`,
+        );
+      }
+
+      switch (pathType) {
+        case AssetPathType.ENCODED_VIDEO:
+          await this.assetRepository.save({ id, encodedVideoPath: pathValue });
+          break;
+
+        case AssetPathType.JPEG_THUMBNAIL:
+          await this.assetRepository.save({ id, resizePath: pathValue });
+          break;
+
+        case AssetPathType.WEBP_THUMBNAIL:
+          await this.assetRepository.save({ id, webpPath: pathValue });
+          break;
+
+        case AssetPathType.ORIGINAL:
+          await this.assetRepository.save({ id, originalPath: pathValue });
+          break;
+
+        case AssetPathType.SIDECAR:
+          await this.assetRepository.save({ id, sidecarPath: pathValue });
+          break;
+
+        case PersonPathType.FACE:
+          await this.personRepository.update({ id, thumbnailPath: pathValue });
+          break;
+
+        case UserPathType.PROFILE:
+          await this.userRepository.update(id, { profileImagePath: pathValue });
+          break;
+      }
+    }
+  }
+
+  async getFileReport() {
+    const fullPath = (filename: string) => resolve(filename);
+    const hasFile = (items: Set<string>, filename: string) => items.has(filename) || items.has(fullPath(filename));
+    const crawl = async (folder: StorageFolder) =>
+      new Set(await this.storageRepository.crawl({ pathsToCrawl: [StorageCore.getBaseFolder(folder)] }));
+
+    const uploadFiles = await crawl(StorageFolder.UPLOAD);
+    const libraryFiles = await crawl(StorageFolder.LIBRARY);
+    const thumbFiles = await crawl(StorageFolder.THUMBNAILS);
+    const videoFiles = await crawl(StorageFolder.ENCODED_VIDEO);
+    const profileFiles = await crawl(StorageFolder.PROFILE);
+    const allFiles = new Set<string>();
+    for (const list of [libraryFiles, thumbFiles, videoFiles, profileFiles, uploadFiles]) {
+      for (const item of list) {
+        allFiles.add(item);
+      }
+    }
+
+    const track = (filename: string | null) => {
+      if (!filename) {
+        return;
+      }
+      allFiles.delete(filename);
+      allFiles.delete(fullPath(filename));
+    };
+
+    this.logger.log(
+      `Found ${libraryFiles.size} original files, ${thumbFiles.size} thumbnails, ${videoFiles.size} encoded videos, ${profileFiles.size} profile files`,
+    );
+    const pagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (options) =>
+      this.assetRepository.getAll(options, { withDeleted: true }),
+    );
+
+    let assetCount = 0;
+
+    const orphans: FileReportItemDto[] = [];
+    for await (const assets of pagination) {
+      assetCount += assets.length;
+      for (const { id, originalPath, resizePath, encodedVideoPath, webpPath, isExternal, checksum } of assets) {
+        for (const file of [originalPath, resizePath, encodedVideoPath, webpPath]) {
+          track(file);
+        }
+
+        const entity = { entityId: id, entityType: PathEntityType.ASSET, checksum: checksum.toString('base64') };
+        if (
+          originalPath &&
+          !hasFile(libraryFiles, originalPath) &&
+          !hasFile(uploadFiles, originalPath) &&
+          // Android motion assets
+          !hasFile(videoFiles, originalPath) &&
+          // ignore external library assets
+          !isExternal
+        ) {
+          orphans.push({ ...entity, pathType: AssetPathType.ORIGINAL, pathValue: originalPath });
+        }
+        if (resizePath && !hasFile(thumbFiles, resizePath)) {
+          orphans.push({ ...entity, pathType: AssetPathType.JPEG_THUMBNAIL, pathValue: resizePath });
+        }
+        if (webpPath && !hasFile(thumbFiles, webpPath)) {
+          orphans.push({ ...entity, pathType: AssetPathType.WEBP_THUMBNAIL, pathValue: webpPath });
+        }
+        if (encodedVideoPath && !hasFile(videoFiles, encodedVideoPath)) {
+          orphans.push({ ...entity, pathType: AssetPathType.WEBP_THUMBNAIL, pathValue: encodedVideoPath });
+        }
+      }
+    }
+
+    const users = await this.userRepository.getList();
+    for (const { id, profileImagePath } of users) {
+      track(profileImagePath);
+
+      const entity = { entityId: id, entityType: PathEntityType.USER };
+      if (profileImagePath && !hasFile(profileFiles, profileImagePath)) {
+        orphans.push({ ...entity, pathType: UserPathType.PROFILE, pathValue: profileImagePath });
+      }
+    }
+
+    const people = await this.personRepository.getAll();
+    for (const { id, thumbnailPath } of people) {
+      track(thumbnailPath);
+      const entity = { entityId: id, entityType: PathEntityType.PERSON };
+      if (thumbnailPath && !hasFile(thumbFiles, thumbnailPath)) {
+        orphans.push({ ...entity, pathType: PersonPathType.FACE, pathValue: thumbnailPath });
+      }
+    }
+
+    this.logger.log(`Found ${assetCount} assets, ${users.length} users, ${people.length} people`);
+
+    const extras: string[] = [];
+    for (const file of allFiles) {
+      extras.push(file);
+    }
+
+    // send as absolute paths
+    for (const orphan of orphans) {
+      orphan.pathValue = fullPath(orphan.pathValue);
+    }
+
+    return { orphans, extras };
+  }
 }