chore: remove audit file report (#17994)

server/src/controllers/file-report.controller.ts

@@ -1,29 +0,0 @@
-import { Body, Controller, Get, Post } from '@nestjs/common';
-import { ApiTags } from '@nestjs/swagger';
-import { FileChecksumDto, FileChecksumResponseDto, FileReportDto, FileReportFixDto } from 'src/dtos/audit.dto';
-import { Authenticated } from 'src/middleware/auth.guard';
-import { AuditService } from 'src/services/audit.service';
-
-@ApiTags('File Reports')
-@Controller('reports')
-export class ReportController {
-  constructor(private service: AuditService) {}
-
-  @Get()
-  @Authenticated({ admin: true })
-  getAuditFiles(): Promise<FileReportDto> {
-    return this.service.getFileReport();
-  }
-
-  @Post('checksum')
-  @Authenticated({ admin: true })
-  getFileChecksums(@Body() dto: FileChecksumDto): Promise<FileChecksumResponseDto[]> {
-    return this.service.getChecksums(dto);
-  }
-
-  @Post('fix')
-  @Authenticated({ admin: true })
-  fixAuditFiles(@Body() dto: FileReportFixDto): Promise<void> {
-    return this.service.fixItems(dto.items);
-  }
-}

server/src/controllers/index.ts

@@ -8,7 +8,6 @@ import { AuthController } from 'src/controllers/auth.controller';
 import { DownloadController } from 'src/controllers/download.controller';
 import { DuplicateController } from 'src/controllers/duplicate.controller';
 import { FaceController } from 'src/controllers/face.controller';
-import { ReportController } from 'src/controllers/file-report.controller';
 import { JobController } from 'src/controllers/job.controller';
 import { LibraryController } from 'src/controllers/library.controller';
 import { MapController } from 'src/controllers/map.controller';
@@ -53,7 +52,6 @@ export const controllers = [
   OAuthController,
   PartnerController,
   PersonController,
-  ReportController,
   SearchController,
   ServerController,
   SessionController,

server/src/dtos/audit.dto.ts

@@ -1,73 +0,0 @@
-import { ApiProperty } from '@nestjs/swagger';
-import { Type } from 'class-transformer';
-import { IsArray, IsEnum, IsString, IsUUID, ValidateNested } from 'class-validator';
-import { AssetPathType, EntityType, PathType, PersonPathType, UserPathType } from 'src/enum';
-import { Optional, ValidateDate, ValidateUUID } from 'src/validation';
-
-const PathEnum = Object.values({ ...AssetPathType, ...PersonPathType, ...UserPathType });
-
-export class AuditDeletesDto {
-  @ValidateDate()
-  after!: Date;
-
-  @ApiProperty({ enum: EntityType, enumName: 'EntityType' })
-  @IsEnum(EntityType)
-  entityType!: EntityType;
-
-  @Optional()
-  @IsUUID('4')
-  @ApiProperty({ format: 'uuid' })
-  userId?: string;
-}
-
-export enum PathEntityType {
-  ASSET = 'asset',
-  PERSON = 'person',
-  USER = 'user',
-}
-
-export class AuditDeletesResponseDto {
-  needsFullSync!: boolean;
-  ids!: string[];
-}
-
-export class FileReportDto {
-  orphans!: FileReportItemDto[];
-  extras!: string[];
-}
-
-export class FileChecksumDto {
-  @IsString({ each: true })
-  filenames!: string[];
-}
-
-export class FileChecksumResponseDto {
-  filename!: string;
-  checksum!: string;
-}
-
-export class FileReportFixDto {
-  @IsArray()
-  @ValidateNested({ each: true })
-  @Type(() => FileReportItemDto)
-  items!: FileReportItemDto[];
-}
-
-// used both as request and response dto
-export class FileReportItemDto {
-  @ValidateUUID()
-  entityId!: string;
-
-  @ApiProperty({ enumName: 'PathEntityType', enum: PathEntityType })
-  @IsEnum(PathEntityType)
-  entityType!: PathEntityType;
-
-  @ApiProperty({ enumName: 'PathType', enum: PathEnum })
-  @IsEnum(PathEnum)
-  pathType!: PathType;
-
-  @IsString()
-  pathValue!: string;
-
-  checksum?: string;
-}

server/src/services/audit.service.spec.ts

@@ -1,6 +1,4 @@
-import { BadRequestException } from '@nestjs/common';
-import { FileReportItemDto } from 'src/dtos/audit.dto';
-import { AssetFileType, AssetPathType, JobStatus, PersonPathType, UserPathType } from 'src/enum';
+import { JobStatus } from 'src/enum';
 import { AuditService } from 'src/services/audit.service';
 import { newTestService, ServiceMocks } from 'test/utils';
 
@@ -25,148 +23,4 @@ describe(AuditService.name, () => {
       expect(mocks.audit.removeBefore).toHaveBeenCalledWith(expect.any(Date));
     });
   });
-
-  describe('getChecksums', () => {
-    it('should fail if the file is not in the immich path', async () => {
-      await expect(sut.getChecksums({ filenames: ['foo/bar'] })).rejects.toBeInstanceOf(BadRequestException);
-
-      expect(mocks.crypto.hashFile).not.toHaveBeenCalled();
-    });
-
-    it('should get checksum for valid file', async () => {
-      await expect(sut.getChecksums({ filenames: ['./upload/my-file.jpg'] })).resolves.toEqual([
-        { filename: './upload/my-file.jpg', checksum: expect.any(String) },
-      ]);
-
-      expect(mocks.crypto.hashFile).toHaveBeenCalledWith('./upload/my-file.jpg');
-    });
-  });
-
-  describe('fixItems', () => {
-    it('should fail if the file is not in the immich path', async () => {
-      await expect(
-        sut.fixItems([
-          { entityId: 'my-id', pathType: AssetPathType.ORIGINAL, pathValue: 'foo/bar' } as FileReportItemDto,
-        ]),
-      ).rejects.toBeInstanceOf(BadRequestException);
-
-      expect(mocks.asset.update).not.toHaveBeenCalled();
-      expect(mocks.asset.upsertFile).not.toHaveBeenCalled();
-      expect(mocks.person.update).not.toHaveBeenCalled();
-      expect(mocks.user.update).not.toHaveBeenCalled();
-    });
-
-    it('should update encoded video path', async () => {
-      await sut.fixItems([
-        {
-          entityId: 'my-id',
-          pathType: AssetPathType.ENCODED_VIDEO,
-          pathValue: './upload/my-video.mp4',
-        } as FileReportItemDto,
-      ]);
-
-      expect(mocks.asset.update).toHaveBeenCalledWith({ id: 'my-id', encodedVideoPath: './upload/my-video.mp4' });
-      expect(mocks.asset.upsertFile).not.toHaveBeenCalled();
-      expect(mocks.person.update).not.toHaveBeenCalled();
-      expect(mocks.user.update).not.toHaveBeenCalled();
-    });
-
-    it('should update preview path', async () => {
-      await sut.fixItems([
-        {
-          entityId: 'my-id',
-          pathType: AssetPathType.PREVIEW,
-          pathValue: './upload/my-preview.png',
-        } as FileReportItemDto,
-      ]);
-
-      expect(mocks.asset.upsertFile).toHaveBeenCalledWith({
-        assetId: 'my-id',
-        type: AssetFileType.PREVIEW,
-        path: './upload/my-preview.png',
-      });
-      expect(mocks.asset.update).not.toHaveBeenCalled();
-      expect(mocks.person.update).not.toHaveBeenCalled();
-      expect(mocks.user.update).not.toHaveBeenCalled();
-    });
-
-    it('should update thumbnail path', async () => {
-      await sut.fixItems([
-        {
-          entityId: 'my-id',
-          pathType: AssetPathType.THUMBNAIL,
-          pathValue: './upload/my-thumbnail.webp',
-        } as FileReportItemDto,
-      ]);
-
-      expect(mocks.asset.upsertFile).toHaveBeenCalledWith({
-        assetId: 'my-id',
-        type: AssetFileType.THUMBNAIL,
-        path: './upload/my-thumbnail.webp',
-      });
-      expect(mocks.asset.update).not.toHaveBeenCalled();
-      expect(mocks.person.update).not.toHaveBeenCalled();
-      expect(mocks.user.update).not.toHaveBeenCalled();
-    });
-
-    it('should update original path', async () => {
-      await sut.fixItems([
-        {
-          entityId: 'my-id',
-          pathType: AssetPathType.ORIGINAL,
-          pathValue: './upload/my-original.png',
-        } as FileReportItemDto,
-      ]);
-
-      expect(mocks.asset.update).toHaveBeenCalledWith({ id: 'my-id', originalPath: './upload/my-original.png' });
-      expect(mocks.asset.upsertFile).not.toHaveBeenCalled();
-      expect(mocks.person.update).not.toHaveBeenCalled();
-      expect(mocks.user.update).not.toHaveBeenCalled();
-    });
-
-    it('should update sidecar path', async () => {
-      await sut.fixItems([
-        {
-          entityId: 'my-id',
-          pathType: AssetPathType.SIDECAR,
-          pathValue: './upload/my-sidecar.xmp',
-        } as FileReportItemDto,
-      ]);
-
-      expect(mocks.asset.update).toHaveBeenCalledWith({ id: 'my-id', sidecarPath: './upload/my-sidecar.xmp' });
-      expect(mocks.asset.upsertFile).not.toHaveBeenCalled();
-      expect(mocks.person.update).not.toHaveBeenCalled();
-      expect(mocks.user.update).not.toHaveBeenCalled();
-    });
-
-    it('should update face path', async () => {
-      await sut.fixItems([
-        {
-          entityId: 'my-id',
-          pathType: PersonPathType.FACE,
-          pathValue: './upload/my-face.jpg',
-        } as FileReportItemDto,
-      ]);
-
-      expect(mocks.person.update).toHaveBeenCalledWith({ id: 'my-id', thumbnailPath: './upload/my-face.jpg' });
-      expect(mocks.asset.update).not.toHaveBeenCalled();
-      expect(mocks.asset.upsertFile).not.toHaveBeenCalled();
-      expect(mocks.user.update).not.toHaveBeenCalled();
-    });
-
-    it('should update profile path', async () => {
-      await sut.fixItems([
-        {
-          entityId: 'my-id',
-          pathType: UserPathType.PROFILE,
-          pathValue: './upload/my-profile-pic.jpg',
-        } as FileReportItemDto,
-      ]);
-
-      expect(mocks.user.update).toHaveBeenCalledWith('my-id', { profileImagePath: './upload/my-profile-pic.jpg' });
-      expect(mocks.asset.update).not.toHaveBeenCalled();
-      expect(mocks.asset.upsertFile).not.toHaveBeenCalled();
-      expect(mocks.person.update).not.toHaveBeenCalled();
-    });
-  });
 });

server/src/services/audit.service.ts

@@ -1,23 +1,9 @@
-import { BadRequestException, Injectable } from '@nestjs/common';
+import { Injectable } from '@nestjs/common';
 import { DateTime } from 'luxon';
-import { resolve } from 'node:path';
-import { AUDIT_LOG_MAX_DURATION, JOBS_ASSET_PAGINATION_SIZE } from 'src/constants';
-import { StorageCore } from 'src/cores/storage.core';
+import { AUDIT_LOG_MAX_DURATION } from 'src/constants';
 import { OnJob } from 'src/decorators';
-import { FileChecksumDto, FileChecksumResponseDto, FileReportItemDto, PathEntityType } from 'src/dtos/audit.dto';
-import {
-  AssetFileType,
-  AssetPathType,
-  JobName,
-  JobStatus,
-  PersonPathType,
-  QueueName,
-  StorageFolder,
-  UserPathType,
-} from 'src/enum';
+import { JobName, JobStatus, QueueName } from 'src/enum';
 import { BaseService } from 'src/services/base.service';
-import { getAssetFiles } from 'src/utils/asset.util';
-import { usePagination } from 'src/utils/pagination';
 
 @Injectable()
 export class AuditService extends BaseService {
@@ -26,187 +12,4 @@ export class AuditService extends BaseService {
     await this.auditRepository.removeBefore(DateTime.now().minus(AUDIT_LOG_MAX_DURATION).toJSDate());
     return JobStatus.SUCCESS;
   }
-
-  async getChecksums(dto: FileChecksumDto) {
-    const results: FileChecksumResponseDto[] = [];
-    for (const filename of dto.filenames) {
-      if (!StorageCore.isImmichPath(filename)) {
-        throw new BadRequestException(
-          `Could not get the checksum of ${filename} because the file isn't accessible by Immich`,
-        );
-      }
-
-      const checksum = await this.cryptoRepository.hashFile(filename);
-      results.push({ filename, checksum: checksum.toString('base64') });
-    }
-    return results;
-  }
-
-  async fixItems(items: FileReportItemDto[]) {
-    for (const { entityId: id, pathType, pathValue } of items) {
-      if (!StorageCore.isImmichPath(pathValue)) {
-        throw new BadRequestException(
-          `Could not fix item ${id} with path ${pathValue} because the file isn't accessible by Immich`,
-        );
-      }
-
-      switch (pathType) {
-        case AssetPathType.ENCODED_VIDEO: {
-          await this.assetRepository.update({ id, encodedVideoPath: pathValue });
-          break;
-        }
-
-        case AssetPathType.PREVIEW: {
-          await this.assetRepository.upsertFile({ assetId: id, type: AssetFileType.PREVIEW, path: pathValue });
-          break;
-        }
-
-        case AssetPathType.THUMBNAIL: {
-          await this.assetRepository.upsertFile({ assetId: id, type: AssetFileType.THUMBNAIL, path: pathValue });
-          break;
-        }
-
-        case AssetPathType.ORIGINAL: {
-          await this.assetRepository.update({ id, originalPath: pathValue });
-          break;
-        }
-
-        case AssetPathType.SIDECAR: {
-          await this.assetRepository.update({ id, sidecarPath: pathValue });
-          break;
-        }
-
-        case PersonPathType.FACE: {
-          await this.personRepository.update({ id, thumbnailPath: pathValue });
-          break;
-        }
-
-        case UserPathType.PROFILE: {
-          await this.userRepository.update(id, { profileImagePath: pathValue });
-          break;
-        }
-      }
-    }
-  }
-
-  private fullPath(filename: string) {
-    return resolve(filename);
-  }
-
-  async getFileReport() {
-    const hasFile = (items: Set<string>, filename: string) => items.has(filename) || items.has(this.fullPath(filename));
-    const crawl = async (folder: StorageFolder) =>
-      new Set(
-        await this.storageRepository.crawl({
-          includeHidden: true,
-          pathsToCrawl: [StorageCore.getBaseFolder(folder)],
-        }),
-      );
-
-    const uploadFiles = await crawl(StorageFolder.UPLOAD);
-    const libraryFiles = await crawl(StorageFolder.LIBRARY);
-    const thumbFiles = await crawl(StorageFolder.THUMBNAILS);
-    const videoFiles = await crawl(StorageFolder.ENCODED_VIDEO);
-    const profileFiles = await crawl(StorageFolder.PROFILE);
-    const allFiles = new Set<string>();
-    for (const list of [libraryFiles, thumbFiles, videoFiles, profileFiles, uploadFiles]) {
-      for (const item of list) {
-        allFiles.add(item);
-      }
-    }
-
-    const track = (filename: string | null | undefined) => {
-      if (!filename) {
-        return;
-      }
-      allFiles.delete(filename);
-      allFiles.delete(this.fullPath(filename));
-    };
-
-    this.logger.log(
-      `Found ${libraryFiles.size} original files, ${thumbFiles.size} thumbnails, ${videoFiles.size} encoded videos, ${profileFiles.size} profile files`,
-    );
-    const pagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (options) =>
-      this.assetRepository.getAll(options, { withDeleted: true, withArchived: true }),
-    );
-
-    let assetCount = 0;
-
-    const orphans: FileReportItemDto[] = [];
-    for await (const assets of pagination) {
-      assetCount += assets.length;
-      for (const { id, files, originalPath, encodedVideoPath, isExternal, checksum } of assets) {
-        const { fullsizeFile, previewFile, thumbnailFile } = getAssetFiles(files);
-        for (const file of [
-          originalPath,
-          fullsizeFile?.path,
-          previewFile?.path,
-          encodedVideoPath,
-          thumbnailFile?.path,
-        ]) {
-          track(file);
-        }
-
-        const entity = { entityId: id, entityType: PathEntityType.ASSET, checksum: checksum.toString('base64') };
-        if (
-          originalPath &&
-          !hasFile(libraryFiles, originalPath) &&
-          !hasFile(uploadFiles, originalPath) &&
-          // Android motion assets
-          !hasFile(videoFiles, originalPath) &&
-          // ignore external library assets
-          !isExternal
-        ) {
-          orphans.push({ ...entity, pathType: AssetPathType.ORIGINAL, pathValue: originalPath });
-        }
-        if (previewFile && !hasFile(thumbFiles, previewFile.path)) {
-          orphans.push({ ...entity, pathType: AssetPathType.PREVIEW, pathValue: previewFile.path });
-        }
-        if (thumbnailFile && !hasFile(thumbFiles, thumbnailFile.path)) {
-          orphans.push({ ...entity, pathType: AssetPathType.THUMBNAIL, pathValue: thumbnailFile.path });
-        }
-        if (encodedVideoPath && !hasFile(videoFiles, encodedVideoPath)) {
-          orphans.push({ ...entity, pathType: AssetPathType.THUMBNAIL, pathValue: encodedVideoPath });
-        }
-      }
-    }
-
-    const users = await this.userRepository.getList();
-    for (const { id, profileImagePath } of users) {
-      track(profileImagePath);
-
-      const entity = { entityId: id, entityType: PathEntityType.USER };
-      if (profileImagePath && !hasFile(profileFiles, profileImagePath)) {
-        orphans.push({ ...entity, pathType: UserPathType.PROFILE, pathValue: profileImagePath });
-      }
-    }
-
-    let peopleCount = 0;
-    for await (const { id, thumbnailPath } of this.personRepository.getAll()) {
-      track(thumbnailPath);
-      const entity = { entityId: id, entityType: PathEntityType.PERSON };
-      if (thumbnailPath && !hasFile(thumbFiles, thumbnailPath)) {
-        orphans.push({ ...entity, pathType: PersonPathType.FACE, pathValue: thumbnailPath });
-      }
-
-      if (peopleCount === JOBS_ASSET_PAGINATION_SIZE) {
-        this.logger.log(`Found ${assetCount} assets, ${users.length} users, ${peopleCount} people`);
-        peopleCount = 0;
-      }
-    }
-
-    this.logger.log(`Found ${assetCount} assets, ${users.length} users, ${peopleCount} people`);
-
-    const extras: string[] = [];
-    for (const file of allFiles) {
-      extras.push(file);
-    }
-
-    // send as absolute paths
-    for (const orphan of orphans) {
-      orphan.pathValue = this.fullPath(orphan.pathValue);
-    }
-
-    return { orphans, extras };
-  }
 }