feat(server): immich checksum header (#9229)

* feat: dedupe by checksum header

* chore: open api

server/src/services/asset-v1.service.ts

@@ -37,6 +37,7 @@ import { IUserRepository } from 'src/interfaces/user.interface';
 import { UploadFile } from 'src/services/asset.service';
 import { CacheControl, ImmichFileResponse, getLivePhotoMotionFilename } from 'src/utils/file';
 import { mimeTypes } from 'src/utils/mime-types';
+import { fromChecksum } from 'src/utils/request';
 import { QueryFailedError } from 'typeorm';
 
 @Injectable()
@@ -164,14 +165,7 @@ export class AssetServiceV1 {
   }
 
   async bulkUploadCheck(auth: AuthDto, dto: AssetBulkUploadCheckDto): Promise<AssetBulkUploadCheckResponseDto> {
-    // support base64 and hex checksums
-    for (const asset of dto.assets) {
-      if (asset.checksum.length === 28) {
-        asset.checksum = Buffer.from(asset.checksum, 'base64').toString('hex');
-      }
-    }
-
-    const checksums: Buffer[] = dto.assets.map((asset) => Buffer.from(asset.checksum, 'hex'));
+    const checksums: Buffer[] = dto.assets.map((asset) => fromChecksum(asset.checksum));
     const results = await this.assetRepositoryV1.getAssetsByChecksums(auth.user.id, checksums);
     const checksumMap: Record<string, string> = {};
 
@@ -181,7 +175,7 @@ export class AssetServiceV1 {
 
     return {
       results: dto.assets.map(({ id, checksum }) => {
-        const duplicate = checksumMap[checksum];
+        const duplicate = checksumMap[fromChecksum(checksum).toString('hex')];
         if (duplicate) {
           return {
             id,

server/src/services/asset.service.spec.ts

@@ -29,6 +29,8 @@ import { newSystemConfigRepositoryMock } from 'test/repositories/system-config.r
 import { newUserRepositoryMock } from 'test/repositories/user.repository.mock';
 import { Mocked, vitest } from 'vitest';
 
+const file1 = Buffer.from('d2947b871a706081be194569951b7db246907957', 'hex');
+
 const stats: AssetStats = {
   [AssetType.IMAGE]: 10,
   [AssetType.VIDEO]: 23,
@@ -198,6 +200,31 @@ describe(AssetService.name, () => {
     mockGetById([assetStub.livePhotoStillAsset, assetStub.livePhotoMotionAsset]);
   });
 
+  describe('getUploadAssetIdByChecksum', () => {
+    it('should handle a non-existent asset', async () => {
+      await expect(sut.getUploadAssetIdByChecksum(authStub.admin, file1.toString('hex'))).resolves.toBeUndefined();
+      expect(assetMock.getUploadAssetIdByChecksum).toHaveBeenCalledWith(authStub.admin.user.id, file1);
+    });
+
+    it('should find an existing asset', async () => {
+      assetMock.getUploadAssetIdByChecksum.mockResolvedValue('asset-id');
+      await expect(sut.getUploadAssetIdByChecksum(authStub.admin, file1.toString('hex'))).resolves.toEqual({
+        id: 'asset-id',
+        duplicate: true,
+      });
+      expect(assetMock.getUploadAssetIdByChecksum).toHaveBeenCalledWith(authStub.admin.user.id, file1);
+    });
+
+    it('should find an existing asset by base64', async () => {
+      assetMock.getUploadAssetIdByChecksum.mockResolvedValue('asset-id');
+      await expect(sut.getUploadAssetIdByChecksum(authStub.admin, file1.toString('base64'))).resolves.toEqual({
+        id: 'asset-id',
+        duplicate: true,
+      });
+      expect(assetMock.getUploadAssetIdByChecksum).toHaveBeenCalledWith(authStub.admin.user.id, file1);
+    });
+  });
+
   describe('canUpload', () => {
     it('should require an authenticated user', () => {
       expect(() => sut.canUploadFile(uploadFile.nullAuth)).toThrowError(UnauthorizedException);

server/src/services/asset.service.ts

@@ -12,6 +12,7 @@ import {
   SanitizedAssetResponseDto,
   mapAsset,
 } from 'src/dtos/asset-response.dto';
+import { AssetFileUploadResponseDto } from 'src/dtos/asset-v1-response.dto';
 import {
   AssetBulkDeleteDto,
   AssetBulkUpdateDto,
@@ -47,6 +48,7 @@ import { ISystemConfigRepository } from 'src/interfaces/system-config.interface'
 import { IUserRepository } from 'src/interfaces/user.interface';
 import { mimeTypes } from 'src/utils/mime-types';
 import { usePagination } from 'src/utils/pagination';
+import { fromChecksum } from 'src/utils/request';
 
 export interface UploadRequest {
   auth: AuthDto | null;
@@ -83,6 +85,19 @@ export class AssetService {
     this.configCore = SystemConfigCore.create(configRepository, this.logger);
   }
 
+  async getUploadAssetIdByChecksum(auth: AuthDto, checksum?: string): Promise<AssetFileUploadResponseDto | undefined> {
+    if (!checksum) {
+      return;
+    }
+
+    const assetId = await this.assetRepository.getUploadAssetIdByChecksum(auth.user.id, fromChecksum(checksum));
+    if (!assetId) {
+      return;
+    }
+
+    return { id: assetId, duplicate: true };
+  }
+
   canUploadFile({ auth, fieldName, file }: UploadRequest): true {
     this.access.requireUploadAccess(auth);