refactor(server): download assets (#3032)

* refactor: download assets

* chore: open api

* chore: finish tests, make size configurable

* chore: defualt to 4GiB

* chore: open api

* fix: optional archive size

* fix: bugs

* chore: cleanup

server/src/domain/access/access.core.ts

@@ -16,6 +16,7 @@ export enum Permission {
   ALBUM_UPDATE = 'album.update',
   ALBUM_DELETE = 'album.delete',
   ALBUM_SHARE = 'album.share',
+  ALBUM_DOWNLOAD = 'album.download',
 
   LIBRARY_READ = 'library.read',
   LIBRARY_DOWNLOAD = 'library.download',
@@ -68,6 +69,10 @@ export class AccessCore {
         // TODO: fix this to not use authUser.id for shared link access control
         return this.repository.asset.hasOwnerAccess(authUser.id, id);
 
+      case Permission.ALBUM_DOWNLOAD: {
+        return !!authUser.isAllowDownload && (await this.repository.album.hasSharedLinkAccess(sharedLinkId, id));
+      }
+
       // case Permission.ALBUM_READ:
       //   return this.repository.album.hasSharedLinkAccess(sharedLinkId, id);
 
@@ -122,6 +127,12 @@ export class AccessCore {
       case Permission.ALBUM_SHARE:
         return this.repository.album.hasOwnerAccess(authUser.id, id);
 
+      case Permission.ALBUM_DOWNLOAD:
+        return (
+          (await this.repository.album.hasOwnerAccess(authUser.id, id)) ||
+          (await this.repository.album.hasSharedAlbumAccess(authUser.id, id))
+        );
+
       case Permission.LIBRARY_READ:
         return authUser.id === id || (await this.repository.library.hasPartnerAccess(authUser.id, id));
 

server/src/domain/asset/asset.repository.ts

@@ -44,6 +44,8 @@ export const IAssetRepository = 'IAssetRepository';
 export interface IAssetRepository {
   getByDate(ownerId: string, date: Date): Promise<AssetEntity[]>;
   getByIds(ids: string[]): Promise<AssetEntity[]>;
+  getByAlbumId(pagination: PaginationOptions, albumId: string): Paginated<AssetEntity>;
+  getByUserId(pagination: PaginationOptions, userId: string): Paginated<AssetEntity>;
   getWithout(pagination: PaginationOptions, property: WithoutProperty): Paginated<AssetEntity>;
   getWith(pagination: PaginationOptions, property: WithProperty): Paginated<AssetEntity>;
   getFirstAssetForAlbumId(albumId: string): Promise<AssetEntity | null>;

server/src/domain/asset/asset.service.spec.ts

@@ -1,21 +1,48 @@
-import { assetEntityStub, authStub, newAssetRepositoryMock } from '@test';
+import { BadRequestException } from '@nestjs/common';
+import {
+  assetEntityStub,
+  authStub,
+  IAccessRepositoryMock,
+  newAccessRepositoryMock,
+  newAssetRepositoryMock,
+  newStorageRepositoryMock,
+} from '@test';
 import { when } from 'jest-when';
-import { AssetService, IAssetRepository, mapAsset } from '.';
+import { Readable } from 'stream';
+import { IStorageRepository } from '../storage';
+import { IAssetRepository } from './asset.repository';
+import { AssetService } from './asset.service';
+import { DownloadResponseDto } from './index';
+import { mapAsset } from './response-dto';
+
+const downloadResponse: DownloadResponseDto = {
+  totalSize: 105_000,
+  archives: [
+    {
+      assetIds: ['asset-id', 'asset-id'],
+      size: 105_000,
+    },
+  ],
+};
 
 describe(AssetService.name, () => {
   let sut: AssetService;
+  let accessMock: IAccessRepositoryMock;
   let assetMock: jest.Mocked<IAssetRepository>;
+  let storageMock: jest.Mocked<IStorageRepository>;
 
   it('should work', () => {
     expect(sut).toBeDefined();
   });
 
   beforeEach(async () => {
+    accessMock = newAccessRepositoryMock();
     assetMock = newAssetRepositoryMock();
-    sut = new AssetService(assetMock);
+    storageMock = newStorageRepositoryMock();
+    sut = new AssetService(accessMock, assetMock, storageMock);
   });
 
-  describe('get map markers', () => {
+  describe('getMapMarkers', () => {
     it('should get geo information of assets', async () => {
       assetMock.getMapMarkers.mockResolvedValue(
         [assetEntityStub.withLocation].map((asset) => ({
@@ -76,25 +103,191 @@ describe(AssetService.name, () => {
         [authStub.admin.id, new Date('2021-06-15T05:00:00.000Z')],
       ]);
     });
+
+    it('should set the title correctly', async () => {
+      when(assetMock.getByDate)
+        .calledWith(authStub.admin.id, new Date('2022-06-15T00:00:00.000Z'))
+        .mockResolvedValue([assetEntityStub.image]);
+      when(assetMock.getByDate)
+        .calledWith(authStub.admin.id, new Date('2021-06-15T00:00:00.000Z'))
+        .mockResolvedValue([assetEntityStub.video]);
+
+      await expect(sut.getMemoryLane(authStub.admin, { timestamp: new Date(2023, 5, 15), years: 2 })).resolves.toEqual([
+        { title: '1 year since...', assets: [mapAsset(assetEntityStub.image)] },
+        { title: '2 years since...', assets: [mapAsset(assetEntityStub.video)] },
+      ]);
+
+      expect(assetMock.getByDate).toHaveBeenCalledTimes(2);
+      expect(assetMock.getByDate.mock.calls).toEqual([
+        [authStub.admin.id, new Date('2022-06-15T00:00:00.000Z')],
+        [authStub.admin.id, new Date('2021-06-15T00:00:00.000Z')],
+      ]);
+    });
   });
 
-  it('should set the title correctly', async () => {
-    when(assetMock.getByDate)
-      .calledWith(authStub.admin.id, new Date('2022-06-15T00:00:00.000Z'))
-      .mockResolvedValue([assetEntityStub.image]);
-    when(assetMock.getByDate)
-      .calledWith(authStub.admin.id, new Date('2021-06-15T00:00:00.000Z'))
-      .mockResolvedValue([assetEntityStub.video]);
+  describe('downloadFile', () => {
+    it('should require the asset.download permission', async () => {
+      accessMock.asset.hasOwnerAccess.mockResolvedValue(false);
+      accessMock.asset.hasAlbumAccess.mockResolvedValue(false);
+      accessMock.asset.hasPartnerAccess.mockResolvedValue(false);
 
-    await expect(sut.getMemoryLane(authStub.admin, { timestamp: new Date(2023, 5, 15), years: 2 })).resolves.toEqual([
-      { title: '1 year since...', assets: [mapAsset(assetEntityStub.image)] },
-      { title: '2 years since...', assets: [mapAsset(assetEntityStub.video)] },
-    ]);
+      await expect(sut.downloadFile(authStub.admin, 'asset-1')).rejects.toBeInstanceOf(BadRequestException);
 
-    expect(assetMock.getByDate).toHaveBeenCalledTimes(2);
-    expect(assetMock.getByDate.mock.calls).toEqual([
-      [authStub.admin.id, new Date('2022-06-15T00:00:00.000Z')],
-      [authStub.admin.id, new Date('2021-06-15T00:00:00.000Z')],
-    ]);
+      expect(accessMock.asset.hasOwnerAccess).toHaveBeenCalledWith(authStub.admin.id, 'asset-1');
+      expect(accessMock.asset.hasAlbumAccess).toHaveBeenCalledWith(authStub.admin.id, 'asset-1');
+      expect(accessMock.asset.hasPartnerAccess).toHaveBeenCalledWith(authStub.admin.id, 'asset-1');
+    });
+
+    it('should throw an error if the asset is not found', async () => {
+      accessMock.asset.hasOwnerAccess.mockResolvedValue(true);
+      assetMock.getByIds.mockResolvedValue([]);
+
+      await expect(sut.downloadFile(authStub.admin, 'asset-1')).rejects.toBeInstanceOf(BadRequestException);
+
+      expect(assetMock.getByIds).toHaveBeenCalledWith(['asset-1']);
+    });
+
+    it('should download a file', async () => {
+      const stream = new Readable();
+
+      accessMock.asset.hasOwnerAccess.mockResolvedValue(true);
+      assetMock.getByIds.mockResolvedValue([assetEntityStub.image]);
+      storageMock.createReadStream.mockResolvedValue({ stream });
+
+      await expect(sut.downloadFile(authStub.admin, 'asset-1')).resolves.toEqual({ stream });
+
+      expect(storageMock.createReadStream).toHaveBeenCalledWith(
+        assetEntityStub.image.originalPath,
+        assetEntityStub.image.mimeType,
+      );
+    });
+
+    it('should download an archive', async () => {
+      const archiveMock = {
+        addFile: jest.fn(),
+        finalize: jest.fn(),
+        stream: new Readable(),
+      };
+
+      accessMock.asset.hasOwnerAccess.mockResolvedValue(true);
+      assetMock.getByIds.mockResolvedValue([assetEntityStub.noResizePath, assetEntityStub.noWebpPath]);
+      storageMock.createZipStream.mockReturnValue(archiveMock);
+
+      await expect(sut.downloadArchive(authStub.admin, { assetIds: ['asset-1', 'asset-2'] })).resolves.toEqual({
+        stream: archiveMock.stream,
+      });
+
+      expect(archiveMock.addFile).toHaveBeenCalledTimes(2);
+      expect(archiveMock.addFile).toHaveBeenNthCalledWith(1, 'upload/library/IMG_123.jpg', 'IMG_123.jpg');
+      expect(archiveMock.addFile).toHaveBeenNthCalledWith(2, 'upload/library/IMG_456.jpg', 'IMG_456.jpg');
+    });
+
+    it('should handle duplicate file names', async () => {
+      const archiveMock = {
+        addFile: jest.fn(),
+        finalize: jest.fn(),
+        stream: new Readable(),
+      };
+
+      accessMock.asset.hasOwnerAccess.mockResolvedValue(true);
+      assetMock.getByIds.mockResolvedValue([assetEntityStub.noResizePath, assetEntityStub.noResizePath]);
+      storageMock.createZipStream.mockReturnValue(archiveMock);
+
+      await expect(sut.downloadArchive(authStub.admin, { assetIds: ['asset-1', 'asset-2'] })).resolves.toEqual({
+        stream: archiveMock.stream,
+      });
+
+      expect(archiveMock.addFile).toHaveBeenCalledTimes(2);
+      expect(archiveMock.addFile).toHaveBeenNthCalledWith(1, 'upload/library/IMG_123.jpg', 'IMG_123.jpg');
+      expect(archiveMock.addFile).toHaveBeenNthCalledWith(2, 'upload/library/IMG_123.jpg', 'IMG_123+1.jpg');
+    });
+  });
+
+  describe('getDownloadInfo', () => {
+    it('should throw an error for an invalid dto', async () => {
+      await expect(sut.getDownloadInfo(authStub.admin, {})).rejects.toBeInstanceOf(BadRequestException);
+    });
+
+    it('should return a list of archives (assetIds)', async () => {
+      accessMock.asset.hasOwnerAccess.mockResolvedValue(true);
+      assetMock.getByIds.mockResolvedValue([assetEntityStub.image, assetEntityStub.video]);
+
+      const assetIds = ['asset-1', 'asset-2'];
+      await expect(sut.getDownloadInfo(authStub.admin, { assetIds })).resolves.toEqual(downloadResponse);
+
+      expect(assetMock.getByIds).toHaveBeenCalledWith(['asset-1', 'asset-2']);
+    });
+
+    it('should return a list of archives (albumId)', async () => {
+      accessMock.album.hasOwnerAccess.mockResolvedValue(true);
+      assetMock.getByAlbumId.mockResolvedValue({
+        items: [assetEntityStub.image, assetEntityStub.video],
+        hasNextPage: false,
+      });
+
+      await expect(sut.getDownloadInfo(authStub.admin, { albumId: 'album-1' })).resolves.toEqual(downloadResponse);
+
+      expect(accessMock.album.hasOwnerAccess).toHaveBeenCalledWith(authStub.admin.id, 'album-1');
+      expect(assetMock.getByAlbumId).toHaveBeenCalledWith({ take: 2500, skip: 0 }, 'album-1');
+    });
+
+    it('should return a list of archives (userId)', async () => {
+      assetMock.getByUserId.mockResolvedValue({
+        items: [assetEntityStub.image, assetEntityStub.video],
+        hasNextPage: false,
+      });
+
+      await expect(sut.getDownloadInfo(authStub.admin, { userId: authStub.admin.id })).resolves.toEqual(
+        downloadResponse,
+      );
+
+      expect(assetMock.getByUserId).toHaveBeenCalledWith({ take: 2500, skip: 0 }, authStub.admin.id);
+    });
+
+    it('should split archives by size', async () => {
+      assetMock.getByUserId.mockResolvedValue({
+        items: [
+          { ...assetEntityStub.image, id: 'asset-1' },
+          { ...assetEntityStub.video, id: 'asset-2' },
+          { ...assetEntityStub.withLocation, id: 'asset-3' },
+          { ...assetEntityStub.noWebpPath, id: 'asset-4' },
+        ],
+        hasNextPage: false,
+      });
+
+      await expect(
+        sut.getDownloadInfo(authStub.admin, {
+          userId: authStub.admin.id,
+          archiveSize: 30_000,
+        }),
+      ).resolves.toEqual({
+        totalSize: 251_456,
+        archives: [
+          { assetIds: ['asset-1', 'asset-2'], size: 105_000 },
+          { assetIds: ['asset-3', 'asset-4'], size: 146_456 },
+        ],
+      });
+    });
+
+    it('should include the video portion of a live photo', async () => {
+      accessMock.asset.hasOwnerAccess.mockResolvedValue(true);
+      when(assetMock.getByIds)
+        .calledWith([assetEntityStub.livePhotoStillAsset.id])
+        .mockResolvedValue([assetEntityStub.livePhotoStillAsset]);
+      when(assetMock.getByIds)
+        .calledWith([assetEntityStub.livePhotoMotionAsset.id])
+        .mockResolvedValue([assetEntityStub.livePhotoMotionAsset]);
+
+      const assetIds = [assetEntityStub.livePhotoStillAsset.id];
+      await expect(sut.getDownloadInfo(authStub.admin, { assetIds })).resolves.toEqual({
+        totalSize: 125_000,
+        archives: [
+          {
+            assetIds: [assetEntityStub.livePhotoStillAsset.id, assetEntityStub.livePhotoMotionAsset.id],
+            size: 125_000,
+          },
+        ],
+      });
+    });
   });
 });

server/src/domain/asset/asset.service.ts

@@ -1,14 +1,27 @@
-import { Inject } from '@nestjs/common';
+import { BadRequestException, Inject } from '@nestjs/common';
 import { DateTime } from 'luxon';
+import { extname } from 'path';
+import { AssetEntity } from '../../infra/entities/asset.entity';
 import { AuthUserDto } from '../auth';
+import { HumanReadableSize, usePagination } from '../domain.util';
+import { AccessCore, IAccessRepository, Permission } from '../index';
+import { ImmichReadStream, IStorageRepository } from '../storage';
 import { IAssetRepository } from './asset.repository';
-import { MemoryLaneDto } from './dto';
+import { AssetIdsDto, DownloadArchiveInfo, DownloadDto, DownloadResponseDto, MemoryLaneDto } from './dto';
 import { MapMarkerDto } from './dto/map-marker.dto';
 import { mapAsset, MapMarkerResponseDto } from './response-dto';
 import { MemoryLaneResponseDto } from './response-dto/memory-lane-response.dto';
 
 export class AssetService {
-  constructor(@Inject(IAssetRepository) private assetRepository: IAssetRepository) {}
+  private access: AccessCore;
+
+  constructor(
+    @Inject(IAccessRepository) accessRepository: IAccessRepository,
+    @Inject(IAssetRepository) private assetRepository: IAssetRepository,
+    @Inject(IStorageRepository) private storageRepository: IStorageRepository,
+  ) {
+    this.access = new AccessCore(accessRepository);
+  }
 
   getMapMarkers(authUser: AuthUserDto, options: MapMarkerDto): Promise<MapMarkerResponseDto[]> {
     return this.assetRepository.getMapMarkers(authUser.id, options);
@@ -32,4 +45,102 @@ export class AssetService {
 
     return Promise.all(requests).then((results) => results.filter((result) => result.assets.length > 0));
   }
+
+  async downloadFile(authUser: AuthUserDto, id: string): Promise<ImmichReadStream> {
+    await this.access.requirePermission(authUser, Permission.ASSET_DOWNLOAD, id);
+
+    const [asset] = await this.assetRepository.getByIds([id]);
+    if (!asset) {
+      throw new BadRequestException('Asset not found');
+    }
+
+    return this.storageRepository.createReadStream(asset.originalPath, asset.mimeType);
+  }
+
+  async getDownloadInfo(authUser: AuthUserDto, dto: DownloadDto): Promise<DownloadResponseDto> {
+    const targetSize = dto.archiveSize || HumanReadableSize.GiB * 4;
+    const archives: DownloadArchiveInfo[] = [];
+    let archive: DownloadArchiveInfo = { size: 0, assetIds: [] };
+
+    const assetPagination = await this.getDownloadAssets(authUser, dto);
+    for await (const assets of assetPagination) {
+      // motion part of live photos
+      const motionIds = assets.map((asset) => asset.livePhotoVideoId).filter<string>((id): id is string => !!id);
+      if (motionIds.length > 0) {
+        assets.push(...(await this.assetRepository.getByIds(motionIds)));
+      }
+
+      for (const asset of assets) {
+        archive.size += Number(asset.exifInfo?.fileSizeInByte || 0);
+        archive.assetIds.push(asset.id);
+
+        if (archive.size > targetSize) {
+          archives.push(archive);
+          archive = { size: 0, assetIds: [] };
+        }
+      }
+
+      if (archive.assetIds.length > 0) {
+        archives.push(archive);
+      }
+    }
+
+    return {
+      totalSize: archives.reduce((total, item) => (total += item.size), 0),
+      archives,
+    };
+  }
+
+  async downloadArchive(authUser: AuthUserDto, dto: AssetIdsDto): Promise<ImmichReadStream> {
+    await this.access.requirePermission(authUser, Permission.ASSET_DOWNLOAD, dto.assetIds);
+
+    const zip = this.storageRepository.createZipStream();
+    const assets = await this.assetRepository.getByIds(dto.assetIds);
+    const paths: Record<string, boolean> = {};
+
+    for (const { originalPath, originalFileName } of assets) {
+      const ext = extname(originalPath);
+      let filename = `${originalFileName}${ext}`;
+      for (let i = 0; i < 10_000; i++) {
+        if (!paths[filename]) {
+          break;
+        }
+        filename = `${originalFileName}+${i + 1}${ext}`;
+      }
+
+      paths[filename] = true;
+      zip.addFile(originalPath, filename);
+    }
+
+    zip.finalize();
+
+    return { stream: zip.stream };
+  }
+
+  private async getDownloadAssets(authUser: AuthUserDto, dto: DownloadDto): Promise<AsyncGenerator<AssetEntity[]>> {
+    const PAGINATION_SIZE = 2500;
+
+    if (dto.assetIds) {
+      const assetIds = dto.assetIds;
+      await this.access.requirePermission(authUser, Permission.ASSET_DOWNLOAD, assetIds);
+      const assets = await this.assetRepository.getByIds(assetIds);
+      return (async function* () {
+        yield assets;
+      })();
+    }
+
+    if (dto.albumId) {
+      const albumId = dto.albumId;
+      await this.access.requirePermission(authUser, Permission.ALBUM_DOWNLOAD, albumId);
+      return usePagination(PAGINATION_SIZE, (pagination) => this.assetRepository.getByAlbumId(pagination, albumId));
+    }
+
+    if (dto.userId) {
+      const userId = dto.userId;
+      await this.access.requirePermission(authUser, Permission.LIBRARY_DOWNLOAD, userId);
+      return usePagination(PAGINATION_SIZE, (pagination) => this.assetRepository.getByUserId(pagination, userId));
+    }
+
+    throw new BadRequestException('assetIds, albumId, or userId is required');
+  }
 }

server/src/domain/asset/dto/download.dto.ts

@@ -0,0 +1,31 @@
+import { ValidateUUID } from '@app/immich/decorators/validate-uuid.decorator';
+import { ApiProperty } from '@nestjs/swagger';
+import { IsInt, IsOptional, IsPositive } from 'class-validator';
+
+export class DownloadDto {
+  @ValidateUUID({ each: true, optional: true })
+  assetIds?: string[];
+
+  @ValidateUUID({ optional: true })
+  albumId?: string;
+
+  @ValidateUUID({ optional: true })
+  userId?: string;
+
+  @IsInt()
+  @IsPositive()
+  @IsOptional()
+  archiveSize?: number;
+}
+
+export class DownloadResponseDto {
+  @ApiProperty({ type: 'integer' })
+  totalSize!: number;
+  archives!: DownloadArchiveInfo[];
+}
+
+export class DownloadArchiveInfo {
+  @ApiProperty({ type: 'integer' })
+  size!: number;
+  assetIds!: string[];
+}

server/src/domain/asset/dto/index.ts

@@ -1,3 +1,4 @@
 export * from './asset-ids.dto';
+export * from './download.dto';
 export * from './map-marker.dto';
 export * from './memory-lane.dto';

server/src/domain/storage/storage.repository.ts

@@ -1,9 +1,14 @@
-import { ReadStream } from 'fs';
+import { Readable } from 'stream';
 
 export interface ImmichReadStream {
-  stream: ReadStream;
-  type: string;
-  length: number;
+  stream: Readable;
+  type?: string;
+  length?: number;
+}
+
+export interface ImmichZipStream extends ImmichReadStream {
+  addFile: (inputPath: string, filename: string) => void;
+  finalize: () => Promise<void>;
 }
 
 export interface DiskUsage {
@@ -15,7 +20,8 @@ export interface DiskUsage {
 export const IStorageRepository = 'IStorageRepository';
 
 export interface IStorageRepository {
-  createReadStream(filepath: string, mimeType: string): Promise<ImmichReadStream>;
+  createZipStream(): ImmichZipStream;
+  createReadStream(filepath: string, mimeType?: string | null): Promise<ImmichReadStream>;
   unlink(filepath: string): Promise<void>;
   unlinkDir(folder: string, options?: { recursive?: boolean; force?: boolean }): Promise<void>;
   removeEmptyDirs(folder: string): Promise<void>;