feat: use stream for template migrations (#16700)

server/src/repositories/asset.repository.ts

@@ -23,6 +23,7 @@ import {
 } from 'src/entities/asset.entity';
 import { AssetFileType, AssetOrder, AssetStatus, AssetType } from 'src/enum';
 import { AssetSearchOptions, SearchExploreItem, SearchExploreItemSet } from 'src/repositories/search.repository';
+import { StorageAsset } from 'src/types';
 import { anyUuid, asUuid, removeUndefinedKeys, unnest } from 'src/utils/database';
 import { globToSqlPattern } from 'src/utils/misc';
 import { Paginated, PaginationOptions, paginationHelper } from 'src/utils/pagination';
@@ -604,6 +605,38 @@ export class AssetRepository {
       .executeTakeFirst() as Promise<AssetEntity | undefined>;
   }
 
+  private storageTemplateAssetQuery() {
+    return this.db
+      .selectFrom('assets')
+      .innerJoin('exif', 'assets.id', 'exif.assetId')
+      .select([
+        'assets.id',
+        'assets.ownerId',
+        'assets.type',
+        'assets.checksum',
+        'assets.originalPath',
+        'assets.isExternal',
+        'assets.sidecarPath',
+        'assets.originalFileName',
+        'assets.livePhotoVideoId',
+        'assets.fileCreatedAt',
+        'exif.timeZone',
+        'exif.fileSizeInByte',
+      ])
+      .where('assets.deletedAt', 'is', null)
+      .where('assets.fileCreatedAt', 'is not', null);
+  }
+
+  getStorageTemplateAsset(id: string): Promise<StorageAsset | undefined> {
+    return this.storageTemplateAssetQuery().where('assets.id', '=', id).executeTakeFirst() as Promise<
+      StorageAsset | undefined
+    >;
+  }
+
+  streamStorageTemplateAssets() {
+    return this.storageTemplateAssetQuery().stream() as AsyncIterableIterator<StorageAsset>;
+  }
+
   @GenerateSql(
     ...Object.values(WithProperty).map((property) => ({
       name: property,

server/src/services/storage-template.service.spec.ts

@@ -1,12 +1,14 @@
 import { Stats } from 'node:fs';
 import { defaults, SystemConfig } from 'src/config';
-import { AssetEntity } from 'src/entities/asset.entity';
 import { AssetPathType, JobStatus } from 'src/enum';
 import { StorageTemplateService } from 'src/services/storage-template.service';
 import { albumStub } from 'test/fixtures/album.stub';
 import { assetStub } from 'test/fixtures/asset.stub';
 import { userStub } from 'test/fixtures/user.stub';
-import { newTestService, ServiceMocks } from 'test/utils';
+import { makeStream, newTestService, ServiceMocks } from 'test/utils';
+
+const motionAsset = assetStub.storageAsset({});
+const stillAsset = assetStub.storageAsset({ livePhotoVideoId: motionAsset.id });
 
 describe(StorageTemplateService.name, () => {
   let sut: StorageTemplateService;
@@ -91,7 +93,7 @@ describe(StorageTemplateService.name, () => {
   describe('handleMigrationSingle', () => {
     it('should skip when storage template is disabled', async () => {
       mocks.systemMetadata.get.mockResolvedValue({ storageTemplate: { enabled: false } });
-      await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(JobStatus.SKIPPED);
+      await expect(sut.handleMigrationSingle({ id: testAsset.id })).resolves.toBe(JobStatus.SKIPPED);
       expect(mocks.asset.getByIds).not.toHaveBeenCalled();
       expect(mocks.storage.checkFileExists).not.toHaveBeenCalled();
       expect(mocks.storage.rename).not.toHaveBeenCalled();
@@ -104,51 +106,37 @@ describe(StorageTemplateService.name, () => {
 
     it('should migrate single moving picture', async () => {
       mocks.user.get.mockResolvedValue(userStub.user1);
-      const newMotionPicturePath = `upload/library/${userStub.user1.id}/2022/2022-06-19/${assetStub.livePhotoStillAsset.id}.mp4`;
-      const newStillPicturePath = `upload/library/${userStub.user1.id}/2022/2022-06-19/${assetStub.livePhotoStillAsset.id}.jpg`;
+      const newMotionPicturePath = `upload/library/${motionAsset.ownerId}/2022/2022-06-19/${motionAsset.originalFileName}`;
+      const newStillPicturePath = `upload/library/${stillAsset.ownerId}/2022/2022-06-19/${stillAsset.originalFileName}`;
 
-      mocks.asset.getByIds.mockImplementation((ids) => {
-        const assets = [assetStub.livePhotoStillAsset, assetStub.livePhotoMotionAsset];
-        return Promise.resolve(
-          ids.map((id) => assets.find((asset) => asset.id === id)).filter((asset) => !!asset),
-        ) as Promise<AssetEntity[]>;
-      });
+      mocks.asset.getStorageTemplateAsset.mockResolvedValueOnce(stillAsset);
+      mocks.asset.getStorageTemplateAsset.mockResolvedValueOnce(motionAsset);
 
       mocks.move.create.mockResolvedValueOnce({
         id: '123',
-        entityId: assetStub.livePhotoStillAsset.id,
+        entityId: stillAsset.id,
         pathType: AssetPathType.ORIGINAL,
-        oldPath: assetStub.livePhotoStillAsset.originalPath,
+        oldPath: stillAsset.originalPath,
         newPath: newStillPicturePath,
       });
 
       mocks.move.create.mockResolvedValueOnce({
         id: '124',
-        entityId: assetStub.livePhotoMotionAsset.id,
+        entityId: motionAsset.id,
         pathType: AssetPathType.ORIGINAL,
-        oldPath: assetStub.livePhotoMotionAsset.originalPath,
+        oldPath: motionAsset.originalPath,
         newPath: newMotionPicturePath,
       });
 
-      await expect(sut.handleMigrationSingle({ id: assetStub.livePhotoStillAsset.id })).resolves.toBe(
-        JobStatus.SUCCESS,
-      );
+      await expect(sut.handleMigrationSingle({ id: stillAsset.id })).resolves.toBe(JobStatus.SUCCESS);
 
-      expect(mocks.asset.getByIds).toHaveBeenCalledWith([assetStub.livePhotoStillAsset.id], { exifInfo: true });
-      expect(mocks.asset.getByIds).toHaveBeenCalledWith([assetStub.livePhotoMotionAsset.id], { exifInfo: true });
       expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(2);
-      expect(mocks.asset.update).toHaveBeenCalledWith({
-        id: assetStub.livePhotoStillAsset.id,
-        originalPath: newStillPicturePath,
-      });
-      expect(mocks.asset.update).toHaveBeenCalledWith({
-        id: assetStub.livePhotoMotionAsset.id,
-        originalPath: newMotionPicturePath,
-      });
+      expect(mocks.asset.update).toHaveBeenCalledWith({ id: stillAsset.id, originalPath: newStillPicturePath });
+      expect(mocks.asset.update).toHaveBeenCalledWith({ id: motionAsset.id, originalPath: newMotionPicturePath });
     });
 
     it('should use handlebar if condition for album', async () => {
-      const asset = assetStub.image;
+      const asset = assetStub.storageAsset();
       const user = userStub.user1;
       const album = albumStub.oneAsset;
       const config = structuredClone(defaults);
@@ -157,7 +145,7 @@ describe(StorageTemplateService.name, () => {
       sut.onConfigInit({ newConfig: config });
 
       mocks.user.get.mockResolvedValue(user);
-      mocks.asset.getByIds.mockResolvedValueOnce([asset]);
+      mocks.asset.getStorageTemplateAsset.mockResolvedValueOnce(asset);
       mocks.album.getByAssetId.mockResolvedValueOnce([album]);
 
       expect(await sut.handleMigrationSingle({ id: asset.id })).toBe(JobStatus.SUCCESS);
@@ -171,14 +159,14 @@ describe(StorageTemplateService.name, () => {
     });
 
     it('should use handlebar else condition for album', async () => {
-      const asset = assetStub.image;
+      const asset = assetStub.storageAsset();
       const user = userStub.user1;
       const config = structuredClone(defaults);
       config.storageTemplate.template = '{{y}}/{{#if album}}{{album}}{{else}}other//{{MM}}{{/if}}/{{filename}}';
       sut.onConfigInit({ newConfig: config });
 
       mocks.user.get.mockResolvedValue(user);
-      mocks.asset.getByIds.mockResolvedValueOnce([asset]);
+      mocks.asset.getStorageTemplateAsset.mockResolvedValueOnce(asset);
 
       expect(await sut.handleMigrationSingle({ id: asset.id })).toBe(JobStatus.SUCCESS);
 
@@ -193,153 +181,150 @@ describe(StorageTemplateService.name, () => {
 
     it('should migrate previously failed move from original path when it still exists', async () => {
       mocks.user.get.mockResolvedValue(userStub.user1);
-      const previousFailedNewPath = `upload/library/${userStub.user1.id}/2023/Feb/${assetStub.image.id}.jpg`;
-      const newPath = `upload/library/${userStub.user1.id}/2023/2023-02-23/${assetStub.image.id}.jpg`;
 
-      mocks.storage.checkFileExists.mockImplementation((path) =>
-        Promise.resolve(path === assetStub.image.originalPath),
-      );
+      const asset = assetStub.storageAsset();
+      const previousFailedNewPath = `upload/library/${userStub.user1.id}/2023/Feb/${asset.originalFileName}`;
+      const newPath = `upload/library/${userStub.user1.id}/2022/2022-06-19/${asset.originalFileName}`;
+
+      mocks.storage.checkFileExists.mockImplementation((path) => Promise.resolve(path === asset.originalPath));
       mocks.move.getByEntity.mockResolvedValue({
         id: '123',
-        entityId: assetStub.image.id,
+        entityId: asset.id,
         pathType: AssetPathType.ORIGINAL,
-        oldPath: assetStub.image.originalPath,
+        oldPath: asset.originalPath,
         newPath: previousFailedNewPath,
       });
-      mocks.asset.getByIds.mockResolvedValue([assetStub.image]);
+      mocks.asset.getStorageTemplateAsset.mockResolvedValue(asset);
       mocks.move.update.mockResolvedValue({
         id: '123',
-        entityId: assetStub.image.id,
+        entityId: asset.id,
         pathType: AssetPathType.ORIGINAL,
-        oldPath: assetStub.image.originalPath,
+        oldPath: asset.originalPath,
         newPath,
       });
 
-      await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(JobStatus.SUCCESS);
+      await expect(sut.handleMigrationSingle({ id: asset.id })).resolves.toBe(JobStatus.SUCCESS);
 
-      expect(mocks.asset.getByIds).toHaveBeenCalledWith([assetStub.image.id], { exifInfo: true });
+      expect(mocks.asset.getStorageTemplateAsset).toHaveBeenCalledWith(asset.id);
       expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(3);
-      expect(mocks.storage.rename).toHaveBeenCalledWith(assetStub.image.originalPath, newPath);
+      expect(mocks.storage.rename).toHaveBeenCalledWith(asset.originalPath, newPath);
       expect(mocks.move.update).toHaveBeenCalledWith('123', {
         id: '123',
-        oldPath: assetStub.image.originalPath,
+        oldPath: asset.originalPath,
         newPath,
       });
       expect(mocks.asset.update).toHaveBeenCalledWith({
-        id: assetStub.image.id,
+        id: asset.id,
         originalPath: newPath,
       });
     });
 
     it('should migrate previously failed move from previous new path when old path no longer exists, should validate file size still matches before moving', async () => {
       mocks.user.get.mockResolvedValue(userStub.user1);
-      const previousFailedNewPath = `upload/library/${userStub.user1.id}/2023/Feb/${assetStub.image.id}.jpg`;
-      const newPath = `upload/library/${userStub.user1.id}/2023/2023-02-23/${assetStub.image.id}.jpg`;
+
+      const asset = assetStub.storageAsset({ fileSizeInByte: 5000 });
+      const previousFailedNewPath = `upload/library/${asset.ownerId}/2022/June/${asset.originalFileName}`;
+      const newPath = `upload/library/${asset.ownerId}/2022/2022-06-19/${asset.originalFileName}`;
 
       mocks.storage.checkFileExists.mockImplementation((path) => Promise.resolve(path === previousFailedNewPath));
       mocks.storage.stat.mockResolvedValue({ size: 5000 } as Stats);
-      mocks.crypto.hashFile.mockResolvedValue(assetStub.image.checksum);
+      mocks.crypto.hashFile.mockResolvedValue(asset.checksum);
       mocks.move.getByEntity.mockResolvedValue({
         id: '123',
-        entityId: assetStub.image.id,
+        entityId: asset.id,
         pathType: AssetPathType.ORIGINAL,
-        oldPath: assetStub.image.originalPath,
+        oldPath: asset.originalPath,
         newPath: previousFailedNewPath,
       });
-      mocks.asset.getByIds.mockResolvedValue([assetStub.image]);
+      mocks.asset.getStorageTemplateAsset.mockResolvedValue(asset);
       mocks.move.update.mockResolvedValue({
         id: '123',
-        entityId: assetStub.image.id,
+        entityId: asset.id,
         pathType: AssetPathType.ORIGINAL,
         oldPath: previousFailedNewPath,
         newPath,
       });
 
-      await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(JobStatus.SUCCESS);
+      await expect(sut.handleMigrationSingle({ id: asset.id })).resolves.toBe(JobStatus.SUCCESS);
 
-      expect(mocks.asset.getByIds).toHaveBeenCalledWith([assetStub.image.id], { exifInfo: true });
+      expect(mocks.asset.getStorageTemplateAsset).toHaveBeenCalledWith(asset.id);
       expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(3);
       expect(mocks.storage.stat).toHaveBeenCalledWith(previousFailedNewPath);
       expect(mocks.storage.rename).toHaveBeenCalledWith(previousFailedNewPath, newPath);
       expect(mocks.storage.copyFile).not.toHaveBeenCalled();
-      expect(mocks.move.update).toHaveBeenCalledWith('123', {
-        id: '123',
-        oldPath: previousFailedNewPath,
-        newPath,
-      });
-      expect(mocks.asset.update).toHaveBeenCalledWith({
-        id: assetStub.image.id,
-        originalPath: newPath,
-      });
+      expect(mocks.move.update).toHaveBeenCalledWith('123', { id: '123', oldPath: previousFailedNewPath, newPath });
+      expect(mocks.asset.update).toHaveBeenCalledWith({ id: asset.id, originalPath: newPath });
     });
 
     it('should fail move if copying and hash of asset and the new file do not match', async () => {
       mocks.user.get.mockResolvedValue(userStub.user1);
-      const newPath = `upload/library/${userStub.user1.id}/2023/2023-02-23/${assetStub.image.id}.jpg`;
+      const newPath = `upload/library/${userStub.user1.id}/2022/2022-06-19/${testAsset.originalFileName}`;
 
       mocks.storage.rename.mockRejectedValue({ code: 'EXDEV' });
       mocks.storage.stat.mockResolvedValue({ size: 5000 } as Stats);
       mocks.crypto.hashFile.mockResolvedValue(Buffer.from('different-hash', 'utf8'));
-      mocks.asset.getByIds.mockResolvedValue([assetStub.image]);
+      mocks.asset.getStorageTemplateAsset.mockResolvedValue(testAsset);
       mocks.move.create.mockResolvedValue({
         id: '123',
-        entityId: assetStub.image.id,
+        entityId: testAsset.id,
         pathType: AssetPathType.ORIGINAL,
-        oldPath: assetStub.image.originalPath,
+        oldPath: testAsset.originalPath,
         newPath,
       });
 
-      await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(JobStatus.SUCCESS);
+      await expect(sut.handleMigrationSingle({ id: testAsset.id })).resolves.toBe(JobStatus.SUCCESS);
 
-      expect(mocks.asset.getByIds).toHaveBeenCalledWith([assetStub.image.id], { exifInfo: true });
+      expect(mocks.asset.getStorageTemplateAsset).toHaveBeenCalledWith(testAsset.id);
       expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(1);
       expect(mocks.storage.stat).toHaveBeenCalledWith(newPath);
       expect(mocks.move.create).toHaveBeenCalledWith({
-        entityId: assetStub.image.id,
+        entityId: testAsset.id,
         pathType: AssetPathType.ORIGINAL,
-        oldPath: assetStub.image.originalPath,
+        oldPath: testAsset.originalPath,
         newPath,
       });
-      expect(mocks.storage.rename).toHaveBeenCalledWith(assetStub.image.originalPath, newPath);
-      expect(mocks.storage.copyFile).toHaveBeenCalledWith(assetStub.image.originalPath, newPath);
+      expect(mocks.storage.rename).toHaveBeenCalledWith(testAsset.originalPath, newPath);
+      expect(mocks.storage.copyFile).toHaveBeenCalledWith(testAsset.originalPath, newPath);
       expect(mocks.storage.unlink).toHaveBeenCalledWith(newPath);
       expect(mocks.storage.unlink).toHaveBeenCalledTimes(1);
       expect(mocks.asset.update).not.toHaveBeenCalled();
     });
 
+    const testAsset = assetStub.storageAsset();
+
     it.each`
-      failedPathChecksum                     | failedPathSize                              | reason
-      ${assetStub.image.checksum}            | ${500}                                      | ${'file size'}
-      ${Buffer.from('bad checksum', 'utf8')} | ${assetStub.image.exifInfo?.fileSizeInByte} | ${'checksum'}
+      failedPathChecksum                     | failedPathSize              | reason
+      ${testAsset.checksum}                  | ${500}                      | ${'file size'}
+      ${Buffer.from('bad checksum', 'utf8')} | ${testAsset.fileSizeInByte} | ${'checksum'}
     `(
       'should fail to migrate previously failed move from previous new path when old path no longer exists if $reason validation fails',
       async ({ failedPathChecksum, failedPathSize }) => {
         mocks.user.get.mockResolvedValue(userStub.user1);
-        const previousFailedNewPath = `upload/library/${userStub.user1.id}/2023/Feb/${assetStub.image.id}.jpg`;
-        const newPath = `upload/library/${userStub.user1.id}/2023/2023-02-23/${assetStub.image.id}.jpg`;
+        const previousFailedNewPath = `upload/library/${userStub.user1.id}/2023/Feb/${testAsset.originalFileName}`;
+        const newPath = `upload/library/${userStub.user1.id}/2023/2023-02-23/${testAsset.originalFileName}`;
 
         mocks.storage.checkFileExists.mockImplementation((path) => Promise.resolve(previousFailedNewPath === path));
         mocks.storage.stat.mockResolvedValue({ size: failedPathSize } as Stats);
         mocks.crypto.hashFile.mockResolvedValue(failedPathChecksum);
         mocks.move.getByEntity.mockResolvedValue({
           id: '123',
-          entityId: assetStub.image.id,
+          entityId: testAsset.id,
           pathType: AssetPathType.ORIGINAL,
-          oldPath: assetStub.image.originalPath,
+          oldPath: testAsset.originalPath,
           newPath: previousFailedNewPath,
         });
-        mocks.asset.getByIds.mockResolvedValue([assetStub.image]);
+        mocks.asset.getStorageTemplateAsset.mockResolvedValue(testAsset);
         mocks.move.update.mockResolvedValue({
           id: '123',
-          entityId: assetStub.image.id,
+          entityId: testAsset.id,
           pathType: AssetPathType.ORIGINAL,
           oldPath: previousFailedNewPath,
           newPath,
         });
 
-        await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(JobStatus.SUCCESS);
+        await expect(sut.handleMigrationSingle({ id: testAsset.id })).resolves.toBe(JobStatus.SUCCESS);
 
-        expect(mocks.asset.getByIds).toHaveBeenCalledWith([assetStub.image.id], { exifInfo: true });
+        expect(mocks.asset.getStorageTemplateAsset).toHaveBeenCalledWith(testAsset.id);
         expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(3);
         expect(mocks.storage.stat).toHaveBeenCalledWith(previousFailedNewPath);
         expect(mocks.storage.rename).not.toHaveBeenCalled();
@@ -352,29 +337,28 @@ describe(StorageTemplateService.name, () => {
 
   describe('handle template migration', () => {
     it('should handle no assets', async () => {
-      mocks.asset.getAll.mockResolvedValue({
-        items: [],
-        hasNextPage: false,
-      });
+      mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([]));
       mocks.user.getList.mockResolvedValue([]);
 
       await sut.handleMigration();
 
-      expect(mocks.asset.getAll).toHaveBeenCalled();
+      expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
     });
 
     it('should handle an asset with a duplicate destination', async () => {
-      mocks.asset.getAll.mockResolvedValue({
-        items: [assetStub.image],
-        hasNextPage: false,
-      });
+      const asset = assetStub.storageAsset();
+      const oldPath = asset.originalPath;
+      const newPath = `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`;
+      const newPath2 = newPath.replace('.jpg', '+1.jpg');
+
+      mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
       mocks.user.getList.mockResolvedValue([userStub.user1]);
       mocks.move.create.mockResolvedValue({
         id: '123',
-        entityId: assetStub.image.id,
+        entityId: asset.id,
         pathType: AssetPathType.ORIGINAL,
-        oldPath: assetStub.image.originalPath,
-        newPath: 'upload/library/user-id/2023/2023-02-23/asset-id+1.jpg',
+        oldPath,
+        newPath,
       });
 
       mocks.storage.checkFileExists.mockResolvedValueOnce(true);
@@ -382,30 +366,21 @@ describe(StorageTemplateService.name, () => {
 
       await sut.handleMigration();
 
-      expect(mocks.asset.getAll).toHaveBeenCalled();
+      expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
       expect(mocks.storage.checkFileExists).toHaveBeenCalledTimes(2);
-      expect(mocks.asset.update).toHaveBeenCalledWith({
-        id: assetStub.image.id,
-        originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id+1.jpg',
-      });
+      expect(mocks.asset.update).toHaveBeenCalledWith({ id: asset.id, originalPath: newPath2 });
       expect(mocks.user.getList).toHaveBeenCalled();
     });
 
     it('should skip when an asset already matches the template', async () => {
-      mocks.asset.getAll.mockResolvedValue({
-        items: [
-          {
-            ...assetStub.image,
-            originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id.jpg',
-          },
-        ],
-        hasNextPage: false,
-      });
+      const asset = assetStub.storageAsset({ originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id.jpg' });
+
+      mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
       mocks.user.getList.mockResolvedValue([userStub.user1]);
 
       await sut.handleMigration();
 
-      expect(mocks.asset.getAll).toHaveBeenCalled();
+      expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
       expect(mocks.storage.rename).not.toHaveBeenCalled();
       expect(mocks.storage.copyFile).not.toHaveBeenCalled();
       expect(mocks.storage.checkFileExists).not.toHaveBeenCalledTimes(2);
@@ -413,20 +388,14 @@ describe(StorageTemplateService.name, () => {
     });
 
     it('should skip when an asset is probably a duplicate', async () => {
-      mocks.asset.getAll.mockResolvedValue({
-        items: [
-          {
-            ...assetStub.image,
-            originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id+1.jpg',
-          },
-        ],
-        hasNextPage: false,
-      });
+      const asset = assetStub.storageAsset({ originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id+1.jpg' });
+
+      mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
       mocks.user.getList.mockResolvedValue([userStub.user1]);
 
       await sut.handleMigration();
 
-      expect(mocks.asset.getAll).toHaveBeenCalled();
+      expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
       expect(mocks.storage.rename).not.toHaveBeenCalled();
       expect(mocks.storage.copyFile).not.toHaveBeenCalled();
       expect(mocks.storage.checkFileExists).not.toHaveBeenCalledTimes(2);
@@ -434,72 +403,63 @@ describe(StorageTemplateService.name, () => {
     });
 
     it('should move an asset', async () => {
-      mocks.asset.getAll.mockResolvedValue({
-        items: [assetStub.image],
-        hasNextPage: false,
-      });
+      const asset = assetStub.storageAsset();
+      const oldPath = asset.originalPath;
+      const newPath = `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`;
+      mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
       mocks.user.getList.mockResolvedValue([userStub.user1]);
       mocks.move.create.mockResolvedValue({
         id: '123',
         entityId: assetStub.image.id,
         pathType: AssetPathType.ORIGINAL,
         oldPath: assetStub.image.originalPath,
-        newPath: 'upload/library/user-id/2023/2023-02-23/asset-id.jpg',
+        newPath,
       });
 
       await sut.handleMigration();
 
-      expect(mocks.asset.getAll).toHaveBeenCalled();
-      expect(mocks.storage.rename).toHaveBeenCalledWith(
-        '/original/path.jpg',
-        'upload/library/user-id/2023/2023-02-23/asset-id.jpg',
-      );
-      expect(mocks.asset.update).toHaveBeenCalledWith({
-        id: assetStub.image.id,
-        originalPath: 'upload/library/user-id/2023/2023-02-23/asset-id.jpg',
-      });
+      expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
+      expect(mocks.storage.rename).toHaveBeenCalledWith(oldPath, newPath);
+      expect(mocks.asset.update).toHaveBeenCalledWith({ id: asset.id, originalPath: newPath });
     });
 
     it('should use the user storage label', async () => {
-      mocks.asset.getAll.mockResolvedValue({
-        items: [assetStub.image],
-        hasNextPage: false,
-      });
+      const asset = assetStub.storageAsset();
+      mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
       mocks.user.getList.mockResolvedValue([userStub.storageLabel]);
       mocks.move.create.mockResolvedValue({
         id: '123',
-        entityId: assetStub.image.id,
+        entityId: asset.id,
         pathType: AssetPathType.ORIGINAL,
-        oldPath: assetStub.image.originalPath,
-        newPath: 'upload/library/user-id/2023/2023-02-23/asset-id.jpg',
+        oldPath: asset.originalPath,
+        newPath: `upload/library/user-id/2023/2023-02-23/${asset.originalFileName}`,
       });
 
       await sut.handleMigration();
 
-      expect(mocks.asset.getAll).toHaveBeenCalled();
+      expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
       expect(mocks.storage.rename).toHaveBeenCalledWith(
         '/original/path.jpg',
-        'upload/library/label-1/2023/2023-02-23/asset-id.jpg',
+        `upload/library/label-1/2022/2022-06-19/${asset.originalFileName}`,
       );
       expect(mocks.asset.update).toHaveBeenCalledWith({
-        id: assetStub.image.id,
-        originalPath: 'upload/library/label-1/2023/2023-02-23/asset-id.jpg',
+        id: asset.id,
+        originalPath: `upload/library/label-1/2022/2022-06-19/${asset.originalFileName}`,
       });
     });
 
     it('should copy the file if rename fails due to EXDEV (rename across filesystems)', async () => {
-      const newPath = 'upload/library/user-id/2023/2023-02-23/asset-id.jpg';
-      mocks.asset.getAll.mockResolvedValue({
-        items: [assetStub.image],
-        hasNextPage: false,
-      });
+      const asset = assetStub.storageAsset({ originalPath: '/path/to/original.jpg', fileSizeInByte: 5000 });
+      const oldPath = asset.originalPath;
+      const newPath = `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`;
+      mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
       mocks.storage.rename.mockRejectedValue({ code: 'EXDEV' });
       mocks.user.getList.mockResolvedValue([userStub.user1]);
       mocks.move.create.mockResolvedValue({
         id: '123',
-        entityId: assetStub.image.id,
+        entityId: asset.id,
         pathType: AssetPathType.ORIGINAL,
-        oldPath: assetStub.image.originalPath,
+        oldPath,
         newPath,
       });
       mocks.storage.stat.mockResolvedValueOnce({
@@ -510,40 +470,36 @@ describe(StorageTemplateService.name, () => {
         size: 5000,
       } as Stats);
       mocks.storage.stat.mockResolvedValueOnce({
+        size: 5000,
         atime: new Date(),
         mtime: new Date(),
       } as Stats);
-      mocks.crypto.hashFile.mockResolvedValue(assetStub.image.checksum);
+      mocks.crypto.hashFile.mockResolvedValue(asset.checksum);
 
       await sut.handleMigration();
 
-      expect(mocks.asset.getAll).toHaveBeenCalled();
-      expect(mocks.storage.rename).toHaveBeenCalledWith('/original/path.jpg', newPath);
-      expect(mocks.storage.copyFile).toHaveBeenCalledWith('/original/path.jpg', newPath);
+      expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
+      expect(mocks.storage.rename).toHaveBeenCalledWith(oldPath, newPath);
+      expect(mocks.storage.copyFile).toHaveBeenCalledWith(oldPath, newPath);
+      expect(mocks.storage.stat).toHaveBeenCalledWith(oldPath);
       expect(mocks.storage.stat).toHaveBeenCalledWith(newPath);
-      expect(mocks.storage.stat).toHaveBeenCalledWith(assetStub.image.originalPath);
       expect(mocks.storage.utimes).toHaveBeenCalledWith(newPath, expect.any(Date), expect.any(Date));
-      expect(mocks.storage.unlink).toHaveBeenCalledWith(assetStub.image.originalPath);
+      expect(mocks.storage.unlink).toHaveBeenCalledWith(oldPath);
       expect(mocks.storage.unlink).toHaveBeenCalledTimes(1);
-      expect(mocks.asset.update).toHaveBeenCalledWith({
-        id: assetStub.image.id,
-        originalPath: newPath,
-      });
+      expect(mocks.asset.update).toHaveBeenCalledWith({ id: asset.id, originalPath: newPath });
     });
 
     it('should not update the database if the move fails due to incorrect newPath filesize', async () => {
-      mocks.asset.getAll.mockResolvedValue({
-        items: [assetStub.image],
-        hasNextPage: false,
-      });
+      const asset = assetStub.storageAsset();
+      mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
       mocks.storage.rename.mockRejectedValue({ code: 'EXDEV' });
       mocks.user.getList.mockResolvedValue([userStub.user1]);
       mocks.move.create.mockResolvedValue({
         id: '123',
-        entityId: assetStub.image.id,
+        entityId: asset.id,
         pathType: AssetPathType.ORIGINAL,
-        oldPath: assetStub.image.originalPath,
-        newPath: 'upload/library/user-id/2023/2023-02-23/asset-id.jpg',
+        oldPath: asset.originalPath,
+        newPath: `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`,
       });
       mocks.storage.stat.mockResolvedValue({
         size: 100,
@@ -551,41 +507,41 @@ describe(StorageTemplateService.name, () => {
 
       await sut.handleMigration();
 
-      expect(mocks.asset.getAll).toHaveBeenCalled();
+      expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
       expect(mocks.storage.rename).toHaveBeenCalledWith(
         '/original/path.jpg',
-        'upload/library/user-id/2023/2023-02-23/asset-id.jpg',
+        `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`,
       );
       expect(mocks.storage.copyFile).toHaveBeenCalledWith(
         '/original/path.jpg',
-        'upload/library/user-id/2023/2023-02-23/asset-id.jpg',
+        `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`,
+      );
+      expect(mocks.storage.stat).toHaveBeenCalledWith(
+        `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`,
       );
-      expect(mocks.storage.stat).toHaveBeenCalledWith('upload/library/user-id/2023/2023-02-23/asset-id.jpg');
       expect(mocks.asset.update).not.toHaveBeenCalled();
     });
 
     it('should not update the database if the move fails', async () => {
-      mocks.asset.getAll.mockResolvedValue({
-        items: [assetStub.image],
-        hasNextPage: false,
-      });
+      const asset = assetStub.storageAsset();
+      mocks.asset.streamStorageTemplateAssets.mockReturnValue(makeStream([asset]));
       mocks.storage.rename.mockRejectedValue(new Error('Read only system'));
       mocks.storage.copyFile.mockRejectedValue(new Error('Read only system'));
       mocks.move.create.mockResolvedValue({
         id: 'move-123',
-        entityId: '123',
+        entityId: asset.id,
         pathType: AssetPathType.ORIGINAL,
-        oldPath: assetStub.image.originalPath,
+        oldPath: asset.originalPath,
         newPath: '',
       });
       mocks.user.getList.mockResolvedValue([userStub.user1]);
 
       await sut.handleMigration();
 
-      expect(mocks.asset.getAll).toHaveBeenCalled();
+      expect(mocks.asset.streamStorageTemplateAssets).toHaveBeenCalled();
       expect(mocks.storage.rename).toHaveBeenCalledWith(
         '/original/path.jpg',
-        'upload/library/user-id/2023/2023-02-23/asset-id.jpg',
+        `upload/library/user-id/2022/2022-06-19/${asset.originalFileName}`,
       );
       expect(mocks.asset.update).not.toHaveBeenCalled();
     });

server/src/services/storage-template.service.ts

@@ -3,17 +3,14 @@ import handlebar from 'handlebars';
 import { DateTime } from 'luxon';
 import path from 'node:path';
 import sanitize from 'sanitize-filename';
-import { JOBS_ASSET_PAGINATION_SIZE } from 'src/constants';
 import { StorageCore } from 'src/cores/storage.core';
 import { OnEvent, OnJob } from 'src/decorators';
 import { SystemConfigTemplateStorageOptionDto } from 'src/dtos/system-config.dto';
-import { AssetEntity } from 'src/entities/asset.entity';
 import { AssetPathType, AssetType, DatabaseLock, JobName, JobStatus, QueueName, StorageFolder } from 'src/enum';
 import { ArgOf } from 'src/repositories/event.repository';
 import { BaseService } from 'src/services/base.service';
-import { JobOf } from 'src/types';
+import { JobOf, StorageAsset } from 'src/types';
 import { getLivePhotoMotionFilename } from 'src/utils/file';
-import { usePagination } from 'src/utils/pagination';
 
 const storageTokens = {
   secondOptions: ['s', 'ss', 'SSS'],
@@ -53,7 +50,7 @@ export interface MoveAssetMetadata {
 }
 
 interface RenderMetadata {
-  asset: AssetEntity;
+  asset: StorageAsset;
   filename: string;
   extension: string;
   albumName: string | null;
@@ -98,7 +95,7 @@ export class StorageTemplateService extends BaseService {
           originalPath: '/upload/test/IMG_123.jpg',
           type: AssetType.IMAGE,
           id: 'd587e44b-f8c0-4832-9ba3-43268bbf5d4e',
-        } as AssetEntity,
+        } as StorageAsset,
         filename: 'IMG_123',
         extension: 'jpg',
         albumName: 'album',
@@ -121,7 +118,7 @@ export class StorageTemplateService extends BaseService {
       return JobStatus.SKIPPED;
     }
 
-    const [asset] = await this.assetRepository.getByIds([id], { exifInfo: true });
+    const asset = await this.assetRepository.getStorageTemplateAsset(id);
     if (!asset) {
       return JobStatus.FAILED;
     }
@@ -133,7 +130,7 @@ export class StorageTemplateService extends BaseService {
 
     // move motion part of live photo
     if (asset.livePhotoVideoId) {
-      const [livePhotoVideo] = await this.assetRepository.getByIds([asset.livePhotoVideoId], { exifInfo: true });
+      const livePhotoVideo = await this.assetRepository.getStorageTemplateAsset(asset.livePhotoVideoId);
       if (!livePhotoVideo) {
         return JobStatus.FAILED;
       }
@@ -152,19 +149,17 @@ export class StorageTemplateService extends BaseService {
       this.logger.log('Storage template migration disabled, skipping');
       return JobStatus.SKIPPED;
     }
+
     await this.moveRepository.cleanMoveHistory();
-    const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
-      this.assetRepository.getAll(pagination, { withExif: true, withArchived: true }),
-    );
+
+    const assets = this.assetRepository.streamStorageTemplateAssets();
     const users = await this.userRepository.getList();
 
-    for await (const assets of assetPagination) {
-      for (const asset of assets) {
-        const user = users.find((user) => user.id === asset.ownerId);
-        const storageLabel = user?.storageLabel || null;
-        const filename = asset.originalFileName || asset.id;
-        await this.moveAsset(asset, { storageLabel, filename });
-      }
+    for await (const asset of assets) {
+      const user = users.find((user) => user.id === asset.ownerId);
+      const storageLabel = user?.storageLabel || null;
+      const filename = asset.originalFileName || asset.id;
+      await this.moveAsset(asset, { storageLabel, filename });
     }
 
     this.logger.debug('Cleaning up empty directories...');
@@ -182,7 +177,7 @@ export class StorageTemplateService extends BaseService {
     await this.moveRepository.cleanMoveHistorySingle(assetId);
   }
 
-  async moveAsset(asset: AssetEntity, metadata: MoveAssetMetadata) {
+  async moveAsset(asset: StorageAsset, metadata: MoveAssetMetadata) {
     if (asset.isExternal || StorageCore.isAndroidMotionPath(asset.originalPath)) {
       // External assets are not affected by storage template
       // TODO: shouldn't this only apply to external assets?
@@ -190,11 +185,11 @@ export class StorageTemplateService extends BaseService {
     }
 
     return this.databaseRepository.withLock(DatabaseLock.StorageTemplateMigration, async () => {
-      const { id, sidecarPath, originalPath, exifInfo, checksum } = asset;
+      const { id, sidecarPath, originalPath, checksum, fileSizeInByte } = asset;
       const oldPath = originalPath;
       const newPath = await this.getTemplatePath(asset, metadata);
 
-      if (!exifInfo || !exifInfo.fileSizeInByte) {
+      if (!fileSizeInByte) {
         this.logger.error(`Asset ${id} missing exif info, skipping storage template migration`);
         return;
       }
@@ -205,7 +200,7 @@ export class StorageTemplateService extends BaseService {
           pathType: AssetPathType.ORIGINAL,
           oldPath,
           newPath,
-          assetInfo: { sizeInBytes: exifInfo.fileSizeInByte, checksum },
+          assetInfo: { sizeInBytes: fileSizeInByte, checksum },
         });
         if (sidecarPath) {
           await this.storageCore.moveFile({
@@ -221,7 +216,7 @@ export class StorageTemplateService extends BaseService {
     });
   }
 
-  private async getTemplatePath(asset: AssetEntity, metadata: MoveAssetMetadata): Promise<string> {
+  private async getTemplatePath(asset: StorageAsset, metadata: MoveAssetMetadata): Promise<string> {
     const { storageLabel, filename } = metadata;
 
     try {
@@ -344,7 +339,7 @@ export class StorageTemplateService extends BaseService {
     };
 
     const systemTimeZone = Intl.DateTimeFormat().resolvedOptions().timeZone;
-    const zone = asset.exifInfo?.timeZone || systemTimeZone;
+    const zone = asset.timeZone || systemTimeZone;
     const dt = DateTime.fromJSDate(asset.fileCreatedAt, { zone });
 
     for (const token of Object.values(storageTokens).flat()) {

server/src/types.ts

@@ -1,4 +1,5 @@
 import {
+  AssetType,
   DatabaseExtension,
   ExifOrientation,
   ImageFormat,
@@ -438,3 +439,18 @@ export type SyncAck = {
   type: SyncEntityType;
   updateId: string;
 };
+
+export type StorageAsset = {
+  id: string;
+  ownerId: string;
+  livePhotoVideoId: string | null;
+  type: AssetType;
+  isExternal: boolean;
+  checksum: Buffer;
+  timeZone: string | null;
+  fileCreatedAt: Date;
+  originalPath: string;
+  originalFileName: string;
+  sidecarPath: string | null;
+  fileSizeInByte: number | null;
+};

server/test/fixtures/asset.stub.ts

@@ -3,6 +3,7 @@ import { AssetEntity } from 'src/entities/asset.entity';
 import { ExifEntity } from 'src/entities/exif.entity';
 import { StackEntity } from 'src/entities/stack.entity';
 import { AssetFileType, AssetStatus, AssetType } from 'src/enum';
+import { StorageAsset } from 'src/types';
 import { authStub } from 'test/fixtures/auth.stub';
 import { fileStub } from 'test/fixtures/file.stub';
 import { libraryStub } from 'test/fixtures/library.stub';
@@ -40,6 +41,21 @@ export const stackStub = (stackId: string, assets: AssetEntity[]): StackEntity =
 };
 
 export const assetStub = {
+  storageAsset: (asset: Partial<StorageAsset> = {}) => ({
+    id: 'asset-id',
+    ownerId: 'user-id',
+    livePhotoVideoId: null,
+    type: AssetType.IMAGE,
+    isExternal: false,
+    checksum: Buffer.from('file hash'),
+    timeZone: null,
+    fileCreatedAt: new Date('2022-06-19T23:41:36.910Z'),
+    originalPath: '/original/path.jpg',
+    originalFileName: 'IMG_123.jpg',
+    sidecarPath: null,
+    fileSizeInByte: 12_345,
+    ...asset,
+  }),
   noResizePath: Object.freeze<AssetEntity>({
     id: 'asset-id',
     status: AssetStatus.ACTIVE,

server/test/repositories/asset.repository.mock.ts

@@ -44,5 +44,7 @@ export const newAssetRepositoryMock = (): Mocked<RepositoryInterface<AssetReposi
     detectOfflineExternalAssets: vitest.fn(),
     filterNewExternalAssetPaths: vitest.fn(),
     updateByLibraryId: vitest.fn(),
+    streamStorageTemplateAssets: vitest.fn(),
+    getStorageTemplateAsset: vitest.fn(),
   };
 };