immich-app/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2025-12-25 17:24:58 +03:00
Code Issues Packages Projects Releases 18 Wiki Activity

refactor: stream assets for thumbnail job (#17623)

Browse Source
This commit is contained in:
Daniel Dietzler
2025-04-15 19:53:28 +02:00
committed by GitHub
parent b710ad36f3
commit 5bceefce75
4 changed files with 75 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
server/src/services/media.service.spec.ts
View File

@@ -39,6 +39,7 @@ describe(MediaService.name, () => {
describe('handleQueueGenerateThumbnails', () => {
it('should queue all assets', async () => {
mocks.assetJob.streamForThumbnailJob.mockReturnValue(makeStream([assetStub.image]));
mocks.asset.getAll.mockResolvedValue({
items: [assetStub.image],
hasNextPage: false,
@@ -49,8 +50,7 @@ describe(MediaService.name, () => {
await sut.handleQueueGenerateThumbnails({ force: true });
expect(mocks.asset.getAll).toHaveBeenCalled();
expect(mocks.asset.getWithout).not.toHaveBeenCalled();
expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith(true);
expect(mocks.job.queueAll).toHaveBeenCalledWith([
{
name: JobName.GENERATE_THUMBNAILS,
@@ -68,6 +68,7 @@ describe(MediaService.name, () => {
});
it('should queue trashed assets when force is true', async () => {
mocks.assetJob.streamForThumbnailJob.mockReturnValue(makeStream([assetStub.archived]));
mocks.asset.getAll.mockResolvedValue({
items: [assetStub.trashed],
hasNextPage: false,
@@ -76,11 +77,7 @@ describe(MediaService.name, () => {
await sut.handleQueueGenerateThumbnails({ force: true });
expect(mocks.asset.getAll).toHaveBeenCalledWith(
{ skip: 0, take: 1000 },
expect.objectContaining({ withDeleted: true }),
);
expect(mocks.asset.getWithout).not.toHaveBeenCalled();
expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith(true);
expect(mocks.job.queueAll).toHaveBeenCalledWith([
{
name: JobName.GENERATE_THUMBNAILS,
@@ -90,19 +87,12 @@ describe(MediaService.name, () => {
});
it('should queue archived assets when force is true', async () => {
mocks.asset.getAll.mockResolvedValue({
items: [assetStub.archived],
hasNextPage: false,
});
mocks.assetJob.streamForThumbnailJob.mockReturnValue(makeStream([assetStub.archived]));
mocks.person.getAll.mockReturnValue(makeStream());
await sut.handleQueueGenerateThumbnails({ force: true });
expect(mocks.asset.getAll).toHaveBeenCalledWith(
{ skip: 0, take: 1000 },
expect.objectContaining({ withArchived: true }),
);
expect(mocks.asset.getWithout).not.toHaveBeenCalled();
expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith(true);
expect(mocks.job.queueAll).toHaveBeenCalledWith([
{
name: JobName.GENERATE_THUMBNAILS,
@@ -112,18 +102,13 @@ describe(MediaService.name, () => {
});
it('should queue all people with missing thumbnail path', async () => {
mocks.asset.getWithout.mockResolvedValue({
items: [assetStub.image],
hasNextPage: false,
});
mocks.assetJob.streamForThumbnailJob.mockReturnValue(makeStream([assetStub.image]));
mocks.person.getAll.mockReturnValue(makeStream([personStub.noThumbnail, personStub.noThumbnail]));
mocks.person.getRandomFace.mockResolvedValueOnce(faceStub.face1);
await sut.handleQueueGenerateThumbnails({ force: false });
expect(mocks.asset.getAll).not.toHaveBeenCalled();
expect(mocks.asset.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.THUMBNAIL);
expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith(false);
expect(mocks.person.getAll).toHaveBeenCalledWith({ thumbnailPath: '' });
expect(mocks.person.getRandomFace).toHaveBeenCalled();
expect(mocks.person.update).toHaveBeenCalledTimes(1);
@@ -138,15 +123,11 @@ describe(MediaService.name, () => {
});
it('should queue all assets with missing resize path', async () => {
mocks.asset.getWithout.mockResolvedValue({
items: [assetStub.noResizePath],
hasNextPage: false,
});
mocks.assetJob.streamForThumbnailJob.mockReturnValue(makeStream([assetStub.noResizePath]));
mocks.person.getAll.mockReturnValue(makeStream());
await sut.handleQueueGenerateThumbnails({ force: false });
expect(mocks.asset.getAll).not.toHaveBeenCalled();
expect(mocks.asset.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.THUMBNAIL);
expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith(false);
expect(mocks.job.queueAll).toHaveBeenCalledWith([
{
name: JobName.GENERATE_THUMBNAILS,
@@ -158,15 +139,11 @@ describe(MediaService.name, () => {
});
it('should queue all assets with missing webp path', async () => {
mocks.asset.getWithout.mockResolvedValue({
items: [assetStub.noWebpPath],
hasNextPage: false,
});
mocks.assetJob.streamForThumbnailJob.mockReturnValue(makeStream([assetStub.noWebpPath]));
mocks.person.getAll.mockReturnValue(makeStream());
await sut.handleQueueGenerateThumbnails({ force: false });
expect(mocks.asset.getAll).not.toHaveBeenCalled();
expect(mocks.asset.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.THUMBNAIL);
expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith(false);
expect(mocks.job.queueAll).toHaveBeenCalledWith([
{
name: JobName.GENERATE_THUMBNAILS,
@@ -178,15 +155,11 @@ describe(MediaService.name, () => {
});
it('should queue all assets with missing thumbhash', async () => {
mocks.asset.getWithout.mockResolvedValue({
items: [assetStub.noThumbhash],
hasNextPage: false,
});
mocks.assetJob.streamForThumbnailJob.mockReturnValue(makeStream([assetStub.noThumbhash]));
mocks.person.getAll.mockReturnValue(makeStream());
await sut.handleQueueGenerateThumbnails({ force: false });
expect(mocks.asset.getAll).not.toHaveBeenCalled();
expect(mocks.asset.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.THUMBNAIL);
expect(mocks.assetJob.streamForThumbnailJob).toHaveBeenCalledWith(false);
expect(mocks.job.queueAll).toHaveBeenCalledWith([
{
name: JobName.GENERATE_THUMBNAILS,

28
server/src/services/media.service.ts
View File

@@ -51,30 +51,16 @@ export class MediaService extends BaseService {
@OnJob({ name: JobName.QUEUE_GENERATE_THUMBNAILS, queue: QueueName.THUMBNAIL_GENERATION })
async handleQueueGenerateThumbnails({ force }: JobOf<JobName.QUEUE_GENERATE_THUMBNAILS>): Promise<JobStatus> {
const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => {
return force
? this.assetRepository.getAll(pagination, {
isVisible: true,
withDeleted: true,
withArchived: true,
})
: this.assetRepository.getWithout(pagination, WithoutProperty.THUMBNAIL);
});
const thumbJobs: JobItem[] = [];
for await (const asset of this.assetJobRepository.streamForThumbnailJob(!!force)) {
const { previewFile, thumbnailFile } = getAssetFiles(asset.files);
for await (const assets of assetPagination) {
const jobs: JobItem[] = [];
for (const asset of assets) {
const { previewFile, thumbnailFile } = getAssetFiles(asset.files);
if (!previewFile || !thumbnailFile || !asset.thumbhash || force) {
jobs.push({ name: JobName.GENERATE_THUMBNAILS, data: { id: asset.id } });
continue;
}
if (!previewFile || !thumbnailFile || !asset.thumbhash || force) {
thumbJobs.push({ name: JobName.GENERATE_THUMBNAILS, data: { id: asset.id } });
continue;
}
await this.jobRepository.queueAll(jobs);
}
await this.jobRepository.queueAll(thumbJobs);
const jobs: JobItem[] = [];