immich-app/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2025-12-24 17:24:56 +03:00
Code Issues Packages Projects Releases 18 Wiki Activity

refactor: stream for sidecar (#17995)

Browse Source 
* refactor: stream for sidecar

* chore: make sql

---------

Co-authored-by: Jason Rasmussen <jason@rasm.me>
This commit is contained in:
Alex
2025-04-30 10:53:51 -05:00
committed by GitHub
parent 436cff72b5
commit 732b06eec8
6 changed files with 46 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
server/src/services/metadata.service.spec.ts
View File

@@ -5,7 +5,6 @@ import { constants } from 'node:fs/promises';
import { defaults } from 'src/config';
import { MapAsset } from 'src/dtos/asset-response.dto';
import { AssetType, ExifOrientation, ImmichWorker, JobName, JobStatus, SourceType } from 'src/enum';
import { WithoutProperty } from 'src/repositories/asset.repository';
import { ImmichTags } from 'src/repositories/metadata.repository';
import { MetadataService } from 'src/services/metadata.service';
import { assetStub } from 'test/fixtures/asset.stub';
@@ -1346,12 +1345,11 @@ describe(MetadataService.name, () => {
describe('handleQueueSidecar', () => {
it('should queue assets with sidecar files', async () => {
mocks.asset.getAll.mockResolvedValue({ items: [assetStub.sidecar], hasNextPage: false });
mocks.assetJob.streamForSidecar.mockReturnValue(makeStream([assetStub.image]));
await sut.handleQueueSidecar({ force: true });
expect(mocks.assetJob.streamForSidecar).toHaveBeenCalledWith(true);
expect(mocks.asset.getAll).toHaveBeenCalledWith({ take: 1000, skip: 0 });
expect(mocks.asset.getWithout).not.toHaveBeenCalled();
expect(mocks.job.queueAll).toHaveBeenCalledWith([
{
name: JobName.SIDECAR_SYNC,
@@ -1361,12 +1359,11 @@ describe(MetadataService.name, () => {
});
it('should queue assets without sidecar files', async () => {
mocks.asset.getWithout.mockResolvedValue({ items: [assetStub.image], hasNextPage: false });
mocks.assetJob.streamForSidecar.mockReturnValue(makeStream([assetStub.image]));
await sut.handleQueueSidecar({ force: false });
expect(mocks.asset.getWithout).toHaveBeenCalledWith({ take: 1000, skip: 0 }, WithoutProperty.SIDECAR);
expect(mocks.asset.getAll).not.toHaveBeenCalled();
expect(mocks.assetJob.streamForSidecar).toHaveBeenCalledWith(false);
expect(mocks.job.queueAll).toHaveBeenCalledWith([
{
name: JobName.SIDECAR_DISCOVERY,

32
server/src/services/metadata.service.ts
View File

@@ -22,14 +22,12 @@ import {
QueueName,
SourceType,
} from 'src/enum';
import { WithoutProperty } from 'src/repositories/asset.repository';
import { ArgOf } from 'src/repositories/event.repository';
import { ReverseGeocodeResult } from 'src/repositories/map.repository';
import { ImmichTags } from 'src/repositories/metadata.repository';
import { BaseService } from 'src/services/base.service';
import { JobOf } from 'src/types';
import { JobItem, JobOf } from 'src/types';
import { isFaceImportEnabled } from 'src/utils/misc';
import { usePagination } from 'src/utils/pagination';
import { upsertTags } from 'src/utils/tag';
/** look for a date from these tags (in order) */
@@ -289,23 +287,23 @@ export class MetadataService extends BaseService {
}
@OnJob({ name: JobName.QUEUE_SIDECAR, queue: QueueName.SIDECAR })
async handleQueueSidecar(job: JobOf<JobName.QUEUE_SIDECAR>): Promise<JobStatus> {
const { force } = job;
const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => {
return force
? this.assetRepository.getAll(pagination)
: this.assetRepository.getWithout(pagination, WithoutProperty.SIDECAR);
});
async handleQueueSidecar({ force }: JobOf<JobName.QUEUE_SIDECAR>): Promise<JobStatus> {
let jobs: JobItem[] = [];
const queueAll = async () => {
await this.jobRepository.queueAll(jobs);
jobs = [];
};
for await (const assets of assetPagination) {
await this.jobRepository.queueAll(
assets.map((asset) => ({
name: force ? JobName.SIDECAR_SYNC : JobName.SIDECAR_DISCOVERY,
data: { id: asset.id },
})),
);
const assets = this.assetJobRepository.streamForSidecar(force);
for await (const asset of assets) {
jobs.push({ name: force ? JobName.SIDECAR_SYNC : JobName.SIDECAR_DISCOVERY, data: { id: asset.id } });
if (jobs.length >= JOBS_ASSET_PAGINATION_SIZE) {
await queueAll();
}
}
await queueAll();
return JobStatus.SUCCESS;
}