immich-app/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2025-12-28 17:24:56 +03:00
Code Issues Packages Projects Releases 18 Wiki Activity

refactor: migrate person repository to kysely (#15242)

Browse Source 
* refactor: migrate person repository to kysely

* `asVector` begone

* linting

* fix metadata faces

* update test

---------

Co-authored-by: Alex <alex.tran1502@gmail.com>
Co-authored-by: mertalev <101130780+mertalev@users.noreply.github.com>
This commit is contained in:
Daniel Dietzler
2025-01-21 19:12:28 +01:00
committed by GitHub
parent 0c152366ec
commit 332a865ce6
29 changed files with 715 additions and 747 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
server/src/services/audit.service.ts
View File

@@ -201,21 +201,22 @@ export class AuditService extends BaseService {
}
}
const personPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
this.personRepository.getAll(pagination),
);
for await (const people of personPagination) {
for (const { id, thumbnailPath } of people) {
track(thumbnailPath);
const entity = { entityId: id, entityType: PathEntityType.PERSON };
if (thumbnailPath && !hasFile(thumbFiles, thumbnailPath)) {
orphans.push({ ...entity, pathType: PersonPathType.FACE, pathValue: thumbnailPath });
}
let peopleCount = 0;
for await (const { id, thumbnailPath } of this.personRepository.getAll()) {
track(thumbnailPath);
const entity = { entityId: id, entityType: PathEntityType.PERSON };
if (thumbnailPath && !hasFile(thumbFiles, thumbnailPath)) {
orphans.push({ ...entity, pathType: PersonPathType.FACE, pathValue: thumbnailPath });
}
this.logger.log(`Found ${assetCount} assets, ${users.length} users, ${people.length} people`);
if (peopleCount === JOBS_ASSET_PAGINATION_SIZE) {
this.logger.log(`Found ${assetCount} assets, ${users.length} users, ${peopleCount} people`);
peopleCount = 0;
}
}
this.logger.log(`Found ${assetCount} assets, ${users.length} users, ${peopleCount} people`);
const extras: string[] = [];
for (const file of allFiles) {
extras.push(file);

58
server/src/services/media.service.spec.ts
View File

@@ -25,7 +25,7 @@ import { assetStub } from 'test/fixtures/asset.stub';
import { faceStub } from 'test/fixtures/face.stub';
import { probeStub } from 'test/fixtures/media.stub';
import { personStub } from 'test/fixtures/person.stub';
import { newTestService } from 'test/utils';
import { makeStream, newTestService } from 'test/utils';
import { Mocked } from 'vitest';
describe(MediaService.name, () => {
@@ -55,10 +55,8 @@ describe(MediaService.name, () => {
items: [assetStub.image],
hasNextPage: false,
});
personMock.getAll.mockResolvedValue({
items: [personStub.newThumbnail],
hasNextPage: false,
});
personMock.getAll.mockReturnValue(makeStream([personStub.newThumbnail]));
personMock.getFacesByIds.mockResolvedValue([faceStub.face1]);
await sut.handleQueueGenerateThumbnails({ force: true });
@@ -72,7 +70,7 @@ describe(MediaService.name, () => {
},
]);
expect(personMock.getAll).toHaveBeenCalledWith({ skip: 0, take: 1000 }, {});
expect(personMock.getAll).toHaveBeenCalledWith(undefined);
expect(jobMock.queueAll).toHaveBeenCalledWith([
{
name: JobName.GENERATE_PERSON_THUMBNAIL,
@@ -86,10 +84,7 @@ describe(MediaService.name, () => {
items: [assetStub.trashed],
hasNextPage: false,
});
personMock.getAll.mockResolvedValue({
items: [],
hasNextPage: false,
});
personMock.getAll.mockReturnValue(makeStream());
await sut.handleQueueGenerateThumbnails({ force: true });
@@ -111,10 +106,7 @@ describe(MediaService.name, () => {
items: [assetStub.archived],
hasNextPage: false,
});
personMock.getAll.mockResolvedValue({
items: [],
hasNextPage: false,
});
personMock.getAll.mockReturnValue(makeStream());
await sut.handleQueueGenerateThumbnails({ force: true });
@@ -136,10 +128,7 @@ describe(MediaService.name, () => {
items: [assetStub.image],
hasNextPage: false,
});
personMock.getAll.mockResolvedValue({
items: [personStub.noThumbnail, personStub.noThumbnail],
hasNextPage: false,
});
personMock.getAll.mockReturnValue(makeStream([personStub.noThumbnail, personStub.noThumbnail]));
personMock.getRandomFace.mockResolvedValueOnce(faceStub.face1);
await sut.handleQueueGenerateThumbnails({ force: false });
@@ -147,7 +136,7 @@ describe(MediaService.name, () => {
expect(assetMock.getAll).not.toHaveBeenCalled();
expect(assetMock.getWithout).toHaveBeenCalledWith({ skip: 0, take: 1000 }, WithoutProperty.THUMBNAIL);
expect(personMock.getAll).toHaveBeenCalledWith({ skip: 0, take: 1000 }, { where: { thumbnailPath: '' } });
expect(personMock.getAll).toHaveBeenCalledWith({ thumbnailPath: '' });
expect(personMock.getRandomFace).toHaveBeenCalled();
expect(personMock.update).toHaveBeenCalledTimes(1);
expect(jobMock.queueAll).toHaveBeenCalledWith([
@@ -165,11 +154,7 @@ describe(MediaService.name, () => {
items: [assetStub.noResizePath],
hasNextPage: false,
});
personMock.getAll.mockResolvedValue({
items: [],
hasNextPage: false,
});
personMock.getAll.mockReturnValue(makeStream());
await sut.handleQueueGenerateThumbnails({ force: false });
expect(assetMock.getAll).not.toHaveBeenCalled();
@@ -181,7 +166,7 @@ describe(MediaService.name, () => {
},
]);
expect(personMock.getAll).toHaveBeenCalledWith({ skip: 0, take: 1000 }, { where: { thumbnailPath: '' } });
expect(personMock.getAll).toHaveBeenCalledWith({ thumbnailPath: '' });
});
it('should queue all assets with missing webp path', async () => {
@@ -189,11 +174,7 @@ describe(MediaService.name, () => {
items: [assetStub.noWebpPath],
hasNextPage: false,
});
personMock.getAll.mockResolvedValue({
items: [],
hasNextPage: false,
});
personMock.getAll.mockReturnValue(makeStream());
await sut.handleQueueGenerateThumbnails({ force: false });
expect(assetMock.getAll).not.toHaveBeenCalled();
@@ -205,7 +186,7 @@ describe(MediaService.name, () => {
},
]);
expect(personMock.getAll).toHaveBeenCalledWith({ skip: 0, take: 1000 }, { where: { thumbnailPath: '' } });
expect(personMock.getAll).toHaveBeenCalledWith({ thumbnailPath: '' });
});
it('should queue all assets with missing thumbhash', async () => {
@@ -213,11 +194,7 @@ describe(MediaService.name, () => {
items: [assetStub.noThumbhash],
hasNextPage: false,
});
personMock.getAll.mockResolvedValue({
items: [],
hasNextPage: false,
});
personMock.getAll.mockReturnValue(makeStream());
await sut.handleQueueGenerateThumbnails({ force: false });
expect(assetMock.getAll).not.toHaveBeenCalled();
@@ -229,7 +206,7 @@ describe(MediaService.name, () => {
},
]);
expect(personMock.getAll).toHaveBeenCalledWith({ skip: 0, take: 1000 }, { where: { thumbnailPath: '' } });
expect(personMock.getAll).toHaveBeenCalledWith({ thumbnailPath: '' });
});
});
@@ -237,7 +214,7 @@ describe(MediaService.name, () => {
it('should remove empty directories and queue jobs', async () => {
assetMock.getAll.mockResolvedValue({ hasNextPage: false, items: [assetStub.image] });
jobMock.getJobCounts.mockResolvedValue({ active: 1, waiting: 0 } as JobCounts);
personMock.getAll.mockResolvedValue({ hasNextPage: false, items: [personStub.withName] });
personMock.getAll.mockReturnValue(makeStream([personStub.withName]));
await expect(sut.handleQueueMigration()).resolves.toBe(JobStatus.SUCCESS);
@@ -730,10 +707,7 @@ describe(MediaService.name, () => {
items: [assetStub.video],
hasNextPage: false,
});
personMock.getAll.mockResolvedValue({
items: [],
hasNextPage: false,
});
personMock.getAll.mockReturnValue(makeStream());
await sut.handleQueueVideoConversion({ force: true });

38
server/src/services/media.service.ts
View File

@@ -72,23 +72,20 @@ export class MediaService extends BaseService {
}
const jobs: JobItem[] = [];
const personPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
this.personRepository.getAll(pagination, { where: force ? undefined : { thumbnailPath: '' } }),
);
for await (const people of personPagination) {
for (const person of people) {
if (!person.faceAssetId) {
const face = await this.personRepository.getRandomFace(person.id);
if (!face) {
continue;
}
const people = this.personRepository.getAll(force ? undefined : { thumbnailPath: '' });
await this.personRepository.update({ id: person.id, faceAssetId: face.id });
for await (const person of people) {
if (!person.faceAssetId) {
const face = await this.personRepository.getRandomFace(person.id);
if (!face) {
continue;
}
jobs.push({ name: JobName.GENERATE_PERSON_THUMBNAIL, data: { id: person.id } });
await this.personRepository.update({ id: person.id, faceAssetId: face.id });
}
jobs.push({ name: JobName.GENERATE_PERSON_THUMBNAIL, data: { id: person.id } });
}
await this.jobRepository.queueAll(jobs);
@@ -114,16 +111,19 @@ export class MediaService extends BaseService {
);
}
const personPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
this.personRepository.getAll(pagination),
);
let jobs: { name: JobName.MIGRATE_PERSON; data: { id: string } }[] = [];
for await (const people of personPagination) {
await this.jobRepository.queueAll(
people.map((person) => ({ name: JobName.MIGRATE_PERSON, data: { id: person.id } })),
);
for await (const person of this.personRepository.getAll()) {
jobs.push({ name: JobName.MIGRATE_PERSON, data: { id: person.id } });
if (jobs.length === JOBS_ASSET_PAGINATION_SIZE) {
await this.jobRepository.queueAll(jobs);
jobs = [];
}
}
await this.jobRepository.queueAll(jobs);
return JobStatus.SUCCESS;
}

4
server/src/services/metadata.service.spec.ts
View File

@@ -1086,7 +1086,9 @@ describe(MetadataService.name, () => {
],
[],
);
expect(personMock.updateAll).toHaveBeenCalledWith([{ id: 'random-uuid', faceAssetId: 'random-uuid' }]);
expect(personMock.updateAll).toHaveBeenCalledWith([
{ id: 'random-uuid', ownerId: 'admin-id', faceAssetId: 'random-uuid' },
]);
expect(jobMock.queueAll).toHaveBeenCalledWith([
{
name: JobName.GENERATE_PERSON_THUMBNAIL,

10
server/src/services/metadata.service.ts
View File

@@ -509,11 +509,11 @@ export class MetadataService extends BaseService {
return;
}
const facesToAdd: Partial<AssetFaceEntity>[] = [];
const facesToAdd: (Partial<AssetFaceEntity> & { assetId: string })[] = [];
const existingNames = await this.personRepository.getDistinctNames(asset.ownerId, { withHidden: true });
const existingNameMap = new Map(existingNames.map(({ id, name }) => [name.toLowerCase(), id]));
const missing: Partial<PersonEntity>[] = [];
const missingWithFaceAsset: Partial<PersonEntity>[] = [];
const missing: (Partial<PersonEntity> & { ownerId: string })[] = [];
const missingWithFaceAsset: { id: string; ownerId: string; faceAssetId: string }[] = [];
for (const region of tags.RegionInfo.RegionList) {
if (!region.Name) {
continue;
@@ -540,7 +540,7 @@ export class MetadataService extends BaseService {
facesToAdd.push(face);
if (!existingNameMap.has(loweredName)) {
missing.push({ id: personId, ownerId: asset.ownerId, name: region.Name });
missingWithFaceAsset.push({ id: personId, faceAssetId: face.id });
missingWithFaceAsset.push({ id: personId, ownerId: asset.ownerId, faceAssetId: face.id });
}
}
@@ -557,7 +557,7 @@ export class MetadataService extends BaseService {
}
if (facesToAdd.length > 0) {
this.logger.debug(`Creating ${facesToAdd} faces from metadata for asset ${asset.id}`);
this.logger.debug(`Creating ${facesToAdd.length} faces from metadata for asset ${asset.id}`);
}
if (facesToRemove.length > 0 || facesToAdd.length > 0) {

70
server/src/services/person.service.spec.ts
View File

@@ -20,8 +20,7 @@ import { faceStub } from 'test/fixtures/face.stub';
import { personStub } from 'test/fixtures/person.stub';
import { systemConfigStub } from 'test/fixtures/system-config.stub';
import { IAccessRepositoryMock } from 'test/repositories/access.repository.mock';
import { newTestService } from 'test/utils';
import { IsNull } from 'typeorm';
import { makeStream, newTestService } from 'test/utils';
import { Mocked } from 'vitest';
const responseDto: PersonResponseDto = {
@@ -46,7 +45,7 @@ const face = {
imageHeight: 500,
imageWidth: 400,
};
const faceSearch = { faceId, embedding: [1, 2, 3, 4] };
const faceSearch = { faceId, embedding: '[1, 2, 3, 4]' };
const detectFaceMock: DetectedFaces = {
faces: [
{
@@ -495,14 +494,8 @@ describe(PersonService.name, () => {
});
it('should delete existing people and faces if forced', async () => {
personMock.getAll.mockResolvedValue({
items: [faceStub.face1.person, personStub.randomPerson],
hasNextPage: false,
});
personMock.getAllFaces.mockResolvedValue({
items: [faceStub.face1],
hasNextPage: false,
});
personMock.getAll.mockReturnValue(makeStream([faceStub.face1.person, personStub.randomPerson]));
personMock.getAllFaces.mockReturnValue(makeStream([faceStub.face1]));
assetMock.getAll.mockResolvedValue({
items: [assetStub.image],
hasNextPage: false,
@@ -544,18 +537,12 @@ describe(PersonService.name, () => {
it('should queue missing assets', async () => {
jobMock.getJobCounts.mockResolvedValue({ active: 1, waiting: 0, paused: 0, completed: 0, failed: 0, delayed: 0 });
personMock.getAllFaces.mockResolvedValue({
items: [faceStub.face1],
hasNextPage: false,
});
personMock.getAllFaces.mockReturnValue(makeStream([faceStub.face1]));
personMock.getAllWithoutFaces.mockResolvedValue([]);
await sut.handleQueueRecognizeFaces({});
expect(personMock.getAllFaces).toHaveBeenCalledWith(
{ skip: 0, take: 1000 },
{ where: { personId: IsNull(), sourceType: SourceType.MACHINE_LEARNING } },
);
expect(personMock.getAllFaces).toHaveBeenCalledWith({ personId: null, sourceType: SourceType.MACHINE_LEARNING });
expect(jobMock.queueAll).toHaveBeenCalledWith([
{
name: JobName.FACIAL_RECOGNITION,
@@ -569,19 +556,13 @@ describe(PersonService.name, () => {
it('should queue all assets', async () => {
jobMock.getJobCounts.mockResolvedValue({ active: 1, waiting: 0, paused: 0, completed: 0, failed: 0, delayed: 0 });
personMock.getAll.mockResolvedValue({
items: [],
hasNextPage: false,
});
personMock.getAllFaces.mockResolvedValue({
items: [faceStub.face1],
hasNextPage: false,
});
personMock.getAll.mockReturnValue(makeStream());
personMock.getAllFaces.mockReturnValue(makeStream([faceStub.face1]));
personMock.getAllWithoutFaces.mockResolvedValue([]);
await sut.handleQueueRecognizeFaces({ force: true });
expect(personMock.getAllFaces).toHaveBeenCalledWith({ skip: 0, take: 1000 }, {});
expect(personMock.getAllFaces).toHaveBeenCalledWith(undefined);
expect(jobMock.queueAll).toHaveBeenCalledWith([
{
name: JobName.FACIAL_RECOGNITION,
@@ -595,26 +576,17 @@ describe(PersonService.name, () => {
it('should run nightly if new face has been added since last run', async () => {
personMock.getLatestFaceDate.mockResolvedValue(new Date().toISOString());
personMock.getAllFaces.mockResolvedValue({
items: [faceStub.face1],
hasNextPage: false,
});
personMock.getAllFaces.mockReturnValue(makeStream([faceStub.face1]));
jobMock.getJobCounts.mockResolvedValue({ active: 1, waiting: 0, paused: 0, completed: 0, failed: 0, delayed: 0 });
personMock.getAll.mockResolvedValue({
items: [],
hasNextPage: false,
});
personMock.getAllFaces.mockResolvedValue({
items: [faceStub.face1],
hasNextPage: false,
});
personMock.getAll.mockReturnValue(makeStream());
personMock.getAllFaces.mockReturnValue(makeStream([faceStub.face1]));
personMock.getAllWithoutFaces.mockResolvedValue([]);
await sut.handleQueueRecognizeFaces({ force: true, nightly: true });
expect(systemMock.get).toHaveBeenCalledWith(SystemMetadataKey.FACIAL_RECOGNITION_STATE);
expect(personMock.getLatestFaceDate).toHaveBeenCalledOnce();
expect(personMock.getAllFaces).toHaveBeenCalledWith({ skip: 0, take: 1000 }, {});
expect(personMock.getAllFaces).toHaveBeenCalledWith(undefined);
expect(jobMock.queueAll).toHaveBeenCalledWith([
{
name: JobName.FACIAL_RECOGNITION,
@@ -631,10 +603,7 @@ describe(PersonService.name, () => {
systemMock.get.mockResolvedValue({ lastRun: lastRun.toISOString() });
personMock.getLatestFaceDate.mockResolvedValue(new Date(lastRun.getTime() - 1).toISOString());
personMock.getAllFaces.mockResolvedValue({
items: [faceStub.face1],
hasNextPage: false,
});
personMock.getAllFaces.mockReturnValue(makeStream([faceStub.face1]));
personMock.getAllWithoutFaces.mockResolvedValue([]);
await sut.handleQueueRecognizeFaces({ force: true, nightly: true });
@@ -648,15 +617,8 @@ describe(PersonService.name, () => {
it('should delete existing people if forced', async () => {
jobMock.getJobCounts.mockResolvedValue({ active: 1, waiting: 0, paused: 0, completed: 0, failed: 0, delayed: 0 });
personMock.getAll.mockResolvedValue({
items: [faceStub.face1.person, personStub.randomPerson],
hasNextPage: false,
});
personMock.getAllFaces.mockResolvedValue({
items: [faceStub.face1],
hasNextPage: false,
});
personMock.getAll.mockReturnValue(makeStream([faceStub.face1.person, personStub.randomPerson]));
personMock.getAllFaces.mockReturnValue(makeStream([faceStub.face1]));
personMock.getAllWithoutFaces.mockResolvedValue([personStub.randomPerson]);
await sut.handleQueueRecognizeFaces({ force: true });

25
server/src/services/person.service.ts
View File

@@ -50,7 +50,6 @@ import { ImmichFileResponse } from 'src/utils/file';
import { mimeTypes } from 'src/utils/mime-types';
import { isFaceImportEnabled, isFacialRecognitionEnabled } from 'src/utils/misc';
import { usePagination } from 'src/utils/pagination';
import { IsNull } from 'typeorm';
@Injectable()
export class PersonService extends BaseService {
@@ -306,7 +305,7 @@ export class PersonService extends BaseService {
);
this.logger.debug(`${faces.length} faces detected in ${previewFile.path}`);
const facesToAdd: (Partial<AssetFaceEntity> & { id: string })[] = [];
const facesToAdd: (Partial<AssetFaceEntity> & { id: string; assetId: string })[] = [];
const embeddings: FaceSearchEntity[] = [];
const mlFaceIds = new Set<string>();
for (const face of asset.faces) {
@@ -414,18 +413,22 @@ export class PersonService extends BaseService {
}
const lastRun = new Date().toISOString();
const facePagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
this.personRepository.getAllFaces(pagination, {
where: force ? undefined : { personId: IsNull(), sourceType: SourceType.MACHINE_LEARNING },
}),
const facePagination = this.personRepository.getAllFaces(
force ? undefined : { personId: null, sourceType: SourceType.MACHINE_LEARNING },
);
for await (const page of facePagination) {
await this.jobRepository.queueAll(
page.map((face) => ({ name: JobName.FACIAL_RECOGNITION, data: { id: face.id, deferred: false } })),
);
let jobs: { name: JobName.FACIAL_RECOGNITION; data: { id: string; deferred: false } }[] = [];
for await (const face of facePagination) {
jobs.push({ name: JobName.FACIAL_RECOGNITION, data: { id: face.id, deferred: false } });
if (jobs.length === JOBS_ASSET_PAGINATION_SIZE) {
await this.jobRepository.queueAll(jobs);
jobs = [];
}
}
await this.jobRepository.queueAll(jobs);
await this.systemMetadataRepository.set(SystemMetadataKey.FACIAL_RECOGNITION_STATE, { lastRun });
return JobStatus.SUCCESS;
@@ -441,7 +444,7 @@ export class PersonService extends BaseService {
const face = await this.personRepository.getFaceByIdWithAssets(
id,
{ person: true, asset: true, faceSearch: true },
{ id: true, personId: true, sourceType: true, faceSearch: { embedding: true } },
{ id: true, personId: true, sourceType: true, faceSearch: true },
);
if (!face || !face.asset) {
this.logger.warn(`Face ${id} not found`);

8
server/src/services/smart-info.service.spec.ts
View File

@@ -284,7 +284,7 @@ describe(SmartInfoService.name, () => {
});
it('should save the returned objects', async () => {
machineLearningMock.encodeImage.mockResolvedValue([0.01, 0.02, 0.03]);
machineLearningMock.encodeImage.mockResolvedValue('[0.01, 0.02, 0.03]');
expect(await sut.handleEncodeClip({ id: assetStub.image.id })).toEqual(JobStatus.SUCCESS);
@@ -293,7 +293,7 @@ describe(SmartInfoService.name, () => {
'/uploads/user-id/thumbs/path.jpg',
expect.objectContaining({ modelName: 'ViT-B-32__openai' }),
);
expect(searchMock.upsert).toHaveBeenCalledWith(assetStub.image.id, [0.01, 0.02, 0.03]);
expect(searchMock.upsert).toHaveBeenCalledWith(assetStub.image.id, '[0.01, 0.02, 0.03]');
});
it('should skip invisible assets', async () => {
@@ -315,7 +315,7 @@ describe(SmartInfoService.name, () => {
});
it('should wait for database', async () => {
machineLearningMock.encodeImage.mockResolvedValue([0.01, 0.02, 0.03]);
machineLearningMock.encodeImage.mockResolvedValue('[0.01, 0.02, 0.03]');
databaseMock.isBusy.mockReturnValue(true);
expect(await sut.handleEncodeClip({ id: assetStub.image.id })).toEqual(JobStatus.SUCCESS);
@@ -326,7 +326,7 @@ describe(SmartInfoService.name, () => {
'/uploads/user-id/thumbs/path.jpg',
expect.objectContaining({ modelName: 'ViT-B-32__openai' }),
);
expect(searchMock.upsert).toHaveBeenCalledWith(assetStub.image.id, [0.01, 0.02, 0.03]);
expect(searchMock.upsert).toHaveBeenCalledWith(assetStub.image.id, '[0.01, 0.02, 0.03]');
});
});