immich-app/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2025-12-27 09:14:55 +03:00
Code Issues Packages Projects Releases 18 Wiki Activity

feat(server): Import face regions from metadata (#6455)

Browse Source 
* feat: faces-from-metadata - Import face regions from metadata

Implements immich-app#1692.
- OpenAPI spec changes to accomodate metadata face import configs. New settings to enable the feature.
- Updates admin UI compoments
- ML faces detection/recognition & Exif/Metadata faces compatibility

Signed-off-by: BugFest <bugfest.dev@pm.me>

* chore(web): remove unused file confirm-enable-import-faces

* chore(web): format metadata-settings

* fix(server): faces-from-metadata tests and format

* fix(server): code refinements, nullable face asset sourceType

* fix(server): Add RegionInfo to ImmichTags interface

* fix(server): deleteAllFaces sourceType param can be undefined

* fix(server): exiftool-vendored 27.0.0 moves readArgs into ExifToolOptions

* fix(server): rename isImportFacesFromMetadataEnabled to isFaceImportEnabled

* fix(server): simplify sourceType conditional

* fix(server): small fixes

* fix(server): handling sourceType

* fix(server): sourceType enum

* fix(server): refactor metadata applyTaggedFaces

* fix(server): create/update signature changes

* fix(server): reduce computational cost of Person.getManyByName

* fix(server): use faceList instead of faceSet

* fix(server): Skip regions without Name defined

* fix(mobile): Update open-api (face assets feature changes)

* fix(server): Face-Person reconciliation with map/index

* fix(server): tags.RegionInfo.AppliedToDimensions must be defined to process face-region

* fix(server): fix shared-link.service.ts format

* fix(mobile): Update open-api after branch update

* simplify

* fix(server): minor fixes

* fix(server): person create/update methods type enforcement

* fix(server): style fixes

* fix(server): remove unused metadata code

* fix(server): metadata faces unit tests

* fix(server): top level config metadata category

* fix(server): rename upsertFaces to replaceFaces

* fix(server): remove sourceType when unnecessary

* fix(server): sourceType as ENUM

* fix(server): format fixes

* fix(server): fix tests after sourceType ENUM change

* fix(server): remove unnecessary JobItem cast

* fix(server): fix asset enum imports

* fix(open-api): add metadata config

* fix(mobile): update open-api after metadata open-api spec changes

* fix(web): update web/api metadata config

* fix(server): remove duplicated sourceType def

* fix(server): update generated sql queries

* fix(e2e): tests for metadata face import feature

* fix(web): Fix check:typescript

* fix(e2e): update subproject ref

* fix(server): revert format changes to pass format checks after ci

* fix(mobile): update open-api

* fix(server,movile,open-api,mobile): sourceType as DB data type

* fix(e2e): upload face asset after enabling metadata face import

* fix(web): simplify metadata admin settings and i18n keys

* Update person.repository.ts

Co-authored-by: Jason Rasmussen <jason@rasm.me>

* fix(server): asset_faces.sourceType column not nullable

* fix(server): simplified syntax

* fix(e2e): use SDK for everything except the endpoint being tested

* fix(e2e): fix test format

* chore: clean up

* chore: clean up

* chore: update e2e/test-assets

---------

Signed-off-by: BugFest <bugfest.dev@pm.me>
Co-authored-by: mertalev <101130780+mertalev@users.noreply.github.com>
Co-authored-by: Jason Rasmussen <jason@rasm.me>
This commit is contained in:
BugFest
2024-09-05 00:23:58 +02:00
committed by GitHub
parent 720412645f
commit 77e6a6d78b
48 changed files with 1058 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
server/src/services/audit.service.ts
View File

@@ -115,7 +115,7 @@ export class AuditService {
}
case PersonPathType.FACE: {
await this.personRepository.update({ id, thumbnailPath: pathValue });
await this.personRepository.update([{ id, thumbnailPath: pathValue }]);
break;
}

2
server/src/services/media.service.ts
View File

@@ -117,7 +117,7 @@ export class MediaService {
continue;
}
await this.personRepository.update({ id: person.id, faceAssetId: face.id });
await this.personRepository.update([{ id: person.id, faceAssetId: face.id }]);
}
jobs.push({ name: JobName.GENERATE_PERSON_THUMBNAIL, data: { id: person.id } });

121
server/src/services/metadata.service.spec.ts
View File

@@ -3,7 +3,7 @@ import { randomBytes } from 'node:crypto';
import { Stats } from 'node:fs';
import { constants } from 'node:fs/promises';
import { ExifEntity } from 'src/entities/exif.entity';
import { AssetType } from 'src/enum';
import { AssetType, SourceType } from 'src/enum';
import { IAlbumRepository } from 'src/interfaces/album.interface';
import { IAssetRepository, WithoutProperty } from 'src/interfaces/asset.interface';
import { ICryptoRepository } from 'src/interfaces/crypto.interface';
@@ -24,6 +24,8 @@ import { MetadataService, Orientation } from 'src/services/metadata.service';
import { assetStub } from 'test/fixtures/asset.stub';
import { fileStub } from 'test/fixtures/file.stub';
import { probeStub } from 'test/fixtures/media.stub';
import { metadataStub } from 'test/fixtures/metadata.stub';
import { personStub } from 'test/fixtures/person.stub';
import { tagStub } from 'test/fixtures/tag.stub';
import { newAlbumRepositoryMock } from 'test/repositories/album.repository.mock';
import { newAssetRepositoryMock } from 'test/repositories/asset.repository.mock';
@@ -956,6 +958,123 @@ describe(MetadataService.name, () => {
}),
);
});
it('should skip importing metadata when the feature is disabled', async () => {
assetMock.getByIds.mockResolvedValue([assetStub.primaryImage]);
systemMock.get.mockResolvedValue({ metadata: { faces: { import: false } } });
metadataMock.readTags.mockResolvedValue(metadataStub.withFace);
await sut.handleMetadataExtraction({ id: assetStub.image.id });
expect(personMock.getDistinctNames).not.toHaveBeenCalled();
});
it('should skip importing metadata face for assets without tags.RegionInfo', async () => {
assetMock.getByIds.mockResolvedValue([assetStub.primaryImage]);
systemMock.get.mockResolvedValue({ metadata: { faces: { import: true } } });
metadataMock.readTags.mockResolvedValue(metadataStub.empty);
await sut.handleMetadataExtraction({ id: assetStub.image.id });
expect(personMock.getDistinctNames).not.toHaveBeenCalled();
});
it('should skip importing faces without name', async () => {
assetMock.getByIds.mockResolvedValue([assetStub.primaryImage]);
systemMock.get.mockResolvedValue({ metadata: { faces: { import: true } } });
metadataMock.readTags.mockResolvedValue(metadataStub.withFaceNoName);
personMock.getDistinctNames.mockResolvedValue([]);
personMock.create.mockResolvedValue([]);
personMock.replaceFaces.mockResolvedValue([]);
personMock.update.mockResolvedValue([]);
await sut.handleMetadataExtraction({ id: assetStub.image.id });
expect(personMock.create).toHaveBeenCalledWith([]);
expect(personMock.replaceFaces).toHaveBeenCalledWith(assetStub.primaryImage.id, [], SourceType.EXIF);
expect(personMock.update).toHaveBeenCalledWith([]);
});
it('should skip importing faces with empty name', async () => {
assetMock.getByIds.mockResolvedValue([assetStub.primaryImage]);
systemMock.get.mockResolvedValue({ metadata: { faces: { import: true } } });
metadataMock.readTags.mockResolvedValue(metadataStub.withFaceEmptyName);
personMock.getDistinctNames.mockResolvedValue([]);
personMock.create.mockResolvedValue([]);
personMock.replaceFaces.mockResolvedValue([]);
personMock.update.mockResolvedValue([]);
await sut.handleMetadataExtraction({ id: assetStub.image.id });
expect(personMock.create).toHaveBeenCalledWith([]);
expect(personMock.replaceFaces).toHaveBeenCalledWith(assetStub.primaryImage.id, [], SourceType.EXIF);
expect(personMock.update).toHaveBeenCalledWith([]);
});
it('should apply metadata face tags creating new persons', async () => {
assetMock.getByIds.mockResolvedValue([assetStub.primaryImage]);
systemMock.get.mockResolvedValue({ metadata: { faces: { import: true } } });
metadataMock.readTags.mockResolvedValue(metadataStub.withFace);
personMock.getDistinctNames.mockResolvedValue([]);
personMock.create.mockResolvedValue([personStub.withName]);
personMock.replaceFaces.mockResolvedValue(['face-asset-uuid']);
personMock.update.mockResolvedValue([personStub.withName]);
await sut.handleMetadataExtraction({ id: assetStub.primaryImage.id });
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.primaryImage.id]);
expect(personMock.getDistinctNames).toHaveBeenCalledWith(assetStub.primaryImage.ownerId, { withHidden: true });
expect(personMock.create).toHaveBeenCalledWith([expect.objectContaining({ name: personStub.withName.name })]);
expect(personMock.replaceFaces).toHaveBeenCalledWith(
assetStub.primaryImage.id,
[
{
id: 'random-uuid',
assetId: assetStub.primaryImage.id,
personId: 'random-uuid',
imageHeight: 100,
imageWidth: 100,
boundingBoxX1: 0,
boundingBoxX2: 10,
boundingBoxY1: 0,
boundingBoxY2: 10,
sourceType: SourceType.EXIF,
},
],
SourceType.EXIF,
);
expect(personMock.update).toHaveBeenCalledWith([{ id: 'random-uuid', faceAssetId: 'random-uuid' }]);
expect(jobMock.queueAll).toHaveBeenCalledWith([
{
name: JobName.GENERATE_PERSON_THUMBNAIL,
data: { id: personStub.withName.id },
},
]);
});
it('should assign metadata face tags to existing persons', async () => {
assetMock.getByIds.mockResolvedValue([assetStub.primaryImage]);
systemMock.get.mockResolvedValue({ metadata: { faces: { import: true } } });
metadataMock.readTags.mockResolvedValue(metadataStub.withFace);
personMock.getDistinctNames.mockResolvedValue([{ id: personStub.withName.id, name: personStub.withName.name }]);
personMock.create.mockResolvedValue([]);
personMock.replaceFaces.mockResolvedValue(['face-asset-uuid']);
personMock.update.mockResolvedValue([personStub.withName]);
await sut.handleMetadataExtraction({ id: assetStub.primaryImage.id });
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.primaryImage.id]);
expect(personMock.getDistinctNames).toHaveBeenCalledWith(assetStub.primaryImage.ownerId, { withHidden: true });
expect(personMock.create).toHaveBeenCalledWith([]);
expect(personMock.replaceFaces).toHaveBeenCalledWith(
assetStub.primaryImage.id,
[
{
id: 'random-uuid',
assetId: assetStub.primaryImage.id,
personId: personStub.withName.id,
imageHeight: 100,
imageWidth: 100,
boundingBoxX1: 0,
boundingBoxX2: 10,
boundingBoxY1: 0,
boundingBoxY2: 10,
sourceType: SourceType.EXIF,
},
],
SourceType.EXIF,
);
expect(personMock.update).toHaveBeenCalledWith([]);
expect(jobMock.queueAll).toHaveBeenCalledWith([]);
});
});
describe('handleQueueSidecar', () => {

71
server/src/services/metadata.service.ts
View File

@@ -9,9 +9,11 @@ import { SystemConfig } from 'src/config';
import { StorageCore } from 'src/cores/storage.core';
import { SystemConfigCore } from 'src/cores/system-config.core';
import { OnEmit } from 'src/decorators';
import { AssetFaceEntity } from 'src/entities/asset-face.entity';
import { AssetEntity } from 'src/entities/asset.entity';
import { ExifEntity } from 'src/entities/exif.entity';
import { AssetType } from 'src/enum';
import { PersonEntity } from 'src/entities/person.entity';
import { AssetType, SourceType } from 'src/enum';
import { IAlbumRepository } from 'src/interfaces/album.interface';
import { IAssetRepository, WithoutProperty } from 'src/interfaces/asset.interface';
import { ICryptoRepository } from 'src/interfaces/crypto.interface';
@@ -37,6 +39,7 @@ import { IStorageRepository } from 'src/interfaces/storage.interface';
import { ISystemMetadataRepository } from 'src/interfaces/system-metadata.interface';
import { ITagRepository } from 'src/interfaces/tag.interface';
import { IUserRepository } from 'src/interfaces/user.interface';
import { isFaceImportEnabled } from 'src/utils/misc';
import { usePagination } from 'src/utils/pagination';
import { upsertTags } from 'src/utils/tag';
@@ -104,7 +107,7 @@ export class MetadataService {
@Inject(IMediaRepository) private mediaRepository: IMediaRepository,
@Inject(IMetadataRepository) private repository: IMetadataRepository,
@Inject(IMoveRepository) moveRepository: IMoveRepository,
@Inject(IPersonRepository) personRepository: IPersonRepository,
@Inject(IPersonRepository) private personRepository: IPersonRepository,
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
@Inject(ISystemMetadataRepository) systemMetadataRepository: ISystemMetadataRepository,
@Inject(ITagRepository) private tagRepository: ITagRepository,
@@ -215,6 +218,7 @@ export class MetadataService {
}
async handleMetadataExtraction({ id }: IEntityJob): Promise<JobStatus> {
const { metadata } = await this.configCore.getConfig({ withCache: true });
const [asset] = await this.assetRepository.getByIds([id]);
if (!asset) {
return JobStatus.FAILED;
@@ -253,6 +257,10 @@ export class MetadataService {
metadataExtractedAt: new Date(),
});
if (isFaceImportEnabled(metadata)) {
await this.applyTaggedFaces(asset, exifTags);
}
return JobStatus.SUCCESS;
}
@@ -512,6 +520,65 @@ export class MetadataService {
}
}
private async applyTaggedFaces(asset: AssetEntity, tags: ImmichTags) {
if (!tags.RegionInfo?.AppliedToDimensions || tags.RegionInfo.RegionList.length === 0) {
return;
}
const discoveredFaces: Partial<AssetFaceEntity>[] = [];
const existingNames = await this.personRepository.getDistinctNames(asset.ownerId, { withHidden: true });
const existingNameMap = new Map(existingNames.map(({ id, name }) => [name.toLowerCase(), id]));
const missing: Partial<PersonEntity>[] = [];
const missingWithFaceAsset: Partial<PersonEntity>[] = [];
for (const region of tags.RegionInfo.RegionList) {
if (!region.Name) {
continue;
}
const imageWidth = tags.RegionInfo.AppliedToDimensions.W;
const imageHeight = tags.RegionInfo.AppliedToDimensions.H;
const loweredName = region.Name.toLowerCase();
const personId = existingNameMap.get(loweredName) || this.cryptoRepository.randomUUID();
const face = {
id: this.cryptoRepository.randomUUID(),
personId,
assetId: asset.id,
imageWidth,
imageHeight,
boundingBoxX1: Math.floor((region.Area.X - region.Area.W / 2) * imageWidth),
boundingBoxY1: Math.floor((region.Area.Y - region.Area.H / 2) * imageHeight),
boundingBoxX2: Math.floor((region.Area.X + region.Area.W / 2) * imageWidth),
boundingBoxY2: Math.floor((region.Area.Y + region.Area.H / 2) * imageHeight),
sourceType: SourceType.EXIF,
};
discoveredFaces.push(face);
if (!existingNameMap.has(loweredName)) {
missing.push({ id: personId, ownerId: asset.ownerId, name: region.Name });
missingWithFaceAsset.push({ id: personId, faceAssetId: face.id });
}
}
if (missing.length > 0) {
this.logger.debug(`Creating missing persons: ${missing.map((p) => `${p.name}/${p.id}`)}`);
}
const newPersons = await this.personRepository.create(missing);
const faceIds = await this.personRepository.replaceFaces(asset.id, discoveredFaces, SourceType.EXIF);
this.logger.debug(`Created ${faceIds.length} faces for asset ${asset.id}`);
await this.personRepository.update(missingWithFaceAsset);
await this.jobRepository.queueAll(
newPersons.map((person) => ({
name: JobName.GENERATE_PERSON_THUMBNAIL,
data: { id: person.id },
})),
);
}
private async exifData(
asset: AssetEntity,
): Promise<{ exifData: ExifEntityWithoutGeocodeAndTypeOrm; exifTags: ImmichTags }> {

93
server/src/services/person.service.spec.ts
View File

@@ -3,7 +3,7 @@ import { Colorspace } from 'src/config';
import { BulkIdErrorReason } from 'src/dtos/asset-ids.response.dto';
import { PersonResponseDto, mapFaces, mapPerson } from 'src/dtos/person.dto';
import { AssetFaceEntity } from 'src/entities/asset-face.entity';
import { SystemMetadataKey } from 'src/enum';
import { SourceType, SystemMetadataKey } from 'src/enum';
import { IAssetRepository, WithoutProperty } from 'src/interfaces/asset.interface';
import { ICryptoRepository } from 'src/interfaces/crypto.interface';
import { IJobRepository, JobName, JobStatus } from 'src/interfaces/job.interface';
@@ -241,18 +241,18 @@ describe(PersonService.name, () => {
});
it("should update a person's name", async () => {
personMock.update.mockResolvedValue(personStub.withName);
personMock.update.mockResolvedValue([personStub.withName]);
personMock.getAssets.mockResolvedValue([assetStub.image]);
accessMock.person.checkOwnerAccess.mockResolvedValue(new Set(['person-1']));
await expect(sut.update(authStub.admin, 'person-1', { name: 'Person 1' })).resolves.toEqual(responseDto);
expect(personMock.update).toHaveBeenCalledWith({ id: 'person-1', name: 'Person 1' });
expect(personMock.update).toHaveBeenCalledWith([{ id: 'person-1', name: 'Person 1' }]);
expect(accessMock.person.checkOwnerAccess).toHaveBeenCalledWith(authStub.admin.user.id, new Set(['person-1']));
});
it("should update a person's date of birth", async () => {
personMock.update.mockResolvedValue(personStub.withBirthDate);
personMock.update.mockResolvedValue([personStub.withBirthDate]);
personMock.getAssets.mockResolvedValue([assetStub.image]);
accessMock.person.checkOwnerAccess.mockResolvedValue(new Set(['person-1']));
@@ -264,25 +264,25 @@ describe(PersonService.name, () => {
isHidden: false,
updatedAt: expect.any(Date),
});
expect(personMock.update).toHaveBeenCalledWith({ id: 'person-1', birthDate: '1976-06-30' });
expect(personMock.update).toHaveBeenCalledWith([{ id: 'person-1', birthDate: '1976-06-30' }]);
expect(jobMock.queue).not.toHaveBeenCalled();
expect(jobMock.queueAll).not.toHaveBeenCalled();
expect(accessMock.person.checkOwnerAccess).toHaveBeenCalledWith(authStub.admin.user.id, new Set(['person-1']));
});
it('should update a person visibility', async () => {
personMock.update.mockResolvedValue(personStub.withName);
personMock.update.mockResolvedValue([personStub.withName]);
personMock.getAssets.mockResolvedValue([assetStub.image]);
accessMock.person.checkOwnerAccess.mockResolvedValue(new Set(['person-1']));
await expect(sut.update(authStub.admin, 'person-1', { isHidden: false })).resolves.toEqual(responseDto);
expect(personMock.update).toHaveBeenCalledWith({ id: 'person-1', isHidden: false });
expect(personMock.update).toHaveBeenCalledWith([{ id: 'person-1', isHidden: false }]);
expect(accessMock.person.checkOwnerAccess).toHaveBeenCalledWith(authStub.admin.user.id, new Set(['person-1']));
});
it("should update a person's thumbnailPath", async () => {
personMock.update.mockResolvedValue(personStub.withName);
personMock.update.mockResolvedValue([personStub.withName]);
personMock.getFacesByIds.mockResolvedValue([faceStub.face1]);
accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set([assetStub.image.id]));
accessMock.person.checkOwnerAccess.mockResolvedValue(new Set(['person-1']));
@@ -291,7 +291,7 @@ describe(PersonService.name, () => {
sut.update(authStub.admin, 'person-1', { featureFaceAssetId: faceStub.face1.assetId }),
).resolves.toEqual(responseDto);
expect(personMock.update).toHaveBeenCalledWith({ id: 'person-1', faceAssetId: faceStub.face1.id });
expect(personMock.update).toHaveBeenCalledWith([{ id: 'person-1', faceAssetId: faceStub.face1.id }]);
expect(personMock.getFacesByIds).toHaveBeenCalledWith([
{
assetId: faceStub.face1.assetId,
@@ -441,11 +441,11 @@ describe(PersonService.name, () => {
describe('createPerson', () => {
it('should create a new person', async () => {
personMock.create.mockResolvedValue(personStub.primaryPerson);
personMock.create.mockResolvedValue([personStub.primaryPerson]);
await expect(sut.create(authStub.admin, {})).resolves.toBe(personStub.primaryPerson);
expect(personMock.create).toHaveBeenCalledWith({ ownerId: authStub.admin.user.id });
expect(personMock.create).toHaveBeenCalledWith([{ ownerId: authStub.admin.user.id }]);
});
});
@@ -496,6 +496,7 @@ describe(PersonService.name, () => {
items: [personStub.withName],
hasNextPage: false,
});
personMock.getAllWithoutFaces.mockResolvedValue([]);
await sut.handleQueueDetectFaces({ force: true });
@@ -510,7 +511,7 @@ describe(PersonService.name, () => {
it('should delete existing people and faces if forced', async () => {
personMock.getAll.mockResolvedValue({
items: [faceStub.face1.person],
items: [faceStub.face1.person, personStub.randomPerson],
hasNextPage: false,
});
personMock.getAllFaces.mockResolvedValue({
@@ -521,6 +522,7 @@ describe(PersonService.name, () => {
items: [assetStub.image],
hasNextPage: false,
});
personMock.getAllWithoutFaces.mockResolvedValue([personStub.randomPerson]);
await sut.handleQueueDetectFaces({ force: true });
@@ -531,8 +533,8 @@ describe(PersonService.name, () => {
data: { id: assetStub.image.id },
},
]);
expect(personMock.delete).toHaveBeenCalledWith([faceStub.face1.person]);
expect(storageMock.unlink).toHaveBeenCalledWith(faceStub.face1.person.thumbnailPath);
expect(personMock.delete).toHaveBeenCalledWith([personStub.randomPerson]);
expect(storageMock.unlink).toHaveBeenCalledWith(personStub.randomPerson.thumbnailPath);
});
});
@@ -561,10 +563,14 @@ describe(PersonService.name, () => {
items: [faceStub.face1],
hasNextPage: false,
});
personMock.getAllWithoutFaces.mockResolvedValue([]);
await sut.handleQueueRecognizeFaces({});
expect(personMock.getAllFaces).toHaveBeenCalledWith({ skip: 0, take: 1000 }, { where: { personId: IsNull() } });
expect(personMock.getAllFaces).toHaveBeenCalledWith(
{ skip: 0, take: 1000 },
{ where: { personId: IsNull(), sourceType: IsNull() } },
);
expect(jobMock.queueAll).toHaveBeenCalledWith([
{
name: JobName.FACIAL_RECOGNITION,
@@ -586,6 +592,7 @@ describe(PersonService.name, () => {
items: [faceStub.face1],
hasNextPage: false,
});
personMock.getAllWithoutFaces.mockResolvedValue([]);
await sut.handleQueueRecognizeFaces({ force: true });
@@ -616,6 +623,8 @@ describe(PersonService.name, () => {
items: [faceStub.face1],
hasNextPage: false,
});
personMock.getAllWithoutFaces.mockResolvedValue([]);
await sut.handleQueueRecognizeFaces({ force: true, nightly: true });
expect(systemMock.get).toHaveBeenCalledWith(SystemMetadataKey.FACIAL_RECOGNITION_STATE);
@@ -641,6 +650,7 @@ describe(PersonService.name, () => {
items: [faceStub.face1],
hasNextPage: false,
});
personMock.getAllWithoutFaces.mockResolvedValue([]);
await sut.handleQueueRecognizeFaces({ force: true, nightly: true });
@@ -654,7 +664,7 @@ describe(PersonService.name, () => {
it('should delete existing people and faces if forced', async () => {
jobMock.getJobCounts.mockResolvedValue({ active: 1, waiting: 0, paused: 0, completed: 0, failed: 0, delayed: 0 });
personMock.getAll.mockResolvedValue({
items: [faceStub.face1.person],
items: [faceStub.face1.person, personStub.randomPerson],
hasNextPage: false,
});
personMock.getAllFaces.mockResolvedValue({
@@ -662,17 +672,19 @@ describe(PersonService.name, () => {
hasNextPage: false,
});
personMock.getAllWithoutFaces.mockResolvedValue([personStub.randomPerson]);
await sut.handleQueueRecognizeFaces({ force: true });
expect(personMock.getAllFaces).toHaveBeenCalledWith({ skip: 0, take: 1000 }, {});
expect(personMock.deleteAllFaces).toHaveBeenCalledWith({ sourceType: SourceType.MACHINE_LEARNING });
expect(jobMock.queueAll).toHaveBeenCalledWith([
{
name: JobName.FACIAL_RECOGNITION,
data: { id: faceStub.face1.id, deferred: false },
},
]);
expect(personMock.delete).toHaveBeenCalledWith([faceStub.face1.person]);
expect(storageMock.unlink).toHaveBeenCalledWith(faceStub.face1.person.thumbnailPath);
expect(personMock.delete).toHaveBeenCalledWith([personStub.randomPerson]);
expect(storageMock.unlink).toHaveBeenCalledWith(personStub.randomPerson.thumbnailPath);
});
});
@@ -807,7 +819,7 @@ describe(PersonService.name, () => {
systemMock.get.mockResolvedValue({ machineLearning: { facialRecognition: { minFaces: 1 } } });
searchMock.searchFaces.mockResolvedValue(faces);
personMock.getFaceByIdWithAssets.mockResolvedValue(faceStub.noPerson1);
personMock.create.mockResolvedValue(faceStub.primaryFace1.person);
personMock.create.mockResolvedValue([faceStub.primaryFace1.person]);
await sut.handleRecognizeFaces({ id: faceStub.noPerson1.id });
@@ -832,14 +844,16 @@ describe(PersonService.name, () => {
systemMock.get.mockResolvedValue({ machineLearning: { facialRecognition: { minFaces: 1 } } });
searchMock.searchFaces.mockResolvedValue(faces);
personMock.getFaceByIdWithAssets.mockResolvedValue(faceStub.noPerson1);
personMock.create.mockResolvedValue(personStub.withName);
personMock.create.mockResolvedValue([personStub.withName]);
await sut.handleRecognizeFaces({ id: faceStub.noPerson1.id });
expect(personMock.create).toHaveBeenCalledWith({
ownerId: faceStub.noPerson1.asset.ownerId,
faceAssetId: faceStub.noPerson1.id,
});
expect(personMock.create).toHaveBeenCalledWith([
{
ownerId: faceStub.noPerson1.asset.ownerId,
faceAssetId: faceStub.noPerson1.id,
},
]);
expect(personMock.reassignFaces).toHaveBeenCalledWith({
faceIds: [faceStub.noPerson1.id],
newPersonId: personStub.withName.id,
@@ -851,7 +865,7 @@ describe(PersonService.name, () => {
searchMock.searchFaces.mockResolvedValue(faces);
personMock.getFaceByIdWithAssets.mockResolvedValue(faceStub.noPerson1);
personMock.create.mockResolvedValue(personStub.withName);
personMock.create.mockResolvedValue([personStub.withName]);
await sut.handleRecognizeFaces({ id: faceStub.noPerson1.id });
@@ -870,7 +884,7 @@ describe(PersonService.name, () => {
systemMock.get.mockResolvedValue({ machineLearning: { facialRecognition: { minFaces: 3 } } });
searchMock.searchFaces.mockResolvedValue(faces);
personMock.getFaceByIdWithAssets.mockResolvedValue(faceStub.noPerson1);
personMock.create.mockResolvedValue(personStub.withName);
personMock.create.mockResolvedValue([personStub.withName]);
await sut.handleRecognizeFaces({ id: faceStub.noPerson1.id });
@@ -892,7 +906,7 @@ describe(PersonService.name, () => {
systemMock.get.mockResolvedValue({ machineLearning: { facialRecognition: { minFaces: 3 } } });
searchMock.searchFaces.mockResolvedValueOnce(faces).mockResolvedValueOnce([]);
personMock.getFaceByIdWithAssets.mockResolvedValue(faceStub.noPerson1);
personMock.create.mockResolvedValue(personStub.withName);
personMock.create.mockResolvedValue([personStub.withName]);
await sut.handleRecognizeFaces({ id: faceStub.noPerson1.id, deferred: true });
@@ -965,10 +979,12 @@ describe(PersonService.name, () => {
processInvalidImages: false,
},
);
expect(personMock.update).toHaveBeenCalledWith({
id: 'person-1',
thumbnailPath: 'upload/thumbs/admin_id/pe/rs/person-1.jpeg',
});
expect(personMock.update).toHaveBeenCalledWith([
{
id: 'person-1',
thumbnailPath: 'upload/thumbs/admin_id/pe/rs/person-1.jpeg',
},
]);
});
it('should generate a thumbnail without going negative', async () => {
@@ -1087,7 +1103,7 @@ describe(PersonService.name, () => {
it('should merge two people with smart merge', async () => {
personMock.getById.mockResolvedValueOnce(personStub.randomPerson);
personMock.getById.mockResolvedValueOnce(personStub.primaryPerson);
personMock.update.mockResolvedValue({ ...personStub.randomPerson, name: personStub.primaryPerson.name });
personMock.update.mockResolvedValue([{ ...personStub.randomPerson, name: personStub.primaryPerson.name }]);
accessMock.person.checkOwnerAccess.mockResolvedValueOnce(new Set(['person-3']));
accessMock.person.checkOwnerAccess.mockResolvedValueOnce(new Set(['person-1']));
@@ -1100,10 +1116,12 @@ describe(PersonService.name, () => {
oldPersonId: personStub.primaryPerson.id,
});
expect(personMock.update).toHaveBeenCalledWith({
id: personStub.randomPerson.id,
name: personStub.primaryPerson.name,
});
expect(personMock.update).toHaveBeenCalledWith([
{
id: personStub.randomPerson.id,
name: personStub.primaryPerson.name,
},
]);
expect(accessMock.person.checkOwnerAccess).toHaveBeenCalledWith(authStub.admin.user.id, new Set(['person-1']));
});
@@ -1177,6 +1195,7 @@ describe(PersonService.name, () => {
id: faceStub.face1.id,
imageHeight: 1024,
imageWidth: 1024,
sourceType: SourceType.MACHINE_LEARNING,
person: mapPerson(personStub.withName),
});
});

62
server/src/services/person.service.ts
View File

@@ -25,7 +25,7 @@ import { AssetFaceEntity } from 'src/entities/asset-face.entity';
import { AssetEntity } from 'src/entities/asset.entity';
import { PersonPathType } from 'src/entities/move.entity';
import { PersonEntity } from 'src/entities/person.entity';
import { AssetType, Permission, SystemMetadataKey } from 'src/enum';
import { AssetType, Permission, SourceType, SystemMetadataKey } from 'src/enum';
import { IAccessRepository } from 'src/interfaces/access.interface';
import { IAssetRepository, WithoutProperty } from 'src/interfaces/asset.interface';
import { ICryptoRepository } from 'src/interfaces/crypto.interface';
@@ -53,7 +53,7 @@ import { checkAccess, requireAccess } from 'src/utils/access';
import { getAssetFiles } from 'src/utils/asset.util';
import { CacheControl, ImmichFileResponse } from 'src/utils/file';
import { mimeTypes } from 'src/utils/mime-types';
import { isFacialRecognitionEnabled } from 'src/utils/misc';
import { isFaceImportEnabled, isFacialRecognitionEnabled } from 'src/utils/misc';
import { usePagination } from 'src/utils/pagination';
import { IsNull } from 'typeorm';
@@ -173,10 +173,7 @@ export class PersonService {
const assetFace = await this.repository.getRandomFace(personId);
if (assetFace !== null) {
await this.repository.update({
id: personId,
faceAssetId: assetFace.id,
});
await this.repository.update([{ id: personId, faceAssetId: assetFace.id }]);
jobs.push({ name: JobName.GENERATE_PERSON_THUMBNAIL, data: { id: personId } });
}
}
@@ -214,13 +211,16 @@ export class PersonService {
return assets.map((asset) => mapAsset(asset));
}
create(auth: AuthDto, dto: PersonCreateDto): Promise<PersonResponseDto> {
return this.repository.create({
ownerId: auth.user.id,
name: dto.name,
birthDate: dto.birthDate,
isHidden: dto.isHidden,
});
async create(auth: AuthDto, dto: PersonCreateDto): Promise<PersonResponseDto> {
const [created] = await this.repository.create([
{
ownerId: auth.user.id,
name: dto.name,
birthDate: dto.birthDate,
isHidden: dto.isHidden,
},
]);
return created;
}
async update(auth: AuthDto, id: string, dto: PersonUpdateDto): Promise<PersonResponseDto> {
@@ -239,7 +239,7 @@ export class PersonService {
faceId = face.id;
}
const person = await this.repository.update({ id, faceAssetId: faceId, name, birthDate, isHidden });
const [person] = await this.repository.update([{ id, faceAssetId: faceId, name, birthDate, isHidden }]);
if (assetId) {
await this.jobRepository.queue({ name: JobName.GENERATE_PERSON_THUMBNAIL, data: { id } });
@@ -296,8 +296,8 @@ export class PersonService {
}
if (force) {
await this.deleteAllPeople();
await this.repository.deleteAllFaces();
await this.repository.deleteAllFaces({ sourceType: SourceType.MACHINE_LEARNING });
await this.handlePersonCleanup();
}
const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) => {
@@ -339,11 +339,7 @@ export class PersonService {
return JobStatus.FAILED;
}
if (!asset.isVisible) {
return JobStatus.SKIPPED;
}
if (!asset.isVisible) {
if (!asset.isVisible || asset.faces.length > 0) {
return JobStatus.SKIPPED;
}
@@ -408,7 +404,8 @@ export class PersonService {
const { waiting } = await this.jobRepository.getJobCounts(QueueName.FACIAL_RECOGNITION);
if (force) {
await this.deleteAllPeople();
await this.repository.deleteAllFaces({ sourceType: SourceType.MACHINE_LEARNING });
await this.handlePersonCleanup();
} else if (waiting) {
this.logger.debug(
`Skipping facial recognition queueing because ${waiting} job${waiting > 1 ? 's are' : ' is'} already queued`,
@@ -418,7 +415,9 @@ export class PersonService {
const lastRun = new Date().toISOString();
const facePagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
this.repository.getAllFaces(pagination, { where: force ? undefined : { personId: IsNull() } }),
this.repository.getAllFaces(pagination, {
where: force ? undefined : { personId: IsNull(), sourceType: IsNull() },
}),
);
for await (const page of facePagination) {
@@ -441,13 +440,18 @@ export class PersonService {
const face = await this.repository.getFaceByIdWithAssets(
id,
{ person: true, asset: true, faceSearch: true },
{ id: true, personId: true, faceSearch: { embedding: true } },
{ id: true, personId: true, sourceType: true, faceSearch: { embedding: true } },
);
if (!face || !face.asset) {
this.logger.warn(`Face ${id} not found`);
return JobStatus.FAILED;
}
if (face.sourceType !== SourceType.MACHINE_LEARNING) {
this.logger.warn(`Skipping face ${id} due to source ${face.sourceType}`);
return JobStatus.SKIPPED;
}
if (!face.faceSearch?.embedding) {
this.logger.warn(`Face ${id} does not have an embedding`);
return JobStatus.FAILED;
@@ -497,7 +501,7 @@ export class PersonService {
if (isCore && !personId) {
this.logger.log(`Creating new person for face ${id}`);
const newPerson = await this.repository.create({ ownerId: face.asset.ownerId, faceAssetId: face.id });
const [newPerson] = await this.repository.create([{ ownerId: face.asset.ownerId, faceAssetId: face.id }]);
await this.jobRepository.queue({ name: JobName.GENERATE_PERSON_THUMBNAIL, data: { id: newPerson.id } });
personId = newPerson.id;
}
@@ -522,8 +526,8 @@ export class PersonService {
}
async handleGeneratePersonThumbnail(data: IEntityJob): Promise<JobStatus> {
const { machineLearning, image } = await this.configCore.getConfig({ withCache: true });
if (!isFacialRecognitionEnabled(machineLearning)) {
const { machineLearning, metadata, image } = await this.configCore.getConfig({ withCache: true });
if (!isFacialRecognitionEnabled(machineLearning) && !isFaceImportEnabled(metadata)) {
return JobStatus.SKIPPED;
}
@@ -573,7 +577,7 @@ export class PersonService {
} as const;
await this.mediaRepository.generateThumbnail(inputPath, thumbnailPath, thumbnailOptions);
await this.repository.update({ id: person.id, thumbnailPath });
await this.repository.update([{ id: person.id, thumbnailPath }]);
return JobStatus.SUCCESS;
}
@@ -620,7 +624,7 @@ export class PersonService {
}
if (Object.keys(update).length > 0) {
primaryPerson = await this.repository.update({ id: primaryPerson.id, ...update });
[primaryPerson] = await this.repository.update([{ id: primaryPerson.id, ...update }]);
}
const mergeName = mergePerson.name || mergePerson.id;

1
server/src/services/server.service.spec.ts
View File

@@ -160,6 +160,7 @@ describe(ServerService.name, () => {
smartSearch: true,
duplicateDetection: true,
facialRecognition: true,
importFaces: false,
map: true,
reverseGeocoding: true,
oauth: false,

3
server/src/services/server.service.ts
View File

@@ -90,7 +90,7 @@ export class ServerService {
}
async getFeatures(): Promise<ServerFeaturesDto> {
const { reverseGeocoding, map, machineLearning, trash, oauth, passwordLogin, notifications } =
const { reverseGeocoding, metadata, map, machineLearning, trash, oauth, passwordLogin, notifications } =
await this.configCore.getConfig({ withCache: false });
return {
@@ -99,6 +99,7 @@ export class ServerService {
duplicateDetection: isDuplicateDetectionEnabled(machineLearning),
map: map.enabled,
reverseGeocoding: reverseGeocoding.enabled,
importFaces: metadata.faces.import,
sidecar: true,
search: true,
trash: trash.enabled,

5
server/src/services/system-config.service.spec.ts
View File

@@ -74,6 +74,11 @@ const updatedConfig = Object.freeze<SystemConfig>({
enabled: true,
level: LogLevel.LOG,
},
metadata: {
faces: {
import: false,
},
},
machineLearning: {
enabled: true,
url: 'http://immich-machine-learning:3003',