immich-app/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2025-12-20 09:15:35 +03:00
Code Issues Packages Projects Releases 18 Wiki Activity

feat: use pgvecto.rs (#3605)

Browse Source
This commit is contained in:
Jason Rasmussen
2023-12-08 11:15:46 -05:00
committed by GitHub
parent 429ad28810
commit 1e99ba8167
99 changed files with 1935 additions and 2583 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
server/src/domain/album/album.service.spec.ts
View File

@@ -12,7 +12,6 @@ import {
} from '@test';
import _ from 'lodash';
import { BulkIdErrorReason } from '../asset';
import { JobName } from '../job';
import { IAlbumRepository, IAssetRepository, IJobRepository, IUserRepository } from '../repositories';
import { AlbumService } from './album.service';
@@ -188,11 +187,6 @@ describe(AlbumService.name, () => {
assetIds: ['123'],
});
expect(jobMock.queue).toHaveBeenCalledWith({
name: JobName.SEARCH_INDEX_ALBUM,
data: { ids: [albumStub.empty.id] },
});
expect(albumMock.create).toHaveBeenCalledWith({
ownerId: authStub.admin.id,
albumName: albumStub.empty.albumName,
@@ -270,10 +264,6 @@ describe(AlbumService.name, () => {
id: 'album-4',
albumName: 'new album name',
});
expect(jobMock.queue).toHaveBeenCalledWith({
name: JobName.SEARCH_INDEX_ALBUM,
data: { ids: [albumStub.oneAsset.id] },
});
});
});

5
server/src/domain/album/album.service.ts
View File

@@ -4,7 +4,6 @@ import { AccessCore, Permission } from '../access';
import { BulkIdErrorReason, BulkIdResponseDto, BulkIdsDto } from '../asset';
import { AuthUserDto } from '../auth';
import { setUnion } from '../domain.util';
import { JobName } from '../job';
import {
AlbumAssetCount,
AlbumInfoOptions,
@@ -131,7 +130,6 @@ export class AlbumService {
albumThumbnailAssetId: dto.assetIds?.[0] || null,
});
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ALBUM, data: { ids: [album.id] } });
return mapAlbumWithAssets(album);
}
@@ -154,8 +152,6 @@ export class AlbumService {
isActivityEnabled: dto.isActivityEnabled,
});
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ALBUM, data: { ids: [updatedAlbum.id] } });
return mapAlbumWithoutAssets(updatedAlbum);
}
@@ -165,7 +161,6 @@ export class AlbumService {
const album = await this.findOrFail(id, { withAssets: false });
await this.albumRepository.delete(album);
await this.jobRepository.queue({ name: JobName.SEARCH_REMOVE_ALBUM, data: { ids: [id] } });
}
async addAssets(authUser: AuthUserDto, id: string, dto: BulkIdsDto): Promise<BulkIdResponseDto[]> {

38
server/src/domain/asset/asset.service.spec.ts
View File

@@ -794,14 +794,7 @@ describe(AssetService.name, () => {
await sut.deleteAll(authStub.user1, { ids: ['asset1', 'asset2'], force: false });
expect(assetMock.softDeleteAll).toHaveBeenCalledWith(['asset1', 'asset2']);
expect(jobMock.queue.mock.calls).toEqual([
[
{
name: JobName.SEARCH_REMOVE_ASSET,
data: { ids: ['asset1', 'asset2'] },
},
],
]);
expect(jobMock.queue.mock.calls).toEqual([]);
});
});
@@ -820,14 +813,7 @@ describe(AssetService.name, () => {
await sut.restoreAll(authStub.user1, { ids: ['asset1', 'asset2'] });
expect(assetMock.restoreAll).toHaveBeenCalledWith(['asset1', 'asset2']);
expect(jobMock.queue.mock.calls).toEqual([
[
{
name: JobName.SEARCH_INDEX_ASSET,
data: { ids: ['asset1', 'asset2'] },
},
],
]);
expect(jobMock.queue.mock.calls).toEqual([]);
});
});
@@ -853,19 +839,6 @@ describe(AssetService.name, () => {
await sut.handleAssetDeletion({ id: assetWithFace.id });
expect(jobMock.queue.mock.calls).toEqual([
[
{
name: JobName.SEARCH_REMOVE_FACE,
data: { assetId: faceStub.face1.assetId, personId: faceStub.face1.personId },
},
],
[
{
name: JobName.SEARCH_REMOVE_FACE,
data: { assetId: faceStub.mergeFace1.assetId, personId: faceStub.mergeFace1.personId },
},
],
[{ name: JobName.SEARCH_REMOVE_ASSET, data: { ids: [assetWithFace.id] } }],
[
{
name: JobName.DELETE_FILES,
@@ -907,9 +880,7 @@ describe(AssetService.name, () => {
await sut.handleAssetDeletion({ id: assetStub.readOnly.id });
expect(jobMock.queue.mock.calls).toEqual([
[{ name: JobName.SEARCH_REMOVE_ASSET, data: { ids: [assetStub.readOnly.id] } }],
]);
expect(jobMock.queue.mock.calls).toEqual([]);
expect(assetMock.remove).toHaveBeenCalledWith(assetStub.readOnly);
});
@@ -934,7 +905,6 @@ describe(AssetService.name, () => {
expect(assetMock.remove).toHaveBeenCalledWith(assetStub.external);
expect(jobMock.queue.mock.calls).toEqual([
[{ name: JobName.SEARCH_REMOVE_ASSET, data: { ids: [assetStub.external.id] } }],
[
{
name: JobName.DELETE_FILES,
@@ -955,9 +925,7 @@ describe(AssetService.name, () => {
await sut.handleAssetDeletion({ id: assetStub.livePhotoStillAsset.id });
expect(jobMock.queue.mock.calls).toEqual([
[{ name: JobName.SEARCH_REMOVE_ASSET, data: { ids: [assetStub.livePhotoStillAsset.id] } }],
[{ name: JobName.ASSET_DELETION, data: { id: assetStub.livePhotoMotionAsset.id } }],
[{ name: JobName.SEARCH_REMOVE_ASSET, data: { ids: [assetStub.livePhotoMotionAsset.id] } }],
[
{
name: JobName.DELETE_FILES,

20
server/src/domain/asset/asset.service.ts
View File

@@ -397,7 +397,6 @@ export class AssetService {
await this.updateMetadata({ id, description, dateTimeOriginal, latitude, longitude });
const asset = await this.assetRepository.save({ id, ...rest });
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ASSET, data: { ids: [id] } });
return mapAsset(asset);
}
@@ -426,7 +425,10 @@ export class AssetService {
await this.updateMetadata({ id, dateTimeOriginal, latitude, longitude });
}
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ASSET, data: { ids } });
for (const id of ids) {
await this.updateMetadata({ id, dateTimeOriginal, latitude, longitude });
}
await this.assetRepository.updateAll(ids, options);
this.communicationRepository.send(CommunicationEvent.ASSET_UPDATE, authUser.id, ids);
}
@@ -463,16 +465,6 @@ export class AssetService {
return false;
}
if (asset.faces) {
await Promise.all(
asset.faces.map(
({ assetId, personId }) =>
personId != null &&
this.jobRepository.queue({ name: JobName.SEARCH_REMOVE_FACE, data: { assetId, personId } }),
),
);
}
// Replace the parent of the stack children with a new asset
if (asset.stack && asset.stack.length != 0) {
const stackIds = asset.stack.map((a) => a.id);
@@ -482,7 +474,6 @@ export class AssetService {
}
await this.assetRepository.remove(asset);
await this.jobRepository.queue({ name: JobName.SEARCH_REMOVE_ASSET, data: { ids: [asset.id] } });
this.communicationRepository.send(CommunicationEvent.ASSET_DELETE, asset.ownerId, id);
// TODO refactor this to use cascades
@@ -513,7 +504,6 @@ export class AssetService {
}
} else {
await this.assetRepository.softDeleteAll(ids);
await this.jobRepository.queue({ name: JobName.SEARCH_REMOVE_ASSET, data: { ids } });
this.communicationRepository.send(CommunicationEvent.ASSET_TRASH, authUser.id, ids);
}
}
@@ -527,7 +517,6 @@ export class AssetService {
for await (const assets of assetPagination) {
const ids = assets.map((a) => a.id);
await this.assetRepository.restoreAll(ids);
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ASSET, data: { ids } });
this.communicationRepository.send(CommunicationEvent.ASSET_RESTORE, authUser.id, ids);
}
return;
@@ -547,7 +536,6 @@ export class AssetService {
const { ids } = dto;
await this.access.requirePermission(authUser, Permission.ASSET_RESTORE, ids);
await this.assetRepository.restoreAll(ids);
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ASSET, data: { ids } });
this.communicationRepository.send(CommunicationEvent.ASSET_RESTORE, authUser.id, ids);
}

7
server/src/domain/domain.config.ts
View File

@@ -13,16 +13,11 @@ export const immichAppConfig: ConfigModuleOptions = {
envFilePath: '.env',
isGlobal: true,
validationSchema: Joi.object({
NODE_ENV: Joi.string().required().valid('development', 'production', 'staging').default('development'),
NODE_ENV: Joi.string().optional().valid('development', 'production', 'staging').default('development'),
DB_USERNAME: WHEN_DB_URL_SET,
DB_PASSWORD: WHEN_DB_URL_SET,
DB_DATABASE_NAME: WHEN_DB_URL_SET,
DB_URL: Joi.string().optional(),
TYPESENSE_API_KEY: Joi.when('TYPESENSE_ENABLED', {
is: 'false',
then: Joi.string().optional(),
otherwise: Joi.string().required(),
}),
DISABLE_REVERSE_GEOCODING: Joi.boolean().optional().valid(true, false).default(false),
REVERSE_GEOCODING_PRECISION: Joi.number().optional().valid(0, 1, 2, 3).default(3),
LOG_LEVEL: Joi.string().optional().valid('simple', 'verbose', 'debug', 'log', 'warn', 'error').default('log'),

10
server/src/domain/domain.module.ts
View File

@@ -1,4 +1,4 @@
import { DynamicModule, Global, Module, ModuleMetadata, OnApplicationShutdown, Provider } from '@nestjs/common';
import { DynamicModule, Global, Module, ModuleMetadata, Provider } from '@nestjs/common';
import { ActivityService } from './activity';
import { AlbumService } from './album';
import { APIKeyService } from './api-key';
@@ -54,9 +54,7 @@ const providers: Provider[] = [
@Global()
@Module({})
export class DomainModule implements OnApplicationShutdown {
constructor(private searchService: SearchService) {}
export class DomainModule {
static register(options: Pick<ModuleMetadata, 'imports'>): DynamicModule {
return {
module: DomainModule,
@@ -65,8 +63,4 @@ export class DomainModule implements OnApplicationShutdown {
exports: [...providers],
};
}
onApplicationShutdown() {
this.searchService.teardown();
}
}

26
server/src/domain/job/job.constants.ts
View File

@@ -78,17 +78,6 @@ export enum JobName {
DELETE_FILES = 'delete-files',
CLEAN_OLD_AUDIT_LOGS = 'clean-old-audit-logs',
// search
SEARCH_INDEX_ASSETS = 'search-index-assets',
SEARCH_INDEX_ASSET = 'search-index-asset',
SEARCH_INDEX_FACE = 'search-index-face',
SEARCH_INDEX_FACES = 'search-index-faces',
SEARCH_INDEX_ALBUMS = 'search-index-albums',
SEARCH_INDEX_ALBUM = 'search-index-album',
SEARCH_REMOVE_ALBUM = 'search-remove-album',
SEARCH_REMOVE_ASSET = 'search-remove-asset',
SEARCH_REMOVE_FACE = 'search-remove-face',
// clip
QUEUE_ENCODE_CLIP = 'queue-clip-encode',
ENCODE_CLIP = 'clip-encode',
@@ -151,21 +140,6 @@ export const JOBS_TO_QUEUE: Record<JobName, QueueName> = {
[JobName.QUEUE_ENCODE_CLIP]: QueueName.CLIP_ENCODING,
[JobName.ENCODE_CLIP]: QueueName.CLIP_ENCODING,
// search - albums
[JobName.SEARCH_INDEX_ALBUMS]: QueueName.SEARCH,
[JobName.SEARCH_INDEX_ALBUM]: QueueName.SEARCH,
[JobName.SEARCH_REMOVE_ALBUM]: QueueName.SEARCH,
// search - assets
[JobName.SEARCH_INDEX_ASSETS]: QueueName.SEARCH,
[JobName.SEARCH_INDEX_ASSET]: QueueName.SEARCH,
[JobName.SEARCH_REMOVE_ASSET]: QueueName.SEARCH,
// search - faces
[JobName.SEARCH_INDEX_FACES]: QueueName.SEARCH,
[JobName.SEARCH_INDEX_FACE]: QueueName.SEARCH,
[JobName.SEARCH_REMOVE_FACE]: QueueName.SEARCH,
// XMP sidecars
[JobName.QUEUE_SIDECAR]: QueueName.SIDECAR,
[JobName.SIDECAR_DISCOVERY]: QueueName.SIDECAR,

5
server/src/domain/job/job.interface.ts
View File

@@ -2,11 +2,6 @@ export interface IBaseJob {
force?: boolean;
}
export interface IAssetFaceJob extends IBaseJob {
assetId: string;
personId: string;
}
export interface IEntityJob extends IBaseJob {
id: string;
source?: 'upload' | 'sidecar-write';

43
server/src/domain/job/job.service.spec.ts
View File

@@ -1,4 +1,4 @@
import { SystemConfig } from '@app/infra/entities';
import { SystemConfig, SystemConfigKey } from '@app/infra/entities';
import { BadRequestException } from '@nestjs/common';
import {
assetStub,
@@ -18,7 +18,7 @@ import {
JobHandler,
JobItem,
} from '../repositories';
import { SystemConfigCore } from '../system-config/system-config.core';
import { FeatureFlag, SystemConfigCore } from '../system-config/system-config.core';
import { JobCommand, JobName, QueueName } from './job.constants';
import { JobService } from './job.service';
@@ -271,7 +271,7 @@ describe(JobService.name, () => {
},
{
item: { name: JobName.LINK_LIVE_PHOTOS, data: { id: 'asset-1' } },
jobs: [JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE, JobName.SEARCH_INDEX_ASSET],
jobs: [JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE],
},
{
item: { name: JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE, data: { id: 'asset-1', source: 'upload' } },
@@ -281,6 +281,10 @@ describe(JobService.name, () => {
item: { name: JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE, data: { id: 'asset-1' } },
jobs: [],
},
{
item: { name: JobName.GENERATE_PERSON_THUMBNAIL, data: { id: 'asset-1' } },
jobs: [],
},
{
item: { name: JobName.GENERATE_JPEG_THUMBNAIL, data: { id: 'asset-1' } },
jobs: [
@@ -315,15 +319,15 @@ describe(JobService.name, () => {
},
{
item: { name: JobName.CLASSIFY_IMAGE, data: { id: 'asset-1' } },
jobs: [JobName.SEARCH_INDEX_ASSET],
jobs: [],
},
{
item: { name: JobName.ENCODE_CLIP, data: { id: 'asset-1' } },
jobs: [JobName.SEARCH_INDEX_ASSET],
jobs: [],
},
{
item: { name: JobName.RECOGNIZE_FACES, data: { id: 'asset-1' } },
jobs: [JobName.SEARCH_INDEX_ASSET],
jobs: [],
},
];
@@ -357,5 +361,32 @@ describe(JobService.name, () => {
expect(jobMock.queue).not.toHaveBeenCalled();
});
}
const featureTests: Array<{ queue: QueueName; feature: FeatureFlag; configKey: SystemConfigKey }> = [
{
queue: QueueName.CLIP_ENCODING,
feature: FeatureFlag.CLIP_ENCODE,
configKey: SystemConfigKey.MACHINE_LEARNING_CLIP_ENABLED,
},
{
queue: QueueName.OBJECT_TAGGING,
feature: FeatureFlag.TAG_IMAGE,
configKey: SystemConfigKey.MACHINE_LEARNING_CLASSIFICATION_ENABLED,
},
{
queue: QueueName.RECOGNIZE_FACES,
feature: FeatureFlag.FACIAL_RECOGNITION,
configKey: SystemConfigKey.MACHINE_LEARNING_FACIAL_RECOGNITION_ENABLED,
},
];
for (const { queue, feature, configKey } of featureTests) {
it(`should throw an error if attempting to queue ${queue} when ${feature} is disabled`, async () => {
configMock.load.mockResolvedValue([{ key: configKey, value: false }]);
jobMock.getQueueStatus.mockResolvedValue({ isActive: false, isPaused: false });
await expect(sut.handleCommand(queue, { command: JobCommand.START, force: false })).rejects.toThrow();
});
}
});
});

10
server/src/domain/job/job.service.ts
View File

@@ -236,15 +236,5 @@ export class JobService {
}
}
}
// In addition to the above jobs, all of these should queue `SEARCH_INDEX_ASSET`
switch (item.name) {
case JobName.CLASSIFY_IMAGE:
case JobName.ENCODE_CLIP:
case JobName.RECOGNIZE_FACES:
case JobName.LINK_LIVE_PHOTOS:
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ASSET, data: { ids: [item.data.id] } });
break;
}
}
}

91
server/src/domain/person/person.service.spec.ts
View File

@@ -12,7 +12,7 @@ import {
newMediaRepositoryMock,
newMoveRepositoryMock,
newPersonRepositoryMock,
newSearchRepositoryMock,
newSmartInfoRepositoryMock,
newStorageRepositoryMock,
newSystemConfigRepositoryMock,
personStub,
@@ -26,12 +26,12 @@ import {
IMediaRepository,
IMoveRepository,
IPersonRepository,
ISearchRepository,
ISmartInfoRepository,
IStorageRepository,
ISystemConfigRepository,
WithoutProperty,
} from '../repositories';
import { PersonResponseDto, mapFaces } from './person.dto';
import { PersonResponseDto, mapFaces, mapPerson } from './person.dto';
import { PersonService } from './person.service';
const responseDto: PersonResponseDto = {
@@ -61,33 +61,6 @@ const detectFaceMock = {
score: 0.2,
};
const faceSearch = {
noMatch: {
total: 0,
count: 0,
page: 1,
items: [],
distances: [],
facets: [],
},
oneMatch: {
total: 1,
count: 1,
page: 1,
items: [faceStub.face1],
distances: [0.1],
facets: [],
},
oneRemoteMatch: {
total: 1,
count: 1,
page: 1,
items: [faceStub.face1],
distances: [0.8],
facets: [],
},
};
describe(PersonService.name, () => {
let accessMock: IAccessRepositoryMock;
let assetMock: jest.Mocked<IAssetRepository>;
@@ -97,8 +70,8 @@ describe(PersonService.name, () => {
let mediaMock: jest.Mocked<IMediaRepository>;
let moveMock: jest.Mocked<IMoveRepository>;
let personMock: jest.Mocked<IPersonRepository>;
let searchMock: jest.Mocked<ISearchRepository>;
let storageMock: jest.Mocked<IStorageRepository>;
let smartInfoMock: jest.Mocked<ISmartInfoRepository>;
let sut: PersonService;
beforeEach(async () => {
@@ -110,8 +83,8 @@ describe(PersonService.name, () => {
moveMock = newMoveRepositoryMock();
mediaMock = newMediaRepositoryMock();
personMock = newPersonRepositoryMock();
searchMock = newSearchRepositoryMock();
storageMock = newStorageRepositoryMock();
smartInfoMock = newSmartInfoRepositoryMock();
sut = new PersonService(
accessMock,
assetMock,
@@ -119,10 +92,10 @@ describe(PersonService.name, () => {
moveMock,
mediaMock,
personMock,
searchMock,
configMock,
storageMock,
jobMock,
smartInfoMock,
);
mediaMock.crop.mockResolvedValue(croppedFace);
@@ -283,10 +256,6 @@ describe(PersonService.name, () => {
expect(personMock.getById).toHaveBeenCalledWith('person-1');
expect(personMock.update).toHaveBeenCalledWith({ id: 'person-1', name: 'Person 1' });
expect(jobMock.queue).toHaveBeenCalledWith({
name: JobName.SEARCH_INDEX_ASSET,
data: { ids: [assetStub.image.id] },
});
expect(accessMock.person.checkOwnerAccess).toHaveBeenCalledWith(authStub.admin.id, new Set(['person-1']));
});
@@ -320,10 +289,6 @@ describe(PersonService.name, () => {
expect(personMock.getById).toHaveBeenCalledWith('person-1');
expect(personMock.update).toHaveBeenCalledWith({ id: 'person-1', isHidden: false });
expect(jobMock.queue).toHaveBeenCalledWith({
name: JobName.SEARCH_INDEX_ASSET,
data: { ids: [assetStub.image.id] },
});
expect(accessMock.person.checkOwnerAccess).toHaveBeenCalledWith(authStub.admin.id, new Set(['person-1']));
});
@@ -508,6 +473,17 @@ describe(PersonService.name, () => {
});
});
describe('handlePersonDelete', () => {
it('should delete person', async () => {
personMock.getById.mockResolvedValue(personStub.withName);
await sut.handlePersonDelete({ id: personStub.withName.id });
expect(personMock.delete).toHaveBeenCalledWith(personStub.withName);
expect(storageMock.unlink).toHaveBeenCalledWith(personStub.withName.thumbnailPath);
});
});
describe('handlePersonCleanup', () => {
it('should delete people without faces', async () => {
personMock.getAllWithoutFaces.mockResolvedValue([personStub.noName]);
@@ -547,7 +523,7 @@ describe(PersonService.name, () => {
hasNextPage: false,
});
personMock.getAll.mockResolvedValue([personStub.withName]);
searchMock.deleteAllFaces.mockResolvedValue(100);
personMock.deleteAll.mockResolvedValue(5);
await sut.handleQueueRecognizeFaces({ force: true });
@@ -626,7 +602,7 @@ describe(PersonService.name, () => {
it('should match existing people', async () => {
machineLearningMock.detectFaces.mockResolvedValue([detectFaceMock]);
searchMock.searchFaces.mockResolvedValue(faceSearch.oneMatch);
smartInfoMock.searchFaces.mockResolvedValue([faceStub.face1]);
assetMock.getByIds.mockResolvedValue([assetStub.image]);
await sut.handleRecognizeFaces({ id: assetStub.image.id });
@@ -645,7 +621,7 @@ describe(PersonService.name, () => {
it('should create a new person', async () => {
machineLearningMock.detectFaces.mockResolvedValue([detectFaceMock]);
searchMock.searchFaces.mockResolvedValue(faceSearch.oneRemoteMatch);
smartInfoMock.searchFaces.mockResolvedValue([]);
personMock.create.mockResolvedValue(personStub.noName);
assetMock.getByIds.mockResolvedValue([assetStub.image]);
personMock.createFace.mockResolvedValue(faceStub.primaryFace1);
@@ -664,10 +640,6 @@ describe(PersonService.name, () => {
imageHeight: 500,
imageWidth: 400,
});
expect(jobMock.queue.mock.calls).toEqual([
[{ name: JobName.SEARCH_INDEX_FACE, data: { personId: 'person-1', assetId: 'asset-id' } }],
[{ name: JobName.GENERATE_PERSON_THUMBNAIL, data: { id: 'person-1' } }],
]);
});
});
describe('handleGeneratePersonThumbnail', () => {
@@ -873,4 +845,27 @@ describe(PersonService.name, () => {
expect(accessMock.person.checkOwnerAccess).toHaveBeenCalledWith(authStub.admin.id, new Set(['person-1']));
});
});
describe('mapFace', () => {
it('should map a face', () => {
expect(mapFaces(faceStub.face1, personStub.withName.owner)).toEqual({
boundingBoxX1: 0,
boundingBoxX2: 1,
boundingBoxY1: 0,
boundingBoxY2: 1,
id: 'assetFaceId',
imageHeight: 1024,
imageWidth: 1024,
person: mapPerson(personStub.withName),
});
});
it('should not map person if person is null', () => {
expect(mapFaces({ ...faceStub.face1, person: null }, authStub.user1).person).toBeNull();
});
it('should not map person if person does not match auth user id', () => {
expect(mapFaces(faceStub.face1, authStub.user1).person).toBeNull();
});
});
});

46
server/src/domain/person/person.service.ts
View File

@@ -9,7 +9,6 @@ import { usePagination } from '../domain.util';
import { IBaseJob, IEntityJob, JOBS_ASSET_PAGINATION_SIZE, JobName } from '../job';
import { FACE_THUMBNAIL_SIZE } from '../media';
import {
AssetFaceId,
CropOptions,
IAccessRepository,
IAssetRepository,
@@ -18,7 +17,7 @@ import {
IMediaRepository,
IMoveRepository,
IPersonRepository,
ISearchRepository,
ISmartInfoRepository,
IStorageRepository,
ISystemConfigRepository,
ImmichReadStream,
@@ -56,10 +55,10 @@ export class PersonService {
@Inject(IMoveRepository) moveRepository: IMoveRepository,
@Inject(IMediaRepository) private mediaRepository: IMediaRepository,
@Inject(IPersonRepository) private repository: IPersonRepository,
@Inject(ISearchRepository) private searchRepository: ISearchRepository,
@Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
@Inject(IStorageRepository) private storageRepository: IStorageRepository,
@Inject(IJobRepository) private jobRepository: IJobRepository,
@Inject(ISmartInfoRepository) private smartInfoRepository: ISmartInfoRepository,
) {
this.access = AccessCore.create(accessRepository);
this.configCore = SystemConfigCore.create(configRepository);
@@ -198,11 +197,6 @@ export class PersonService {
if (name !== undefined || birthDate !== undefined || isHidden !== undefined) {
person = await this.repository.update({ id, name, birthDate, isHidden });
if (this.needsSearchIndexUpdate(dto)) {
const assets = await this.repository.getAssets(id);
const ids = assets.map((asset) => asset.id);
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ASSET, data: { ids } });
}
}
if (assetId) {
@@ -281,8 +275,7 @@ export class PersonService {
for (const person of people) {
await this.jobRepository.queue({ name: JobName.PERSON_DELETE, data: { id: person.id } });
}
const faces = await this.searchRepository.deleteAllFaces();
this.logger.debug(`Deleted ${people} people and ${faces} faces`);
this.logger.debug(`Deleted ${people.length} people`);
}
for await (const assets of assetPagination) {
@@ -318,20 +311,17 @@ export class PersonService {
);
this.logger.debug(`${faces.length} faces detected in ${asset.resizePath}`);
this.logger.verbose(faces.map((face) => ({ ...face, embedding: `float[${face.embedding.length}]` })));
this.logger.verbose(faces.map((face) => ({ ...face, embedding: `vector(${face.embedding.length})` })));
for (const { embedding, ...rest } of faces) {
const faceSearchResult = await this.searchRepository.searchFaces(embedding, { ownerId: asset.ownerId });
let personId: string | null = null;
// try to find a matching face and link to the associated person
// The closer to 0, the better the match. Range is from 0 to 2
if (faceSearchResult.total && faceSearchResult.distances[0] <= machineLearning.facialRecognition.maxDistance) {
this.logger.verbose(`Match face with distance ${faceSearchResult.distances[0]}`);
personId = faceSearchResult.items[0].personId;
}
const matches = await this.smartInfoRepository.searchFaces({
ownerId: asset.ownerId,
embedding,
numResults: 1,
maxDistance: machineLearning.facialRecognition.maxDistance,
});
let personId = matches[0]?.personId || null;
let newPerson: PersonEntity | null = null;
if (!personId) {
this.logger.debug('No matches, creating a new person.');
@@ -350,8 +340,6 @@ export class PersonService {
boundingBoxY1: rest.boundingBox.y1,
boundingBoxY2: rest.boundingBox.y2,
});
const faceId: AssetFaceId = { assetId: asset.id, personId };
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_FACE, data: faceId });
if (newPerson) {
await this.repository.update({ id: personId, faceAssetId: face.id });
@@ -489,21 +477,9 @@ export class PersonService {
}
}
// Re-index all faces in typesense for up-to-date search results
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_FACES });
return results;
}
/**
* Returns true if the given person update is going to require an update of the search index.
* @param dto the Person going to be updated
* @private
*/
private needsSearchIndexUpdate(dto: PersonUpdateDto): boolean {
return dto.name !== undefined || dto.isHidden !== undefined;
}
private async findOrFail(id: string) {
const person = await this.repository.getById(id);
if (!person) {

30
server/src/domain/repositories/asset.repository.ts
View File

@@ -1,3 +1,4 @@
import { SearchExploreItem } from '@app/domain';
import { AssetEntity, AssetJobStatusEntity, AssetType, ExifEntity } from '@app/infra/entities';
import { FindOptionsRelations } from 'typeorm';
import { Paginated, PaginationOptions } from '../domain.util';
@@ -105,8 +106,7 @@ export enum TimeBucketSize {
MONTH = 'MONTH',
}
export interface TimeBucketOptions {
size: TimeBucketSize;
export interface AssetBuilderOptions {
isArchived?: boolean;
isFavorite?: boolean;
isTrashed?: boolean;
@@ -114,6 +114,12 @@ export interface TimeBucketOptions {
personId?: string;
userIds?: string[];
withStacked?: boolean;
exifInfo?: boolean;
assetType?: AssetType;
}
export interface TimeBucketOptions extends AssetBuilderOptions {
size: TimeBucketSize;
}
export interface TimeBucketItem {
@@ -142,6 +148,21 @@ export interface MonthDay {
month: number;
}
export interface AssetExploreFieldOptions {
maxFields: number;
minAssetsPerField: number;
}
export interface AssetExploreOptions extends AssetExploreFieldOptions {
relation: keyof AssetEntity;
relatedField: string;
unnest?: boolean;
}
export interface MetadataSearchOptions {
numResults: number;
}
export const IAssetRepository = 'IAssetRepository';
export interface IAssetRepository {
@@ -152,7 +173,7 @@ export interface IAssetRepository {
getByChecksum(userId: string, checksum: Buffer): Promise<AssetEntity | null>;
getByAlbumId(pagination: PaginationOptions, albumId: string): Paginated<AssetEntity>;
getByUserId(pagination: PaginationOptions, userId: string, options?: AssetSearchOptions): Paginated<AssetEntity>;
getById(id: string): Promise<AssetEntity | null>;
getById(id: string, relations?: FindOptionsRelations<AssetEntity>): Promise<AssetEntity | null>;
getWithout(pagination: PaginationOptions, property: WithoutProperty): Paginated<AssetEntity>;
getWith(pagination: PaginationOptions, property: WithProperty, libraryId?: string): Paginated<AssetEntity>;
getRandom(userId: string, count: number): Promise<AssetEntity[]>;
@@ -176,4 +197,7 @@ export interface IAssetRepository {
upsertExif(exif: Partial<ExifEntity>): Promise<void>;
upsertJobStatus(jobStatus: Partial<AssetJobStatusEntity>): Promise<void>;
search(options: AssetSearchOptions): Promise<AssetEntity[]>;
getAssetIdByCity(userId: string, options: AssetExploreFieldOptions): Promise<SearchExploreItem<string>>;
getAssetIdByTag(userId: string, options: AssetExploreFieldOptions): Promise<SearchExploreItem<string>>;
searchMetadata(query: string, userId: string, options: MetadataSearchOptions): Promise<AssetEntity[]>;
}

15
server/src/domain/repositories/job.repository.ts
View File

@@ -2,9 +2,7 @@ import { JobName, QueueName } from '../job/job.constants';
import {
IAssetDeletionJob,
IAssetFaceJob,
IBaseJob,
IBulkEntityJob,
IDeleteFilesJob,
IEntityJob,
ILibraryFileJob,
@@ -96,18 +94,7 @@ export type JobItem =
| { name: JobName.LIBRARY_REMOVE_OFFLINE; data: IEntityJob }
| { name: JobName.LIBRARY_DELETE; data: IEntityJob }
| { name: JobName.LIBRARY_QUEUE_SCAN_ALL; data: IBaseJob }
| { name: JobName.LIBRARY_QUEUE_CLEANUP; data: IBaseJob }
// Search
| { name: JobName.SEARCH_INDEX_ASSETS; data?: IBaseJob }
| { name: JobName.SEARCH_INDEX_ASSET; data: IBulkEntityJob }
| { name: JobName.SEARCH_INDEX_FACES; data?: IBaseJob }
| { name: JobName.SEARCH_INDEX_FACE; data: IAssetFaceJob }
| { name: JobName.SEARCH_INDEX_ALBUMS; data?: IBaseJob }
| { name: JobName.SEARCH_INDEX_ALBUM; data: IBulkEntityJob }
| { name: JobName.SEARCH_REMOVE_ASSET; data: IBulkEntityJob }
| { name: JobName.SEARCH_REMOVE_ALBUM; data: IBulkEntityJob }
| { name: JobName.SEARCH_REMOVE_FACE; data: IAssetFaceJob };
| { name: JobName.LIBRARY_QUEUE_CLEANUP; data: IBaseJob };
export type JobHandler<T = any> = (data: T) => boolean | Promise<boolean>;
export type JobItemHandler = (item: JobItem) => Promise<void>;

2
server/src/domain/repositories/person.repository.ts
View File

@@ -41,9 +41,7 @@ export interface IPersonRepository {
update(entity: Partial<PersonEntity>): Promise<PersonEntity>;
delete(entity: PersonEntity): Promise<PersonEntity | null>;
deleteAll(): Promise<number>;
getStatistics(personId: string): Promise<PersonStatistics>;
getAllFaces(): Promise<AssetFaceEntity[]>;
getFacesByIds(ids: AssetFaceId[]): Promise<AssetFaceEntity[]>;
getRandomFace(personId: string): Promise<AssetFaceEntity | null>;

55
server/src/domain/repositories/search.repository.ts
View File

@@ -1,20 +1,10 @@
import { AlbumEntity, AssetEntity, AssetFaceEntity, AssetType } from '@app/infra/entities';
export enum SearchCollection {
ASSETS = 'assets',
ALBUMS = 'albums',
FACES = 'faces',
}
import { AssetType } from '@app/infra/entities';
export enum SearchStrategy {
CLIP = 'CLIP',
TEXT = 'TEXT',
}
export interface SearchFaceFilter {
ownerId: string;
}
export interface SearchFilter {
id?: string;
userId: string;
@@ -55,43 +45,12 @@ export interface SearchFacet {
}>;
}
export type SearchExploreItemSet<T> = Array<{
value: string;
data: T;
}>;
export interface SearchExploreItem<T> {
fieldName: string;
items: Array<{
value: string;
data: T;
}>;
}
export type OwnedFaceEntity = Pick<AssetFaceEntity, 'assetId' | 'personId' | 'embedding'> & {
/** computed as assetId|personId */
id: string;
/** copied from asset.id */
ownerId: string;
};
export type SearchCollectionIndexStatus = Record<SearchCollection, boolean>;
export const ISearchRepository = 'ISearchRepository';
export interface ISearchRepository {
setup(): Promise<void>;
checkMigrationStatus(): Promise<SearchCollectionIndexStatus>;
importAlbums(items: AlbumEntity[], done: boolean): Promise<void>;
importAssets(items: AssetEntity[], done: boolean): Promise<void>;
importFaces(items: OwnedFaceEntity[], done: boolean): Promise<void>;
deleteAlbums(ids: string[]): Promise<void>;
deleteAssets(ids: string[]): Promise<void>;
deleteFaces(ids: string[]): Promise<void>;
deleteAllFaces(): Promise<number>;
updateCLIPField(num_dim: number): Promise<void>;
searchAlbums(query: string, filters: SearchFilter): Promise<SearchResult<AlbumEntity>>;
searchAssets(query: string, filters: SearchFilter): Promise<SearchResult<AssetEntity>>;
vectorSearch(query: number[], filters: SearchFilter): Promise<SearchResult<AssetEntity>>;
searchFaces(query: number[], filters: SearchFaceFilter): Promise<SearchResult<AssetFaceEntity>>;
explore(userId: string): Promise<SearchExploreItem<AssetEntity>[]>;
items: SearchExploreItemSet<T>;
}

18
server/src/domain/repositories/smart-info.repository.ts
View File

@@ -1,7 +1,19 @@
import { SmartInfoEntity } from '@app/infra/entities';
import { AssetEntity, AssetFaceEntity, SmartInfoEntity } from '@app/infra/entities';
export const ISmartInfoRepository = 'ISmartInfoRepository';
export interface ISmartInfoRepository {
upsert(info: Partial<SmartInfoEntity>): Promise<void>;
export type Embedding = number[];
export interface EmbeddingSearch {
ownerId: string;
embedding: Embedding;
numResults: number;
maxDistance?: number;
}
export interface ISmartInfoRepository {
init(modelName: string): Promise<void>;
searchCLIP(search: EmbeddingSearch): Promise<AssetEntity[]>;
searchFaces(search: EmbeddingSearch): Promise<AssetFaceEntity[]>;
upsert(smartInfo: Partial<SmartInfoEntity>, embedding?: Embedding): Promise<void>;
}

54
server/src/domain/search/dto/search.dto.ts
View File

@@ -1,6 +1,6 @@
import { AssetType } from '@app/infra/entities';
import { Transform } from 'class-transformer';
import { IsArray, IsBoolean, IsEnum, IsNotEmpty, IsString } from 'class-validator';
import { IsBoolean, IsEnum, IsNotEmpty, IsString } from 'class-validator';
import { Optional, toBoolean } from '../../domain.util';
export class SearchDto {
@@ -23,58 +23,6 @@ export class SearchDto {
@Optional()
type?: AssetType;
@IsBoolean()
@Optional()
@Transform(toBoolean)
isFavorite?: boolean;
@IsBoolean()
@Optional()
@Transform(toBoolean)
isArchived?: boolean;
@IsString()
@IsNotEmpty()
@Optional()
'exifInfo.city'?: string;
@IsString()
@IsNotEmpty()
@Optional()
'exifInfo.state'?: string;
@IsString()
@IsNotEmpty()
@Optional()
'exifInfo.country'?: string;
@IsString()
@IsNotEmpty()
@Optional()
'exifInfo.make'?: string;
@IsString()
@IsNotEmpty()
@Optional()
'exifInfo.model'?: string;
@IsString()
@IsNotEmpty()
@Optional()
'exifInfo.projectionType'?: string;
@IsString({ each: true })
@IsArray()
@Optional()
@Transform(({ value }) => value.split(','))
'smartInfo.objects'?: string[];
@IsString({ each: true })
@IsArray()
@Optional()
@Transform(({ value }) => value.split(','))
'smartInfo.tags'?: string[];
@IsBoolean()
@Optional()
@Transform(toBoolean)

418
server/src/domain/search/search.service.spec.ts
View File

@@ -1,29 +1,20 @@
import { BadRequestException } from '@nestjs/common';
import { SystemConfigKey } from '@app/infra/entities';
import {
albumStub,
assetStub,
asyncTick,
authStub,
faceStub,
newAlbumRepositoryMock,
newAssetRepositoryMock,
newJobRepositoryMock,
newMachineLearningRepositoryMock,
newPersonRepositoryMock,
newSearchRepositoryMock,
newSmartInfoRepositoryMock,
newSystemConfigRepositoryMock,
searchStub,
personStub,
} from '@test';
import { plainToInstance } from 'class-transformer';
import { mapAsset } from '../asset';
import { JobName } from '../job';
import {
IAlbumRepository,
IAssetRepository,
IJobRepository,
IMachineLearningRepository,
IPersonRepository,
ISearchRepository,
ISmartInfoRepository,
ISystemConfigRepository,
} from '../repositories';
import { SearchDto } from './dto';
@@ -33,401 +24,126 @@ jest.useFakeTimers();
describe(SearchService.name, () => {
let sut: SearchService;
let albumMock: jest.Mocked<IAlbumRepository>;
let assetMock: jest.Mocked<IAssetRepository>;
let configMock: jest.Mocked<ISystemConfigRepository>;
let jobMock: jest.Mocked<IJobRepository>;
let personMock: jest.Mocked<IPersonRepository>;
let machineMock: jest.Mocked<IMachineLearningRepository>;
let searchMock: jest.Mocked<ISearchRepository>;
let personMock: jest.Mocked<IPersonRepository>;
let smartInfoMock: jest.Mocked<ISmartInfoRepository>;
beforeEach(async () => {
albumMock = newAlbumRepositoryMock();
beforeEach(() => {
assetMock = newAssetRepositoryMock();
configMock = newSystemConfigRepositoryMock();
jobMock = newJobRepositoryMock();
personMock = newPersonRepositoryMock();
machineMock = newMachineLearningRepositoryMock();
searchMock = newSearchRepositoryMock();
sut = new SearchService(albumMock, assetMock, jobMock, machineMock, personMock, searchMock, configMock);
searchMock.checkMigrationStatus.mockResolvedValue({ assets: false, albums: false, faces: false });
delete process.env.TYPESENSE_ENABLED;
await sut.init();
});
const disableSearch = () => {
searchMock.setup.mockClear();
searchMock.checkMigrationStatus.mockClear();
jobMock.queue.mockClear();
process.env.TYPESENSE_ENABLED = 'false';
};
afterEach(() => {
sut.teardown();
personMock = newPersonRepositoryMock();
smartInfoMock = newSmartInfoRepositoryMock();
sut = new SearchService(configMock, machineMock, personMock, smartInfoMock, assetMock);
});
it('should work', () => {
expect(sut).toBeDefined();
});
describe('request dto', () => {
it('should convert smartInfo.tags to a string list', () => {
const instance = plainToInstance(SearchDto, { 'smartInfo.tags': 'a,b,c' });
expect(instance['smartInfo.tags']).toEqual(['a', 'b', 'c']);
});
describe('searchPerson', () => {
it('should pass options to search', async () => {
const { name } = personStub.withName;
it('should handle empty smartInfo.tags', () => {
const instance = plainToInstance(SearchDto, {});
expect(instance['smartInfo.tags']).toBeUndefined();
});
await sut.searchPerson(authStub.user1, { name, withHidden: false });
it('should convert smartInfo.objects to a string list', () => {
const instance = plainToInstance(SearchDto, { 'smartInfo.objects': 'a,b,c' });
expect(instance['smartInfo.objects']).toEqual(['a', 'b', 'c']);
});
expect(personMock.getByName).toHaveBeenCalledWith(authStub.user1.id, name, { withHidden: false });
it('should handle empty smartInfo.objects', () => {
const instance = plainToInstance(SearchDto, {});
expect(instance['smartInfo.objects']).toBeUndefined();
});
});
await sut.searchPerson(authStub.user1, { name, withHidden: true });
describe(`init`, () => {
it('should skip when search is disabled', async () => {
disableSearch();
await sut.init();
expect(searchMock.setup).not.toHaveBeenCalled();
expect(searchMock.checkMigrationStatus).not.toHaveBeenCalled();
expect(jobMock.queue).not.toHaveBeenCalled();
});
it('should skip schema migration if not needed', async () => {
await sut.init();
expect(searchMock.setup).toHaveBeenCalled();
expect(jobMock.queue).not.toHaveBeenCalled();
});
it('should do schema migration if needed', async () => {
searchMock.checkMigrationStatus.mockResolvedValue({ assets: true, albums: true, faces: true });
await sut.init();
expect(searchMock.setup).toHaveBeenCalled();
expect(jobMock.queue.mock.calls).toEqual([
[{ name: JobName.SEARCH_INDEX_ASSETS }],
[{ name: JobName.SEARCH_INDEX_ALBUMS }],
[{ name: JobName.SEARCH_INDEX_FACES }],
]);
expect(personMock.getByName).toHaveBeenCalledWith(authStub.user1.id, name, { withHidden: true });
});
});
describe('getExploreData', () => {
it('should throw bad request exception if search is disabled', async () => {
disableSearch();
await expect(sut.getExploreData(authStub.admin)).rejects.toBeInstanceOf(BadRequestException);
expect(searchMock.explore).not.toHaveBeenCalled();
});
it('should get assets by city and tag', async () => {
assetMock.getAssetIdByCity.mockResolvedValueOnce({
fieldName: 'exifInfo.city',
items: [{ value: 'Paris', data: assetStub.image.id }],
});
assetMock.getAssetIdByTag.mockResolvedValueOnce({
fieldName: 'smartInfo.tags',
items: [{ value: 'train', data: assetStub.imageFrom2015.id }],
});
assetMock.getByIds.mockResolvedValueOnce([assetStub.image, assetStub.imageFrom2015]);
const expectedResponse = [
{ fieldName: 'exifInfo.city', items: [{ value: 'Paris', data: mapAsset(assetStub.image) }] },
{ fieldName: 'smartInfo.tags', items: [{ value: 'train', data: mapAsset(assetStub.imageFrom2015) }] },
];
it('should return explore data if feature flag SEARCH is set', async () => {
searchMock.explore.mockResolvedValue([{ fieldName: 'name', items: [{ value: 'image', data: assetStub.image }] }]);
assetMock.getByIds.mockResolvedValue([assetStub.image]);
const result = await sut.getExploreData(authStub.user1);
await expect(sut.getExploreData(authStub.admin)).resolves.toEqual([
{
fieldName: 'name',
items: [{ value: 'image', data: mapAsset(assetStub.image) }],
},
]);
expect(searchMock.explore).toHaveBeenCalledWith(authStub.admin.id);
expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
expect(result).toEqual(expectedResponse);
});
});
describe('search', () => {
// it('should throw an error is search is disabled', async () => {
// sut['enabled'] = false;
it('should throw an error if query is missing', async () => {
await expect(sut.search(authStub.user1, { q: '' })).rejects.toThrow('Missing query');
});
// await expect(sut.search(authStub.admin, {})).rejects.toBeInstanceOf(BadRequestException);
// expect(searchMock.searchAlbums).not.toHaveBeenCalled();
// expect(searchMock.searchAssets).not.toHaveBeenCalled();
// });
it('should search assets and albums using text search', async () => {
searchMock.searchAssets.mockResolvedValue(searchStub.withImage);
searchMock.searchAlbums.mockResolvedValue(searchStub.emptyResults);
assetMock.getByIds.mockResolvedValue([assetStub.image]);
await expect(sut.search(authStub.admin, {})).resolves.toEqual({
it('should search by metadata if `clip` option is false', async () => {
const dto: SearchDto = { q: 'test query', clip: false };
assetMock.searchMetadata.mockResolvedValueOnce([assetStub.image]);
const expectedResponse = {
albums: {
total: 0,
count: 0,
page: 1,
items: [],
facets: [],
distances: [],
},
assets: {
total: 1,
count: 1,
page: 1,
items: [mapAsset(assetStub.image)],
facets: [],
distances: [],
},
});
};
// expect(searchMock.searchAssets).toHaveBeenCalledWith('*', { userId: authStub.admin.id });
expect(searchMock.searchAlbums).toHaveBeenCalledWith('*', { userId: authStub.admin.id });
const result = await sut.search(authStub.user1, dto);
expect(result).toEqual(expectedResponse);
expect(assetMock.searchMetadata).toHaveBeenCalledWith(dto.q, authStub.user1.id, { numResults: 250 });
expect(smartInfoMock.searchCLIP).not.toHaveBeenCalled();
});
it('should search assets and albums using vector search', async () => {
searchMock.vectorSearch.mockResolvedValue(searchStub.emptyResults);
searchMock.searchAlbums.mockResolvedValue(searchStub.emptyResults);
machineMock.encodeText.mockResolvedValue([123]);
await expect(sut.search(authStub.admin, { clip: true, query: 'foo' })).resolves.toEqual({
it('should search by CLIP if `clip` option is true', async () => {
const dto: SearchDto = { q: 'test query', clip: true };
const embedding = [1, 2, 3];
smartInfoMock.searchCLIP.mockResolvedValueOnce([assetStub.image]);
machineMock.encodeText.mockResolvedValueOnce(embedding);
const expectedResponse = {
albums: {
total: 0,
count: 0,
page: 1,
items: [],
facets: [],
distances: [],
},
assets: {
total: 0,
count: 0,
page: 1,
items: [],
total: 1,
count: 1,
items: [mapAsset(assetStub.image)],
facets: [],
distances: [],
},
});
};
expect(machineMock.encodeText).toHaveBeenCalledWith(expect.any(String), { text: 'foo' }, expect.any(Object));
expect(searchMock.vectorSearch).toHaveBeenCalledWith([123], {
userId: authStub.admin.id,
clip: true,
query: 'foo',
});
expect(searchMock.searchAlbums).toHaveBeenCalledWith('foo', {
userId: authStub.admin.id,
clip: true,
query: 'foo',
});
});
});
const result = await sut.search(authStub.user1, dto);
describe('handleIndexAssets', () => {
it('should call done, even when there are no assets', async () => {
await sut.handleIndexAssets();
expect(searchMock.importAssets).toHaveBeenCalledWith([], true);
expect(result).toEqual(expectedResponse);
expect(smartInfoMock.searchCLIP).toHaveBeenCalledWith({ ownerId: authStub.user1.id, embedding, numResults: 100 });
expect(assetMock.searchMetadata).not.toHaveBeenCalled();
});
it('should index all the assets', async () => {
assetMock.getAll.mockResolvedValue({
items: [assetStub.image],
hasNextPage: false,
});
it('should throw an error if clip is requested but disabled', async () => {
const dto: SearchDto = { q: 'test query', clip: true };
configMock.load
.mockResolvedValueOnce([{ key: SystemConfigKey.MACHINE_LEARNING_ENABLED, value: false }])
.mockResolvedValueOnce([{ key: SystemConfigKey.MACHINE_LEARNING_CLIP_ENABLED, value: false }]);
await sut.handleIndexAssets();
expect(searchMock.importAssets.mock.calls).toEqual([
[[assetStub.image], false],
[[], true],
]);
});
it('should skip if search is disabled', async () => {
sut['enabled'] = false;
await sut.handleIndexAssets();
expect(searchMock.importAssets).not.toHaveBeenCalled();
expect(searchMock.importAlbums).not.toHaveBeenCalled();
});
});
describe('handleIndexAsset', () => {
it('should skip if search is disabled', () => {
sut['enabled'] = false;
sut.handleIndexAsset({ ids: [assetStub.image.id] });
});
it('should index the asset', () => {
sut.handleIndexAsset({ ids: [assetStub.image.id] });
});
});
describe('handleIndexAlbums', () => {
it('should skip if search is disabled', async () => {
sut['enabled'] = false;
await sut.handleIndexAlbums();
});
it('should index all the albums', async () => {
albumMock.getAll.mockResolvedValue([albumStub.empty]);
await sut.handleIndexAlbums();
expect(searchMock.importAlbums).toHaveBeenCalledWith([albumStub.empty], true);
});
});
describe('handleIndexAlbum', () => {
it('should skip if search is disabled', () => {
sut['enabled'] = false;
sut.handleIndexAlbum({ ids: [albumStub.empty.id] });
});
it('should index the album', () => {
sut.handleIndexAlbum({ ids: [albumStub.empty.id] });
});
});
describe('handleRemoveAlbum', () => {
it('should skip if search is disabled', () => {
sut['enabled'] = false;
sut.handleRemoveAlbum({ ids: ['album1'] });
});
it('should remove the album', () => {
sut.handleRemoveAlbum({ ids: ['album1'] });
});
});
describe('handleRemoveAsset', () => {
it('should skip if search is disabled', () => {
sut['enabled'] = false;
sut.handleRemoveAsset({ ids: ['asset1'] });
});
it('should remove the asset', () => {
sut.handleRemoveAsset({ ids: ['asset1'] });
});
});
describe('handleIndexFaces', () => {
it('should call done, even when there are no faces', async () => {
personMock.getAllFaces.mockResolvedValue([]);
await sut.handleIndexFaces();
expect(searchMock.importFaces).toHaveBeenCalledWith([], true);
});
it('should index all the faces', async () => {
personMock.getAllFaces.mockResolvedValue([faceStub.face1]);
await sut.handleIndexFaces();
expect(searchMock.importFaces.mock.calls).toEqual([
[
[
{
id: 'asset-id|person-1',
ownerId: 'user-id',
assetId: 'asset-id',
personId: 'person-1',
embedding: [1, 2, 3, 4],
},
],
false,
],
[[], true],
]);
});
it('should skip if search is disabled', async () => {
sut['enabled'] = false;
await sut.handleIndexFaces();
expect(searchMock.importFaces).not.toHaveBeenCalled();
});
});
describe('handleIndexAsset', () => {
it('should skip if search is disabled', async () => {
sut['enabled'] = false;
await sut.handleIndexFace({ assetId: 'asset-1', personId: 'person-1' });
expect(searchMock.importFaces).not.toHaveBeenCalled();
expect(personMock.getFacesByIds).not.toHaveBeenCalled();
});
it('should index the face', async () => {
personMock.getFacesByIds.mockResolvedValue([faceStub.face1]);
await sut.handleIndexFace({ assetId: 'asset-1', personId: 'person-1' });
expect(personMock.getFacesByIds).toHaveBeenCalledWith([{ assetId: 'asset-1', personId: 'person-1' }]);
});
});
describe('handleRemoveFace', () => {
it('should skip if search is disabled', () => {
sut['enabled'] = false;
sut.handleRemoveFace({ assetId: 'asset-1', personId: 'person-1' });
});
it('should remove the face', () => {
sut.handleRemoveFace({ assetId: 'asset-1', personId: 'person-1' });
});
});
describe('flush', () => {
it('should flush queued album updates', async () => {
albumMock.getByIds.mockResolvedValue([albumStub.empty]);
sut.handleIndexAlbum({ ids: ['album1'] });
jest.runOnlyPendingTimers();
await asyncTick(4);
expect(albumMock.getByIds).toHaveBeenCalledWith(['album1']);
expect(searchMock.importAlbums).toHaveBeenCalledWith([albumStub.empty], false);
});
it('should flush queued album deletes', async () => {
sut.handleRemoveAlbum({ ids: ['album1'] });
jest.runOnlyPendingTimers();
await asyncTick(4);
expect(searchMock.deleteAlbums).toHaveBeenCalledWith(['album1']);
});
it('should flush queued asset updates', async () => {
assetMock.getByIds.mockResolvedValue([assetStub.image]);
sut.handleIndexAsset({ ids: ['asset1'] });
jest.runOnlyPendingTimers();
await asyncTick(4);
expect(assetMock.getByIds).toHaveBeenCalledWith(['asset1']);
expect(searchMock.importAssets).toHaveBeenCalledWith([assetStub.image], false);
});
it('should flush queued asset deletes', async () => {
sut.handleRemoveAsset({ ids: ['asset1'] });
jest.runOnlyPendingTimers();
await asyncTick(4);
expect(searchMock.deleteAssets).toHaveBeenCalledWith(['asset1']);
await expect(sut.search(authStub.user1, dto)).rejects.toThrow('CLIP is not enabled');
await expect(sut.search(authStub.user1, dto)).rejects.toThrow('CLIP is not enabled');
});
});
});

385
server/src/domain/search/search.service.ts
View File

@@ -1,396 +1,99 @@
import { AlbumEntity, AssetEntity, AssetFaceEntity } from '@app/infra/entities';
import { AssetEntity } from '@app/infra/entities';
import { Inject, Injectable, Logger } from '@nestjs/common';
import { mapAlbumWithAssets } from '../album';
import { AssetResponseDto, mapAsset } from '../asset';
import { AuthUserDto } from '../auth';
import { usePagination } from '../domain.util';
import { IAssetFaceJob, IBulkEntityJob, JOBS_ASSET_PAGINATION_SIZE, JobName } from '../job';
import { PersonResponseDto } from '../person/person.dto';
import { PersonResponseDto } from '../person';
import {
AssetFaceId,
IAlbumRepository,
IAssetRepository,
IJobRepository,
IMachineLearningRepository,
IPersonRepository,
ISearchRepository,
ISmartInfoRepository,
ISystemConfigRepository,
OwnedFaceEntity,
SearchCollection,
SearchExploreItem,
SearchResult,
SearchStrategy,
} from '../repositories';
import { FeatureFlag, SystemConfigCore } from '../system-config';
import { SearchDto, SearchPeopleDto } from './dto';
import { SearchResponseDto } from './response-dto';
interface SyncQueue {
upsert: Set<string>;
delete: Set<string>;
}
@Injectable()
export class SearchService {
private logger = new Logger(SearchService.name);
private enabled = false;
private timer: NodeJS.Timeout | null = null;
private configCore: SystemConfigCore;
private albumQueue: SyncQueue = {
upsert: new Set(),
delete: new Set(),
};
private assetQueue: SyncQueue = {
upsert: new Set(),
delete: new Set(),
};
private faceQueue: SyncQueue = {
upsert: new Set(),
delete: new Set(),
};
constructor(
@Inject(IAlbumRepository) private albumRepository: IAlbumRepository,
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
@Inject(IJobRepository) private jobRepository: IJobRepository,
@Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
@Inject(IMachineLearningRepository) private machineLearning: IMachineLearningRepository,
@Inject(IPersonRepository) private personRepository: IPersonRepository,
@Inject(ISearchRepository) private searchRepository: ISearchRepository,
@Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
@Inject(ISmartInfoRepository) private smartInfoRepository: ISmartInfoRepository,
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
) {
this.configCore = SystemConfigCore.create(configRepository);
}
teardown() {
if (this.timer) {
clearInterval(this.timer);
this.timer = null;
}
}
async init() {
this.enabled = await this.configCore.hasFeature(FeatureFlag.SEARCH);
if (!this.enabled) {
return;
}
this.logger.log('Running bootstrap');
await this.searchRepository.setup();
const migrationStatus = await this.searchRepository.checkMigrationStatus();
if (migrationStatus[SearchCollection.ASSETS]) {
this.logger.debug('Queueing job to re-index all assets');
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ASSETS });
}
if (migrationStatus[SearchCollection.ALBUMS]) {
this.logger.debug('Queueing job to re-index all albums');
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_ALBUMS });
}
if (migrationStatus[SearchCollection.FACES]) {
this.logger.debug('Queueing job to re-index all faces');
await this.jobRepository.queue({ name: JobName.SEARCH_INDEX_FACES });
}
this.timer = setInterval(() => this.flush(), 5_000);
async searchPerson(authUser: AuthUserDto, dto: SearchPeopleDto): Promise<PersonResponseDto[]> {
return this.personRepository.getByName(authUser.id, dto.name, { withHidden: dto.withHidden });
}
async getExploreData(authUser: AuthUserDto): Promise<SearchExploreItem<AssetResponseDto>[]> {
await this.configCore.requireFeature(FeatureFlag.SEARCH);
const results = await this.searchRepository.explore(authUser.id);
const lookup = await this.getLookupMap(
results.reduce(
(ids: string[], result: SearchExploreItem<AssetEntity>) => [
...ids,
...result.items.map((item) => item.data.id),
],
[],
),
);
const options = { maxFields: 12, minAssetsPerField: 5 };
const results = await Promise.all([
this.assetRepository.getAssetIdByCity(authUser.id, options),
this.assetRepository.getAssetIdByTag(authUser.id, options),
]);
const assetIds = new Set<string>(results.flatMap((field) => field.items.map((item) => item.data)));
const assets = await this.assetRepository.getByIds(Array.from(assetIds));
const assetMap = new Map<string, AssetResponseDto>(assets.map((asset) => [asset.id, mapAsset(asset)]));
return results.map(({ fieldName, items }) => ({
fieldName,
items: items
.map(({ value, data }) => ({ value, data: lookup[data.id] }))
.filter(({ data }) => !!data)
.map(({ value, data }) => ({ value, data: mapAsset(data) })),
items: items.map(({ value, data }) => ({ value, data: assetMap.get(data) as AssetResponseDto })),
}));
}
async search(authUser: AuthUserDto, dto: SearchDto): Promise<SearchResponseDto> {
const { machineLearning } = await this.configCore.getConfig();
await this.configCore.requireFeature(FeatureFlag.SEARCH);
const query = dto.q || dto.query || '*';
const query = dto.q || dto.query;
if (!query) {
throw new Error('Missing query');
}
const hasClip = machineLearning.enabled && machineLearning.clip.enabled;
const strategy = dto.clip && hasClip ? SearchStrategy.CLIP : SearchStrategy.TEXT;
const filters = { userId: authUser.id, ...dto };
if (dto.clip && !hasClip) {
throw new Error('CLIP is not enabled');
}
const strategy = dto.clip ? SearchStrategy.CLIP : SearchStrategy.TEXT;
let assets: AssetEntity[] = [];
let assets: SearchResult<AssetEntity>;
switch (strategy) {
case SearchStrategy.CLIP:
const {
machineLearning: { clip },
} = await this.configCore.getConfig();
const embedding = await this.machineLearning.encodeText(machineLearning.url, { text: query }, clip);
assets = await this.searchRepository.vectorSearch(embedding, filters);
const embedding = await this.machineLearning.encodeText(
machineLearning.url,
{ text: query },
machineLearning.clip,
);
assets = await this.smartInfoRepository.searchCLIP({ ownerId: authUser.id, embedding, numResults: 100 });
break;
case SearchStrategy.TEXT:
assets = await this.assetRepository.searchMetadata(query, authUser.id, { numResults: 250 });
default:
assets = await this.searchRepository.searchAssets(query, filters);
break;
}
const albums = await this.searchRepository.searchAlbums(query, filters);
const lookup = await this.getLookupMap(assets.items.map((asset) => asset.id));
return {
albums: { ...albums, items: albums.items.map(mapAlbumWithAssets) },
albums: {
total: 0,
count: 0,
items: [],
facets: [],
},
assets: {
...assets,
items: assets.items
.map((item) => lookup[item.id])
.filter((item) => !!item)
.map((asset) => mapAsset(asset)),
total: assets.length,
count: assets.length,
items: assets.map((asset) => mapAsset(asset)),
facets: [],
},
};
}
searchPerson(authUser: AuthUserDto, dto: SearchPeopleDto): Promise<PersonResponseDto[]> {
return this.personRepository.getByName(authUser.id, dto.name, { withHidden: dto.withHidden });
}
async handleIndexAlbums() {
if (!this.enabled) {
return false;
}
const albums = this.patchAlbums(await this.albumRepository.getAll());
this.logger.log(`Indexing ${albums.length} albums`);
await this.searchRepository.importAlbums(albums, true);
return true;
}
async handleIndexAssets() {
if (!this.enabled) {
return false;
}
// TODO: do this in batches based on searchIndexVersion
const assetPagination = usePagination(JOBS_ASSET_PAGINATION_SIZE, (pagination) =>
this.assetRepository.getAll(pagination, { isVisible: true }),
);
for await (const assets of assetPagination) {
this.logger.debug(`Indexing ${assets.length} assets`);
const patchedAssets = this.patchAssets(assets);
await this.searchRepository.importAssets(patchedAssets, false);
}
await this.searchRepository.importAssets([], true);
this.logger.debug('Finished re-indexing all assets');
return false;
}
async handleIndexFaces() {
if (!this.enabled) {
return false;
}
await this.searchRepository.deleteAllFaces();
// TODO: do this in batches based on searchIndexVersion
const faces = this.patchFaces(await this.personRepository.getAllFaces());
this.logger.log(`Indexing ${faces.length} faces`);
const chunkSize = 1000;
for (let i = 0; i < faces.length; i += chunkSize) {
await this.searchRepository.importFaces(faces.slice(i, i + chunkSize), false);
}
await this.searchRepository.importFaces([], true);
this.logger.debug('Finished re-indexing all faces');
return true;
}
handleIndexAlbum({ ids }: IBulkEntityJob) {
if (!this.enabled) {
return false;
}
for (const id of ids) {
this.albumQueue.upsert.add(id);
}
return true;
}
handleIndexAsset({ ids }: IBulkEntityJob) {
if (!this.enabled) {
return false;
}
for (const id of ids) {
this.assetQueue.upsert.add(id);
}
return true;
}
async handleIndexFace({ assetId, personId }: IAssetFaceJob) {
if (!this.enabled) {
return false;
}
// immediately push to typesense
await this.searchRepository.importFaces(await this.idsToFaces([{ assetId, personId }]), false);
return true;
}
handleRemoveAlbum({ ids }: IBulkEntityJob) {
if (!this.enabled) {
return false;
}
for (const id of ids) {
this.albumQueue.delete.add(id);
}
return true;
}
handleRemoveAsset({ ids }: IBulkEntityJob) {
if (!this.enabled) {
return false;
}
for (const id of ids) {
this.assetQueue.delete.add(id);
}
return true;
}
handleRemoveFace({ assetId, personId }: IAssetFaceJob) {
if (!this.enabled) {
return false;
}
this.faceQueue.delete.add(this.asKey({ assetId, personId }));
return true;
}
private async flush() {
if (this.albumQueue.upsert.size > 0) {
const ids = [...this.albumQueue.upsert.keys()];
const items = await this.idsToAlbums(ids);
this.logger.debug(`Flushing ${items.length} album upserts`);
await this.searchRepository.importAlbums(items, false);
this.albumQueue.upsert.clear();
}
if (this.albumQueue.delete.size > 0) {
const ids = [...this.albumQueue.delete.keys()];
this.logger.debug(`Flushing ${ids.length} album deletes`);
await this.searchRepository.deleteAlbums(ids);
this.albumQueue.delete.clear();
}
if (this.assetQueue.upsert.size > 0) {
const ids = [...this.assetQueue.upsert.keys()];
const items = await this.idsToAssets(ids);
this.logger.debug(`Flushing ${items.length} asset upserts`);
await this.searchRepository.importAssets(items, false);
this.assetQueue.upsert.clear();
}
if (this.assetQueue.delete.size > 0) {
const ids = [...this.assetQueue.delete.keys()];
this.logger.debug(`Flushing ${ids.length} asset deletes`);
await this.searchRepository.deleteAssets(ids);
this.assetQueue.delete.clear();
}
if (this.faceQueue.upsert.size > 0) {
const ids = [...this.faceQueue.upsert.keys()].map((key) => this.asParts(key));
const items = await this.idsToFaces(ids);
this.logger.debug(`Flushing ${items.length} face upserts`);
await this.searchRepository.importFaces(items, false);
this.faceQueue.upsert.clear();
}
if (this.faceQueue.delete.size > 0) {
const ids = [...this.faceQueue.delete.keys()];
this.logger.debug(`Flushing ${ids.length} face deletes`);
await this.searchRepository.deleteFaces(ids);
this.faceQueue.delete.clear();
}
}
private async idsToAlbums(ids: string[]): Promise<AlbumEntity[]> {
const entities = await this.albumRepository.getByIds(ids);
return this.patchAlbums(entities);
}
private async idsToAssets(ids: string[]): Promise<AssetEntity[]> {
const entities = await this.assetRepository.getByIds(ids);
return this.patchAssets(entities.filter((entity) => entity.isVisible));
}
private async idsToFaces(ids: AssetFaceId[]): Promise<OwnedFaceEntity[]> {
return this.patchFaces(await this.personRepository.getFacesByIds(ids));
}
private patchAssets(assets: AssetEntity[]): AssetEntity[] {
return assets;
}
private patchAlbums(albums: AlbumEntity[]): AlbumEntity[] {
return albums.map((entity) => ({ ...entity, assets: [] }));
}
private patchFaces(faces: AssetFaceEntity[]): OwnedFaceEntity[] {
const results: OwnedFaceEntity[] = [];
for (const face of faces) {
if (face.personId) {
results.push({
id: this.asKey(face as AssetFaceId),
ownerId: face.asset.ownerId,
assetId: face.assetId,
personId: face.personId,
embedding: face.embedding,
});
}
}
return results;
}
private asKey(face: AssetFaceId): string {
return `${face.assetId}|${face.personId}`;
}
private asParts(key: string): AssetFaceId {
const [assetId, personId] = key.split('|');
return { assetId, personId };
}
private async getLookupMap(assetIds: string[]) {
const assets = await this.assetRepository.getByIds(assetIds);
const lookup: Record<string, AssetEntity> = {};
for (const asset of assets) {
lookup[asset.id] = asset;
}
return lookup;
}
}

107
server/src/domain/smart-info/smart-info.constant.ts Normal file
View File

@@ -0,0 +1,107 @@
export type ModelInfo = {
dimSize: number;
};
export const CLIP_MODEL_INFO: Record<string, ModelInfo> = {
RN50__openai: {
dimSize: 1024,
},
RN50__yfcc15m: {
dimSize: 1024,
},
RN50__cc12m: {
dimSize: 1024,
},
RN101__openai: {
dimSize: 512,
},
RN101__yfcc15m: {
dimSize: 512,
},
RN50x4__openai: {
dimSize: 640,
},
RN50x16__openai: {
dimSize: 768,
},
RN50x64__openai: {
dimSize: 1024,
},
'ViT-B-32__openai': {
dimSize: 512,
},
'ViT-B-32__laion2b_e16': {
dimSize: 512,
},
'ViT-B-32__laion400m_e31': {
dimSize: 512,
},
'ViT-B-32__laion400m_e32': {
dimSize: 512,
},
'ViT-B-32__laion2b-s34b-b79k': {
dimSize: 512,
},
'ViT-B-16__openai': {
dimSize: 512,
},
'ViT-B-16__laion400m_e31': {
dimSize: 512,
},
'ViT-B-16__laion400m_e32': {
dimSize: 512,
},
'ViT-B-16-plus-240__laion400m_e31': {
dimSize: 640,
},
'ViT-B-16-plus-240__laion400m_e32': {
dimSize: 640,
},
'ViT-L-14__openai': {
dimSize: 768,
},
'ViT-L-14__laion400m_e31': {
dimSize: 768,
},
'ViT-L-14__laion400m_e32': {
dimSize: 768,
},
'ViT-L-14__laion2b-s32b-b82k': {
dimSize: 768,
},
'ViT-L-14-336__openai': {
dimSize: 768,
},
'ViT-H-14__laion2b-s32b-b79k': {
dimSize: 1024,
},
'ViT-g-14__laion2b-s12b-b42k': {
dimSize: 1024,
},
'LABSE-Vit-L-14': {
dimSize: 768,
},
'XLM-Roberta-Large-Vit-B-32': {
dimSize: 512,
},
'XLM-Roberta-Large-Vit-B-16Plus': {
dimSize: 640,
},
'XLM-Roberta-Large-Vit-L-14': {
dimSize: 768,
},
};
export function cleanModelName(modelName: string): string {
const tokens = modelName.split('/');
return tokens[tokens.length - 1].replace(/:/g, '_');
}
export function getCLIPModelInfo(modelName: string): ModelInfo {
const modelInfo = CLIP_MODEL_INFO[cleanModelName(modelName)];
if (!modelInfo) {
throw new Error(`Unknown CLIP model: ${modelName}`);
}
return modelInfo;
}

28
server/src/domain/smart-info/smart-info.service.spec.ts
View File

@@ -16,6 +16,7 @@ import {
ISystemConfigRepository,
WithoutProperty,
} from '../repositories';
import { cleanModelName, getCLIPModelInfo } from './smart-info.constant';
import { SmartInfoService } from './smart-info.service';
const asset = {
@@ -195,10 +196,29 @@ describe(SmartInfoService.name, () => {
{ imagePath: 'path/to/resize.ext' },
{ enabled: true, modelName: 'ViT-B-32__openai' },
);
expect(smartMock.upsert).toHaveBeenCalledWith({
assetId: 'asset-1',
clipEmbedding: [0.01, 0.02, 0.03],
});
expect(smartMock.upsert).toHaveBeenCalledWith(
{
assetId: 'asset-1',
},
[0.01, 0.02, 0.03],
);
});
});
describe('cleanModelName', () => {
it('should clean name', () => {
expect(cleanModelName('ViT-B-32::openai')).toEqual('ViT-B-32__openai');
expect(cleanModelName('M-CLIP/XLM-Roberta-Large-Vit-L-14')).toEqual('XLM-Roberta-Large-Vit-L-14');
});
});
describe('getCLIPModelInfo', () => {
it('should return the model info', () => {
expect(getCLIPModelInfo('ViT-B-32__openai')).toEqual({ dimSize: 512 });
});
it('should throw an error if the model is not present', () => {
expect(() => getCLIPModelInfo('test-model')).toThrow('Unknown CLIP model: test-model');
});
});
});

26
server/src/domain/smart-info/smart-info.service.ts
View File

@@ -1,6 +1,7 @@
import { Inject, Injectable } from '@nestjs/common';
import { Inject, Injectable, Logger } from '@nestjs/common';
import { setTimeout } from 'timers/promises';
import { usePagination } from '../domain.util';
import { IBaseJob, IEntityJob, JOBS_ASSET_PAGINATION_SIZE, JobName } from '../job';
import { IBaseJob, IEntityJob, JOBS_ASSET_PAGINATION_SIZE, JobName, QueueName } from '../job';
import {
IAssetRepository,
IJobRepository,
@@ -14,6 +15,7 @@ import { SystemConfigCore } from '../system-config';
@Injectable()
export class SmartInfoService {
private configCore: SystemConfigCore;
private logger = new Logger(SmartInfoService.name);
constructor(
@Inject(IAssetRepository) private assetRepository: IAssetRepository,
@@ -25,6 +27,24 @@ export class SmartInfoService {
this.configCore = SystemConfigCore.create(configRepository);
}
async init() {
await this.jobRepository.pause(QueueName.CLIP_ENCODING);
let { isActive } = await this.jobRepository.getQueueStatus(QueueName.CLIP_ENCODING);
while (isActive) {
this.logger.verbose('Waiting for CLIP encoding queue to stop...');
await setTimeout(1000).then(async () => {
({ isActive } = await this.jobRepository.getQueueStatus(QueueName.CLIP_ENCODING));
});
}
const { machineLearning } = await this.configCore.getConfig();
await this.repository.init(machineLearning.clip.modelName);
await this.jobRepository.resume(QueueName.CLIP_ENCODING);
}
async handleQueueObjectTagging({ force }: IBaseJob) {
const { machineLearning } = await this.configCore.getConfig();
if (!machineLearning.enabled || !machineLearning.classification.enabled) {
@@ -105,7 +125,7 @@ export class SmartInfoService {
machineLearning.clip,
);
await this.repository.upsert({ assetId: asset.id, clipEmbedding: clipEmbedding });
await this.repository.upsert({ assetId: asset.id }, clipEmbedding);
return true;
}

2
server/src/domain/system-config/system-config.core.ts
View File

@@ -210,7 +210,7 @@ export class SystemConfigCore {
[FeatureFlag.MAP]: config.map.enabled,
[FeatureFlag.REVERSE_GEOCODING]: config.reverseGeocoding.enabled,
[FeatureFlag.SIDECAR]: true,
[FeatureFlag.SEARCH]: process.env.TYPESENSE_ENABLED !== 'false',
[FeatureFlag.SEARCH]: true,
[FeatureFlag.TRASH]: config.trash.enabled,
// TODO: use these instead of `POST oauth/config`

10
server/src/domain/system-config/system-config.service.spec.ts
View File

@@ -13,7 +13,12 @@ import {
import { BadRequestException } from '@nestjs/common';
import { newCommunicationRepositoryMock, newJobRepositoryMock, newSystemConfigRepositoryMock } from '@test';
import { JobName, QueueName } from '../job';
import { ICommunicationRepository, IJobRepository, ISystemConfigRepository } from '../repositories';
import {
ICommunicationRepository,
IJobRepository,
ISmartInfoRepository,
ISystemConfigRepository,
} from '../repositories';
import { defaults, SystemConfigValidator } from './system-config.core';
import { SystemConfigService } from './system-config.service';
@@ -133,13 +138,14 @@ describe(SystemConfigService.name, () => {
let configMock: jest.Mocked<ISystemConfigRepository>;
let communicationMock: jest.Mocked<ICommunicationRepository>;
let jobMock: jest.Mocked<IJobRepository>;
let smartInfoMock: jest.Mocked<ISmartInfoRepository>;
beforeEach(async () => {
delete process.env.IMMICH_CONFIG_FILE;
configMock = newSystemConfigRepositoryMock();
communicationMock = newCommunicationRepositoryMock();
jobMock = newJobRepositoryMock();
sut = new SystemConfigService(configMock, communicationMock, jobMock);
sut = new SystemConfigService(configMock, communicationMock, jobMock, smartInfoMock);
});
it('should work', () => {

17
server/src/domain/system-config/system-config.service.ts
View File

@@ -1,6 +1,12 @@
import { Inject, Injectable } from '@nestjs/common';
import { JobName } from '../job';
import { CommunicationEvent, ICommunicationRepository, IJobRepository, ISystemConfigRepository } from '../repositories';
import {
CommunicationEvent,
ICommunicationRepository,
IJobRepository,
ISmartInfoRepository,
ISystemConfigRepository,
} from '../repositories';
import { SystemConfigDto, mapConfig } from './dto/system-config.dto';
import { SystemConfigTemplateStorageOptionDto } from './response-dto/system-config-template-storage-option.dto';
import {
@@ -22,6 +28,7 @@ export class SystemConfigService {
@Inject(ISystemConfigRepository) private repository: ISystemConfigRepository,
@Inject(ICommunicationRepository) private communicationRepository: ICommunicationRepository,
@Inject(IJobRepository) private jobRepository: IJobRepository,
@Inject(ISmartInfoRepository) private smartInfoRepository: ISmartInfoRepository,
) {
this.core = SystemConfigCore.create(repository);
}
@@ -41,10 +48,14 @@ export class SystemConfigService {
}
async updateConfig(dto: SystemConfigDto): Promise<SystemConfigDto> {
const config = await this.core.updateConfig(dto);
const oldConfig = await this.core.getConfig();
const newConfig = await this.core.updateConfig(dto);
await this.jobRepository.queue({ name: JobName.SYSTEM_CONFIG_CHANGE });
this.communicationRepository.broadcast(CommunicationEvent.CONFIG_UPDATE, {});
return mapConfig(config);
if (oldConfig.machineLearning.clip.modelName !== newConfig.machineLearning.clip.modelName) {
await this.smartInfoRepository.init(newConfig.machineLearning.clip.modelName);
}
return mapConfig(newConfig);
}
async refreshConfig() {

2
server/src/immich/api-v1/asset/asset-repository.ts
View File

@@ -1,6 +1,6 @@
import { AssetCreate } from '@app/domain';
import { AssetEntity } from '@app/infra/entities';
import OptionalBetween from '@app/infra/utils/optional-between.util';
import { OptionalBetween } from '@app/infra/infra.utils';
import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { In } from 'typeorm/find-options/operator/In';

8
server/src/immich/app.module.ts
View File

@@ -1,7 +1,7 @@
import { DomainModule } from '@app/domain';
import { InfraModule } from '@app/infra';
import { AssetEntity } from '@app/infra/entities';
import { Module, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
import { Module, OnModuleInit } from '@nestjs/common';
import { APP_GUARD, APP_INTERCEPTOR } from '@nestjs/core';
import { ScheduleModule } from '@nestjs/schedule';
import { TypeOrmModule } from '@nestjs/typeorm';
@@ -73,14 +73,10 @@ import { ErrorInterceptor, FileUploadInterceptor } from './interceptors';
FileUploadInterceptor,
],
})
export class AppModule implements OnModuleInit, OnModuleDestroy {
export class AppModule implements OnModuleInit {
constructor(private appService: AppService) {}
async onModuleInit() {
await this.appService.init();
}
async onModuleDestroy() {
await this.appService.destroy();
}
}

9
server/src/immich/app.service.ts
View File

@@ -1,4 +1,4 @@
import { JobService, LibraryService, ONE_HOUR, SearchService, ServerInfoService, StorageService } from '@app/domain';
import { JobService, LibraryService, ONE_HOUR, ServerInfoService, StorageService } from '@app/domain';
import { Injectable, Logger } from '@nestjs/common';
import { Cron, CronExpression, Interval } from '@nestjs/schedule';
@@ -9,7 +9,6 @@ export class AppService {
constructor(
private jobService: JobService,
private libraryService: LibraryService,
private searchService: SearchService,
private storageService: StorageService,
private serverService: ServerInfoService,
) {}
@@ -26,13 +25,7 @@ export class AppService {
async init() {
this.storageService.init();
await this.searchService.init();
await this.serverService.handleVersionCheck();
this.logger.log(`Feature Flags: ${JSON.stringify(await this.serverService.getFeatures(), null, 2)}`);
await this.libraryService.init();
}
async destroy() {
this.searchService.teardown();
}
}

4
server/src/immich/main.ts
View File

@@ -1,5 +1,5 @@
import { envName, getLogLevels, isDev, serverVersion } from '@app/domain';
import { RedisIoAdapter } from '@app/infra';
import { RedisIoAdapter, enablePrefilter } from '@app/infra';
import { Logger } from '@nestjs/common';
import { NestFactory } from '@nestjs/core';
import { NestExpressApplication } from '@nestjs/platform-express';
@@ -29,6 +29,8 @@ export async function bootstrap() {
app.useStaticAssets('www');
app.use(indexFallback(excludePaths));
await enablePrefilter();
const server = await app.listen(port);
server.requestTimeout = 30 * 60 * 1000;

41
server/src/infra/database-locks.ts Normal file
View File

@@ -0,0 +1,41 @@
import { dataSource } from '@app/infra';
import AsyncLock from 'async-lock';
export enum DatabaseLock {
GeodataImport = 100,
CLIPDimSize = 512,
}
export async function acquireLock(lock: DatabaseLock): Promise<void> {
return dataSource.query('SELECT pg_advisory_lock($1)', [lock]);
}
export async function releaseLock(lock: DatabaseLock): Promise<void> {
return dataSource.query('SELECT pg_advisory_unlock($1)', [lock]);
}
export const asyncLock = new AsyncLock();
export function RequireLock<T>(
lock: DatabaseLock,
): (target: any, propertyKey: string, descriptor: PropertyDescriptor) => void {
return function (target: any, propertyKey: string, descriptor: PropertyDescriptor): void {
const originalMethod = descriptor.value;
descriptor.value = async function (...args: any[]): Promise<T> {
if (!dataSource.isInitialized) {
await dataSource.initialize();
}
let res;
await asyncLock.acquire(DatabaseLock[lock], async () => {
try {
await acquireLock(lock);
res = await originalMethod.apply(this, args);
} finally {
await releaseLock(lock);
}
});
return res as any;
};
};
}

7
server/src/infra/database.config.ts
View File

@@ -25,3 +25,10 @@ export const databaseConfig: PostgresConnectionOptions = {
// this export is used by TypeORM commands in package.json#scripts
export const dataSource = new DataSource(databaseConfig);
export async function enablePrefilter() {
if (!dataSource.isInitialized) {
await dataSource.initialize();
}
await dataSource.query(`SET vectors.enable_prefilter = on`);
}

11
server/src/infra/entities/asset-face.entity.ts
View File

@@ -2,7 +2,7 @@ import { Column, Entity, Index, ManyToOne, PrimaryGeneratedColumn } from 'typeor
import { AssetEntity } from './asset.entity';
import { PersonEntity } from './person.entity';
@Entity('asset_faces')
@Entity('asset_faces', { synchronize: false })
@Index(['personId', 'assetId'])
export class AssetFaceEntity {
@PrimaryGeneratedColumn('uuid')
@@ -14,12 +14,9 @@ export class AssetFaceEntity {
@Column({ nullable: true, type: 'uuid' })
personId!: string | null;
@Column({
type: 'float4',
array: true,
nullable: true,
})
embedding!: number[] | null;
@Index('face_index', { synchronize: false })
@Column({ type: 'float4', array: true, select: false })
embedding!: number[];
@Column({ default: 0, type: 'int' })
imageWidth!: number;

4
server/src/infra/entities/asset.entity.ts
View File

@@ -20,6 +20,7 @@ import { ExifEntity } from './exif.entity';
import { LibraryEntity } from './library.entity';
import { SharedLinkEntity } from './shared-link.entity';
import { SmartInfoEntity } from './smart-info.entity';
import { SmartSearchEntity } from './smart-search.entity';
import { TagEntity } from './tag.entity';
import { UserEntity } from './user.entity';
@@ -137,6 +138,9 @@ export class AssetEntity {
@OneToOne(() => SmartInfoEntity, (smartInfoEntity) => smartInfoEntity.asset)
smartInfo?: SmartInfoEntity;
@OneToOne(() => SmartSearchEntity, (smartSearchEntity) => smartSearchEntity.asset)
smartSearch?: SmartSearchEntity;
@ManyToMany(() => TagEntity, (tag) => tag.assets, { cascade: true })
@JoinTable({ name: 'tag_asset' })
tags!: TagEntity[];

2
server/src/infra/entities/exif.entity.ts
View File

@@ -46,6 +46,7 @@ export class ExifEntity {
@Column({ type: 'varchar', nullable: true })
projectionType!: string | null;
@Index('exif_city')
@Column({ type: 'varchar', nullable: true })
city!: string | null;
@@ -98,6 +99,7 @@ export class ExifEntity {
@Column({
type: 'tsvector',
generatedType: 'STORED',
select: false,
asExpression: `TO_TSVECTOR('english',
COALESCE(make, '') || ' ' ||
COALESCE(model, '') || ' ' ||

3
server/src/infra/entities/index.ts
View File

@@ -15,6 +15,7 @@ import { PartnerEntity } from './partner.entity';
import { PersonEntity } from './person.entity';
import { SharedLinkEntity } from './shared-link.entity';
import { SmartInfoEntity } from './smart-info.entity';
import { SmartSearchEntity } from './smart-search.entity';
import { SystemConfigEntity } from './system-config.entity';
import { SystemMetadataEntity } from './system-metadata.entity';
import { TagEntity } from './tag.entity';
@@ -38,6 +39,7 @@ export * from './partner.entity';
export * from './person.entity';
export * from './shared-link.entity';
export * from './smart-info.entity';
export * from './smart-search.entity';
export * from './system-config.entity';
export * from './system-metadata.entity';
export * from './tag.entity';
@@ -61,6 +63,7 @@ export const databaseEntities = [
PersonEntity,
SharedLinkEntity,
SmartInfoEntity,
SmartSearchEntity,
SystemConfigEntity,
SystemMetadataEntity,
TagEntity,

9
server/src/infra/entities/smart-info.entity.ts
View File

@@ -1,7 +1,7 @@
import { Column, Entity, JoinColumn, OneToOne, PrimaryColumn } from 'typeorm';
import { AssetEntity } from './asset.entity';
@Entity('smart_info')
@Entity('smart_info', { synchronize: false })
export class SmartInfoEntity {
@OneToOne(() => AssetEntity, { onDelete: 'CASCADE', nullable: true })
@JoinColumn({ name: 'assetId', referencedColumnName: 'id' })
@@ -15,11 +15,4 @@ export class SmartInfoEntity {
@Column({ type: 'text', array: true, nullable: true })
objects!: string[] | null;
@Column({
type: 'float4',
array: true,
nullable: true,
})
clipEmbedding!: number[] | null;
}

20
server/src/infra/entities/smart-search.entity.ts Normal file
View File

@@ -0,0 +1,20 @@
import { Column, Entity, Index, JoinColumn, OneToOne, PrimaryColumn } from 'typeorm';
import { AssetEntity } from './asset.entity';
@Entity('smart_search', { synchronize: false })
export class SmartSearchEntity {
@OneToOne(() => AssetEntity, { onDelete: 'CASCADE', nullable: true })
@JoinColumn({ name: 'assetId', referencedColumnName: 'id' })
asset?: AssetEntity;
@PrimaryColumn()
assetId!: string;
@Index('clip_index', { synchronize: false })
@Column({
type: 'float4',
array: true,
select: false,
})
embedding!: number[];
}

1
server/src/infra/index.ts
View File

@@ -1,3 +1,4 @@
export * from './database-locks';
export * from './database.config';
export * from './infra.config';
export * from './infra.module';

34
server/src/infra/infra.config.ts
View File

@@ -2,7 +2,6 @@ import { QueueName } from '@app/domain';
import { RegisterQueueOptions } from '@nestjs/bullmq';
import { QueueOptions } from 'bullmq';
import { RedisOptions } from 'ioredis';
import { ConfigurationOptions } from 'typesense/lib/Typesense/Configuration';
function parseRedisConfig(): RedisOptions {
if (process.env.IMMICH_TEST_ENV == 'true') {
@@ -41,36 +40,3 @@ export const bullConfig: QueueOptions = {
};
export const bullQueues: RegisterQueueOptions[] = Object.values(QueueName).map((name) => ({ name }));
function parseTypeSenseConfig(): ConfigurationOptions {
const typesenseURL = process.env.TYPESENSE_URL;
const common = {
apiKey: process.env.TYPESENSE_API_KEY as string,
numRetries: 15,
retryIntervalSeconds: 4,
connectionTimeoutSeconds: 10,
};
if (typesenseURL && typesenseURL.startsWith('ha://')) {
try {
const decodedString = Buffer.from(typesenseURL.slice(5), 'base64').toString();
return {
nodes: JSON.parse(decodedString),
...common,
};
} catch (error) {
throw new Error(`Failed to decode typesense options: ${error}`);
}
}
return {
nodes: [
{
host: process.env.TYPESENSE_HOST || 'typesense',
port: Number(process.env.TYPESENSE_PORT) || 8108,
protocol: process.env.TYPESENSE_PROTOCOL || 'http',
},
],
...common,
};
}
export const typesenseConfig: ConfigurationOptions = parseTypeSenseConfig();

3
server/src/infra/infra.module.ts
View File

@@ -15,7 +15,6 @@ import {
IMoveRepository,
IPartnerRepository,
IPersonRepository,
ISearchRepository,
IServerInfoRepository,
ISharedLinkRepository,
ISmartInfoRepository,
@@ -59,7 +58,6 @@ import {
SystemConfigRepository,
SystemMetadataRepository,
TagRepository,
TypesenseRepository,
UserRepository,
UserTokenRepository,
} from './repositories';
@@ -80,7 +78,6 @@ const providers: Provider[] = [
{ provide: IMoveRepository, useClass: MoveRepository },
{ provide: IPartnerRepository, useClass: PartnerRepository },
{ provide: IPersonRepository, useClass: PersonRepository },
{ provide: ISearchRepository, useClass: TypesenseRepository },
{ provide: IServerInfoRepository, useClass: ServerInfoRepository },
{ provide: ISharedLinkRepository, useClass: SharedLinkRepository },
{ provide: ISmartInfoRepository, useClass: SmartInfoRepository },

42
server/src/infra/infra.utils.ts Normal file
View File

@@ -0,0 +1,42 @@
import { Paginated, PaginationOptions } from '@app/domain';
import { Between, FindOneOptions, LessThanOrEqual, MoreThanOrEqual, ObjectLiteral, Repository } from 'typeorm';
/**
* Allows optional values unlike the regular Between and uses MoreThanOrEqual
* or LessThanOrEqual when only one parameter is specified.
*/
export function OptionalBetween<T>(from?: T, to?: T) {
if (from && to) {
return Between(from, to);
} else if (from) {
return MoreThanOrEqual(from);
} else if (to) {
return LessThanOrEqual(to);
}
}
export async function paginate<Entity extends ObjectLiteral>(
repository: Repository<Entity>,
paginationOptions: PaginationOptions,
searchOptions?: FindOneOptions<Entity>,
): Paginated<Entity> {
const items = await repository.find({
...searchOptions,
// Take one more item to check if there's a next page
take: paginationOptions.take + 1,
skip: paginationOptions.skip,
});
const hasNextPage = items.length > paginationOptions.take;
items.splice(paginationOptions.take);
return { items, hasNextPage };
}
export const asVector = (embedding: number[], quote = false) =>
quote ? `'[${embedding.join(',')}]'` : `[${embedding.join(',')}]`;
export const isValidInteger = (value: number, options: { min?: number; max?: number }): value is number => {
const { min = Number.MIN_SAFE_INTEGER, max = Number.MAX_SAFE_INTEGER } = options;
return Number.isInteger(value) && value >= min && value <= max;
};

51
server/src/infra/migrations/1700713871511-UsePgVectors.ts Normal file
View File

@@ -0,0 +1,51 @@
import { MigrationInterface, QueryRunner } from 'typeorm';
export class UsePgVectors1700713871511 implements MigrationInterface {
name = 'UsePgVectors1700713871511';
public async up(queryRunner: QueryRunner): Promise<void> {
const faceDimQuery = await queryRunner.query(`
SELECT CARDINALITY(embedding::real[]) as dimsize
FROM asset_faces
LIMIT 1`);
const clipDimQuery = await queryRunner.query(`
SELECT CARDINALITY("clipEmbedding"::real[]) as dimsize
FROM smart_info
LIMIT 1`);
const faceDimSize = faceDimQuery?.[0]?.['dimsize'] ?? 512;
const clipDimSize = clipDimQuery?.[0]?.['dimsize'] ?? 512;
await queryRunner.query('CREATE EXTENSION IF NOT EXISTS vectors');
await queryRunner.query(`
ALTER TABLE asset_faces
ALTER COLUMN embedding SET NOT NULL,
ALTER COLUMN embedding TYPE vector(${faceDimSize})`);
await queryRunner.query(`
CREATE TABLE smart_search (
"assetId" uuid PRIMARY KEY NOT NULL REFERENCES assets(id) ON DELETE CASCADE,
embedding vector(${clipDimSize}) NOT NULL )`);
await queryRunner.query(`
INSERT INTO smart_search("assetId", embedding)
SELECT si."assetId", si."clipEmbedding"
FROM smart_info si
WHERE "clipEmbedding" IS NOT NULL`);
await queryRunner.query(`ALTER TABLE smart_info DROP COLUMN IF EXISTS "clipEmbedding"`);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE asset_faces ALTER COLUMN embedding TYPE real array`);
await queryRunner.query(`ALTER TABLE smart_info ADD COLUMN IF NOT EXISTS "clipEmbedding" TYPE real array`);
await queryRunner.query(`
INSERT INTO smart_info
("assetId", "clipEmbedding")
SELECT s."assetId", s.embedding
FROM smart_search s
ON CONFLICT (s."assetId") DO UPDATE SET "clipEmbedding" = s.embedding`);
await queryRunner.query(`DROP TABLE IF EXISTS smart_search`);
}
}

19
server/src/infra/migrations/1700713994428-AddCLIPEmbeddingIndex.ts Normal file
View File

@@ -0,0 +1,19 @@
import { MigrationInterface, QueryRunner } from 'typeorm';
export class AddCLIPEmbeddingIndex1700713994428 implements MigrationInterface {
name = 'AddCLIPEmbeddingIndex1700713994428';
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`
CREATE INDEX IF NOT EXISTS clip_index ON smart_search
USING vectors (embedding cosine_ops) WITH (options = $$
[indexing.hnsw]
m = 16
ef_construction = 300
$$);`);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`DROP INDEX IF EXISTS clip_index`);
}
}

19
server/src/infra/migrations/1700714033632-AddFaceEmbeddingIndex.ts Normal file
View File

@@ -0,0 +1,19 @@
import { MigrationInterface, QueryRunner } from 'typeorm';
export class AddFaceEmbeddingIndex1700714033632 implements MigrationInterface {
name = 'AddFaceEmbeddingIndex1700714033632';
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`
CREATE INDEX IF NOT EXISTS face_index ON asset_faces
USING vectors (embedding cosine_ops) WITH (options = $$
[indexing.hnsw]
m = 16
ef_construction = 300
$$);`);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`DROP INDEX IF EXISTS face_index`);
}
}

13
server/src/infra/migrations/1700714072055-AddSmartInfoTagsIndex.ts Normal file
View File

@@ -0,0 +1,13 @@
import { MigrationInterface, QueryRunner } from 'typeorm';
export class AddSmartInfoTagsIndex1700714072055 implements MigrationInterface {
name = 'AddSmartInfoTagsIndex1700714072055';
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`CREATE INDEX IF NOT EXISTS si_tags ON smart_info USING GIN (tags);`);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`DROP INDEX IF EXISTS si_tags;`);
}
}

37
server/src/infra/migrations/1700714140297-CreateSmartInfoTextSearchIndex.ts Normal file
View File

@@ -0,0 +1,37 @@
import { MigrationInterface, QueryRunner } from 'typeorm';
export class CreateSmartInfoTextSearchIndex1700714140297 implements MigrationInterface {
name = 'CreateSmartInfoTextSearchIndex1700714140297';
public async up(queryRunner: QueryRunner): Promise<void> {
// https://dba.stackexchange.com/a/164081
await queryRunner.query(`
CREATE OR REPLACE FUNCTION f_concat_ws(text, text[])
RETURNS text
LANGUAGE sql IMMUTABLE PARALLEL SAFE AS
'SELECT array_to_string($2, $1)'`);
await queryRunner.query(`
ALTER TABLE smart_info ADD "smartInfoTextSearchableColumn" tsvector
GENERATED ALWAYS AS (
TO_TSVECTOR(
'english',
f_concat_ws(
' '::text,
COALESCE(tags, array[]::text[]) || COALESCE(objects, array[]::text[])
)
)
)
STORED NOT NULL`);
await queryRunner.query(`
CREATE INDEX smart_info_text_searchable_idx
ON smart_info
USING GIN ("smartInfoTextSearchableColumn")`);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`DROP FUNCTION IF EXISTS immutable_concat_ws`);
await queryRunner.query(`ALTER TABLE smart_info DROP IF EXISTS "smartInfoTextSearchableColumn"`);
}
}

14
server/src/infra/migrations/1701665867595-AddExifCityIndex.ts Normal file
View File

@@ -0,0 +1,14 @@
import { MigrationInterface, QueryRunner } from "typeorm";
export class AddExifCityIndex1701665867595 implements MigrationInterface {
name = 'AddExifCityIndex1701665867595'
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`CREATE INDEX "exif_city" ON "exif" ("city") `);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`DROP INDEX "public"."exif_city"`);
}
}

151
server/src/infra/repositories/asset.repository.ts
View File

@@ -1,5 +1,7 @@
import {
AssetBuilderOptions,
AssetCreate,
AssetExploreFieldOptions,
AssetSearchOptions,
AssetStats,
AssetStatsOptions,
@@ -7,24 +9,25 @@ import {
LivePhotoSearchOptions,
MapMarker,
MapMarkerSearchOptions,
MetadataSearchOptions,
MonthDay,
Paginated,
PaginationOptions,
SearchExploreItem,
TimeBucketItem,
TimeBucketOptions,
TimeBucketSize,
WithoutProperty,
WithProperty,
WithoutProperty,
} from '@app/domain';
import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import _ from 'lodash';
import { DateTime } from 'luxon';
import { And, FindOptionsRelations, FindOptionsWhere, In, IsNull, LessThan, Not, Repository } from 'typeorm';
import { AssetEntity, AssetJobStatusEntity, AssetType, ExifEntity } from '../entities';
import { AssetEntity, AssetJobStatusEntity, AssetType, ExifEntity, SmartInfoEntity } from '../entities';
import { DummyValue, GenerateSql } from '../infra.util';
import OptionalBetween from '../utils/optional-between.util';
import { paginate } from '../utils/pagination.util';
import { OptionalBetween, paginate } from '../infra.utils';
const DEFAULT_SEARCH_SIZE = 250;
@@ -44,6 +47,7 @@ export class AssetRepository implements IAssetRepository {
@InjectRepository(AssetEntity) private repository: Repository<AssetEntity>,
@InjectRepository(ExifEntity) private exifRepository: Repository<ExifEntity>,
@InjectRepository(AssetJobStatusEntity) private jobStatusRepository: Repository<AssetJobStatusEntity>,
@InjectRepository(SmartInfoEntity) private smartInfoRepository: Repository<SmartInfoEntity>,
) {}
async upsertExif(exif: Partial<ExifEntity>): Promise<void> {
@@ -356,16 +360,20 @@ export class AssetRepository implements IAssetRepository {
}
@GenerateSql({ params: [DummyValue.UUID] })
getById(id: string): Promise<AssetEntity | null> {
return this.repository.findOne({
where: { id },
relations: {
getById(id: string, relations: FindOptionsRelations<AssetEntity>): Promise<AssetEntity | null> {
if (!relations) {
relations = {
faces: {
person: true,
},
library: true,
stack: true,
},
};
}
return this.repository.findOne({
where: { id },
relations,
// We are specifically asking for this asset. Return it even if it is soft deleted
withDeleted: true,
});
@@ -472,13 +480,13 @@ export class AssetRepository implements IAssetRepository {
case WithoutProperty.CLIP_ENCODING:
relations = {
smartInfo: true,
smartSearch: true,
};
where = {
isVisible: true,
resizePath: Not(IsNull()),
smartInfo: {
clipEmbedding: IsNull(),
smartSearch: {
embedding: IsNull(),
},
};
break;
@@ -689,15 +697,82 @@ export class AssetRepository implements IAssetRepository {
);
}
private getBuilder(options: TimeBucketOptions) {
const { isArchived, isFavorite, isTrashed, albumId, personId, userIds, withStacked } = options;
@GenerateSql({ params: [DummyValue.UUID, { minAssetsPerField: 5, maxFields: 12 }] })
async getAssetIdByCity(
ownerId: string,
{ minAssetsPerField, maxFields }: AssetExploreFieldOptions,
): Promise<SearchExploreItem<string>> {
const cte = this.exifRepository
.createQueryBuilder('e')
.select('city')
.groupBy('city')
.having('count(city) >= :minAssetsPerField', { minAssetsPerField })
.orderBy('random()')
.limit(maxFields);
const items = await this.getBuilder({
userIds: [ownerId],
exifInfo: false,
assetType: AssetType.IMAGE,
isArchived: false,
})
.select('c.city', 'value')
.addSelect('asset.id', 'data')
.distinctOn(['c.city'])
.innerJoin('exif', 'e', 'asset.id = e."assetId"')
.addCommonTableExpression(cte, 'cities')
.innerJoin('cities', 'c', 'c.city = e.city')
.limit(maxFields)
.getRawMany();
return { fieldName: 'exifInfo.city', items };
}
@GenerateSql({ params: [DummyValue.UUID, { minAssetsPerField: 5, maxFields: 12 }] })
async getAssetIdByTag(
ownerId: string,
{ minAssetsPerField, maxFields }: AssetExploreFieldOptions,
): Promise<SearchExploreItem<string>> {
const cte = this.smartInfoRepository
.createQueryBuilder('si')
.select('unnest(tags)', 'tag')
.groupBy('tag')
.having('count(*) >= :minAssetsPerField', { minAssetsPerField })
.orderBy('random()')
.limit(maxFields);
const items = await this.getBuilder({
userIds: [ownerId],
exifInfo: false,
assetType: AssetType.IMAGE,
isArchived: false,
})
.select('unnest(si.tags)', 'value')
.addSelect('asset.id', 'data')
.distinctOn(['unnest(si.tags)'])
.innerJoin('smart_info', 'si', 'asset.id = si."assetId"')
.addCommonTableExpression(cte, 'random_tags')
.innerJoin('random_tags', 't', 'si.tags @> ARRAY[t.tag]')
.limit(maxFields)
.getRawMany();
return { fieldName: 'smartInfo.tags', items };
}
private getBuilder(options: AssetBuilderOptions) {
const { isArchived, isFavorite, isTrashed, albumId, personId, userIds, withStacked, exifInfo, assetType } = options;
let builder = this.repository
.createQueryBuilder('asset')
.where('asset.isVisible = true')
.andWhere('asset.fileCreatedAt < NOW()')
.leftJoinAndSelect('asset.exifInfo', 'exifInfo')
.leftJoinAndSelect('asset.stack', 'stack');
.andWhere('asset.fileCreatedAt < NOW()');
if (assetType !== undefined) {
builder = builder.andWhere('asset.type = :assetType', { assetType });
}
if (exifInfo !== false) {
builder = builder.leftJoinAndSelect('asset.exifInfo', 'exifInfo').leftJoinAndSelect('asset.stack', 'stack');
}
if (albumId) {
builder = builder.leftJoin('asset.albums', 'album').andWhere('album.id = :albumId', { albumId });
@@ -732,4 +807,46 @@ export class AssetRepository implements IAssetRepository {
return builder;
}
async searchMetadata(query: string, ownerId: string, { numResults }: MetadataSearchOptions): Promise<AssetEntity[]> {
const rows = await this.repository
.createQueryBuilder('assets')
.select('assets.*')
.addSelect('e.country', 'country')
.addSelect('e.state', 'state')
.addSelect('e.city', 'city')
.addSelect('e.description', 'description')
.addSelect('e.model', 'model')
.addSelect('e.make', 'make')
.addSelect('COALESCE(si.tags, array[]::text[])', 'tags')
.addSelect('COALESCE(si.objects, array[]::text[])', 'objects')
.innerJoin('smart_info', 'si', 'si."assetId" = assets."id"')
.innerJoin('exif', 'e', 'assets."id" = e."assetId"')
.where('a.ownerId = :ownerId', { ownerId })
.where(
'(e."exifTextSearchableColumn" || si."smartInfoTextSearchableColumn") @@ PLAINTO_TSQUERY(\'english\', :query)',
{ query },
)
.limit(numResults)
.getRawMany();
return rows.map(
({ tags, objects, country, state, city, description, model, make, ...assetInfo }) =>
({
exifInfo: {
country,
state,
city,
description,
model,
make,
},
smartInfo: {
tags,
objects,
},
...assetInfo,
}) as AssetEntity,
);
}
}

1
server/src/infra/repositories/index.ts
View File

@@ -21,6 +21,5 @@ export * from './smart-info.repository';
export * from './system-config.repository';
export * from './system-metadata.repository';
export * from './tag.repository';
export * from './typesense.repository';
export * from './user-token.repository';
export * from './user.repository';

7
server/src/infra/repositories/metadata.repository.ts
View File

@@ -5,8 +5,8 @@ import {
ISystemMetadataRepository,
ReverseGeocodeResult,
} from '@app/domain';
import { DatabaseLock, RequireLock } from '@app/infra';
import { GeodataAdmin1Entity, GeodataAdmin2Entity, GeodataPlacesEntity, SystemMetadataKey } from '@app/infra/entities';
import { DatabaseLock } from '@app/infra/utils/database-locks';
import { Inject, Logger } from '@nestjs/common';
import { InjectDataSource, InjectRepository } from '@nestjs/typeorm';
import { DefaultReadTaskOptions, exiftool, Tags } from 'exiftool-vendored';
@@ -33,16 +33,14 @@ export class MetadataRepository implements IMetadataRepository {
private logger = new Logger(MetadataRepository.name);
@RequireLock(DatabaseLock.GeodataImport)
async init(): Promise<void> {
this.logger.log('Initializing metadata repository');
const geodataDate = await readFile('/usr/src/resources/geodata-date.txt', 'utf8');
await this.geodataPlacesRepository.query('SELECT pg_advisory_lock($1)', [DatabaseLock.GeodataImport]);
const geocodingMetadata = await this.systemMetadataRepository.get(SystemMetadataKey.REVERSE_GEOCODING_STATE);
if (geocodingMetadata?.lastUpdate === geodataDate) {
await this.dataSource.query('SELECT pg_advisory_unlock($1)', [DatabaseLock.GeodataImport]);
return;
}
@@ -72,7 +70,6 @@ export class MetadataRepository implements IMetadataRepository {
lastImportFileName: CITIES_FILE,
});
await this.dataSource.query('SELECT pg_advisory_unlock($1)', [DatabaseLock.GeodataImport]);
this.logger.log('Geodata import completed');
}

12
server/src/infra/repositories/person.repository.ts
View File

@@ -10,6 +10,7 @@ import { InjectRepository } from '@nestjs/typeorm';
import { In, Repository } from 'typeorm';
import { AssetEntity, AssetFaceEntity, PersonEntity } from '../entities';
import { DummyValue, GenerateSql } from '../infra.util';
import { asVector } from '../infra.utils';
export class PersonRepository implements IPersonRepository {
constructor(
@@ -215,8 +216,15 @@ export class PersonRepository implements IPersonRepository {
return this.personRepository.save(entity);
}
createFace(entity: Partial<AssetFaceEntity>): Promise<AssetFaceEntity> {
return this.assetFaceRepository.save(entity);
async createFace(entity: AssetFaceEntity): Promise<AssetFaceEntity> {
if (!entity.personId) {
throw new Error('Person ID is required to create a face');
}
if (!entity.embedding) {
throw new Error('Embedding is required to create a face');
}
await this.assetFaceRepository.insert({ ...entity, embedding: () => asVector(entity.embedding, true) });
return this.assetFaceRepository.findOneByOrFail({ assetId: entity.assetId, personId: entity.personId });
}
async update(entity: Partial<PersonEntity>): Promise<PersonEntity> {

169
server/src/infra/repositories/smart-info.repository.ts
View File

@@ -1,14 +1,171 @@
import { ISmartInfoRepository } from '@app/domain';
import { Injectable } from '@nestjs/common';
import { Embedding, EmbeddingSearch, ISmartInfoRepository } from '@app/domain';
import { getCLIPModelInfo } from '@app/domain/smart-info/smart-info.constant';
import { DatabaseLock, RequireLock, asyncLock } from '@app/infra';
import { AssetEntity, AssetFaceEntity, SmartInfoEntity, SmartSearchEntity } from '@app/infra/entities';
import { Injectable, Logger } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { Repository } from 'typeorm';
import { SmartInfoEntity } from '../entities';
import { DummyValue, GenerateSql } from '../infra.util';
import { asVector, isValidInteger } from '../infra.utils';
@Injectable()
export class SmartInfoRepository implements ISmartInfoRepository {
constructor(@InjectRepository(SmartInfoEntity) private repository: Repository<SmartInfoEntity>) {}
private logger = new Logger(SmartInfoRepository.name);
async upsert(info: Partial<SmartInfoEntity>): Promise<void> {
await this.repository.upsert(info, { conflictPaths: ['assetId'] });
constructor(
@InjectRepository(SmartInfoEntity) private repository: Repository<SmartInfoEntity>,
@InjectRepository(AssetEntity) private assetRepository: Repository<AssetEntity>,
@InjectRepository(AssetFaceEntity) private assetFaceRepository: Repository<AssetFaceEntity>,
@InjectRepository(SmartSearchEntity) private smartSearchRepository: Repository<SmartSearchEntity>,
) {}
async init(modelName: string): Promise<void> {
const { dimSize } = getCLIPModelInfo(modelName);
if (dimSize == null) {
throw new Error(`Invalid CLIP model name: ${modelName}`);
}
const curDimSize = await this.getDimSize();
this.logger.verbose(`Current database CLIP dimension size is ${curDimSize}`);
if (dimSize != curDimSize) {
this.logger.log(`Dimension size of model ${modelName} is ${dimSize}, but database expects ${curDimSize}.`);
await this.updateDimSize(dimSize);
}
}
@GenerateSql({
params: [{ ownerId: DummyValue.UUID, embedding: Array.from({ length: 512 }, Math.random), numResults: 100 }],
})
async searchCLIP({ ownerId, embedding, numResults }: EmbeddingSearch): Promise<AssetEntity[]> {
if (!isValidInteger(numResults, { min: 1 })) {
throw new Error(`Invalid value for 'numResults': ${numResults}`);
}
let results: AssetEntity[] = [];
await this.assetRepository.manager.transaction(async (manager) => {
await manager.query(`SET LOCAL vectors.k = '${numResults}'`);
results = await manager
.createQueryBuilder(AssetEntity, 'a')
.innerJoin('a.smartSearch', 's')
.where('a.ownerId = :ownerId')
.leftJoinAndSelect('a.exifInfo', 'e')
.orderBy('s.embedding <=> :embedding')
.setParameters({ ownerId, embedding: asVector(embedding) })
.limit(numResults)
.getMany();
});
return results;
}
@GenerateSql({
params: [
{
ownerId: DummyValue.UUID,
embedding: Array.from({ length: 512 }, Math.random),
numResults: 100,
maxDistance: 0.6,
},
],
})
async searchFaces({ ownerId, embedding, numResults, maxDistance }: EmbeddingSearch): Promise<AssetFaceEntity[]> {
if (!isValidInteger(numResults, { min: 1 })) {
throw new Error(`Invalid value for 'numResults': ${numResults}`);
}
let results: AssetFaceEntity[] = [];
await this.assetRepository.manager.transaction(async (manager) => {
await manager.query(`SET LOCAL vectors.k = '${numResults}'`);
const cte = manager
.createQueryBuilder(AssetFaceEntity, 'faces')
.addSelect('1 + (faces.embedding <=> :embedding)', 'distance')
.innerJoin('faces.asset', 'asset')
.where('asset.ownerId = :ownerId')
.orderBy(`faces.embedding <=> :embedding`)
.setParameters({ ownerId, embedding: asVector(embedding) })
.limit(numResults);
results = await manager
.createQueryBuilder()
.select('res.*')
.addCommonTableExpression(cte, 'cte')
.from('cte', 'res')
.where('res.distance <= :maxDistance', { maxDistance })
.getRawMany();
});
return this.assetFaceRepository.create(results);
}
async upsert(smartInfo: Partial<SmartInfoEntity>, embedding?: Embedding): Promise<void> {
await this.repository.upsert(smartInfo, { conflictPaths: ['assetId'] });
if (!smartInfo.assetId || !embedding) {
return;
}
await this.upsertEmbedding(smartInfo.assetId, embedding);
}
private async upsertEmbedding(assetId: string, embedding: number[]): Promise<void> {
if (asyncLock.isBusy(DatabaseLock[DatabaseLock.CLIPDimSize])) {
this.logger.verbose(`Waiting for CLIP dimension size to be updated`);
await asyncLock.acquire(DatabaseLock[DatabaseLock.CLIPDimSize], () => {});
}
await this.smartSearchRepository.upsert(
{ assetId, embedding: () => asVector(embedding, true) },
{ conflictPaths: ['assetId'] },
);
}
@RequireLock(DatabaseLock.CLIPDimSize)
private async updateDimSize(dimSize: number): Promise<void> {
if (!isValidInteger(dimSize, { min: 1, max: 2 ** 16 })) {
throw new Error(`Invalid CLIP dimension size: ${dimSize}`);
}
const curDimSize = await this.getDimSize();
if (curDimSize === dimSize) {
return;
}
this.logger.log(`Updating database CLIP dimension size to ${dimSize}.`);
await this.smartSearchRepository.manager.transaction(async (manager) => {
await manager.query(`DROP TABLE smart_search`);
await manager.query(`
CREATE TABLE smart_search (
"assetId" uuid PRIMARY KEY REFERENCES assets(id) ON DELETE CASCADE,
embedding vector(${dimSize}) NOT NULL )`);
await manager.query(`
CREATE INDEX clip_index ON smart_search
USING vectors (embedding cosine_ops) WITH (options = $$
[indexing.hnsw]
m = 16
ef_construction = 300
$$)`);
});
this.logger.log(`Successfully updated database CLIP dimension size from ${curDimSize} to ${dimSize}.`);
}
private async getDimSize(): Promise<number> {
const res = await this.smartSearchRepository.manager.query(`
SELECT atttypmod as dimsize
FROM pg_attribute f
JOIN pg_class c ON c.oid = f.attrelid
WHERE c.relkind = 'r'::char
AND f.attnum > 0
AND c.relname = 'smart_search'
AND f.attname = 'embedding'`);
const dimSize = res[0]['dimsize'];
if (!isValidInteger(dimSize, { min: 1, max: 2 ** 16 })) {
throw new Error(`Could not retrieve CLIP dimension size`);
}
return dimSize;
}
}

503
server/src/infra/repositories/typesense.repository.ts
View File

@@ -1,503 +0,0 @@
import {
ISearchRepository,
OwnedFaceEntity,
SearchCollection,
SearchCollectionIndexStatus,
SearchExploreItem,
SearchFaceFilter,
SearchFilter,
SearchResult,
} from '@app/domain';
import { Injectable, Logger } from '@nestjs/common';
import _, { Dictionary } from 'lodash';
import { catchError, filter, firstValueFrom, from, map, mergeMap, of, toArray } from 'rxjs';
import { Client } from 'typesense';
import { CollectionCreateSchema } from 'typesense/lib/Typesense/Collections';
import { DocumentSchema, SearchResponse } from 'typesense/lib/Typesense/Documents';
import { AlbumEntity, AssetEntity, AssetFaceEntity } from '../entities';
import { typesenseConfig } from '../infra.config';
import { albumSchema, assetSchema, faceSchema } from '../typesense-schemas';
function removeNil<T extends Dictionary<any>>(item: T): T {
_.forOwn(item, (value, key) => {
if (_.isNil(value) || (_.isObject(value) && !_.isDate(value) && _.isEmpty(removeNil(value)))) {
delete item[key];
}
});
return item;
}
interface MultiSearchError {
code: number;
error: string;
}
interface CustomAssetEntity extends AssetEntity {
geo?: [number, number];
motion?: boolean;
people?: string[];
}
const schemaMap: Record<SearchCollection, CollectionCreateSchema> = {
[SearchCollection.ASSETS]: assetSchema,
[SearchCollection.ALBUMS]: albumSchema,
[SearchCollection.FACES]: faceSchema,
};
const schemas = Object.entries(schemaMap) as [SearchCollection, CollectionCreateSchema][];
@Injectable()
export class TypesenseRepository implements ISearchRepository {
private logger = new Logger(TypesenseRepository.name);
private _client: Client | null = null;
private _updateCLIPLock = false;
private get client(): Client {
if (!this._client) {
throw new Error('Typesense client not available (no apiKey was provided)');
}
return this._client;
}
constructor() {
if (!typesenseConfig.apiKey) {
return;
}
this._client = new Client(typesenseConfig);
}
async setup(): Promise<void> {
const collections = await this.client.collections().retrieve();
for (const collection of collections) {
this.logger.debug(`${collection.name} collection has ${collection.num_documents} documents`);
// await this.client.collections(collection.name).delete();
}
// upsert collections
for (const [collectionName, schema] of schemas) {
const collection = await this.client
.collections(schema.name)
.retrieve()
.catch(() => null);
if (!collection) {
this.logger.log(`Creating schema: ${collectionName}/${schema.name}`);
await this.client.collections().create(schema);
} else {
this.logger.log(`Schema up to date: ${collectionName}/${schema.name}`);
}
}
}
async checkMigrationStatus(): Promise<SearchCollectionIndexStatus> {
const migrationMap: SearchCollectionIndexStatus = {
[SearchCollection.ASSETS]: false,
[SearchCollection.ALBUMS]: false,
[SearchCollection.FACES]: false,
};
// check if alias is using the current schema
const { aliases } = await this.client.aliases().retrieve();
this.logger.log(`Alias mapping: ${JSON.stringify(aliases)}`);
for (const [aliasName, schema] of schemas) {
const match = aliases.find((alias) => alias.name === aliasName);
if (!match || match.collection_name !== schema.name) {
migrationMap[aliasName] = true;
}
}
this.logger.log(`Collections needing migration: ${JSON.stringify(migrationMap)}`);
return migrationMap;
}
async importAlbums(items: AlbumEntity[], done: boolean): Promise<void> {
await this.import(SearchCollection.ALBUMS, items, done);
}
async importAssets(items: AssetEntity[], done: boolean): Promise<void> {
await this.import(SearchCollection.ASSETS, items, done);
}
async importFaces(items: OwnedFaceEntity[], done: boolean): Promise<void> {
await this.import(SearchCollection.FACES, items, done);
}
private async import(
collection: SearchCollection,
items: AlbumEntity[] | AssetEntity[] | OwnedFaceEntity[],
done: boolean,
): Promise<void> {
try {
if (items.length > 0) {
await this.client.collections(schemaMap[collection].name).documents().import(this.patch(collection, items), {
action: 'upsert',
dirty_values: 'coerce_or_drop',
});
}
if (done) {
await this.updateAlias(collection);
}
} catch (error: any) {
await this.handleError(error);
}
}
async explore(userId: string): Promise<SearchExploreItem<AssetEntity>[]> {
const common = {
q: '*',
filter_by: [this.buildFilterBy('ownerId', userId, true), this.buildFilterBy('isArchived', false)].join(' && '),
per_page: 100,
};
const asset$ = this.client.collections<AssetEntity>(assetSchema.name).documents();
const { facet_counts: facets } = await asset$.search({
...common,
query_by: 'originalFileName',
facet_by: 'exifInfo.city,smartInfo.objects',
max_facet_values: 12,
});
return firstValueFrom(
from(facets || []).pipe(
mergeMap(
(facet) =>
from(facet.counts).pipe(
mergeMap((count) => {
const config = {
...common,
query_by: 'originalFileName',
filter_by: [common.filter_by, this.buildFilterBy(facet.field_name, count.value, true)].join(' && '),
per_page: 1,
};
this.logger.verbose(`Explore subquery: "filter_by:${config.filter_by}" (count:${count.count})`);
return from(asset$.search(config)).pipe(
catchError((error: any) => {
this.logger.warn(`Explore subquery error: ${error}`, error?.stack);
return of({ hits: [] });
}),
map((result) => ({
value: count.value,
data: result.hits?.[0]?.document as AssetEntity,
})),
filter((item) => !!item.data),
);
}, 5),
toArray(),
map((items) => ({
fieldName: facet.field_name as string,
items,
})),
),
3,
),
toArray(),
),
);
}
async deleteAlbums(ids: string[]): Promise<void> {
await this.delete(SearchCollection.ALBUMS, ids);
}
async deleteAssets(ids: string[]): Promise<void> {
await this.delete(SearchCollection.ASSETS, ids);
}
async deleteFaces(ids: string[]): Promise<void> {
await this.delete(SearchCollection.FACES, ids);
}
async deleteAllFaces(): Promise<number> {
const records = await this.client.collections(faceSchema.name).documents().delete({ filter_by: 'ownerId:!=null' });
return records.num_deleted;
}
async deleteAllAssets(): Promise<number> {
const records = await this.client.collections(assetSchema.name).documents().delete({ filter_by: 'ownerId:!=null' });
return records.num_deleted;
}
async updateCLIPField(num_dim: number): Promise<void> {
const clipField = assetSchema.fields?.find((field) => field.name === 'smartInfo.clipEmbedding');
if (clipField && !this._updateCLIPLock) {
try {
this._updateCLIPLock = true;
clipField.num_dim = num_dim;
await this.deleteAllAssets();
await this.client
.collections(assetSchema.name)
.update({ fields: [{ name: 'smartInfo.clipEmbedding', drop: true } as any, clipField] });
this.logger.log(`Successfully updated CLIP dimensions to ${num_dim}`);
} catch (err: any) {
this.logger.error(`Error while updating CLIP field: ${err.message}`);
} finally {
this._updateCLIPLock = false;
}
}
}
async delete(collection: SearchCollection, ids: string[]): Promise<void> {
await this.client
.collections(schemaMap[collection].name)
.documents()
.delete({ filter_by: this.buildFilterBy('id', ids, true) });
}
async searchAlbums(query: string, filters: SearchFilter): Promise<SearchResult<AlbumEntity>> {
const results = await this.client
.collections<AlbumEntity>(albumSchema.name)
.documents()
.search({
q: query,
query_by: ['albumName', 'description'].join(','),
filter_by: this.getAlbumFilters(filters),
});
return this.asResponse(results, filters.debug);
}
async searchAssets(query: string, filters: SearchFilter): Promise<SearchResult<AssetEntity>> {
const results = await this.client
.collections<AssetEntity>(assetSchema.name)
.documents()
.search({
q: query,
query_by: [
'originalFileName',
'exifInfo.country',
'exifInfo.state',
'exifInfo.city',
'exifInfo.description',
'exifInfo.model',
'exifInfo.make',
'smartInfo.tags',
'smartInfo.objects',
'people',
].join(','),
per_page: 250,
facet_by: this.getFacetFieldNames(SearchCollection.ASSETS),
filter_by: this.getAssetFilters(filters),
sort_by: filters.recent ? 'createdAt:desc' : undefined,
});
return this.asResponse(results, filters.debug);
}
async searchFaces(input: number[], filters: SearchFaceFilter): Promise<SearchResult<AssetFaceEntity>> {
const { results } = await this.client.multiSearch.perform({
searches: [
{
collection: faceSchema.name,
q: '*',
vector_query: `embedding:([${input.join(',')}], k:5)`,
per_page: 5,
filter_by: this.buildFilterBy('ownerId', filters.ownerId, true),
} as any,
],
});
return this.asResponse(results[0] as SearchResponse<AssetFaceEntity>);
}
async vectorSearch(input: number[], filters: SearchFilter): Promise<SearchResult<AssetEntity>> {
const { results } = await this.client.multiSearch.perform({
searches: [
{
collection: assetSchema.name,
q: '*',
vector_query: `smartInfo.clipEmbedding:([${input.join(',')}], k:100)`,
per_page: 100,
facet_by: this.getFacetFieldNames(SearchCollection.ASSETS),
filter_by: this.getAssetFilters(filters),
} as any,
],
});
return this.asResponse(results[0] as SearchResponse<AssetEntity>, filters.debug);
}
private asResponse<T extends DocumentSchema>(
resultsOrError: SearchResponse<T> | MultiSearchError,
debug?: boolean,
): SearchResult<T> {
const { error, code } = resultsOrError as MultiSearchError;
if (error) {
throw new Error(`Typesense multi-search error: ${code} - ${error}`);
}
const results = resultsOrError as SearchResponse<T>;
return {
page: results.page,
total: results.found,
count: results.out_of,
items: (results.hits || []).map((hit) => hit.document),
distances: (results.hits || []).map((hit: any) => hit.vector_distance),
facets: (results.facet_counts || []).map((facet) => ({
counts: facet.counts.map((item) => ({ count: item.count, value: item.value })),
fieldName: facet.field_name as string,
})),
debug: debug ? results : undefined,
} as SearchResult<T>;
}
private async handleError(error: any) {
this.logger.error('Unable to index documents');
const results = error.importResults || [];
let dimsChanged = false;
for (const result of results) {
try {
result.document = JSON.parse(result.document);
if (result.error.includes('Field `smartInfo.clipEmbedding` must have')) {
dimsChanged = true;
this.logger.warn(
`CLIP embedding dimensions have changed, now ${result.document.smartInfo.clipEmbedding.length} dims. Updating schema...`,
);
await this.updateCLIPField(result.document.smartInfo.clipEmbedding.length);
break;
}
if (result.document?.smartInfo?.clipEmbedding) {
result.document.smartInfo.clipEmbedding = '<truncated>';
}
} catch (err: any) {
this.logger.error(`Error while updating CLIP field: ${(err.message, err.stack)}`);
}
}
if (!dimsChanged) {
this.logger.log(JSON.stringify(results, null, 2));
}
}
private async updateAlias(collection: SearchCollection) {
const schema = schemaMap[collection];
const alias = await this.client
.aliases(collection)
.retrieve()
.catch(() => null);
// update alias to current collection
this.logger.log(`Using new schema: ${alias?.collection_name || '(unset)'} => ${schema.name}`);
await this.client.aliases().upsert(collection, { collection_name: schema.name });
// delete previous collection
if (alias && alias.collection_name !== schema.name) {
this.logger.log(`Deleting old schema: ${alias.collection_name}`);
await this.client.collections(alias.collection_name).delete();
}
}
private patch(collection: SearchCollection, items: AssetEntity[] | AlbumEntity[] | OwnedFaceEntity[]) {
return items.map((item) => {
switch (collection) {
case SearchCollection.ASSETS:
return this.patchAsset(item as AssetEntity);
case SearchCollection.ALBUMS:
return this.patchAlbum(item as AlbumEntity);
case SearchCollection.FACES:
return this.patchFace(item as OwnedFaceEntity);
}
});
}
private patchAlbum(album: AlbumEntity): AlbumEntity {
return removeNil(album);
}
private patchAsset(asset: AssetEntity): CustomAssetEntity {
let custom = asset as CustomAssetEntity;
const lat = asset.exifInfo?.latitude;
const lng = asset.exifInfo?.longitude;
if (lat && lng && lat !== 0 && lng !== 0) {
custom = { ...custom, geo: [lat, lng] };
}
const people = asset.faces
?.filter((face) => !face.person?.isHidden && face.person?.name)
.map((face) => face.person?.name)
.filter((name) => name !== undefined) as string[];
if (people.length) {
custom = { ...custom, people };
}
return removeNil({ ...custom, motion: !!asset.livePhotoVideoId });
}
private patchFace(face: OwnedFaceEntity): OwnedFaceEntity {
return removeNil(face);
}
private getFacetFieldNames(collection: SearchCollection) {
return (schemaMap[collection].fields || [])
.filter((field) => field.facet)
.map((field) => field.name)
.join(',');
}
private getAlbumFilters(filters: SearchFilter) {
const { userId } = filters;
const _filters = [this.buildFilterBy('ownerId', userId, true)];
if (filters.id) {
_filters.push(this.buildFilterBy('id', filters.id, true));
}
for (const item of albumSchema.fields || []) {
const value = filters[item.name as keyof SearchFilter];
if (item.facet && value !== undefined) {
_filters.push(this.buildFilterBy(item.name, value));
}
}
const result = _filters.join(' && ');
this.logger.debug(`Album filters are: ${result}`);
return result;
}
private getAssetFilters(filters: SearchFilter) {
const { userId } = filters;
const _filters = [this.buildFilterBy('ownerId', userId, true), this.buildFilterBy('isArchived', false)];
if (filters.id) {
_filters.push(this.buildFilterBy('id', filters.id, true));
}
for (const item of assetSchema.fields || []) {
const value = filters[item.name as keyof SearchFilter];
if (item.facet && value !== undefined) {
_filters.push(this.buildFilterBy(item.name, value));
}
}
const result = _filters.join(' && ');
this.logger.debug(`Asset filters are: ${result}`);
return result;
}
private buildFilterBy(key: string, values: boolean | string | string[], exact?: boolean) {
const token = exact ? ':=' : ':';
const _values = (Array.isArray(values) ? values : [values]).map((value) => {
if (typeof value === 'boolean' || value === 'true' || value === 'false') {
return value;
}
return '`' + value + '`';
});
const value = _values.length > 1 ? `[${_values.join(',')}]` : _values[0];
return `${key}${token}${value}`;
}
}

5
server/src/infra/sql-generator/index.ts
View File

@@ -1,3 +1,4 @@
import { ISystemConfigRepository } from '@app/domain';
import { INestApplication } from '@nestjs/common';
import { Reflector } from '@nestjs/core';
import { Test } from '@nestjs/testing';
@@ -18,6 +19,7 @@ import {
PartnerRepository,
PersonRepository,
SharedLinkRepository,
SmartInfoRepository,
SystemConfigRepository,
SystemMetadataRepository,
TagRepository,
@@ -38,6 +40,7 @@ const repositories = [
PartnerRepository,
PersonRepository,
SharedLinkRepository,
SmartInfoRepository,
SystemConfigRepository,
SystemMetadataRepository,
TagRepository,
@@ -82,7 +85,7 @@ class SqlGenerator {
}),
TypeOrmModule.forFeature(databaseEntities),
],
providers: repositories,
providers: [{ provide: ISystemConfigRepository, useClass: SystemConfigRepository }, ...repositories],
}).compile();
this.app = await moduleFixture.createNestApplication().init();

80
server/src/infra/sql/asset.repository.sql
View File

@@ -57,8 +57,7 @@ SELECT
"AssetEntity__AssetEntity_exifInfo"."profileDescription" AS "AssetEntity__AssetEntity_exifInfo_profileDescription",
"AssetEntity__AssetEntity_exifInfo"."colorspace" AS "AssetEntity__AssetEntity_exifInfo_colorspace",
"AssetEntity__AssetEntity_exifInfo"."bitsPerSample" AS "AssetEntity__AssetEntity_exifInfo_bitsPerSample",
"AssetEntity__AssetEntity_exifInfo"."fps" AS "AssetEntity__AssetEntity_exifInfo_fps",
"AssetEntity__AssetEntity_exifInfo"."exifTextSearchableColumn" AS "AssetEntity__AssetEntity_exifInfo_exifTextSearchableColumn"
"AssetEntity__AssetEntity_exifInfo"."fps" AS "AssetEntity__AssetEntity_exifInfo_fps"
FROM
"assets" "AssetEntity"
LEFT JOIN "exif" "AssetEntity__AssetEntity_exifInfo" ON "AssetEntity__AssetEntity_exifInfo"."assetId" = "AssetEntity"."id"
@@ -133,8 +132,7 @@ SELECT
"exifInfo"."profileDescription" AS "exifInfo_profileDescription",
"exifInfo"."colorspace" AS "exifInfo_colorspace",
"exifInfo"."bitsPerSample" AS "exifInfo_bitsPerSample",
"exifInfo"."fps" AS "exifInfo_fps",
"exifInfo"."exifTextSearchableColumn" AS "exifInfo_exifTextSearchableColumn"
"exifInfo"."fps" AS "exifInfo_fps"
FROM
"assets" "entity"
LEFT JOIN "exif" "exifInfo" ON "exifInfo"."assetId" = "entity"."id"
@@ -217,11 +215,9 @@ SELECT
"AssetEntity__AssetEntity_exifInfo"."colorspace" AS "AssetEntity__AssetEntity_exifInfo_colorspace",
"AssetEntity__AssetEntity_exifInfo"."bitsPerSample" AS "AssetEntity__AssetEntity_exifInfo_bitsPerSample",
"AssetEntity__AssetEntity_exifInfo"."fps" AS "AssetEntity__AssetEntity_exifInfo_fps",
"AssetEntity__AssetEntity_exifInfo"."exifTextSearchableColumn" AS "AssetEntity__AssetEntity_exifInfo_exifTextSearchableColumn",
"AssetEntity__AssetEntity_smartInfo"."assetId" AS "AssetEntity__AssetEntity_smartInfo_assetId",
"AssetEntity__AssetEntity_smartInfo"."tags" AS "AssetEntity__AssetEntity_smartInfo_tags",
"AssetEntity__AssetEntity_smartInfo"."objects" AS "AssetEntity__AssetEntity_smartInfo_objects",
"AssetEntity__AssetEntity_smartInfo"."clipEmbedding" AS "AssetEntity__AssetEntity_smartInfo_clipEmbedding",
"AssetEntity__AssetEntity_tags"."id" AS "AssetEntity__AssetEntity_tags_id",
"AssetEntity__AssetEntity_tags"."type" AS "AssetEntity__AssetEntity_tags_type",
"AssetEntity__AssetEntity_tags"."name" AS "AssetEntity__AssetEntity_tags_name",
@@ -230,7 +226,6 @@ SELECT
"AssetEntity__AssetEntity_faces"."id" AS "AssetEntity__AssetEntity_faces_id",
"AssetEntity__AssetEntity_faces"."assetId" AS "AssetEntity__AssetEntity_faces_assetId",
"AssetEntity__AssetEntity_faces"."personId" AS "AssetEntity__AssetEntity_faces_personId",
"AssetEntity__AssetEntity_faces"."embedding" AS "AssetEntity__AssetEntity_faces_embedding",
"AssetEntity__AssetEntity_faces"."imageWidth" AS "AssetEntity__AssetEntity_faces_imageWidth",
"AssetEntity__AssetEntity_faces"."imageHeight" AS "AssetEntity__AssetEntity_faces_imageHeight",
"AssetEntity__AssetEntity_faces"."boundingBoxX1" AS "AssetEntity__AssetEntity_faces_boundingBoxX1",
@@ -439,7 +434,6 @@ FROM
"AssetEntity__AssetEntity_faces"."id" AS "AssetEntity__AssetEntity_faces_id",
"AssetEntity__AssetEntity_faces"."assetId" AS "AssetEntity__AssetEntity_faces_assetId",
"AssetEntity__AssetEntity_faces"."personId" AS "AssetEntity__AssetEntity_faces_personId",
"AssetEntity__AssetEntity_faces"."embedding" AS "AssetEntity__AssetEntity_faces_embedding",
"AssetEntity__AssetEntity_faces"."imageWidth" AS "AssetEntity__AssetEntity_faces_imageWidth",
"AssetEntity__AssetEntity_faces"."imageHeight" AS "AssetEntity__AssetEntity_faces_imageHeight",
"AssetEntity__AssetEntity_faces"."boundingBoxX1" AS "AssetEntity__AssetEntity_faces_boundingBoxX1",
@@ -612,3 +606,73 @@ ORDER BY
"AssetEntity"."createdAt" ASC
LIMIT
11
-- AssetRepository.getAssetIdByCity
WITH
"cities" AS (
SELECT
city
FROM
"exif" "e"
GROUP BY
city
HAVING
count(city) >= $1
ORDER BY
random() ASC
LIMIT
12
)
SELECT DISTINCT
ON (c.city) "asset"."id" AS "data",
c.city AS "value"
FROM
"assets" "asset"
INNER JOIN "exif" "e" ON "asset"."id" = e."assetId"
INNER JOIN "cities" "c" ON c.city = "e"."city"
WHERE
(
"asset"."isVisible" = true
AND "asset"."fileCreatedAt" < NOW()
AND "asset"."type" = $2
AND "asset"."ownerId" IN ($3)
AND "asset"."isArchived" = $4
)
AND ("asset"."deletedAt" IS NULL)
LIMIT
12
-- AssetRepository.getAssetIdByTag
WITH
"random_tags" AS (
SELECT
unnest(tags) AS "tag"
FROM
"smart_info" "si"
GROUP BY
tag
HAVING
count(*) >= $1
ORDER BY
random() ASC
LIMIT
12
)
SELECT DISTINCT
ON (unnest("si"."tags")) "asset"."id" AS "data",
unnest("si"."tags") AS "value"
FROM
"assets" "asset"
INNER JOIN "smart_info" "si" ON "asset"."id" = si."assetId"
INNER JOIN "random_tags" "t" ON "si"."tags" @> ARRAY[t.tag]
WHERE
(
"asset"."isVisible" = true
AND "asset"."fileCreatedAt" < NOW()
AND "asset"."type" = $2
AND "asset"."ownerId" IN ($3)
AND "asset"."isArchived" = $4
)
AND ("asset"."deletedAt" IS NULL)
LIMIT
12

10
server/src/infra/sql/person.repository.sql
View File

@@ -12,7 +12,6 @@ SELECT
"AssetFaceEntity"."id" AS "AssetFaceEntity_id",
"AssetFaceEntity"."assetId" AS "AssetFaceEntity_assetId",
"AssetFaceEntity"."personId" AS "AssetFaceEntity_personId",
"AssetFaceEntity"."embedding" AS "AssetFaceEntity_embedding",
"AssetFaceEntity"."imageWidth" AS "AssetFaceEntity_imageWidth",
"AssetFaceEntity"."imageHeight" AS "AssetFaceEntity_imageHeight",
"AssetFaceEntity"."boundingBoxX1" AS "AssetFaceEntity_boundingBoxX1",
@@ -138,7 +137,6 @@ SELECT
"AssetFaceEntity"."id" AS "AssetFaceEntity_id",
"AssetFaceEntity"."assetId" AS "AssetFaceEntity_assetId",
"AssetFaceEntity"."personId" AS "AssetFaceEntity_personId",
"AssetFaceEntity"."embedding" AS "AssetFaceEntity_embedding",
"AssetFaceEntity"."imageWidth" AS "AssetFaceEntity_imageWidth",
"AssetFaceEntity"."imageHeight" AS "AssetFaceEntity_imageHeight",
"AssetFaceEntity"."boundingBoxX1" AS "AssetFaceEntity_boundingBoxX1",
@@ -169,7 +167,6 @@ FROM
"AssetFaceEntity"."id" AS "AssetFaceEntity_id",
"AssetFaceEntity"."assetId" AS "AssetFaceEntity_assetId",
"AssetFaceEntity"."personId" AS "AssetFaceEntity_personId",
"AssetFaceEntity"."embedding" AS "AssetFaceEntity_embedding",
"AssetFaceEntity"."imageWidth" AS "AssetFaceEntity_imageWidth",
"AssetFaceEntity"."imageHeight" AS "AssetFaceEntity_imageHeight",
"AssetFaceEntity"."boundingBoxX1" AS "AssetFaceEntity_boundingBoxX1",
@@ -205,7 +202,6 @@ FROM
"AssetFaceEntity"."id" AS "AssetFaceEntity_id",
"AssetFaceEntity"."assetId" AS "AssetFaceEntity_assetId",
"AssetFaceEntity"."personId" AS "AssetFaceEntity_personId",
"AssetFaceEntity"."embedding" AS "AssetFaceEntity_embedding",
"AssetFaceEntity"."imageWidth" AS "AssetFaceEntity_imageWidth",
"AssetFaceEntity"."imageHeight" AS "AssetFaceEntity_imageHeight",
"AssetFaceEntity"."boundingBoxX1" AS "AssetFaceEntity_boundingBoxX1",
@@ -351,7 +347,6 @@ FROM
"AssetEntity__AssetEntity_faces"."id" AS "AssetEntity__AssetEntity_faces_id",
"AssetEntity__AssetEntity_faces"."assetId" AS "AssetEntity__AssetEntity_faces_assetId",
"AssetEntity__AssetEntity_faces"."personId" AS "AssetEntity__AssetEntity_faces_personId",
"AssetEntity__AssetEntity_faces"."embedding" AS "AssetEntity__AssetEntity_faces_embedding",
"AssetEntity__AssetEntity_faces"."imageWidth" AS "AssetEntity__AssetEntity_faces_imageWidth",
"AssetEntity__AssetEntity_faces"."imageHeight" AS "AssetEntity__AssetEntity_faces_imageHeight",
"AssetEntity__AssetEntity_faces"."boundingBoxX1" AS "AssetEntity__AssetEntity_faces_boundingBoxX1",
@@ -393,8 +388,7 @@ FROM
"AssetEntity__AssetEntity_exifInfo"."profileDescription" AS "AssetEntity__AssetEntity_exifInfo_profileDescription",
"AssetEntity__AssetEntity_exifInfo"."colorspace" AS "AssetEntity__AssetEntity_exifInfo_colorspace",
"AssetEntity__AssetEntity_exifInfo"."bitsPerSample" AS "AssetEntity__AssetEntity_exifInfo_bitsPerSample",
"AssetEntity__AssetEntity_exifInfo"."fps" AS "AssetEntity__AssetEntity_exifInfo_fps",
"AssetEntity__AssetEntity_exifInfo"."exifTextSearchableColumn" AS "AssetEntity__AssetEntity_exifInfo_exifTextSearchableColumn"
"AssetEntity__AssetEntity_exifInfo"."fps" AS "AssetEntity__AssetEntity_exifInfo_fps"
FROM
"assets" "AssetEntity"
LEFT JOIN "asset_faces" "AssetEntity__AssetEntity_faces" ON "AssetEntity__AssetEntity_faces"."assetId" = "AssetEntity"."id"
@@ -421,7 +415,6 @@ SELECT
"AssetFaceEntity"."id" AS "AssetFaceEntity_id",
"AssetFaceEntity"."assetId" AS "AssetFaceEntity_assetId",
"AssetFaceEntity"."personId" AS "AssetFaceEntity_personId",
"AssetFaceEntity"."embedding" AS "AssetFaceEntity_embedding",
"AssetFaceEntity"."imageWidth" AS "AssetFaceEntity_imageWidth",
"AssetFaceEntity"."imageHeight" AS "AssetFaceEntity_imageHeight",
"AssetFaceEntity"."boundingBoxX1" AS "AssetFaceEntity_boundingBoxX1",
@@ -473,7 +466,6 @@ SELECT
"AssetFaceEntity"."id" AS "AssetFaceEntity_id",
"AssetFaceEntity"."assetId" AS "AssetFaceEntity_assetId",
"AssetFaceEntity"."personId" AS "AssetFaceEntity_personId",
"AssetFaceEntity"."embedding" AS "AssetFaceEntity_embedding",
"AssetFaceEntity"."imageWidth" AS "AssetFaceEntity_imageWidth",
"AssetFaceEntity"."imageHeight" AS "AssetFaceEntity_imageHeight",
"AssetFaceEntity"."boundingBoxX1" AS "AssetFaceEntity_boundingBoxX1",

2
server/src/infra/sql/shared.link.repository.sql
View File

@@ -77,7 +77,6 @@ FROM
"9b1d35b344d838023994a3233afd6ffe098be6d8"."colorspace" AS "9b1d35b344d838023994a3233afd6ffe098be6d8_colorspace",
"9b1d35b344d838023994a3233afd6ffe098be6d8"."bitsPerSample" AS "9b1d35b344d838023994a3233afd6ffe098be6d8_bitsPerSample",
"9b1d35b344d838023994a3233afd6ffe098be6d8"."fps" AS "9b1d35b344d838023994a3233afd6ffe098be6d8_fps",
"9b1d35b344d838023994a3233afd6ffe098be6d8"."exifTextSearchableColumn" AS "e18de9deffa83f81ac3c43b5e8c2f08dba727bf8",
"SharedLinkEntity__SharedLinkEntity_album"."id" AS "SharedLinkEntity__SharedLinkEntity_album_id",
"SharedLinkEntity__SharedLinkEntity_album"."ownerId" AS "SharedLinkEntity__SharedLinkEntity_album_ownerId",
"SharedLinkEntity__SharedLinkEntity_album"."albumName" AS "SharedLinkEntity__SharedLinkEntity_album_albumName",
@@ -143,7 +142,6 @@ FROM
"d9f2f4dd8920bad1d6907cdb1d699732daff3c2f"."colorspace" AS "d9f2f4dd8920bad1d6907cdb1d699732daff3c2f_colorspace",
"d9f2f4dd8920bad1d6907cdb1d699732daff3c2f"."bitsPerSample" AS "d9f2f4dd8920bad1d6907cdb1d699732daff3c2f_bitsPerSample",
"d9f2f4dd8920bad1d6907cdb1d699732daff3c2f"."fps" AS "d9f2f4dd8920bad1d6907cdb1d699732daff3c2f_fps",
"d9f2f4dd8920bad1d6907cdb1d699732daff3c2f"."exifTextSearchableColumn" AS "96535c8046de591cca9b8c5825e6c5db502b0e6a",
"6d7fd45329a05fd86b3dbcacde87fe76e33a422d"."id" AS "6d7fd45329a05fd86b3dbcacde87fe76e33a422d_id",
"6d7fd45329a05fd86b3dbcacde87fe76e33a422d"."name" AS "6d7fd45329a05fd86b3dbcacde87fe76e33a422d_name",
"6d7fd45329a05fd86b3dbcacde87fe76e33a422d"."avatarColor" AS "6d7fd45329a05fd86b3dbcacde87fe76e33a422d_avatarColor",

111
server/src/infra/sql/smart.info.repository.sql Normal file
View File

@@ -0,0 +1,111 @@
-- NOTE: This file is auto generated by ./sql-generator
-- SmartInfoRepository.searchCLIP
START TRANSACTION
SET
LOCAL vectors.k = '100'
SELECT
"a"."id" AS "a_id",
"a"."deviceAssetId" AS "a_deviceAssetId",
"a"."ownerId" AS "a_ownerId",
"a"."libraryId" AS "a_libraryId",
"a"."deviceId" AS "a_deviceId",
"a"."type" AS "a_type",
"a"."originalPath" AS "a_originalPath",
"a"."resizePath" AS "a_resizePath",
"a"."webpPath" AS "a_webpPath",
"a"."thumbhash" AS "a_thumbhash",
"a"."encodedVideoPath" AS "a_encodedVideoPath",
"a"."createdAt" AS "a_createdAt",
"a"."updatedAt" AS "a_updatedAt",
"a"."deletedAt" AS "a_deletedAt",
"a"."fileCreatedAt" AS "a_fileCreatedAt",
"a"."localDateTime" AS "a_localDateTime",
"a"."fileModifiedAt" AS "a_fileModifiedAt",
"a"."isFavorite" AS "a_isFavorite",
"a"."isArchived" AS "a_isArchived",
"a"."isExternal" AS "a_isExternal",
"a"."isReadOnly" AS "a_isReadOnly",
"a"."isOffline" AS "a_isOffline",
"a"."checksum" AS "a_checksum",
"a"."duration" AS "a_duration",
"a"."isVisible" AS "a_isVisible",
"a"."livePhotoVideoId" AS "a_livePhotoVideoId",
"a"."originalFileName" AS "a_originalFileName",
"a"."sidecarPath" AS "a_sidecarPath",
"a"."stackParentId" AS "a_stackParentId",
"e"."assetId" AS "e_assetId",
"e"."description" AS "e_description",
"e"."exifImageWidth" AS "e_exifImageWidth",
"e"."exifImageHeight" AS "e_exifImageHeight",
"e"."fileSizeInByte" AS "e_fileSizeInByte",
"e"."orientation" AS "e_orientation",
"e"."dateTimeOriginal" AS "e_dateTimeOriginal",
"e"."modifyDate" AS "e_modifyDate",
"e"."timeZone" AS "e_timeZone",
"e"."latitude" AS "e_latitude",
"e"."longitude" AS "e_longitude",
"e"."projectionType" AS "e_projectionType",
"e"."city" AS "e_city",
"e"."livePhotoCID" AS "e_livePhotoCID",
"e"."state" AS "e_state",
"e"."country" AS "e_country",
"e"."make" AS "e_make",
"e"."model" AS "e_model",
"e"."lensModel" AS "e_lensModel",
"e"."fNumber" AS "e_fNumber",
"e"."focalLength" AS "e_focalLength",
"e"."iso" AS "e_iso",
"e"."exposureTime" AS "e_exposureTime",
"e"."profileDescription" AS "e_profileDescription",
"e"."colorspace" AS "e_colorspace",
"e"."bitsPerSample" AS "e_bitsPerSample",
"e"."fps" AS "e_fps"
FROM
"assets" "a"
INNER JOIN "smart_search" "s" ON "s"."assetId" = "a"."id"
LEFT JOIN "exif" "e" ON "e"."assetId" = "a"."id"
WHERE
("a"."ownerId" = $1)
AND ("a"."deletedAt" IS NULL)
ORDER BY
"s"."embedding" <= > $2 ASC
LIMIT
100
COMMIT
-- SmartInfoRepository.searchFaces
START TRANSACTION
SET
LOCAL vectors.k = '100'
WITH
"cte" AS (
SELECT
"faces"."id" AS "faces_id",
"faces"."assetId" AS "faces_assetId",
"faces"."personId" AS "faces_personId",
"faces"."imageWidth" AS "faces_imageWidth",
"faces"."imageHeight" AS "faces_imageHeight",
"faces"."boundingBoxX1" AS "faces_boundingBoxX1",
"faces"."boundingBoxY1" AS "faces_boundingBoxY1",
"faces"."boundingBoxX2" AS "faces_boundingBoxX2",
"faces"."boundingBoxY2" AS "faces_boundingBoxY2",
1 + ("faces"."embedding" <= > $1) AS "distance"
FROM
"asset_faces" "faces"
INNER JOIN "assets" "asset" ON "asset"."id" = "faces"."assetId"
AND ("asset"."deletedAt" IS NULL)
WHERE
"asset"."ownerId" = $2
ORDER BY
"faces"."embedding" <= > $3 ASC
LIMIT
100
)
SELECT
res.*
FROM
"cte" "res"
WHERE
res.distance <= $4
COMMIT

14
server/src/infra/typesense-schemas/album.schema.ts
View File

@@ -1,14 +0,0 @@
import { CollectionCreateSchema } from 'typesense/lib/Typesense/Collections';
export const albumSchemaVersion = 2;
export const albumSchema: CollectionCreateSchema = {
name: `albums-v${albumSchemaVersion}`,
fields: [
{ name: 'ownerId', type: 'string', facet: false },
{ name: 'albumName', type: 'string', facet: false, sort: true },
{ name: 'description', type: 'string', facet: false },
{ name: 'createdAt', type: 'string', facet: false, sort: true },
{ name: 'updatedAt', type: 'string', facet: false, sort: true },
],
default_sorting_field: 'createdAt',
};

42
server/src/infra/typesense-schemas/asset.schema.ts
View File

@@ -1,42 +0,0 @@
import { CollectionCreateSchema } from 'typesense/lib/Typesense/Collections';
export const assetSchemaVersion = 10;
export const assetSchema: CollectionCreateSchema = {
name: `assets-v${assetSchemaVersion}`,
fields: [
// asset
{ name: 'ownerId', type: 'string', facet: false },
{ name: 'type', type: 'string', facet: true },
{ name: 'originalPath', type: 'string', facet: false },
{ name: 'createdAt', type: 'string', facet: false, sort: true },
{ name: 'updatedAt', type: 'string', facet: false, sort: true },
{ name: 'fileCreatedAt', type: 'string', facet: false, sort: true },
{ name: 'fileModifiedAt', type: 'string', facet: false, sort: true },
{ name: 'isFavorite', type: 'bool', facet: true },
{ name: 'isArchived', type: 'bool', facet: true },
{ name: 'originalFileName', type: 'string', facet: false, optional: true },
// exif
{ name: 'exifInfo.city', type: 'string', facet: true, optional: true },
{ name: 'exifInfo.country', type: 'string', facet: true, optional: true },
{ name: 'exifInfo.state', type: 'string', facet: true, optional: true },
{ name: 'exifInfo.description', type: 'string', facet: false, optional: true },
{ name: 'exifInfo.make', type: 'string', facet: true, optional: true },
{ name: 'exifInfo.model', type: 'string', facet: true, optional: true },
{ name: 'exifInfo.orientation', type: 'string', optional: true },
{ name: 'exifInfo.projectionType', type: 'string', facet: true, optional: true },
// smart info
{ name: 'smartInfo.objects', type: 'string[]', facet: true, optional: true },
{ name: 'smartInfo.tags', type: 'string[]', facet: true, optional: true },
{ name: 'smartInfo.clipEmbedding', type: 'float[]', facet: false, optional: true, num_dim: 512 },
// computed
{ name: 'geo', type: 'geopoint', facet: false, optional: true },
{ name: 'motion', type: 'bool', facet: true },
{ name: 'people', type: 'string[]', facet: true, optional: true },
],
token_separators: ['.', '-', '_'],
enable_nested_fields: true,
default_sorting_field: 'fileCreatedAt',
};

12
server/src/infra/typesense-schemas/face.schema.ts
View File

@@ -1,12 +0,0 @@
import { CollectionCreateSchema } from 'typesense/lib/Typesense/Collections';
export const faceSchemaVersion = 1;
export const faceSchema: CollectionCreateSchema = {
name: `faces-v${faceSchemaVersion}`,
fields: [
{ name: 'ownerId', type: 'string', facet: false },
{ name: 'assetId', type: 'string', facet: false },
{ name: 'personId', type: 'string', facet: false },
{ name: 'embedding', type: 'float[]', facet: false, num_dim: 512 },
],
};

3
server/src/infra/typesense-schemas/index.ts
View File

@@ -1,3 +0,0 @@
export * from './album.schema';
export * from './asset.schema';
export * from './face.schema';

3
server/src/infra/utils/database-locks.ts
View File

@@ -1,3 +0,0 @@
export enum DatabaseLock {
GeodataImport = 100,
}

15
server/src/infra/utils/optional-between.util.ts
View File

@@ -1,15 +0,0 @@
import { Between, LessThanOrEqual, MoreThanOrEqual } from 'typeorm';
/**
* Allows optional values unlike the regular Between and uses MoreThanOrEqual
* or LessThanOrEqual when only one parameter is specified.
*/
export default function OptionalBetween<T>(from?: T, to?: T) {
if (from && to) {
return Between(from, to);
} else if (from) {
return MoreThanOrEqual(from);
} else if (to) {
return LessThanOrEqual(to);
}
}

20
server/src/infra/utils/pagination.util.ts
View File

@@ -1,20 +0,0 @@
import { Paginated, PaginationOptions } from '@app/domain';
import { FindOneOptions, ObjectLiteral, Repository } from 'typeorm';
export async function paginate<Entity extends ObjectLiteral>(
repository: Repository<Entity>,
paginationOptions: PaginationOptions,
searchOptions?: FindOneOptions<Entity>,
): Paginated<Entity> {
const items = await repository.find({
...searchOptions,
// Take one more item to check if there's a next page
take: paginationOptions.take + 1,
skip: paginationOptions.skip,
});
const hasNextPage = items.length > paginationOptions.take;
items.splice(paginationOptions.take);
return { items, hasNextPage };
}

12
server/src/microservices/app.service.ts
View File

@@ -8,7 +8,6 @@ import {
MediaService,
MetadataService,
PersonService,
SearchService,
ServerInfoService,
SmartInfoService,
StorageService,
@@ -31,7 +30,6 @@ export class AppService {
private mediaService: MediaService,
private metadataService: MetadataService,
private personService: PersonService,
private searchService: SearchService,
private serverInfoService: ServerInfoService,
private smartInfoService: SmartInfoService,
private storageTemplateService: StorageTemplateService,
@@ -52,15 +50,6 @@ export class AppService {
[JobName.CLASSIFY_IMAGE]: (data) => this.smartInfoService.handleClassifyImage(data),
[JobName.QUEUE_ENCODE_CLIP]: (data) => this.smartInfoService.handleQueueEncodeClip(data),
[JobName.ENCODE_CLIP]: (data) => this.smartInfoService.handleEncodeClip(data),
[JobName.SEARCH_INDEX_ALBUMS]: () => this.searchService.handleIndexAlbums(),
[JobName.SEARCH_INDEX_ASSETS]: () => this.searchService.handleIndexAssets(),
[JobName.SEARCH_INDEX_FACES]: () => this.searchService.handleIndexFaces(),
[JobName.SEARCH_INDEX_ALBUM]: (data) => this.searchService.handleIndexAlbum(data),
[JobName.SEARCH_INDEX_ASSET]: (data) => this.searchService.handleIndexAsset(data),
[JobName.SEARCH_INDEX_FACE]: (data) => this.searchService.handleIndexFace(data),
[JobName.SEARCH_REMOVE_ALBUM]: (data) => this.searchService.handleRemoveAlbum(data),
[JobName.SEARCH_REMOVE_ASSET]: (data) => this.searchService.handleRemoveAsset(data),
[JobName.SEARCH_REMOVE_FACE]: (data) => this.searchService.handleRemoveFace(data),
[JobName.STORAGE_TEMPLATE_MIGRATION]: () => this.storageTemplateService.handleMigration(),
[JobName.STORAGE_TEMPLATE_MIGRATION_SINGLE]: (data) => this.storageTemplateService.handleMigrationSingle(data),
[JobName.QUEUE_MIGRATION]: () => this.mediaService.handleQueueMigration(),
@@ -94,7 +83,6 @@ export class AppService {
});
await this.metadataService.init();
await this.searchService.init();
}
async teardown() {

3
server/src/microservices/main.ts
View File

@@ -1,5 +1,5 @@
import { envName, getLogLevels, serverVersion } from '@app/domain';
import { RedisIoAdapter } from '@app/infra';
import { RedisIoAdapter, enablePrefilter } from '@app/infra';
import { Logger } from '@nestjs/common';
import { NestFactory } from '@nestjs/core';
import { AppService } from './app.service';
@@ -12,6 +12,7 @@ export async function bootstrap() {
const app = await NestFactory.create(MicroservicesModule, { logger: getLogLevels() });
app.useWebSocketAdapter(new RedisIoAdapter(app));
await enablePrefilter();
await app.get(AppService).init();
await app.listen(port);