immich-app/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2025-12-26 09:14:58 +03:00
Code Issues Packages Projects Releases 18 Wiki Activity

Merge branch 'fix/sidecar-check-job' of https://github.com/immich-app/immich into feat/sidecar-asset-files

Browse Source
This commit is contained in:
Jonathan Jogenfors
2025-08-27 13:51:14 +02:00
parent 430af9a145 f3b0e8a5e6
commit 65f63be564
196 changed files with 14871 additions and 3011 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
server/src/dtos/album.dto.ts
View File

@@ -65,10 +65,6 @@ export class AlbumsAddAssetsDto {
export class AlbumsAddAssetsResponseDto {
success!: boolean;
@ApiProperty({ type: 'integer' })
albumSuccessCount!: number;
@ApiProperty({ type: 'integer' })
assetSuccessCount!: number;
@ValidateEnum({ enum: BulkIdErrorReason, name: 'BulkIdErrorReason', optional: true })
error?: BulkIdErrorReason;
}

8
server/src/dtos/asset-response.dto.ts
View File

@@ -37,6 +37,13 @@ export class SanitizedAssetResponseDto {
}
export class AssetResponseDto extends SanitizedAssetResponseDto {
@ApiProperty({
type: 'string',
format: 'date-time',
description: 'The UTC timestamp when the asset was originally uploaded to Immich.',
example: '2024-01-15T20:30:00.000Z',
})
createdAt!: Date;
deviceAssetId!: string;
deviceId!: string;
ownerId!: string;
@@ -189,6 +196,7 @@ export function mapAsset(entity: MapAsset, options: AssetMapOptions = {}): Asset
return {
id: entity.id,
createdAt: entity.createdAt,
deviceAssetId: entity.deviceAssetId,
ownerId: entity.ownerId,
owner: entity.owner ? mapUser(entity.owner) : undefined,

3
server/src/enum.ts
View File

@@ -567,8 +567,7 @@ export enum JobName {
SendMail = 'SendMail',
SidecarQueueAll = 'SidecarQueueAll',
SidecarDiscovery = 'SidecarDiscovery',
SidecarSync = 'SidecarSync',
SidecarCheck = 'SidecarCheck',
SidecarWrite = 'SidecarWrite',
SmartSearchQueueAll = 'SmartSearchQueueAll',

12
server/src/queries/asset.job.repository.sql
View File

@@ -58,6 +58,18 @@ where
limit
$3
-- AssetJobRepository.getForSidecarCheckJob
select
"id",
"sidecarPath",
"originalPath"
from
"asset"
where
"asset"."id" = $1::uuid
limit
$2
-- AssetJobRepository.streamForThumbnailJob
select
"asset"."id",

2
server/src/queries/asset.repository.sql
View File

@@ -277,7 +277,7 @@ with
epoch
from
(
asset."localDateTime" - asset."fileCreatedAt" at time zone 'UTC'
asset."localDateTime" AT TIME ZONE 'UTC' - asset."fileCreatedAt" at time zone 'UTC'
)
)::real / 3600 as "localOffsetHours",
"asset"."ownerId",

6
server/src/queries/shared.link.repository.sql
View File

@@ -38,7 +38,11 @@ from
select
"album".*,
coalesce(
json_agg("assets") filter (
json_agg(
"assets"
order by
"assets"."fileCreatedAt" asc
) filter (
where
"assets"."id" is not null
),

8
server/src/repositories/album.repository.ts
View File

@@ -321,6 +321,14 @@ export class AlbumRepository {
.execute();
}
@Chunked({ chunkSize: 30_000 })
async addAssetIdsToAlbums(values: { albumsId: string; assetsId: string }[]): Promise<void> {
if (values.length === 0) {
return;
}
await this.db.insertInto('album_asset').values(values).execute();
}
/**
* Makes sure all thumbnails for albums are updated by:
* - Removing thumbnails from albums without assets

20
server/src/repositories/asset-job.repository.ts
View File

@@ -39,9 +39,9 @@ export class AssetJobRepository {
return this.db
.selectFrom('asset')
.where('asset.id', '=', asUuid(id))
.select((eb) => [
'id',
'originalPath',
.select(['id', 'originalPath'])
.select((eb) => withFiles(eb, AssetFileType.Sidecar))
.select((eb) =>
jsonArrayFrom(
eb
.selectFrom('tag')
@@ -49,8 +49,18 @@ export class AssetJobRepository {
.innerJoin('tag_asset', 'tag.id', 'tag_asset.tagsId')
.whereRef('asset.id', '=', 'tag_asset.assetsId'),
).as('tags'),
])
.select((eb) => withFiles(eb, AssetFileType.Sidecar))
)
.limit(1)
.executeTakeFirst();
}
@GenerateSql({ params: [DummyValue.UUID] })
getForSidecarCheckJob(id: string) {
return this.db
.selectFrom('asset')
.where('asset.id', '=', asUuid(id))
.select(['id', 'originalPath'])
.select(withFiles)
.limit(1)
.executeTakeFirst();
}

2
server/src/repositories/asset.repository.ts
View File

@@ -566,7 +566,7 @@ export class AssetRepository {
sql`asset.type = 'IMAGE'`.as('isImage'),
sql`asset."deletedAt" is not null`.as('isTrashed'),
'asset.livePhotoVideoId',
sql`extract(epoch from (asset."localDateTime" - asset."fileCreatedAt" at time zone 'UTC'))::real / 3600`.as(
sql`extract(epoch from (asset."localDateTime" AT TIME ZONE 'UTC' - asset."fileCreatedAt" at time zone 'UTC'))::real / 3600`.as(
'localOffsetHours',
),
'asset.ownerId',

15
server/src/repositories/person.repository.ts
View File

@@ -1,5 +1,5 @@
import { Injectable } from '@nestjs/common';
import { ExpressionBuilder, Insertable, Kysely, Selectable, sql, Updateable } from 'kysely';
import { ExpressionBuilder, Insertable, Kysely, NotNull, Selectable, sql, Updateable } from 'kysely';
import { jsonObjectFrom } from 'kysely/helpers/postgres';
import { InjectKysely } from 'nestjs-kysely';
import { Chunked, ChunkedArray, DummyValue, GenerateSql } from 'src/decorators';
@@ -68,12 +68,6 @@ const withPerson = (eb: ExpressionBuilder<DB, 'asset_face'>) => {
).as('person');
};
const withAsset = (eb: ExpressionBuilder<DB, 'asset_face'>) => {
return jsonObjectFrom(eb.selectFrom('asset').selectAll('asset').whereRef('asset.id', '=', 'asset_face.assetId')).as(
'asset',
);
};
const withFaceSearch = (eb: ExpressionBuilder<DB, 'asset_face'>) => {
return jsonObjectFrom(
eb.selectFrom('face_search').selectAll('face_search').whereRef('face_search.faceId', '=', 'asset_face.id'),
@@ -481,7 +475,12 @@ export class PersonRepository {
return this.db
.selectFrom('asset_face')
.selectAll('asset_face')
.select(withAsset)
.select((eb) =>
jsonObjectFrom(eb.selectFrom('asset').selectAll('asset').whereRef('asset.id', '=', 'asset_face.assetId')).as(
'asset',
),
)
.$narrowType<{ asset: NotNull }>()
.select(withPerson)
.where('asset_face.assetId', 'in', assetIds)
.where('asset_face.personId', 'in', personIds)

11
server/src/repositories/shared-link.repository.ts
View File

@@ -86,7 +86,16 @@ export class SharedLinkRepository {
(join) => join.onTrue(),
)
.select((eb) =>
eb.fn.coalesce(eb.fn.jsonAgg('assets').filterWhere('assets.id', 'is not', null), sql`'[]'`).as('assets'),
eb.fn
.coalesce(
eb.fn
.jsonAgg('assets')
.orderBy('assets.fileCreatedAt', 'asc')
.filterWhere('assets.id', 'is not', null),
sql`'[]'`,
)
.as('assets'),
)
.select((eb) => eb.fn.toJson('owner').as('owner'))
.groupBy(['album.id', sql`"owner".*`])

116
server/src/services/album.service.spec.ts
View File

@@ -778,9 +778,7 @@ describe(AlbumService.name, () => {
describe('addAssetsToAlbums', () => {
it('should allow the owner to add assets', async () => {
mocks.access.album.checkOwnerAccess
.mockResolvedValueOnce(new Set(['album-123']))
.mockResolvedValueOnce(new Set(['album-321']));
mocks.access.album.checkOwnerAccess.mockResolvedValueOnce(new Set(['album-123', 'album-321']));
mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1', 'asset-2', 'asset-3']));
mocks.album.getById
.mockResolvedValueOnce(_.cloneDeep(albumStub.empty))
@@ -792,7 +790,7 @@ describe(AlbumService.name, () => {
albumIds: ['album-123', 'album-321'],
assetIds: ['asset-1', 'asset-2', 'asset-3'],
}),
).resolves.toEqual({ success: true, albumSuccessCount: 2, assetSuccessCount: 3 });
).resolves.toEqual({ success: true, error: undefined });
expect(mocks.album.update).toHaveBeenCalledTimes(2);
expect(mocks.album.update).toHaveBeenNthCalledWith(1, 'album-123', {
@@ -805,14 +803,18 @@ describe(AlbumService.name, () => {
updatedAt: expect.any(Date),
albumThumbnailAssetId: 'asset-1',
});
expect(mocks.album.addAssetIds).toHaveBeenCalledWith('album-123', ['asset-1', 'asset-2', 'asset-3']);
expect(mocks.album.addAssetIds).toHaveBeenCalledWith('album-321', ['asset-1', 'asset-2', 'asset-3']);
expect(mocks.album.addAssetIdsToAlbums).toHaveBeenCalledWith([
{ albumsId: 'album-123', assetsId: 'asset-1' },
{ albumsId: 'album-123', assetsId: 'asset-2' },
{ albumsId: 'album-123', assetsId: 'asset-3' },
{ albumsId: 'album-321', assetsId: 'asset-1' },
{ albumsId: 'album-321', assetsId: 'asset-2' },
{ albumsId: 'album-321', assetsId: 'asset-3' },
]);
});
it('should not set the thumbnail if the album has one already', async () => {
mocks.access.album.checkOwnerAccess
.mockResolvedValueOnce(new Set(['album-123']))
.mockResolvedValueOnce(new Set(['album-321']));
mocks.access.album.checkOwnerAccess.mockResolvedValueOnce(new Set(['album-123', 'album-321']));
mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1', 'asset-2', 'asset-3']));
mocks.album.getById
.mockResolvedValueOnce(_.cloneDeep({ ...albumStub.empty, albumThumbnailAssetId: 'asset-id' }))
@@ -824,7 +826,7 @@ describe(AlbumService.name, () => {
albumIds: ['album-123', 'album-321'],
assetIds: ['asset-1', 'asset-2', 'asset-3'],
}),
).resolves.toEqual({ success: true, albumSuccessCount: 2, assetSuccessCount: 3 });
).resolves.toEqual({ success: true, error: undefined });
expect(mocks.album.update).toHaveBeenCalledTimes(2);
expect(mocks.album.update).toHaveBeenNthCalledWith(1, 'album-123', {
@@ -837,14 +839,18 @@ describe(AlbumService.name, () => {
updatedAt: expect.any(Date),
albumThumbnailAssetId: 'asset-id',
});
expect(mocks.album.addAssetIds).toHaveBeenCalledWith('album-123', ['asset-1', 'asset-2', 'asset-3']);
expect(mocks.album.addAssetIds).toHaveBeenCalledWith('album-321', ['asset-1', 'asset-2', 'asset-3']);
expect(mocks.album.addAssetIdsToAlbums).toHaveBeenCalledWith([
{ albumsId: 'album-123', assetsId: 'asset-1' },
{ albumsId: 'album-123', assetsId: 'asset-2' },
{ albumsId: 'album-123', assetsId: 'asset-3' },
{ albumsId: 'album-321', assetsId: 'asset-1' },
{ albumsId: 'album-321', assetsId: 'asset-2' },
{ albumsId: 'album-321', assetsId: 'asset-3' },
]);
});
it('should allow a shared user to add assets', async () => {
mocks.access.album.checkSharedAlbumAccess
.mockResolvedValueOnce(new Set(['album-123']))
.mockResolvedValueOnce(new Set(['album-321']));
mocks.access.album.checkSharedAlbumAccess.mockResolvedValueOnce(new Set(['album-123', 'album-321']));
mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1', 'asset-2', 'asset-3']));
mocks.album.getById
.mockResolvedValueOnce(_.cloneDeep(albumStub.sharedWithUser))
@@ -856,7 +862,7 @@ describe(AlbumService.name, () => {
albumIds: ['album-123', 'album-321'],
assetIds: ['asset-1', 'asset-2', 'asset-3'],
}),
).resolves.toEqual({ success: true, albumSuccessCount: 2, assetSuccessCount: 3 });
).resolves.toEqual({ success: true, error: undefined });
expect(mocks.album.update).toHaveBeenCalledTimes(2);
expect(mocks.album.update).toHaveBeenNthCalledWith(1, 'album-123', {
@@ -869,8 +875,14 @@ describe(AlbumService.name, () => {
updatedAt: expect.any(Date),
albumThumbnailAssetId: 'asset-1',
});
expect(mocks.album.addAssetIds).toHaveBeenCalledWith('album-123', ['asset-1', 'asset-2', 'asset-3']);
expect(mocks.album.addAssetIds).toHaveBeenCalledWith('album-321', ['asset-1', 'asset-2', 'asset-3']);
expect(mocks.album.addAssetIdsToAlbums).toHaveBeenCalledWith([
{ albumsId: 'album-123', assetsId: 'asset-1' },
{ albumsId: 'album-123', assetsId: 'asset-2' },
{ albumsId: 'album-123', assetsId: 'asset-3' },
{ albumsId: 'album-321', assetsId: 'asset-1' },
{ albumsId: 'album-321', assetsId: 'asset-2' },
{ albumsId: 'album-321', assetsId: 'asset-3' },
]);
expect(mocks.event.emit).toHaveBeenCalledWith('AlbumUpdate', {
id: 'album-123',
recipientId: 'admin_id',
@@ -896,18 +908,14 @@ describe(AlbumService.name, () => {
}),
).resolves.toEqual({
success: false,
albumSuccessCount: 0,
assetSuccessCount: 0,
error: BulkIdErrorReason.UNKNOWN,
error: BulkIdErrorReason.NO_PERMISSION,
});
expect(mocks.album.update).not.toHaveBeenCalled();
});
it('should not allow a shared link user to add assets to multiple albums', async () => {
mocks.access.album.checkSharedLinkAccess
.mockResolvedValueOnce(new Set(['album-123']))
.mockResolvedValueOnce(new Set());
mocks.access.album.checkSharedLinkAccess.mockResolvedValueOnce(new Set(['album-123']));
mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1', 'asset-2', 'asset-3']));
mocks.album.getById
.mockResolvedValueOnce(_.cloneDeep(albumStub.sharedWithUser))
@@ -919,7 +927,7 @@ describe(AlbumService.name, () => {
albumIds: ['album-123', 'album-321'],
assetIds: ['asset-1', 'asset-2', 'asset-3'],
}),
).resolves.toEqual({ success: true, albumSuccessCount: 1, assetSuccessCount: 3 });
).resolves.toEqual({ success: true, error: undefined });
expect(mocks.album.update).toHaveBeenCalledTimes(1);
expect(mocks.album.update).toHaveBeenNthCalledWith(1, 'album-123', {
@@ -927,22 +935,23 @@ describe(AlbumService.name, () => {
updatedAt: expect.any(Date),
albumThumbnailAssetId: 'asset-1',
});
expect(mocks.album.addAssetIds).toHaveBeenCalledTimes(1);
expect(mocks.album.addAssetIds).toHaveBeenCalledWith('album-123', ['asset-1', 'asset-2', 'asset-3']);
expect(mocks.album.addAssetIdsToAlbums).toHaveBeenCalledWith([
{ albumsId: 'album-123', assetsId: 'asset-1' },
{ albumsId: 'album-123', assetsId: 'asset-2' },
{ albumsId: 'album-123', assetsId: 'asset-3' },
]);
expect(mocks.event.emit).toHaveBeenCalledWith('AlbumUpdate', {
id: 'album-123',
recipientId: 'user-id',
});
expect(mocks.access.album.checkSharedLinkAccess).toHaveBeenCalledWith(
authStub.adminSharedLink.sharedLink?.id,
new Set(['album-123']),
new Set(['album-123', 'album-321']),
);
});
it('should allow adding assets shared via partner sharing', async () => {
mocks.access.album.checkOwnerAccess
.mockResolvedValueOnce(new Set(['album-123']))
.mockResolvedValueOnce(new Set(['album-321']));
mocks.access.album.checkOwnerAccess.mockResolvedValueOnce(new Set(['album-123', 'album-321']));
mocks.access.asset.checkPartnerAccess.mockResolvedValue(new Set(['asset-1', 'asset-2', 'asset-3']));
mocks.album.getById
.mockResolvedValueOnce(_.cloneDeep(albumStub.empty))
@@ -954,7 +963,7 @@ describe(AlbumService.name, () => {
albumIds: ['album-123', 'album-321'],
assetIds: ['asset-1', 'asset-2', 'asset-3'],
}),
).resolves.toEqual({ success: true, albumSuccessCount: 2, assetSuccessCount: 3 });
).resolves.toEqual({ success: true, error: undefined });
expect(mocks.album.update).toHaveBeenCalledTimes(2);
expect(mocks.album.update).toHaveBeenNthCalledWith(1, 'album-123', {
@@ -967,8 +976,14 @@ describe(AlbumService.name, () => {
updatedAt: expect.any(Date),
albumThumbnailAssetId: 'asset-1',
});
expect(mocks.album.addAssetIds).toHaveBeenCalledWith('album-123', ['asset-1', 'asset-2', 'asset-3']);
expect(mocks.album.addAssetIds).toHaveBeenCalledWith('album-321', ['asset-1', 'asset-2', 'asset-3']);
expect(mocks.album.addAssetIdsToAlbums).toHaveBeenCalledWith([
{ albumsId: 'album-123', assetsId: 'asset-1' },
{ albumsId: 'album-123', assetsId: 'asset-2' },
{ albumsId: 'album-123', assetsId: 'asset-3' },
{ albumsId: 'album-321', assetsId: 'asset-1' },
{ albumsId: 'album-321', assetsId: 'asset-2' },
{ albumsId: 'album-321', assetsId: 'asset-3' },
]);
expect(mocks.access.asset.checkPartnerAccess).toHaveBeenCalledWith(
authStub.admin.user.id,
new Set(['asset-1', 'asset-2', 'asset-3']),
@@ -976,23 +991,21 @@ describe(AlbumService.name, () => {
});
it('should skip some duplicate assets', async () => {
mocks.access.album.checkOwnerAccess
.mockResolvedValueOnce(new Set(['album-123']))
.mockResolvedValueOnce(new Set(['album-321']));
mocks.access.album.checkOwnerAccess.mockResolvedValueOnce(new Set(['album-123', 'album-321']));
mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set(['asset-1', 'asset-2', 'asset-3']));
mocks.album.getById
.mockResolvedValueOnce(_.cloneDeep(albumStub.empty))
.mockResolvedValueOnce(_.cloneDeep(albumStub.oneAsset));
mocks.album.getAssetIds
.mockResolvedValueOnce(new Set(['asset-1', 'asset-2', 'asset-3']))
.mockResolvedValueOnce(new Set());
mocks.album.getById
.mockResolvedValueOnce(_.cloneDeep(albumStub.empty))
.mockResolvedValueOnce(_.cloneDeep(albumStub.oneAsset));
await expect(
sut.addAssetsToAlbums(authStub.admin, {
albumIds: ['album-123', 'album-321'],
assetIds: ['asset-1', 'asset-2', 'asset-3'],
}),
).resolves.toEqual({ success: true, albumSuccessCount: 1, assetSuccessCount: 3 });
).resolves.toEqual({ success: true, error: undefined });
expect(mocks.album.update).toHaveBeenCalledTimes(1);
expect(mocks.album.update).toHaveBeenNthCalledWith(1, 'album-321', {
@@ -1000,8 +1013,11 @@ describe(AlbumService.name, () => {
updatedAt: expect.any(Date),
albumThumbnailAssetId: 'asset-1',
});
expect(mocks.album.addAssetIds).toHaveBeenCalledTimes(1);
expect(mocks.album.addAssetIds).toHaveBeenCalledWith('album-321', ['asset-1', 'asset-2', 'asset-3']);
expect(mocks.album.addAssetIdsToAlbums).toHaveBeenCalledWith([
{ albumsId: 'album-321', assetsId: 'asset-1' },
{ albumsId: 'album-321', assetsId: 'asset-2' },
{ albumsId: 'album-321', assetsId: 'asset-3' },
]);
});
it('should skip all duplicate assets', async () => {
@@ -1021,8 +1037,6 @@ describe(AlbumService.name, () => {
}),
).resolves.toEqual({
success: false,
albumSuccessCount: 0,
assetSuccessCount: 0,
error: BulkIdErrorReason.DUPLICATE,
});
@@ -1046,9 +1060,7 @@ describe(AlbumService.name, () => {
}),
).resolves.toEqual({
success: false,
albumSuccessCount: 0,
assetSuccessCount: 0,
error: BulkIdErrorReason.UNKNOWN,
error: BulkIdErrorReason.NO_PERMISSION,
});
expect(mocks.album.update).not.toHaveBeenCalled();
@@ -1076,9 +1088,7 @@ describe(AlbumService.name, () => {
}),
).resolves.toEqual({
success: false,
albumSuccessCount: 0,
assetSuccessCount: 0,
error: BulkIdErrorReason.UNKNOWN,
error: BulkIdErrorReason.NO_PERMISSION,
});
expect(mocks.album.update).not.toHaveBeenCalled();
@@ -1099,9 +1109,7 @@ describe(AlbumService.name, () => {
}),
).resolves.toEqual({
success: false,
albumSuccessCount: 0,
assetSuccessCount: 0,
error: BulkIdErrorReason.UNKNOWN,
error: BulkIdErrorReason.NO_PERMISSION,
});
expect(mocks.access.album.checkSharedLinkAccess).toHaveBeenCalled();

71
server/src/services/album.service.ts
View File

@@ -191,36 +191,57 @@ export class AlbumService extends BaseService {
async addAssetsToAlbums(auth: AuthDto, dto: AlbumsAddAssetsDto): Promise<AlbumsAddAssetsResponseDto> {
const results: AlbumsAddAssetsResponseDto = {
success: false,
albumSuccessCount: 0,
assetSuccessCount: 0,
error: BulkIdErrorReason.DUPLICATE,
};
const successfulAssetIds: Set<string> = new Set();
for (const albumId of dto.albumIds) {
try {
const albumResults = await this.addAssets(auth, albumId, { ids: dto.assetIds });
let success = false;
for (const res of albumResults) {
if (res.success) {
success = true;
results.success = true;
results.error = undefined;
successfulAssetIds.add(res.id);
} else if (results.error && res.error !== BulkIdErrorReason.DUPLICATE) {
results.error = BulkIdErrorReason.UNKNOWN;
}
}
if (success) {
results.albumSuccessCount++;
}
} catch {
if (results.error) {
results.error = BulkIdErrorReason.UNKNOWN;
}
const allowedAlbumIds = await this.checkAccess({
auth,
permission: Permission.AlbumAssetCreate,
ids: dto.albumIds,
});
if (allowedAlbumIds.size === 0) {
results.error = BulkIdErrorReason.NO_PERMISSION;
return results;
}
const allowedAssetIds = await this.checkAccess({ auth, permission: Permission.AssetShare, ids: dto.assetIds });
if (allowedAssetIds.size === 0) {
results.error = BulkIdErrorReason.NO_PERMISSION;
return results;
}
const albumAssetValues: { albumsId: string; assetsId: string }[] = [];
const events: { id: string; recipients: string[] }[] = [];
for (const albumId of allowedAlbumIds) {
const existingAssetIds = await this.albumRepository.getAssetIds(albumId, [...allowedAssetIds]);
const notPresentAssetIds = [...allowedAssetIds].filter((id) => !existingAssetIds.has(id));
if (notPresentAssetIds.length === 0) {
continue;
}
const album = await this.findOrFail(albumId, { withAssets: false });
results.error = undefined;
results.success = true;
for (const assetId of notPresentAssetIds) {
albumAssetValues.push({ albumsId: albumId, assetsId: assetId });
}
await this.albumRepository.update(albumId, {
id: albumId,
updatedAt: new Date(),
albumThumbnailAssetId: album.albumThumbnailAssetId ?? notPresentAssetIds[0],
});
const allUsersExceptUs = [...album.albumUsers.map(({ user }) => user.id), album.owner.id].filter(
(userId) => userId !== auth.user.id,
);
events.push({ id: albumId, recipients: allUsersExceptUs });
}
await this.albumRepository.addAssetIdsToAlbums(albumAssetValues);
for (const event of events) {
for (const recipientId of event.recipients) {
await this.eventRepository.emit('AlbumUpdate', { id: event.id, recipientId });
}
}
results.assetSuccessCount = successfulAssetIds.size;
return results;
}

4
server/src/services/job.service.spec.ts
View File

@@ -238,11 +238,11 @@ describe(JobService.name, () => {
const tests: Array<{ item: JobItem; jobs: JobName[]; stub?: any }> = [
{
item: { name: JobName.SidecarSync, data: { id: 'asset-1' } },
item: { name: JobName.SidecarCheck, data: { id: 'asset-1' } },
jobs: [JobName.AssetExtractMetadata],
},
{
item: { name: JobName.SidecarDiscovery, data: { id: 'asset-1' } },
item: { name: JobName.SidecarCheck, data: { id: 'asset-1' } },
jobs: [JobName.AssetExtractMetadata],
},
{

3
server/src/services/job.service.ts
View File

@@ -309,8 +309,7 @@ export class JobService extends BaseService {
*/
private async onDone(item: JobItem) {
switch (item.name) {
case JobName.SidecarSync:
case JobName.SidecarDiscovery: {
case JobName.SidecarCheck: {
await this.jobRepository.queue({ name: JobName.AssetExtractMetadata, data: item.data });
break;
}

4
server/src/services/library.service.spec.ts
View File

@@ -527,7 +527,7 @@ describe(LibraryService.name, () => {
expect(mocks.job.queueAll).toHaveBeenCalledWith([
{
name: JobName.SidecarDiscovery,
name: JobName.SidecarCheck,
data: {
id: assetStub.external.id,
source: 'upload',
@@ -573,7 +573,7 @@ describe(LibraryService.name, () => {
expect(mocks.job.queueAll).toHaveBeenCalledWith([
{
name: JobName.SidecarDiscovery,
name: JobName.SidecarCheck,
data: {
id: assetStub.image.id,
source: 'upload',

6
server/src/services/library.service.ts
View File

@@ -123,6 +123,10 @@ export class LibraryService extends BaseService {
{
usePolling: false,
ignoreInitial: true,
awaitWriteFinish: {
stabilityThreshold: 5000,
pollInterval: 1000,
},
},
{
onReady: () => _resolve(),
@@ -410,7 +414,7 @@ export class LibraryService extends BaseService {
// We queue a sidecar discovery which, in turn, queues metadata extraction
await this.jobRepository.queueAll(
assetIds.map((assetId) => ({
name: JobName.SidecarDiscovery,
name: JobName.SidecarCheck,
data: { id: assetId, source: 'upload' },
})),
);

1
server/src/services/media.service.spec.ts
View File

@@ -42,7 +42,6 @@ describe(MediaService.name, () => {
mocks.assetJob.streamForThumbnailJob.mockReturnValue(makeStream([assetStub.image]));
mocks.person.getAll.mockReturnValue(makeStream([personStub.newThumbnail]));
mocks.person.getFacesByIds.mockResolvedValue([faceStub.face1]);
await sut.handleQueueGenerateThumbnails({ force: true });

169
server/src/services/metadata.service.spec.ts
View File

@@ -1,7 +1,6 @@
import { BinaryField, ExifDateTime } from 'exiftool-vendored';
import { randomBytes } from 'node:crypto';
import { Stats } from 'node:fs';
import { constants } from 'node:fs/promises';
import { defaults } from 'src/config';
import { MapAsset } from 'src/dtos/asset-response.dto';
import {
@@ -24,6 +23,21 @@ import { tagStub } from 'test/fixtures/tag.stub';
import { factory } from 'test/small.factory';
import { makeStream, newTestService, ServiceMocks } from 'test/utils';
const forSidecarJob = (
asset: {
id?: string;
originalPath?: string;
sidecarPath?: string | null;
} = {},
) => {
return {
id: factory.uuid(),
originalPath: '/path/to/IMG_123.jpg',
sidecarPath: null,
...asset,
};
};
const makeFaceTags = (face: Partial<{ Name: string }> = {}, orientation?: ImmichTags['Orientation']) => ({
Orientation: orientation,
RegionInfo: {
@@ -1473,7 +1487,7 @@ describe(MetadataService.name, () => {
expect(mocks.job.queueAll).toHaveBeenCalledWith([
{
name: JobName.SidecarSync,
name: JobName.SidecarCheck,
data: { id: assetStub.sidecar.id },
},
]);
@@ -1487,159 +1501,66 @@ describe(MetadataService.name, () => {
expect(mocks.assetJob.streamForSidecar).toHaveBeenCalledWith(false);
expect(mocks.job.queueAll).toHaveBeenCalledWith([
{
name: JobName.SidecarDiscovery,
name: JobName.SidecarCheck,
data: { id: assetStub.image.id },
},
]);
});
});
describe('handleSidecarSync', () => {
describe('handleSidecarCheck', () => {
it("should do nothing if asset isn't found", async () => {
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(void 0);
await expect(sut.handleSidecarSync({ id: assetStub.image.id })).resolves.toBe(JobStatus.Failed);
mocks.assetJob.getForSidecarCheckJob.mockResolvedValue(void 0);
await expect(sut.handleSidecarCheck({ id: assetStub.image.id })).resolves.toBeUndefined();
expect(mocks.asset.update).not.toHaveBeenCalled();
});
it('should do nothing if asset has no sidecar file', async () => {
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(removeNonSidecarFiles(assetStub.image));
await expect(sut.handleSidecarSync({ id: assetStub.image.id })).resolves.toBe(JobStatus.Failed);
expect(mocks.asset.update).not.toHaveBeenCalled();
it('should detect a new sidecar at .jpg.xmp', async () => {
const asset = forSidecarJob({ originalPath: '/path/to/IMG_123.jpg' });
mocks.assetJob.getForSidecarCheckJob.mockResolvedValue(asset);
mocks.storage.checkFileExists.mockResolvedValueOnce(true);
await expect(sut.handleSidecarCheck({ id: asset.id })).resolves.toBe(JobStatus.Success);
expect(mocks.asset.update).toHaveBeenCalledWith({ id: asset.id, sidecarPath: `/path/to/IMG_123.jpg.xmp` });
});
it('should set sidecar path if exists (sidecar named photo.ext.xmp)', async () => {
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(removeNonSidecarFiles(assetStub.sidecar));
mocks.storage.checkFileExists.mockResolvedValue(true);
it('should detect a new sidecar at .xmp', async () => {
const asset = forSidecarJob({ originalPath: '/path/to/IMG_123.jpg' });
await expect(sut.handleSidecarSync({ id: assetStub.sidecar.id })).resolves.toBe(JobStatus.Success);
expect(mocks.storage.checkFileExists).toHaveBeenCalledWith(
`${assetStub.sidecar.originalPath}.xmp`,
constants.R_OK,
);
expect(mocks.asset.upsertFile).toHaveBeenCalledWith({
assetId: assetStub.sidecar.id,
path: assetStub.sidecar.files[1].path,
type: AssetFileType.Sidecar,
});
});
it('should set sidecar path if exists (sidecar named photo.xmp)', async () => {
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(
removeNonSidecarFiles(assetStub.sidecarWithoutExt as any),
);
mocks.assetJob.getForSidecarCheckJob.mockResolvedValue(asset);
mocks.storage.checkFileExists.mockResolvedValueOnce(false);
mocks.storage.checkFileExists.mockResolvedValueOnce(true);
await expect(sut.handleSidecarSync({ id: assetStub.sidecarWithoutExt.id })).resolves.toBe(JobStatus.Success);
expect(mocks.storage.checkFileExists).toHaveBeenNthCalledWith(
2,
assetStub.sidecarWithoutExt.files[1].path,
constants.R_OK,
);
expect(mocks.asset.upsertFile).toHaveBeenCalledWith({
assetId: assetStub.sidecarWithoutExt.id,
path: assetStub.sidecarWithoutExt.files[1].path,
type: AssetFileType.Sidecar,
});
});
await expect(sut.handleSidecarCheck({ id: asset.id })).resolves.toBe(JobStatus.Success);
it('should set sidecar path if exists (two sidecars named photo.ext.xmp and photo.xmp, should pick photo.ext.xmp)', async () => {
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(removeNonSidecarFiles(assetStub.sidecar));
mocks.storage.checkFileExists.mockResolvedValueOnce(true);
mocks.storage.checkFileExists.mockResolvedValueOnce(true);
await expect(sut.handleSidecarSync({ id: assetStub.sidecar.id })).resolves.toBe(JobStatus.Success);
expect(mocks.storage.checkFileExists).toHaveBeenNthCalledWith(1, assetStub.sidecar.files[1].path, constants.R_OK);
expect(mocks.storage.checkFileExists).toHaveBeenNthCalledWith(
2,
assetStub.sidecarWithoutExt.files[1].path,
constants.R_OK,
);
expect(mocks.asset.upsertFile).toHaveBeenCalledWith({
assetId: assetStub.sidecar.id,
path: assetStub.sidecar.files[1].path,
type: AssetFileType.Sidecar,
});
expect(mocks.asset.update).toHaveBeenCalledWith({ id: asset.id, sidecarPath: '/path/to/IMG_123.xmp' });
});
it('should unset sidecar path if file does not exist anymore', async () => {
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(removeNonSidecarFiles(assetStub.sidecar));
const asset = forSidecarJob({ originalPath: '/path/to/IMG_123.jpg', sidecarPath: '/path/to/IMG_123.jpg.xmp' });
mocks.assetJob.getForSidecarCheckJob.mockResolvedValue(asset);
mocks.storage.checkFileExists.mockResolvedValue(false);
await expect(sut.handleSidecarSync({ id: assetStub.sidecar.id })).resolves.toBe(JobStatus.Success);
expect(mocks.storage.checkFileExists).toHaveBeenCalledWith(
`${assetStub.sidecar.originalPath}.xmp`,
constants.R_OK,
);
expect(mocks.asset.deleteFile).toHaveBeenCalledWith({
assetId: assetStub.sidecar.id,
type: AssetFileType.Sidecar,
});
});
});
await expect(sut.handleSidecarCheck({ id: asset.id })).resolves.toBe(JobStatus.Success);
describe('handleSidecarDiscovery', () => {
it('should skip hidden assets', async () => {
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(assetStub.livePhotoMotionAsset as any);
await expect(sut.handleSidecarDiscovery({ id: assetStub.livePhotoMotionAsset.id })).resolves.toBe(
JobStatus.Skipped,
);
expect(mocks.storage.checkFileExists).not.toHaveBeenCalled();
expect(mocks.asset.update).not.toHaveBeenCalled();
expect(mocks.asset.upsertFile).not.toHaveBeenCalled();
expect(mocks.asset.update).toHaveBeenCalledWith({ id: asset.id, sidecarPath: null });
});
it('should skip assets that already have a known sidecar', async () => {
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(assetStub.sidecar);
await expect(sut.handleSidecarDiscovery({ id: assetStub.sidecar.id })).resolves.toBe(JobStatus.Skipped);
it('should do nothing if the sidecar file still exists', async () => {
const asset = forSidecarJob({ originalPath: '/path/to/IMG_123.jpg', sidecarPath: '/path/to/IMG_123.jpg' });
expect(mocks.storage.checkFileExists).not.toHaveBeenCalled();
expect(mocks.asset.update).not.toHaveBeenCalled();
expect(mocks.asset.upsertFile).not.toHaveBeenCalled();
});
mocks.assetJob.getForSidecarCheckJob.mockResolvedValue(asset);
mocks.storage.checkFileExists.mockResolvedValueOnce(true);
it('should do nothing when no sidecar is found on disk', async () => {
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(removeNonSidecarFiles(assetStub.image));
mocks.storage.checkFileExists.mockResolvedValue(false);
await expect(sut.handleSidecarDiscovery({ id: assetStub.image.id })).resolves.toBe(JobStatus.Success);
expect(mocks.storage.checkFileExists).toHaveBeenCalledWith('/original/path.jpg.xmp', constants.R_OK);
expect(mocks.storage.checkFileExists).toHaveBeenCalledWith('/original/path.xmp', constants.R_OK);
await expect(sut.handleSidecarCheck({ id: asset.id })).resolves.toBe(JobStatus.Skipped);
expect(mocks.asset.update).not.toHaveBeenCalled();
expect(mocks.asset.upsertFile).not.toHaveBeenCalled();
});
it('should update a image asset when a sidecar is found on disk', async () => {
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(removeNonSidecarFiles(assetStub.image));
mocks.storage.checkFileExists.mockResolvedValue(true);
await expect(sut.handleSidecarDiscovery({ id: assetStub.image.id })).resolves.toBe(JobStatus.Success);
expect(mocks.storage.checkFileExists).toHaveBeenCalledWith('/original/path.jpg.xmp', constants.R_OK);
expect(mocks.storage.checkFileExists).toHaveBeenCalledWith('/original/path.xmp', constants.R_OK);
expect(mocks.asset.upsertFile).toHaveBeenCalledWith({
assetId: assetStub.image.id,
path: '/original/path.jpg.xmp',
type: AssetFileType.Sidecar,
});
});
it('should update a video asset when a sidecar is found', async () => {
mocks.assetJob.getForMetadataExtraction.mockResolvedValue(assetStub.video);
mocks.storage.checkFileExists.mockResolvedValue(true);
await expect(sut.handleSidecarDiscovery({ id: assetStub.video.id })).resolves.toBe(JobStatus.Success);
expect(mocks.storage.checkFileExists).toHaveBeenCalledWith('/original/path.ext.xmp', constants.R_OK);
expect(mocks.storage.checkFileExists).toHaveBeenCalledWith('/original/path.xmp', constants.R_OK);
expect(mocks.asset.upsertFile).toHaveBeenCalledWith({
assetId: assetStub.image.id,
path: '/original/path.ext.xmp',
type: AssetFileType.Sidecar,
});
});
});
describe('handleSidecarWrite', () => {

140
server/src/services/metadata.service.ts
View File

@@ -5,7 +5,7 @@ import _ from 'lodash';
import { Duration } from 'luxon';
import { Stats } from 'node:fs';
import { constants } from 'node:fs/promises';
import path from 'node:path';
import { join, parse } from 'node:path';
import { JOBS_ASSET_PAGINATION_SIZE } from 'src/constants';
import { StorageCore } from 'src/cores/storage.core';
import { Asset, AssetFace, AssetFile } from 'src/database';
@@ -331,7 +331,7 @@ export class MetadataService extends BaseService {
const assets = this.assetJobRepository.streamForSidecar(force);
for await (const asset of assets) {
jobs.push({ name: force ? JobName.SidecarSync : JobName.SidecarDiscovery, data: { id: asset.id } });
jobs.push({ name: JobName.SidecarCheck, data: { id: asset.id } });
if (jobs.length >= JOBS_ASSET_PAGINATION_SIZE) {
await queueAll();
}
@@ -342,14 +342,43 @@ export class MetadataService extends BaseService {
return JobStatus.Success;
}
@OnJob({ name: JobName.SidecarSync, queue: QueueName.Sidecar })
handleSidecarSync({ id }: JobOf<JobName.SidecarSync>): Promise<JobStatus> {
return this.processSidecar(id, true);
}
@OnJob({ name: JobName.SidecarCheck, queue: QueueName.Sidecar })
async handleSidecarCheck({ id }: JobOf<JobName.SidecarCheck>): Promise<JobStatus | undefined> {
const asset = await this.assetJobRepository.getForSidecarCheckJob(id);
if (!asset) {
return;
}
@OnJob({ name: JobName.SidecarDiscovery, queue: QueueName.Sidecar })
handleSidecarDiscovery({ id }: JobOf<JobName.SidecarDiscovery>): Promise<JobStatus> {
return this.processSidecar(id, false);
let sidecarPath = null;
for (const candidate of this.getSidecarCandidates(asset)) {
const exists = await this.storageRepository.checkFileExists(candidate, constants.R_OK);
if (!exists) {
continue;
}
sidecarPath = candidate;
break;
}
const existingSidecar = asset.files ? asset.files.find((file) => file.type === AssetFileType.Sidecar) : null;
const isChanged = sidecarPath !== existingSidecar?.path;
this.logger.debug(
`Sidecar check found old=${existingSidecar?.path}, new=${sidecarPath} will ${isChanged ? 'update' : 'do nothing for'} asset ${asset.id}: ${asset.originalPath}`,
);
if (!isChanged) {
return JobStatus.Skipped;
}
if (sidecarPath === null) {
await this.assetRepository.deleteFile({ assetId: asset.id, type: AssetFileType.Sidecar });
} else {
await this.assetRepository.upsertFile({ assetId: asset.id, type: AssetFileType.Sidecar, path: sidecarPath });
}
return JobStatus.Success;
}
@OnEvent({ name: 'AssetTag' })
@@ -399,6 +428,21 @@ export class MetadataService extends BaseService {
return JobStatus.Success;
}
private getSidecarCandidates({ files, originalPath }: { files: AssetFile[] | null; originalPath: string }) {
const candidates: string[] = [];
const assetPath = parse(originalPath);
candidates.push(
// IMG_123.jpg.xmp
`${originalPath}.xmp`,
// IMG_123.xmp
`${join(assetPath.dir, assetPath.name)}.xmp`,
);
return candidates;
}
private getImageDimensions(exifTags: ImmichTags): { width?: number; height?: number } {
/*
* The "true" values for width and height are a bit hidden, depending on the camera model and file format.
@@ -586,7 +630,7 @@ export class MetadataService extends BaseService {
checksum,
ownerId: asset.ownerId,
originalPath: StorageCore.getAndroidMotionPath(asset, motionAssetId),
originalFileName: `${path.parse(asset.originalFileName).name}.mp4`,
originalFileName: `${parse(asset.originalFileName).name}.mp4`,
visibility: AssetVisibility.Hidden,
deviceAssetId: 'NONE',
deviceId: 'NONE',
@@ -898,80 +942,4 @@ export class MetadataService extends BaseService {
return tags;
}
private async processSidecar(id: string, isSync: boolean): Promise<JobStatus> {
const asset = await this.assetJobRepository.getForMetadataExtraction(id);
if (!asset) {
return JobStatus.Failed;
}
if (isSync) {
// We are performing a sidecar sync
if (asset.files.length === 0) {
return JobStatus.Failed;
}
} else if (!asset.isExternal) {
// We are performing a sidecar discovery and not on an external library
if (asset.visibility === AssetVisibility.Hidden) {
// Skip hidden assets
return JobStatus.Skipped;
} else if (asset.files.length > 1) {
// Skip assets that already have a sidecar
return JobStatus.Skipped;
}
}
let currentSidecar: AssetFile | null;
if (asset.files.length === 0) {
currentSidecar = null;
} else if (asset.files.length === 1) {
currentSidecar = asset.files[0];
} else {
throw new Error(
`Multiple sidecar files found for asset ${asset.id}: ${asset.files.map((file) => file.path).join(', ')}`,
);
}
// XMP sidecars can come in two filename formats. For a photo named photo.ext, the filenames are photo.ext.xmp and photo.xmp
const assetPath = path.parse(asset.originalPath);
const assetPathWithoutExt = path.join(assetPath.dir, assetPath.name);
const sidecarPathWithoutExt = `${assetPathWithoutExt}.xmp`;
const sidecarPathWithExt = `${asset.originalPath}.xmp`;
const [sidecarPathWithExtExists, sidecarPathWithoutExtExists] = await Promise.all([
this.storageRepository.checkFileExists(sidecarPathWithExt, constants.R_OK),
this.storageRepository.checkFileExists(sidecarPathWithoutExt, constants.R_OK),
]);
let sidecarPath = null;
if (sidecarPathWithExtExists) {
// Sidecars with the extension have precedence over those without
sidecarPath = sidecarPathWithExt;
} else if (sidecarPathWithoutExtExists) {
sidecarPath = sidecarPathWithoutExt;
}
if (asset.isExternal) {
if (sidecarPath !== currentSidecar?.path) {
await (sidecarPath
? this.assetRepository.upsertFile({ assetId: asset.id, path: sidecarPath, type: AssetFileType.Sidecar })
: this.assetRepository.deleteFile({ assetId: asset.id, type: AssetFileType.Sidecar }));
}
return JobStatus.Success;
}
if (sidecarPath) {
this.logger.debug(`Detected sidecar at '${sidecarPath}' for asset ${asset.id}: ${asset.originalPath}`);
await this.assetRepository.upsertFile({ assetId: asset.id, path: sidecarPath, type: AssetFileType.Sidecar });
return JobStatus.Success;
}
this.logger.debug(`No sidecar found for asset ${asset.id}: ${asset.originalPath}`);
await this.assetRepository.deleteFile({ assetId: asset.id, type: AssetFileType.Sidecar });
return JobStatus.Success;
}
}

4
server/src/services/person.service.ts
View File

@@ -197,6 +197,10 @@ export class PersonService extends BaseService {
throw new BadRequestException('Invalid assetId for feature face');
}
if (face.asset.isOffline) {
throw new BadRequestException('An offline asset cannot be used for feature face');
}
faceId = face.id;
}

5
server/src/types.ts
View File

@@ -306,8 +306,7 @@ export type JobItem =
// Sidecar Scanning
| { name: JobName.SidecarQueueAll; data: IBaseJob }
| { name: JobName.SidecarDiscovery; data: IEntityJob }
| { name: JobName.SidecarSync; data: IEntityJob }
| { name: JobName.SidecarCheck; data: IEntityJob }
| { name: JobName.SidecarWrite; data: ISidecarWriteJob }
// Facial Recognition
@@ -394,8 +393,8 @@ export interface VectorUpdateResult {
}
export interface ImmichFile extends Express.Multer.File {
/** sha1 hash of file */
uuid: string;
/** sha1 hash of file */
checksum: Buffer;
}