immich-app/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2025-12-25 09:14:58 +03:00
Code Issues Packages Projects Releases 18 Wiki Activity

fix(server): stacked assets for full sync, userIds as array for delta sync (#9100)

Browse Source 
* fix(server): stacked assets for full sync, userIds as array for delta sync

* refactor(server): sync

* fix getDeltaSync after partner removal

---------

Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>
This commit is contained in:
Fynn Petersen-Frey
2024-04-29 05:24:21 +02:00
committed by GitHub
parent fc2e709ad4
commit 32e7cfea3d
22 changed files with 817 additions and 386 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
server/src/services/sync.service.spec.ts
View File

@@ -39,13 +39,12 @@ describe(SyncService.name, () => {
describe('getAllAssetsForUserFullSync', () => {
it('should return a list of all assets owned by the user', async () => {
assetMock.getAllForUserFullSync.mockResolvedValue([assetStub.external, assetStub.hasEncodedVideo]);
await expect(
sut.getAllAssetsForUserFullSync(authStub.user1, { limit: 2, updatedUntil: untilDate }),
).resolves.toEqual([
await expect(sut.getFullSync(authStub.user1, { limit: 2, updatedUntil: untilDate })).resolves.toEqual([
mapAsset(assetStub.external, mapAssetOpts),
mapAsset(assetStub.hasEncodedVideo, mapAssetOpts),
]);
expect(assetMock.getAllForUserFullSync).toHaveBeenCalledWith({
withStacked: true,
ownerId: authStub.user1.user.id,
updatedUntil: untilDate,
limit: 2,
@@ -57,7 +56,7 @@ describe(SyncService.name, () => {
it('should return a response requiring a full sync when partners are out of sync', async () => {
partnerMock.getAll.mockResolvedValue([partnerStub.adminToUser1]);
await expect(
sut.getChangesForDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }),
sut.getDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }),
).resolves.toEqual({ needsFullSync: true, upserted: [], deleted: [] });
expect(assetMock.getChangedDeltaSync).toHaveBeenCalledTimes(0);
expect(auditMock.getAfter).toHaveBeenCalledTimes(0);
@@ -66,7 +65,7 @@ describe(SyncService.name, () => {
it('should return a response requiring a full sync when last sync was too long ago', async () => {
partnerMock.getAll.mockResolvedValue([]);
await expect(
sut.getChangesForDeltaSync(authStub.user1, { updatedAfter: new Date(2000), userIds: [authStub.user1.user.id] }),
sut.getDeltaSync(authStub.user1, { updatedAfter: new Date(2000), userIds: [authStub.user1.user.id] }),
).resolves.toEqual({ needsFullSync: true, upserted: [], deleted: [] });
expect(assetMock.getChangedDeltaSync).toHaveBeenCalledTimes(0);
expect(auditMock.getAfter).toHaveBeenCalledTimes(0);
@@ -78,7 +77,7 @@ describe(SyncService.name, () => {
Array.from<AssetEntity>({ length: 10_000 }).fill(assetStub.image),
);
await expect(
sut.getChangesForDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }),
sut.getDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }),
).resolves.toEqual({ needsFullSync: true, upserted: [], deleted: [] });
expect(assetMock.getChangedDeltaSync).toHaveBeenCalledTimes(1);
expect(auditMock.getAfter).toHaveBeenCalledTimes(0);
@@ -89,7 +88,7 @@ describe(SyncService.name, () => {
assetMock.getChangedDeltaSync.mockResolvedValue([assetStub.image1]);
auditMock.getAfter.mockResolvedValue([assetStub.external.id]);
await expect(
sut.getChangesForDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }),
sut.getDeltaSync(authStub.user1, { updatedAfter: new Date(), userIds: [authStub.user1.user.id] }),
).resolves.toEqual({
needsFullSync: false,
upserted: [mapAsset(assetStub.image1, mapAssetOpts)],

59
server/src/services/sync.service.ts
View File

@@ -1,5 +1,4 @@
import { Inject } from '@nestjs/common';
import _ from 'lodash';
import { DateTime } from 'luxon';
import { AUDIT_LOG_MAX_DURATION } from 'src/constants';
import { AccessCore, Permission } from 'src/cores/access.core';
@@ -11,6 +10,9 @@ import { IAccessRepository } from 'src/interfaces/access.interface';
import { IAssetRepository } from 'src/interfaces/asset.interface';
import { IAuditRepository } from 'src/interfaces/audit.interface';
import { IPartnerRepository } from 'src/interfaces/partner.interface';
import { setIsEqual } from 'src/utils/set';
const FULL_SYNC = { needsFullSync: true, deleted: [], upserted: [] };
export class SyncService {
private access: AccessCore;
@@ -24,52 +26,69 @@ export class SyncService {
this.access = AccessCore.create(accessRepository);
}
async getAllAssetsForUserFullSync(auth: AuthDto, dto: AssetFullSyncDto): Promise<AssetResponseDto[]> {
async getFullSync(auth: AuthDto, dto: AssetFullSyncDto): Promise<AssetResponseDto[]> {
// mobile implementation is faster if this is a single id
const userId = dto.userId || auth.user.id;
await this.access.requirePermission(auth, Permission.TIMELINE_READ, userId);
const assets = await this.assetRepository.getAllForUserFullSync({
ownerId: userId,
// no archived assets for partner user
isArchived: userId === auth.user.id ? undefined : false,
// no stack for partner user
withStacked: userId === auth.user.id ? true : undefined,
lastCreationDate: dto.lastCreationDate,
updatedUntil: dto.updatedUntil,
lastId: dto.lastId,
limit: dto.limit,
});
const options = { auth, stripMetadata: false, withStack: true };
return assets.map((a) => mapAsset(a, options));
return assets.map((a) => mapAsset(a, { auth, stripMetadata: false, withStack: true }));
}
async getChangesForDeltaSync(auth: AuthDto, dto: AssetDeltaSyncDto): Promise<AssetDeltaSyncResponseDto> {
await this.access.requirePermission(auth, Permission.TIMELINE_READ, dto.userIds);
const partner = await this.partnerRepository.getAll(auth.user.id);
const userIds = [auth.user.id, ...partner.filter((p) => p.sharedWithId == auth.user.id).map((p) => p.sharedById)];
userIds.sort();
dto.userIds.sort();
async getDeltaSync(auth: AuthDto, dto: AssetDeltaSyncDto): Promise<AssetDeltaSyncResponseDto> {
// app has not synced in the last 100 days
const duration = DateTime.now().diff(DateTime.fromJSDate(dto.updatedAfter));
if (!_.isEqual(userIds, dto.userIds) || duration > AUDIT_LOG_MAX_DURATION) {
// app does not have the correct partners synced
// or app has not synced in the last 100 days
return { needsFullSync: true, deleted: [], upserted: [] };
if (duration > AUDIT_LOG_MAX_DURATION) {
return FULL_SYNC;
}
const authUserId = auth.user.id;
// app does not have the correct partners synced
const partner = await this.partnerRepository.getAll(authUserId);
const userIds = [authUserId, ...partner.filter((p) => p.sharedWithId == auth.user.id).map((p) => p.sharedById)];
if (!setIsEqual(new Set(userIds), new Set(dto.userIds))) {
return FULL_SYNC;
}
await this.access.requirePermission(auth, Permission.TIMELINE_READ, dto.userIds);
const limit = 10_000;
const upserted = await this.assetRepository.getChangedDeltaSync({ limit, updatedAfter: dto.updatedAfter, userIds });
// too many changes, need to do a full sync
if (upserted.length === limit) {
// too many changes -> do a full sync (paginated) instead
return { needsFullSync: true, deleted: [], upserted: [] };
return FULL_SYNC;
}
const deleted = await this.auditRepository.getAfter(dto.updatedAfter, {
userIds: userIds,
userIds,
entityType: EntityType.ASSET,
action: DatabaseAction.DELETE,
});
const options = { auth, stripMetadata: false, withStack: true };
const result = {
needsFullSync: false,
upserted: upserted.map((a) => mapAsset(a, options)),
upserted: upserted
// do not return archived assets for partner users
.filter((a) => a.ownerId === auth.user.id || (a.ownerId !== auth.user.id && !a.isArchived))
.map((a) =>
mapAsset(a, {
auth,
stripMetadata: false,
// ignore stacks for non partner users
withStack: a.ownerId === authUserId,
}),
),
deleted,
};
return result;