Compare commits

...

2 Commits

Author SHA1 Message Date
mertalev
f5d9bebd84 openapi 2025-05-19 15:20:18 -04:00
mertalev
f16bdb2a01 bulk dedupe server endpoints 2025-05-19 15:19:55 -04:00
13 changed files with 430 additions and 82 deletions

View File

@@ -122,7 +122,9 @@ Class | Method | HTTP request | Description
*DeprecatedApi* | [**getRandom**](doc//DeprecatedApi.md#getrandom) | **GET** /assets/random |
*DownloadApi* | [**downloadArchive**](doc//DownloadApi.md#downloadarchive) | **POST** /download/archive |
*DownloadApi* | [**getDownloadInfo**](doc//DownloadApi.md#getdownloadinfo) | **POST** /download/info |
*DuplicatesApi* | [**deduplicateAll**](doc//DuplicatesApi.md#deduplicateall) | **POST** /duplicates/bulk/deduplicate |
*DuplicatesApi* | [**getAssetDuplicates**](doc//DuplicatesApi.md#getassetduplicates) | **GET** /duplicates |
*DuplicatesApi* | [**keepAll**](doc//DuplicatesApi.md#keepall) | **POST** /duplicates/bulk/keep |
*FacesApi* | [**createFace**](doc//FacesApi.md#createface) | **POST** /faces |
*FacesApi* | [**deleteFace**](doc//FacesApi.md#deleteface) | **DELETE** /faces/{id} |
*FacesApi* | [**getFaces**](doc//FacesApi.md#getfaces) | **GET** /faces |
@@ -327,6 +329,7 @@ Class | Method | HTTP request | Description
- [CreateLibraryDto](doc//CreateLibraryDto.md)
- [CreateProfileImageResponseDto](doc//CreateProfileImageResponseDto.md)
- [DatabaseBackupConfig](doc//DatabaseBackupConfig.md)
- [DeduplicateAllDto](doc//DeduplicateAllDto.md)
- [DownloadArchiveInfo](doc//DownloadArchiveInfo.md)
- [DownloadInfoDto](doc//DownloadInfoDto.md)
- [DownloadResponse](doc//DownloadResponse.md)

View File

@@ -122,6 +122,7 @@ part 'model/create_album_dto.dart';
part 'model/create_library_dto.dart';
part 'model/create_profile_image_response_dto.dart';
part 'model/database_backup_config.dart';
part 'model/deduplicate_all_dto.dart';
part 'model/download_archive_info.dart';
part 'model/download_info_dto.dart';
part 'model/download_response.dart';

View File

@@ -16,6 +16,45 @@ class DuplicatesApi {
final ApiClient apiClient;
/// Performs an HTTP 'POST /duplicates/bulk/deduplicate' operation and returns the [Response].
/// Parameters:
///
/// * [DeduplicateAllDto] deduplicateAllDto (required):
Future<Response> deduplicateAllWithHttpInfo(DeduplicateAllDto deduplicateAllDto,) async {
// ignore: prefer_const_declarations
final apiPath = r'/duplicates/bulk/deduplicate';
// ignore: prefer_final_locals
Object? postBody = deduplicateAllDto;
final queryParams = <QueryParam>[];
final headerParams = <String, String>{};
final formParams = <String, String>{};
const contentTypes = <String>['application/json'];
return apiClient.invokeAPI(
apiPath,
'POST',
queryParams,
postBody,
headerParams,
formParams,
contentTypes.isEmpty ? null : contentTypes.first,
);
}
/// Parameters:
///
/// * [DeduplicateAllDto] deduplicateAllDto (required):
Future<void> deduplicateAll(DeduplicateAllDto deduplicateAllDto,) async {
final response = await deduplicateAllWithHttpInfo(deduplicateAllDto,);
if (response.statusCode >= HttpStatus.badRequest) {
throw ApiException(response.statusCode, await _decodeBodyBytes(response));
}
}
/// Performs an HTTP 'GET /duplicates' operation and returns the [Response].
Future<Response> getAssetDuplicatesWithHttpInfo() async {
// ignore: prefer_const_declarations
@@ -59,4 +98,37 @@ class DuplicatesApi {
}
return null;
}
/// Performs an HTTP 'POST /duplicates/bulk/keep' operation and returns the [Response].
Future<Response> keepAllWithHttpInfo() async {
// ignore: prefer_const_declarations
final apiPath = r'/duplicates/bulk/keep';
// ignore: prefer_final_locals
Object? postBody;
final queryParams = <QueryParam>[];
final headerParams = <String, String>{};
final formParams = <String, String>{};
const contentTypes = <String>[];
return apiClient.invokeAPI(
apiPath,
'POST',
queryParams,
postBody,
headerParams,
formParams,
contentTypes.isEmpty ? null : contentTypes.first,
);
}
Future<void> keepAll() async {
final response = await keepAllWithHttpInfo();
if (response.statusCode >= HttpStatus.badRequest) {
throw ApiException(response.statusCode, await _decodeBodyBytes(response));
}
}
}

View File

@@ -300,6 +300,8 @@ class ApiClient {
return CreateProfileImageResponseDto.fromJson(value);
case 'DatabaseBackupConfig':
return DatabaseBackupConfig.fromJson(value);
case 'DeduplicateAllDto':
return DeduplicateAllDto.fromJson(value);
case 'DownloadArchiveInfo':
return DownloadArchiveInfo.fromJson(value);
case 'DownloadInfoDto':

View File

@@ -0,0 +1,101 @@
//
// AUTO-GENERATED FILE, DO NOT MODIFY!
//
// @dart=2.18
// ignore_for_file: unused_element, unused_import
// ignore_for_file: always_put_required_named_parameters_first
// ignore_for_file: constant_identifier_names
// ignore_for_file: lines_longer_than_80_chars
part of openapi.api;
class DeduplicateAllDto {
/// Returns a new [DeduplicateAllDto] instance.
DeduplicateAllDto({
this.assetIdsToKeep = const [],
});
List<String> assetIdsToKeep;
@override
bool operator ==(Object other) => identical(this, other) || other is DeduplicateAllDto &&
_deepEquality.equals(other.assetIdsToKeep, assetIdsToKeep);
@override
int get hashCode =>
// ignore: unnecessary_parenthesis
(assetIdsToKeep.hashCode);
@override
String toString() => 'DeduplicateAllDto[assetIdsToKeep=$assetIdsToKeep]';
Map<String, dynamic> toJson() {
final json = <String, dynamic>{};
json[r'assetIdsToKeep'] = this.assetIdsToKeep;
return json;
}
/// Returns a new [DeduplicateAllDto] instance and imports its values from
/// [value] if it's a [Map], null otherwise.
// ignore: prefer_constructors_over_static_methods
static DeduplicateAllDto? fromJson(dynamic value) {
upgradeDto(value, "DeduplicateAllDto");
if (value is Map) {
final json = value.cast<String, dynamic>();
return DeduplicateAllDto(
assetIdsToKeep: json[r'assetIdsToKeep'] is Iterable
? (json[r'assetIdsToKeep'] as Iterable).cast<String>().toList(growable: false)
: const [],
);
}
return null;
}
static List<DeduplicateAllDto> listFromJson(dynamic json, {bool growable = false,}) {
final result = <DeduplicateAllDto>[];
if (json is List && json.isNotEmpty) {
for (final row in json) {
final value = DeduplicateAllDto.fromJson(row);
if (value != null) {
result.add(value);
}
}
}
return result.toList(growable: growable);
}
static Map<String, DeduplicateAllDto> mapFromJson(dynamic json) {
final map = <String, DeduplicateAllDto>{};
if (json is Map && json.isNotEmpty) {
json = json.cast<String, dynamic>(); // ignore: parameter_assignments
for (final entry in json.entries) {
final value = DeduplicateAllDto.fromJson(entry.value);
if (value != null) {
map[entry.key] = value;
}
}
}
return map;
}
// maps a json object with a list of DeduplicateAllDto-objects as value to a dart map
static Map<String, List<DeduplicateAllDto>> mapListFromJson(dynamic json, {bool growable = false,}) {
final map = <String, List<DeduplicateAllDto>>{};
if (json is Map && json.isNotEmpty) {
// ignore: parameter_assignments
json = json.cast<String, dynamic>();
for (final entry in json.entries) {
map[entry.key] = DeduplicateAllDto.listFromJson(entry.value, growable: growable,);
}
}
return map;
}
/// The list of required keys that must be present in a JSON.
static const requiredKeys = <String>{
'assetIdsToKeep',
};
}

View File

@@ -2732,6 +2732,66 @@
]
}
},
"/duplicates/bulk/deduplicate": {
"post": {
"operationId": "deduplicateAll",
"parameters": [],
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/DeduplicateAllDto"
}
}
},
"required": true
},
"responses": {
"201": {
"description": ""
}
},
"security": [
{
"bearer": []
},
{
"cookie": []
},
{
"api_key": []
}
],
"tags": [
"Duplicates"
]
}
},
"/duplicates/bulk/keep": {
"post": {
"operationId": "keepAll",
"parameters": [],
"responses": {
"201": {
"description": ""
}
},
"security": [
{
"bearer": []
},
{
"cookie": []
},
{
"api_key": []
}
],
"tags": [
"Duplicates"
]
}
},
"/faces": {
"get": {
"operationId": "getFaces",
@@ -9655,6 +9715,21 @@
],
"type": "object"
},
"DeduplicateAllDto": {
"properties": {
"assetIdsToKeep": {
"items": {
"format": "uuid",
"type": "string"
},
"type": "array"
}
},
"required": [
"assetIdsToKeep"
],
"type": "object"
},
"DownloadArchiveInfo": {
"properties": {
"assetIds": {

View File

@@ -560,6 +560,9 @@ export type DuplicateResponseDto = {
assets: AssetResponseDto[];
duplicateId: string;
};
export type DeduplicateAllDto = {
assetIdsToKeep: string[];
};
export type PersonResponseDto = {
birthDate: string | null;
/** This property was added in v1.126.0 */
@@ -2176,6 +2179,21 @@ export function getAssetDuplicates(opts?: Oazapfts.RequestOpts) {
...opts
}));
}
export function deduplicateAll({ deduplicateAllDto }: {
deduplicateAllDto: DeduplicateAllDto;
}, opts?: Oazapfts.RequestOpts) {
return oazapfts.ok(oazapfts.fetchText("/duplicates/bulk/deduplicate", oazapfts.json({
...opts,
method: "POST",
body: deduplicateAllDto
})));
}
export function keepAll(opts?: Oazapfts.RequestOpts) {
return oazapfts.ok(oazapfts.fetchText("/duplicates/bulk/keep", {
...opts,
method: "POST"
}));
}
export function getFaces({ id }: {
id: string;
}, opts?: Oazapfts.RequestOpts) {

View File

@@ -1,7 +1,8 @@
import { Controller, Get } from '@nestjs/common';
import { Body, Controller, Get, Post } from '@nestjs/common';
import { ApiTags } from '@nestjs/swagger';
import { AuthDto } from 'src/dtos/auth.dto';
import { DuplicateResponseDto } from 'src/dtos/duplicate.dto';
import { DeduplicateAllDto, DuplicateResponseDto } from 'src/dtos/duplicate.dto';
import { Permission } from 'src/enum';
import { Auth, Authenticated } from 'src/middleware/auth.guard';
import { DuplicateService } from 'src/services/duplicate.service';
@@ -15,4 +16,16 @@ export class DuplicateController {
getAssetDuplicates(@Auth() auth: AuthDto): Promise<DuplicateResponseDto[]> {
return this.service.getDuplicates(auth);
}
@Post('/bulk/keep')
@Authenticated({ permission: Permission.ASSET_UPDATE })
async keepAll(@Auth() auth: AuthDto) {
await this.service.keepAll(auth);
}
@Post('/bulk/deduplicate')
@Authenticated({ permission: Permission.ASSET_DELETE })
async deduplicateAll(@Auth() auth: AuthDto, @Body() dto: DeduplicateAllDto) {
await this.service.deduplicateAll(auth, dto);
}
}

View File

@@ -12,3 +12,9 @@ export class ResolveDuplicatesDto {
@ValidateUUID({ each: true })
assetIds!: string[];
}
export class DeduplicateAllDto {
@IsNotEmpty()
@ValidateUUID({ each: true })
assetIdsToKeep!: string[];
}

View File

@@ -146,10 +146,17 @@ export class AssetJobRepository {
@GenerateSql({ params: [], stream: true })
streamForSearchDuplicates(force?: boolean) {
return this.assetsWithPreviews()
.where((eb) => eb.not((eb) => eb.exists(eb.selectFrom('smart_search').whereRef('assetId', '=', 'assets.id'))))
.$if(!force, (qb) => qb.where('job_status.duplicatesDetectedAt', 'is', null))
return this.db
.selectFrom('assets')
.select(['assets.id'])
.where('assets.visibility', '!=', AssetVisibility.HIDDEN)
.where('assets.deletedAt', 'is', null)
.innerJoin('smart_search', 'assets.id', 'smart_search.assetId')
.$if(!force, (qb) =>
qb
.innerJoin('asset_job_status as job_status', 'assetId', 'assets.id')
.where('job_status.duplicatesDetectedAt', 'is', null),
)
.stream();
}

View File

@@ -632,57 +632,100 @@ export class AssetRepository {
@GenerateSql({ params: [DummyValue.UUID] })
getDuplicates(userId: string) {
return (
this.db
.with('duplicates', (qb) =>
return this.db
.with('duplicates', (qb) =>
qb
.selectFrom('assets')
.innerJoin('exif', 'assets.id', 'exif.assetId')
.leftJoinLateral(
(qb) =>
qb
.selectFrom(sql`(select 1)`.as('dummy'))
.selectAll('assets')
.select((eb) => eb.table('exif').as('exifInfo'))
.as('asset'),
(join) => join.onTrue(),
)
.select('assets.duplicateId')
.select((eb) => eb.fn.jsonAgg('asset').$castTo<MapAsset[]>().as('assets'))
.where('assets.ownerId', '=', asUuid(userId))
.where('assets.duplicateId', 'is not', null)
.$narrowType<{ duplicateId: NotNull }>()
.where('assets.deletedAt', 'is', null)
.where('assets.visibility', '!=', AssetVisibility.HIDDEN)
.where('assets.stackId', 'is', null)
.groupBy('assets.duplicateId'),
)
.with('unique', (qb) =>
qb
.selectFrom('duplicates')
.select('duplicateId')
.where((eb) => eb(eb.fn('json_array_length', ['assets']), '=', 1)),
)
.with('removed_unique', (qb) =>
qb
.updateTable('assets')
.set({ duplicateId: null })
.from('unique')
.whereRef('assets.duplicateId', '=', 'unique.duplicateId'),
)
.selectFrom('duplicates')
.selectAll()
.where(({ not, exists }) =>
not(exists((eb) => eb.selectFrom('unique').whereRef('unique.duplicateId', '=', 'duplicates.duplicateId'))),
)
.execute();
}
@GenerateSql({ params: [DummyValue.UUID] })
streamDuplicates(userId: string) {
return this.db
.selectFrom('assets')
.innerJoin('exif', 'assets.id', 'exif.assetId')
.innerJoinLateral(
(qb) =>
qb
.selectFrom('assets')
.leftJoinLateral(
(qb) =>
qb
.selectFrom('exif')
.selectAll('assets')
.select((eb) => eb.table('exif').as('exifInfo'))
.whereRef('exif.assetId', '=', 'assets.id')
.as('asset'),
(join) => join.onTrue(),
)
.select('assets.duplicateId')
.select((eb) =>
eb
.fn('jsonb_agg', [eb.table('asset')])
.$castTo<MapAsset[]>()
.as('assets'),
)
.where('assets.ownerId', '=', asUuid(userId))
.where('assets.duplicateId', 'is not', null)
.$narrowType<{ duplicateId: NotNull }>()
.where('assets.deletedAt', 'is', null)
.where('assets.visibility', '!=', AssetVisibility.HIDDEN)
.where('assets.stackId', 'is', null)
.groupBy('assets.duplicateId'),
)
.with('unique', (qb) =>
qb
.selectFrom('duplicates')
.select('duplicateId')
.where((eb) => eb(eb.fn('jsonb_array_length', ['assets']), '=', 1)),
)
.with('removed_unique', (qb) =>
qb
.updateTable('assets')
.set({ duplicateId: null })
.from('unique')
.whereRef('assets.duplicateId', '=', 'unique.duplicateId'),
)
.selectFrom('duplicates')
.selectAll()
// TODO: compare with filtering by jsonb_array_length > 1
.where(({ not, exists }) =>
not(exists((eb) => eb.selectFrom('unique').whereRef('unique.duplicateId', '=', 'duplicates.duplicateId'))),
)
.execute()
);
.selectFrom(sql`(select 1)`.as('dummy'))
.selectAll('assets')
.select((eb) => eb.table('exif').as('exifInfo'))
.as('asset'),
(join) => join.onTrue(),
)
.select('assets.duplicateId')
.select((eb) => eb.fn.jsonAgg('asset').as('assets'))
.where('assets.ownerId', '=', asUuid(userId))
.where('assets.duplicateId', 'is not', null)
.$narrowType<{ duplicateId: NotNull }>()
.where('assets.deletedAt', 'is', null)
.where('assets.visibility', '!=', AssetVisibility.HIDDEN)
.where('assets.stackId', 'is', null)
.groupBy('assets.duplicateId')
.stream();
}
@GenerateSql({ params: [DummyValue.UUID] })
keepAllDuplicates(userId: string) {
return this.db
.updateTable('assets')
.set({ duplicateId: null })
.where('duplicateId', 'is not', null)
.where('ownerId', '=', userId)
.execute();
}
deduplicateAll(userId: string, keptAssetIds: string[], deduplicatedStatus: AssetStatus) {
return this.db
.with('kept', (qb) =>
// anyUuid ensures the array is passed as a single parameter, so no need to chunk
qb.updateTable('assets').set({ duplicateId: null }).where('id', '=', anyUuid(keptAssetIds)).returning('id'),
)
.updateTable('assets')
.from('kept')
.set({ duplicateId: null, status: deduplicatedStatus })
.whereRef('id', '!=', 'kept.id')
.where('duplicateId', 'is not', null)
.where('ownerId', '=', userId)
.execute();
}
@GenerateSql({ params: [DummyValue.UUID, { minAssetsPerField: 5, maxFields: 12 }] })

View File

@@ -3,8 +3,8 @@ import { JOBS_ASSET_PAGINATION_SIZE } from 'src/constants';
import { OnJob } from 'src/decorators';
import { mapAsset } from 'src/dtos/asset-response.dto';
import { AuthDto } from 'src/dtos/auth.dto';
import { DuplicateResponseDto } from 'src/dtos/duplicate.dto';
import { AssetFileType, AssetVisibility, JobName, JobStatus, QueueName } from 'src/enum';
import { DeduplicateAllDto, DuplicateResponseDto } from 'src/dtos/duplicate.dto';
import { AssetFileType, AssetStatus, AssetVisibility, JobName, JobStatus, QueueName } from 'src/enum';
import { AssetDuplicateResult } from 'src/repositories/search.repository';
import { BaseService } from 'src/services/base.service';
import { JobItem, JobOf } from 'src/types';
@@ -21,6 +21,20 @@ export class DuplicateService extends BaseService {
}));
}
keepAll(auth: AuthDto) {
return this.assetRepository.keepAllDuplicates(auth.user.id);
}
async deduplicateAll(auth: AuthDto, dto: DeduplicateAllDto) {
if (dto.assetIdsToKeep.length === 0) {
return;
}
const { trash } = await this.getConfig({ withCache: false });
const deduplicatedStatus = trash.enabled ? AssetStatus.TRASHED : AssetStatus.DELETED;
return this.assetRepository.deduplicateAll(auth.user.id, dto.assetIdsToKeep, deduplicatedStatus);
}
@OnJob({ name: JobName.QUEUE_DUPLICATE_DETECTION, queue: QueueName.DUPLICATE_DETECTION })
async handleQueueSearchDuplicates({ force }: JobOf<JobName.QUEUE_DUPLICATE_DETECTION>): Promise<JobStatus> {
const { machineLearning } = await this.getConfig({ withCache: false });
@@ -29,20 +43,16 @@ export class DuplicateService extends BaseService {
}
let jobs: JobItem[] = [];
const queueAll = async () => {
await this.jobRepository.queueAll(jobs);
jobs = [];
};
const assets = this.assetJobRepository.streamForSearchDuplicates(force);
for await (const asset of assets) {
jobs.push({ name: JobName.DUPLICATE_DETECTION, data: { id: asset.id } });
if (jobs.length >= JOBS_ASSET_PAGINATION_SIZE) {
await queueAll();
await this.jobRepository.queueAll(jobs);
jobs = [];
}
}
await queueAll();
await this.jobRepository.queueAll(jobs);
return JobStatus.SUCCESS;
}

View File

@@ -15,7 +15,7 @@
import { suggestDuplicate } from '$lib/utils/duplicate-utils';
import { handleError } from '$lib/utils/handle-error';
import type { AssetResponseDto } from '@immich/sdk';
import { deleteAssets, updateAssets } from '@immich/sdk';
import { deduplicateAll, deleteAssets, keepAll, updateAssets } from '@immich/sdk';
import { Button, HStack, IconButton, Text } from '@immich/ui';
import { mdiCheckOutline, mdiInformationOutline, mdiKeyboard, mdiTrashCanOutline } from '@mdi/js';
import { t } from 'svelte-i18n';
@@ -101,33 +101,30 @@
};
const handleDeduplicateAll = async () => {
const idsToKeep = duplicates.map((group) => suggestDuplicate(group.assets)).map((asset) => asset?.id);
const idsToDelete = duplicates.flatMap((group, i) =>
group.assets.map((asset) => asset.id).filter((asset) => asset !== idsToKeep[i]),
);
let assetCount = 0;
const assetIdsToKeep = duplicates.map((group) => suggestDuplicate(group.assets)!.id);
for (const group of duplicates) {
assetCount += group.assets.length;
assetIdsToKeep.push(suggestDuplicate(group.assets)!.id);
}
const dedupedAssetCount = assetCount - assetIdsToKeep.length;
let prompt, confirmText;
if ($featureFlags.trash) {
prompt = $t('bulk_trash_duplicates_confirmation', { values: { count: idsToDelete.length } });
prompt = $t('bulk_trash_duplicates_confirmation', { values: { count: dedupedAssetCount } });
confirmText = $t('confirm');
} else {
prompt = $t('bulk_delete_duplicates_confirmation', { values: { count: idsToDelete.length } });
prompt = $t('bulk_delete_duplicates_confirmation', { values: { count: dedupedAssetCount } });
confirmText = $t('permanently_delete');
}
return withConfirmation(
async () => {
await deleteAssets({ assetBulkDeleteDto: { ids: idsToDelete, force: !$featureFlags.trash } });
await updateAssets({
assetBulkUpdateDto: {
ids: [...idsToDelete, ...idsToKeep.filter((id): id is string => !!id)],
duplicateId: null,
},
});
await deduplicateAll({deduplicateAllDto: { assetIdsToKeep } });
duplicates = [];
deletedNotification(idsToDelete.length);
deletedNotification(dedupedAssetCount);
},
prompt,
confirmText,
@@ -135,10 +132,10 @@
};
const handleKeepAll = async () => {
const ids = duplicates.flatMap((group) => group.assets.map((asset) => asset.id));
const assetCount = duplicates.reduce((acc, cur) => acc + cur.assets.length, 0);
return withConfirmation(
async () => {
await updateAssets({ assetBulkUpdateDto: { ids, duplicateId: null } });
await keepAll();
duplicates = [];
@@ -147,7 +144,7 @@
type: NotificationType.Info,
});
},
$t('bulk_keep_duplicates_confirmation', { values: { count: ids.length } }),
$t('bulk_keep_duplicates_confirmation', { values: { count: assetCount } }),
$t('confirm'),
);
};