feat(server): efficient full app sync (#8755)

* feat(server): efficient full app sync

* add SQL, fix test compile issues

* fix linter warning

* new sync controller+service, add tests

* enable new sync controller+service

* Update server/src/services/sync.service.ts

Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>

---------

Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com>
This commit is contained in:
Fynn Petersen-Frey
2024-04-16 07:26:37 +02:00
committed by GitHub
parent 58e516c766
commit 103cb60a57
26 changed files with 1178 additions and 13 deletions

View File

@@ -24,6 +24,7 @@ doc/AssetBulkUploadCheckDto.md
doc/AssetBulkUploadCheckItem.md
doc/AssetBulkUploadCheckResponseDto.md
doc/AssetBulkUploadCheckResult.md
doc/AssetDeltaSyncResponseDto.md
doc/AssetFaceResponseDto.md
doc/AssetFaceUpdateDto.md
doc/AssetFaceUpdateItem.md
@@ -149,6 +150,7 @@ doc/SharedLinkType.md
doc/SignUpDto.md
doc/SmartInfoResponseDto.md
doc/SmartSearchDto.md
doc/SyncApi.md
doc/SystemConfigApi.md
doc/SystemConfigDto.md
doc/SystemConfigFFmpegDto.md
@@ -218,6 +220,7 @@ lib/api/person_api.dart
lib/api/search_api.dart
lib/api/server_info_api.dart
lib/api/shared_link_api.dart
lib/api/sync_api.dart
lib/api/system_config_api.dart
lib/api/tag_api.dart
lib/api/timeline_api.dart
@@ -248,6 +251,7 @@ lib/model/asset_bulk_upload_check_dto.dart
lib/model/asset_bulk_upload_check_item.dart
lib/model/asset_bulk_upload_check_response_dto.dart
lib/model/asset_bulk_upload_check_result.dart
lib/model/asset_delta_sync_response_dto.dart
lib/model/asset_face_response_dto.dart
lib/model/asset_face_update_dto.dart
lib/model/asset_face_update_item.dart
@@ -427,6 +431,7 @@ test/asset_bulk_upload_check_dto_test.dart
test/asset_bulk_upload_check_item_test.dart
test/asset_bulk_upload_check_response_dto_test.dart
test/asset_bulk_upload_check_result_test.dart
test/asset_delta_sync_response_dto_test.dart
test/asset_face_response_dto_test.dart
test/asset_face_update_dto_test.dart
test/asset_face_update_item_test.dart
@@ -552,6 +557,7 @@ test/shared_link_type_test.dart
test/sign_up_dto_test.dart
test/smart_info_response_dto_test.dart
test/smart_search_dto_test.dart
test/sync_api_test.dart
test/system_config_api_test.dart
test/system_config_dto_test.dart
test/system_config_f_fmpeg_dto_test.dart

View File

@@ -191,6 +191,8 @@ Class | Method | HTTP request | Description
*SharedLinkApi* | [**removeSharedLink**](doc//SharedLinkApi.md#removesharedlink) | **DELETE** /shared-link/{id} |
*SharedLinkApi* | [**removeSharedLinkAssets**](doc//SharedLinkApi.md#removesharedlinkassets) | **DELETE** /shared-link/{id}/assets |
*SharedLinkApi* | [**updateSharedLink**](doc//SharedLinkApi.md#updatesharedlink) | **PATCH** /shared-link/{id} |
*SyncApi* | [**getAllForUserFullSync**](doc//SyncApi.md#getallforuserfullsync) | **GET** /sync/full-sync |
*SyncApi* | [**getDeltaSync**](doc//SyncApi.md#getdeltasync) | **GET** /sync/delta-sync |
*SystemConfigApi* | [**getConfig**](doc//SystemConfigApi.md#getconfig) | **GET** /system-config |
*SystemConfigApi* | [**getConfigDefaults**](doc//SystemConfigApi.md#getconfigdefaults) | **GET** /system-config/defaults |
*SystemConfigApi* | [**getMapStyle**](doc//SystemConfigApi.md#getmapstyle) | **GET** /system-config/map/style.json |
@@ -240,6 +242,7 @@ Class | Method | HTTP request | Description
- [AssetBulkUploadCheckItem](doc//AssetBulkUploadCheckItem.md)
- [AssetBulkUploadCheckResponseDto](doc//AssetBulkUploadCheckResponseDto.md)
- [AssetBulkUploadCheckResult](doc//AssetBulkUploadCheckResult.md)
- [AssetDeltaSyncResponseDto](doc//AssetDeltaSyncResponseDto.md)
- [AssetFaceResponseDto](doc//AssetFaceResponseDto.md)
- [AssetFaceUpdateDto](doc//AssetFaceUpdateDto.md)
- [AssetFaceUpdateItem](doc//AssetFaceUpdateItem.md)

View File

@@ -0,0 +1,17 @@
# openapi.model.AssetDeltaSyncResponseDto
## Load the model package
```dart
import 'package:openapi/api.dart';
```
## Properties
Name | Type | Description | Notes
------------ | ------------- | ------------- | -------------
**deleted** | **List<String>** | | [default to const []]
**needsFullSync** | **bool** | |
**upserted** | [**List<AssetResponseDto>**](AssetResponseDto.md) | | [default to const []]
[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)

135
mobile/openapi/doc/SyncApi.md generated Normal file
View File

@@ -0,0 +1,135 @@
# openapi.api.SyncApi
## Load the API package
```dart
import 'package:openapi/api.dart';
```
All URIs are relative to */api*
Method | HTTP request | Description
------------- | ------------- | -------------
[**getAllForUserFullSync**](SyncApi.md#getallforuserfullsync) | **GET** /sync/full-sync |
[**getDeltaSync**](SyncApi.md#getdeltasync) | **GET** /sync/delta-sync |
# **getAllForUserFullSync**
> List<AssetResponseDto> getAllForUserFullSync(limit, updatedUntil, lastCreationDate, lastId, userId)
### Example
```dart
import 'package:openapi/api.dart';
// TODO Configure API key authorization: cookie
//defaultApiClient.getAuthentication<ApiKeyAuth>('cookie').apiKey = 'YOUR_API_KEY';
// uncomment below to setup prefix (e.g. Bearer) for API key, if needed
//defaultApiClient.getAuthentication<ApiKeyAuth>('cookie').apiKeyPrefix = 'Bearer';
// TODO Configure API key authorization: api_key
//defaultApiClient.getAuthentication<ApiKeyAuth>('api_key').apiKey = 'YOUR_API_KEY';
// uncomment below to setup prefix (e.g. Bearer) for API key, if needed
//defaultApiClient.getAuthentication<ApiKeyAuth>('api_key').apiKeyPrefix = 'Bearer';
// TODO Configure HTTP Bearer authorization: bearer
// Case 1. Use String Token
//defaultApiClient.getAuthentication<HttpBearerAuth>('bearer').setAccessToken('YOUR_ACCESS_TOKEN');
// Case 2. Use Function which generate token.
// String yourTokenGeneratorFunction() { ... }
//defaultApiClient.getAuthentication<HttpBearerAuth>('bearer').setAccessToken(yourTokenGeneratorFunction);
final api_instance = SyncApi();
final limit = 56; // int |
final updatedUntil = 2013-10-20T19:20:30+01:00; // DateTime |
final lastCreationDate = 2013-10-20T19:20:30+01:00; // DateTime |
final lastId = 38400000-8cf0-11bd-b23e-10b96e4ef00d; // String |
final userId = 38400000-8cf0-11bd-b23e-10b96e4ef00d; // String |
try {
final result = api_instance.getAllForUserFullSync(limit, updatedUntil, lastCreationDate, lastId, userId);
print(result);
} catch (e) {
print('Exception when calling SyncApi->getAllForUserFullSync: $e\n');
}
```
### Parameters
Name | Type | Description | Notes
------------- | ------------- | ------------- | -------------
**limit** | **int**| |
**updatedUntil** | **DateTime**| |
**lastCreationDate** | **DateTime**| | [optional]
**lastId** | **String**| | [optional]
**userId** | **String**| | [optional]
### Return type
[**List<AssetResponseDto>**](AssetResponseDto.md)
### Authorization
[cookie](../README.md#cookie), [api_key](../README.md#api_key), [bearer](../README.md#bearer)
### HTTP request headers
- **Content-Type**: Not defined
- **Accept**: application/json
[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md)
# **getDeltaSync**
> AssetDeltaSyncResponseDto getDeltaSync(updatedAfter, userIds)
### Example
```dart
import 'package:openapi/api.dart';
// TODO Configure API key authorization: cookie
//defaultApiClient.getAuthentication<ApiKeyAuth>('cookie').apiKey = 'YOUR_API_KEY';
// uncomment below to setup prefix (e.g. Bearer) for API key, if needed
//defaultApiClient.getAuthentication<ApiKeyAuth>('cookie').apiKeyPrefix = 'Bearer';
// TODO Configure API key authorization: api_key
//defaultApiClient.getAuthentication<ApiKeyAuth>('api_key').apiKey = 'YOUR_API_KEY';
// uncomment below to setup prefix (e.g. Bearer) for API key, if needed
//defaultApiClient.getAuthentication<ApiKeyAuth>('api_key').apiKeyPrefix = 'Bearer';
// TODO Configure HTTP Bearer authorization: bearer
// Case 1. Use String Token
//defaultApiClient.getAuthentication<HttpBearerAuth>('bearer').setAccessToken('YOUR_ACCESS_TOKEN');
// Case 2. Use Function which generate token.
// String yourTokenGeneratorFunction() { ... }
//defaultApiClient.getAuthentication<HttpBearerAuth>('bearer').setAccessToken(yourTokenGeneratorFunction);
final api_instance = SyncApi();
final updatedAfter = 2013-10-20T19:20:30+01:00; // DateTime |
final userIds = []; // List<String> |
try {
final result = api_instance.getDeltaSync(updatedAfter, userIds);
print(result);
} catch (e) {
print('Exception when calling SyncApi->getDeltaSync: $e\n');
}
```
### Parameters
Name | Type | Description | Notes
------------- | ------------- | ------------- | -------------
**updatedAfter** | **DateTime**| |
**userIds** | [**List<String>**](String.md)| | [default to const []]
### Return type
[**AssetDeltaSyncResponseDto**](AssetDeltaSyncResponseDto.md)
### Authorization
[cookie](../README.md#cookie), [api_key](../README.md#api_key), [bearer](../README.md#bearer)
### HTTP request headers
- **Content-Type**: Not defined
- **Accept**: application/json
[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md)

View File

@@ -46,6 +46,7 @@ part 'api/person_api.dart';
part 'api/search_api.dart';
part 'api/server_info_api.dart';
part 'api/shared_link_api.dart';
part 'api/sync_api.dart';
part 'api/system_config_api.dart';
part 'api/tag_api.dart';
part 'api/timeline_api.dart';
@@ -69,6 +70,7 @@ part 'model/asset_bulk_upload_check_dto.dart';
part 'model/asset_bulk_upload_check_item.dart';
part 'model/asset_bulk_upload_check_response_dto.dart';
part 'model/asset_bulk_upload_check_result.dart';
part 'model/asset_delta_sync_response_dto.dart';
part 'model/asset_face_response_dto.dart';
part 'model/asset_face_update_dto.dart';
part 'model/asset_face_update_item.dart';

150
mobile/openapi/lib/api/sync_api.dart generated Normal file
View File

@@ -0,0 +1,150 @@
//
// AUTO-GENERATED FILE, DO NOT MODIFY!
//
// @dart=2.12
// ignore_for_file: unused_element, unused_import
// ignore_for_file: always_put_required_named_parameters_first
// ignore_for_file: constant_identifier_names
// ignore_for_file: lines_longer_than_80_chars
part of openapi.api;
class SyncApi {
SyncApi([ApiClient? apiClient]) : apiClient = apiClient ?? defaultApiClient;
final ApiClient apiClient;
/// Performs an HTTP 'GET /sync/full-sync' operation and returns the [Response].
/// Parameters:
///
/// * [int] limit (required):
///
/// * [DateTime] updatedUntil (required):
///
/// * [DateTime] lastCreationDate:
///
/// * [String] lastId:
///
/// * [String] userId:
Future<Response> getAllForUserFullSyncWithHttpInfo(int limit, DateTime updatedUntil, { DateTime? lastCreationDate, String? lastId, String? userId, }) async {
// ignore: prefer_const_declarations
final path = r'/sync/full-sync';
// ignore: prefer_final_locals
Object? postBody;
final queryParams = <QueryParam>[];
final headerParams = <String, String>{};
final formParams = <String, String>{};
if (lastCreationDate != null) {
queryParams.addAll(_queryParams('', 'lastCreationDate', lastCreationDate));
}
if (lastId != null) {
queryParams.addAll(_queryParams('', 'lastId', lastId));
}
queryParams.addAll(_queryParams('', 'limit', limit));
queryParams.addAll(_queryParams('', 'updatedUntil', updatedUntil));
if (userId != null) {
queryParams.addAll(_queryParams('', 'userId', userId));
}
const contentTypes = <String>[];
return apiClient.invokeAPI(
path,
'GET',
queryParams,
postBody,
headerParams,
formParams,
contentTypes.isEmpty ? null : contentTypes.first,
);
}
/// Parameters:
///
/// * [int] limit (required):
///
/// * [DateTime] updatedUntil (required):
///
/// * [DateTime] lastCreationDate:
///
/// * [String] lastId:
///
/// * [String] userId:
Future<List<AssetResponseDto>?> getAllForUserFullSync(int limit, DateTime updatedUntil, { DateTime? lastCreationDate, String? lastId, String? userId, }) async {
final response = await getAllForUserFullSyncWithHttpInfo(limit, updatedUntil, lastCreationDate: lastCreationDate, lastId: lastId, userId: userId, );
if (response.statusCode >= HttpStatus.badRequest) {
throw ApiException(response.statusCode, await _decodeBodyBytes(response));
}
// When a remote server returns no body with a status of 204, we shall not decode it.
// At the time of writing this, `dart:convert` will throw an "Unexpected end of input"
// FormatException when trying to decode an empty string.
if (response.body.isNotEmpty && response.statusCode != HttpStatus.noContent) {
final responseBody = await _decodeBodyBytes(response);
return (await apiClient.deserializeAsync(responseBody, 'List<AssetResponseDto>') as List)
.cast<AssetResponseDto>()
.toList(growable: false);
}
return null;
}
/// Performs an HTTP 'GET /sync/delta-sync' operation and returns the [Response].
/// Parameters:
///
/// * [DateTime] updatedAfter (required):
///
/// * [List<String>] userIds (required):
Future<Response> getDeltaSyncWithHttpInfo(DateTime updatedAfter, List<String> userIds,) async {
// ignore: prefer_const_declarations
final path = r'/sync/delta-sync';
// ignore: prefer_final_locals
Object? postBody;
final queryParams = <QueryParam>[];
final headerParams = <String, String>{};
final formParams = <String, String>{};
queryParams.addAll(_queryParams('', 'updatedAfter', updatedAfter));
queryParams.addAll(_queryParams('multi', 'userIds', userIds));
const contentTypes = <String>[];
return apiClient.invokeAPI(
path,
'GET',
queryParams,
postBody,
headerParams,
formParams,
contentTypes.isEmpty ? null : contentTypes.first,
);
}
/// Parameters:
///
/// * [DateTime] updatedAfter (required):
///
/// * [List<String>] userIds (required):
Future<AssetDeltaSyncResponseDto?> getDeltaSync(DateTime updatedAfter, List<String> userIds,) async {
final response = await getDeltaSyncWithHttpInfo(updatedAfter, userIds,);
if (response.statusCode >= HttpStatus.badRequest) {
throw ApiException(response.statusCode, await _decodeBodyBytes(response));
}
// When a remote server returns no body with a status of 204, we shall not decode it.
// At the time of writing this, `dart:convert` will throw an "Unexpected end of input"
// FormatException when trying to decode an empty string.
if (response.body.isNotEmpty && response.statusCode != HttpStatus.noContent) {
return await apiClient.deserializeAsync(await _decodeBodyBytes(response), 'AssetDeltaSyncResponseDto',) as AssetDeltaSyncResponseDto;
}
return null;
}
}

View File

@@ -216,6 +216,8 @@ class ApiClient {
return AssetBulkUploadCheckResponseDto.fromJson(value);
case 'AssetBulkUploadCheckResult':
return AssetBulkUploadCheckResult.fromJson(value);
case 'AssetDeltaSyncResponseDto':
return AssetDeltaSyncResponseDto.fromJson(value);
case 'AssetFaceResponseDto':
return AssetFaceResponseDto.fromJson(value);
case 'AssetFaceUpdateDto':

View File

@@ -0,0 +1,116 @@
//
// AUTO-GENERATED FILE, DO NOT MODIFY!
//
// @dart=2.12
// ignore_for_file: unused_element, unused_import
// ignore_for_file: always_put_required_named_parameters_first
// ignore_for_file: constant_identifier_names
// ignore_for_file: lines_longer_than_80_chars
part of openapi.api;
class AssetDeltaSyncResponseDto {
/// Returns a new [AssetDeltaSyncResponseDto] instance.
AssetDeltaSyncResponseDto({
this.deleted = const [],
required this.needsFullSync,
this.upserted = const [],
});
List<String> deleted;
bool needsFullSync;
List<AssetResponseDto> upserted;
@override
bool operator ==(Object other) => identical(this, other) || other is AssetDeltaSyncResponseDto &&
_deepEquality.equals(other.deleted, deleted) &&
other.needsFullSync == needsFullSync &&
_deepEquality.equals(other.upserted, upserted);
@override
int get hashCode =>
// ignore: unnecessary_parenthesis
(deleted.hashCode) +
(needsFullSync.hashCode) +
(upserted.hashCode);
@override
String toString() => 'AssetDeltaSyncResponseDto[deleted=$deleted, needsFullSync=$needsFullSync, upserted=$upserted]';
Map<String, dynamic> toJson() {
final json = <String, dynamic>{};
json[r'deleted'] = this.deleted;
json[r'needsFullSync'] = this.needsFullSync;
json[r'upserted'] = this.upserted;
return json;
}
/// Returns a new [AssetDeltaSyncResponseDto] instance and imports its values from
/// [value] if it's a [Map], null otherwise.
// ignore: prefer_constructors_over_static_methods
static AssetDeltaSyncResponseDto? fromJson(dynamic value) {
if (value is Map) {
final json = value.cast<String, dynamic>();
return AssetDeltaSyncResponseDto(
deleted: json[r'deleted'] is Iterable
? (json[r'deleted'] as Iterable).cast<String>().toList(growable: false)
: const [],
needsFullSync: mapValueOfType<bool>(json, r'needsFullSync')!,
upserted: AssetResponseDto.listFromJson(json[r'upserted']),
);
}
return null;
}
static List<AssetDeltaSyncResponseDto> listFromJson(dynamic json, {bool growable = false,}) {
final result = <AssetDeltaSyncResponseDto>[];
if (json is List && json.isNotEmpty) {
for (final row in json) {
final value = AssetDeltaSyncResponseDto.fromJson(row);
if (value != null) {
result.add(value);
}
}
}
return result.toList(growable: growable);
}
static Map<String, AssetDeltaSyncResponseDto> mapFromJson(dynamic json) {
final map = <String, AssetDeltaSyncResponseDto>{};
if (json is Map && json.isNotEmpty) {
json = json.cast<String, dynamic>(); // ignore: parameter_assignments
for (final entry in json.entries) {
final value = AssetDeltaSyncResponseDto.fromJson(entry.value);
if (value != null) {
map[entry.key] = value;
}
}
}
return map;
}
// maps a json object with a list of AssetDeltaSyncResponseDto-objects as value to a dart map
static Map<String, List<AssetDeltaSyncResponseDto>> mapListFromJson(dynamic json, {bool growable = false,}) {
final map = <String, List<AssetDeltaSyncResponseDto>>{};
if (json is Map && json.isNotEmpty) {
// ignore: parameter_assignments
json = json.cast<String, dynamic>();
for (final entry in json.entries) {
map[entry.key] = AssetDeltaSyncResponseDto.listFromJson(entry.value, growable: growable,);
}
}
return map;
}
/// The list of required keys that must be present in a JSON.
static const requiredKeys = <String>{
'deleted',
'needsFullSync',
'upserted',
};
}

View File

@@ -0,0 +1,37 @@
//
// AUTO-GENERATED FILE, DO NOT MODIFY!
//
// @dart=2.12
// ignore_for_file: unused_element, unused_import
// ignore_for_file: always_put_required_named_parameters_first
// ignore_for_file: constant_identifier_names
// ignore_for_file: lines_longer_than_80_chars
import 'package:openapi/api.dart';
import 'package:test/test.dart';
// tests for AssetDeltaSyncResponseDto
void main() {
// final instance = AssetDeltaSyncResponseDto();
group('test AssetDeltaSyncResponseDto', () {
// List<String> deleted (default value: const [])
test('to test the property `deleted`', () async {
// TODO
});
// bool needsFullSync
test('to test the property `needsFullSync`', () async {
// TODO
});
// List<AssetResponseDto> upserted (default value: const [])
test('to test the property `upserted`', () async {
// TODO
});
});
}

31
mobile/openapi/test/sync_api_test.dart generated Normal file
View File

@@ -0,0 +1,31 @@
//
// AUTO-GENERATED FILE, DO NOT MODIFY!
//
// @dart=2.12
// ignore_for_file: unused_element, unused_import
// ignore_for_file: always_put_required_named_parameters_first
// ignore_for_file: constant_identifier_names
// ignore_for_file: lines_longer_than_80_chars
import 'package:openapi/api.dart';
import 'package:test/test.dart';
/// tests for SyncApi
void main() {
// final instance = SyncApi();
group('tests for SyncApi', () {
//Future<List<AssetResponseDto>> getAllForUserFullSync(int limit, DateTime updatedUntil, { DateTime lastCreationDate, String lastId, String userId }) async
test('test getAllForUserFullSync', () async {
// TODO
});
//Future<AssetDeltaSyncResponseDto> getDeltaSync(DateTime updatedAfter, List<String> userIds) async
test('test getDeltaSync', () async {
// TODO
});
});
}