feat: sync cloudId and eTag on sync

mobile/lib/domain/models/asset/asset_metadata.model.dart

@@ -0,0 +1,44 @@
+enum RemoteAssetMetadataKey {
+  mobileApp("mobile-app");
+
+  final String key;
+
+  const RemoteAssetMetadataKey(this.key);
+}
+
+abstract class RemoteAssetMetadataValue {
+  const RemoteAssetMetadataValue();
+
+  Map<String, dynamic> toJson();
+}
+
+class RemoteAssetMetadataItem {
+  final RemoteAssetMetadataKey key;
+  final RemoteAssetMetadataValue value;
+
+  const RemoteAssetMetadataItem({required this.key, required this.value});
+
+  Map<String, Object?> toJson() {
+    return {'key': key.key, 'value': value};
+  }
+}
+
+class RemoteAssetMobileAppMetadata extends RemoteAssetMetadataValue {
+  final String? cloudId;
+  final String? eTag;
+
+  const RemoteAssetMobileAppMetadata({this.cloudId, this.eTag});
+
+  @override
+  Map<String, dynamic> toJson() {
+    final map = <String, Object?>{};
+    if (cloudId != null) {
+      map["iCloudId"] = cloudId;
+    }
+    if (eTag != null) {
+      map["eTag"] = eTag;
+    }
+
+    return map;
+  }
+}

mobile/lib/domain/models/asset/local_asset.model.dart

@@ -3,6 +3,7 @@ part of 'base_asset.model.dart';
 class LocalAsset extends BaseAsset {
   final String id;
   final String? remoteAssetId;
+  final String? cloudId;
   final int orientation;
 
   final DateTime? adjustmentTime;
@@ -12,6 +13,7 @@ class LocalAsset extends BaseAsset {
   const LocalAsset({
     required this.id,
     String? remoteId,
+    this.cloudId,
     required super.name,
     super.checksum,
     required super.type,
@@ -40,6 +42,9 @@ class LocalAsset extends BaseAsset {
   @override
   String get heroTag => '${id}_${remoteId ?? checksum}';
 
+  String get eTag =>
+      "${createdAt.millisecondsSinceEpoch ~/ 1000}$kUploadETagDelimiter${(adjustmentTime?.millisecondsSinceEpoch ?? 0) ~/ 1000}$kUploadETagDelimiter${latitude ?? 0}$kUploadETagDelimiter${longitude ?? 0}";
+
   bool get hasCoordinates => latitude != null && longitude != null && latitude != 0 && longitude != 0;
 
   @override
@@ -54,11 +59,13 @@ class LocalAsset extends BaseAsset {
    height: ${height ?? "<NA>"},
    durationInSeconds: ${durationInSeconds ?? "<NA>"},
    remoteId: ${remoteId ?? "<NA>"}
+   cloudId: ${cloudId ?? "<NA>"}
+   checksum: ${checksum ?? "<NA>"},
    isFavorite: $isFavorite,
-  orientation: $orientation,
-  adjustmentTime: $adjustmentTime,
-  latitude: ${latitude ?? "<NA>"},
-  longitude: ${longitude ?? "<NA>"},
+   orientation: $orientation,
+   adjustmentTime: $adjustmentTime,
+   latitude: ${latitude ?? "<NA>"},
+   longitude: ${longitude ?? "<NA>"},
  }''';
   }
 
@@ -69,6 +76,7 @@ class LocalAsset extends BaseAsset {
     if (identical(this, other)) return true;
     return super == other &&
         id == other.id &&
+        cloudId == other.cloudId &&
         orientation == other.orientation &&
         adjustmentTime == other.adjustmentTime &&
         latitude == other.latitude &&
@@ -88,6 +96,7 @@ class LocalAsset extends BaseAsset {
   LocalAsset copyWith({
     String? id,
     String? remoteId,
+    String? cloudId,
     String? name,
     String? checksum,
     AssetType? type,
@@ -105,6 +114,7 @@ class LocalAsset extends BaseAsset {
     return LocalAsset(
       id: id ?? this.id,
       remoteId: remoteId ?? this.remoteId,
+      cloudId: cloudId ?? this.cloudId,
       name: name ?? this.name,
       checksum: checksum ?? this.checksum,
       type: type ?? this.type,

mobile/lib/domain/services/hash.service.dart

@@ -40,6 +40,9 @@ class HashService {
     _log.info("Starting hashing of assets");
     final Stopwatch stopwatch = Stopwatch()..start();
     try {
+      // Migrate hashes from cloud ID to local ID so we don't have to re-hash them
+      await _migrateHashes();
+
       // Sorted by backupSelection followed by isCloud
       final localAlbums = await _localAlbumRepository.getBackupAlbums();
 
@@ -75,6 +78,15 @@ class HashService {
     _log.info("Hashing took - ${stopwatch.elapsedMilliseconds}ms");
   }
 
+  Future<void> _migrateHashes() async {
+    final hashMappings = await _localAssetRepository.getHashMappingFromCloudId();
+    if (hashMappings.isEmpty) {
+      return;
+    }
+
+    await _localAssetRepository.updateHashes(hashMappings);
+  }
+
   /// Processes a list of [LocalAsset]s, storing their hash and updating the assets in the DB
   /// with hash for those that were successfully hashed. Hashes are looked up in a table
   /// [LocalAssetHashEntity] by local id. Only missing entries are newly hashed and added to the DB.

mobile/lib/domain/services/local_sync.service.dart

@@ -63,8 +63,9 @@ class LocalSyncService {
 
       final deviceAlbums = await _nativeSyncApi.getAlbums();
       await _localAlbumRepository.updateAll(deviceAlbums.toLocalAlbums());
+      final newAssets = delta.updates.toLocalAssets();
       await _localAlbumRepository.processDelta(
-        updates: delta.updates.toLocalAssets(),
+        updates: newAssets,
         deletes: delta.deletes,
         assetAlbums: delta.assetAlbums,
       );
@@ -92,6 +93,8 @@ class LocalSyncService {
           }
           await updateAlbum(dbAlbum, album);
         }
+
+        await _mapIosCloudIds(newAssets);
       }
       await _nativeSyncApi.checkpointSync();
     } catch (e, s) {
@@ -130,9 +133,12 @@ class LocalSyncService {
     try {
       _log.fine("Adding device album ${album.name}");
 
-      final assets = album.assetCount > 0 ? await _nativeSyncApi.getAssetsForAlbum(album.id) : <PlatformAsset>[];
+      final assets = album.assetCount > 0
+          ? await _nativeSyncApi.getAssetsForAlbum(album.id).then((a) => a.toLocalAssets())
+          : <LocalAsset>[];
 
-      await _localAlbumRepository.upsert(album, toUpsert: assets.toLocalAssets());
+      await _localAlbumRepository.upsert(album, toUpsert: assets);
+      await _mapIosCloudIds(assets);
       _log.fine("Successfully added device album ${album.name}");
     } catch (e, s) {
       _log.warning("Error while adding device album", e, s);
@@ -202,13 +208,16 @@ class LocalSyncService {
         return false;
       }
 
-      final newAssets = await _nativeSyncApi.getAssetsForAlbum(deviceAlbum.id, updatedTimeCond: updatedTime);
+      final newAssets = await _nativeSyncApi
+          .getAssetsForAlbum(deviceAlbum.id, updatedTimeCond: updatedTime)
+          .then((a) => a.toLocalAssets());
 
       await _localAlbumRepository.upsert(
         deviceAlbum.copyWith(backupSelection: dbAlbum.backupSelection),
-        toUpsert: newAssets.toLocalAssets(),
+        toUpsert: newAssets,
       );
 
+      await _mapIosCloudIds(newAssets);
       return true;
     } catch (e, s) {
       _log.warning("Error on fast syncing local album: ${dbAlbum.name}", e, s);
@@ -240,6 +249,7 @@ class LocalSyncService {
       if (dbAlbum.assetCount == 0) {
         _log.fine("Device album ${deviceAlbum.name} is empty. Adding assets to DB.");
         await _localAlbumRepository.upsert(updatedDeviceAlbum, toUpsert: assetsInDevice);
+        await _mapIosCloudIds(assetsInDevice);
         return true;
       }
 
@@ -277,6 +287,7 @@ class LocalSyncService {
       }
 
       await _localAlbumRepository.upsert(updatedDeviceAlbum, toUpsert: assetsToUpsert, toDelete: assetsToDelete);
+      await _mapIosCloudIds(assetsToUpsert);
 
       return true;
     } catch (e, s) {
@@ -285,6 +296,16 @@ class LocalSyncService {
     return true;
   }
 
+  Future<void> _mapIosCloudIds(List<LocalAsset> assets) async {
+    if (!CurrentPlatform.isIOS || assets.isEmpty) {
+      return;
+    }
+
+    final assetIds = assets.map((a) => a.id).toList();
+    final cloudMapping = await _nativeSyncApi.getCloudIdForAssetIds(assetIds);
+    await _localAlbumRepository.updateCloudMapping(cloudMapping);
+  }
+
   bool _assetsEqual(LocalAsset a, LocalAsset b) {
     if (CurrentPlatform.isAndroid) {
       return a.updatedAt.isAtSameMomentAs(b.updatedAt) &&

mobile/lib/domain/services/sync_stream.service.dart

@@ -118,6 +118,10 @@ class SyncStreamService {
         return _syncStreamRepository.deleteAssetsV1(data.cast());
       case SyncEntityType.assetExifV1:
         return _syncStreamRepository.updateAssetsExifV1(data.cast());
+      case SyncEntityType.assetMetadataV1:
+        return _syncStreamRepository.updateAssetsMetadataV1(data.cast());
+      case SyncEntityType.assetMetadataDeleteV1:
+        return _syncStreamRepository.deleteAssetsMetadataV1(data.cast());
       case SyncEntityType.partnerAssetV1:
         return _syncStreamRepository.updateAssetsV1(data.cast(), debugLabel: 'partner');
       case SyncEntityType.partnerAssetBackfillV1:

mobile/lib/domain/utils/background_sync.dart

@@ -1,5 +1,6 @@
 import 'dart:async';
 
+import 'package:immich_mobile/domain/utils/migrate_cloud_ids.dart' as m;
 import 'package:immich_mobile/domain/utils/sync_linked_album.dart';
 import 'package:immich_mobile/providers/infrastructure/sync.provider.dart';
 import 'package:immich_mobile/utils/isolate.dart';
@@ -22,8 +23,13 @@ class BackgroundSyncManager {
   final SyncCallback? onHashingComplete;
   final SyncErrorCallback? onHashingError;
 
+  final SyncCallback? onCloudIdSyncStart;
+  final SyncCallback? onCloudIdSyncComplete;
+  final SyncErrorCallback? onCloudIdSyncError;
+
   Cancelable<bool?>? _syncTask;
   Cancelable<void>? _syncWebsocketTask;
+  Cancelable<void>? _cloudIdSyncTask;
   Cancelable<void>? _deviceAlbumSyncTask;
   Cancelable<void>? _linkedAlbumSyncTask;
   Cancelable<void>? _hashTask;
@@ -38,6 +44,9 @@ class BackgroundSyncManager {
     this.onHashingStart,
     this.onHashingComplete,
     this.onHashingError,
+    this.onCloudIdSyncStart,
+    this.onCloudIdSyncComplete,
+    this.onCloudIdSyncError,
   });
 
   Future<void> cancel() async {
@@ -55,6 +64,12 @@ class BackgroundSyncManager {
     _syncWebsocketTask?.cancel();
     _syncWebsocketTask = null;
 
+    if (_cloudIdSyncTask != null) {
+      futures.add(_cloudIdSyncTask!.future);
+    }
+    _cloudIdSyncTask?.cancel();
+    _cloudIdSyncTask = null;
+
     if (_linkedAlbumSyncTask != null) {
       futures.add(_linkedAlbumSyncTask!.future);
     }
@@ -121,7 +136,6 @@ class BackgroundSyncManager {
         });
   }
 
-  // No need to cancel the task, as it can also be run when the user logs out
   Future<void> hashAssets() {
     if (_hashTask != null) {
       return _hashTask!.future;
@@ -192,6 +206,25 @@ class BackgroundSyncManager {
       _linkedAlbumSyncTask = null;
     });
   }
+
+  Future<void> syncCloudIds() {
+    if (_cloudIdSyncTask != null) {
+      return _cloudIdSyncTask!.future;
+    }
+
+    onCloudIdSyncStart?.call();
+
+    _cloudIdSyncTask = runInIsolateGentle(computation: m.syncCloudIds);
+    return _cloudIdSyncTask!
+        .whenComplete(() {
+          onCloudIdSyncComplete?.call();
+          _cloudIdSyncTask = null;
+        })
+        .catchError((error) {
+          onCloudIdSyncError?.call(error.toString());
+          _cloudIdSyncTask = null;
+        });
+  }
 }
 
 Cancelable<void> _handleWsAssetUploadReadyV1Batch(List<dynamic> batchData) => runInIsolateGentle(

mobile/lib/domain/utils/migrate_cloud_ids.dart

@@ -0,0 +1,82 @@
+import 'package:drift/drift.dart';
+import 'package:hooks_riverpod/hooks_riverpod.dart';
+import 'package:immich_mobile/domain/models/asset/asset_metadata.model.dart';
+import 'package:immich_mobile/infrastructure/repositories/db.repository.dart';
+import 'package:immich_mobile/infrastructure/repositories/local_album.repository.dart';
+import 'package:immich_mobile/platform/native_sync_api.g.dart';
+import 'package:immich_mobile/providers/api.provider.dart';
+import 'package:immich_mobile/providers/infrastructure/db.provider.dart';
+import 'package:immich_mobile/providers/infrastructure/sync.provider.dart';
+import 'package:immich_mobile/providers/user.provider.dart';
+import 'package:logging/logging.dart';
+// ignore: import_rule_openapi
+import 'package:openapi/api.dart';
+
+Future<void> syncCloudIds(ProviderContainer ref) async {
+  final db = ref.read(driftProvider);
+  // Populate cloud IDs for local assets that don't have one yet
+  await _populateCloudIds(db);
+
+  // Wait for remote sync to complete, so we have up-to-date asset metadata entries
+  await ref.read(syncStreamServiceProvider).sync();
+
+  // Fetch the mapping for backed up assets that have a cloud ID locally but do not have a cloud ID on the server
+  final currentUser = ref.read(currentUserProvider);
+  if (currentUser == null) {
+    Logger('migrateCloudIds').warning('Current user is null. Aborting cloudId migration.');
+    return;
+  }
+
+  final mappingsToUpdate = await _fetchCloudIdMappings(db, currentUser.id);
+  final assetApi = ref.read(apiServiceProvider).assetsApi;
+  for (final mapping in mappingsToUpdate) {
+    final mobileMeta = AssetMetadataUpsertItemDto(
+      key: AssetMetadataKey.mobileApp,
+      value: RemoteAssetMobileAppMetadata(cloudId: mapping.cloudId),
+    );
+    try {
+      await assetApi.updateAssetMetadata(mapping.assetId, AssetMetadataUpsertDto(items: [mobileMeta]));
+    } catch (error, stack) {
+      Logger('migrateCloudIds').warning('Failed to update metadata for asset ${mapping.assetId}', error, stack);
+    }
+  }
+}
+
+Future<void> _populateCloudIds(Drift drift) async {
+  final query = drift.localAssetEntity.selectOnly()
+    ..addColumns([drift.localAssetEntity.id])
+    ..where(drift.localAssetEntity.iCloudId.isNull());
+  final ids = await query.map((row) => row.read(drift.localAssetEntity.id)!).get();
+  final cloudMapping = await NativeSyncApi().getCloudIdForAssetIds(ids);
+  await DriftLocalAlbumRepository(drift).updateCloudMapping(cloudMapping);
+}
+
+typedef _CloudIdMapping = ({String assetId, String cloudId});
+
+Future<List<_CloudIdMapping>> _fetchCloudIdMappings(Drift drift, String userId) async {
+  final query =
+      drift.remoteAssetEntity.selectOnly().join([
+          leftOuterJoin(
+            drift.localAssetEntity,
+            drift.localAssetEntity.checksum.equalsExp(drift.remoteAssetEntity.checksum),
+            useColumns: false,
+          ),
+          leftOuterJoin(
+            drift.remoteAssetCloudIdEntity,
+            drift.localAssetEntity.iCloudId.equalsExp(drift.remoteAssetCloudIdEntity.cloudId),
+            useColumns: false,
+          ),
+        ])
+        ..addColumns([drift.remoteAssetEntity.id, drift.localAssetEntity.iCloudId])
+        ..where(
+          drift.localAssetEntity.id.isNotNull() &
+              drift.localAssetEntity.iCloudId.isNotNull() &
+              drift.remoteAssetEntity.ownerId.equals(userId) &
+              drift.remoteAssetCloudIdEntity.cloudId.isNull(),
+        );
+  return query
+      .map(
+        (row) => (assetId: row.read(drift.remoteAssetEntity.id)!, cloudId: row.read(drift.localAssetEntity.iCloudId)!),
+      )
+      .get();
+}