refactor: hashing service (#21997)

* download only backup selected assets

* android impl

* fix tests

* limit concurrent hashing to 16

* extension cleanup

* optimized hashing

* hash only selected albums

* remove concurrency limit

* address review comments

* log more info on failure

* add native cancellation

* small batch size on ios, large on android

* fix: get correct resources

* cleanup getResource

* ios better hash cancellation

* handle graceful cancellation android

* do not trigger multiple hashing ops

* ios: fix circular reference, improve cancellation

* kotlin: more cancellation checks

* no need to create result

* cancel previous task

* avoid race condition

* ensure cancellation gets called

* fix cancellation not happening

---------

Co-authored-by: shenlong-tanwen <139912620+shalong-tanwen@users.noreply.github.com>
Co-authored-by: mertalev <101130780+mertalev@users.noreply.github.com>
Co-authored-by: Alex <alex.tran1502@gmail.com>

mobile/android/app/src/main/kotlin/app/alextran/immich/sync/Messages.g.kt

@@ -209,6 +209,40 @@ data class SyncDelta (
 
   override fun hashCode(): Int = toList().hashCode()
 }
+
+/** Generated class from Pigeon that represents data sent in messages. */
+data class HashResult (
+  val assetId: String,
+  val error: String? = null,
+  val hash: String? = null
+)
+ {
+  companion object {
+    fun fromList(pigeonVar_list: List<Any?>): HashResult {
+      val assetId = pigeonVar_list[0] as String
+      val error = pigeonVar_list[1] as String?
+      val hash = pigeonVar_list[2] as String?
+      return HashResult(assetId, error, hash)
+    }
+  }
+  fun toList(): List<Any?> {
+    return listOf(
+      assetId,
+      error,
+      hash,
+    )
+  }
+  override fun equals(other: Any?): Boolean {
+    if (other !is HashResult) {
+      return false
+    }
+    if (this === other) {
+      return true
+    }
+    return MessagesPigeonUtils.deepEquals(toList(), other.toList())  }
+
+  override fun hashCode(): Int = toList().hashCode()
+}
 private open class MessagesPigeonCodec : StandardMessageCodec() {
   override fun readValueOfType(type: Byte, buffer: ByteBuffer): Any? {
     return when (type) {
@@ -227,6 +261,11 @@ private open class MessagesPigeonCodec : StandardMessageCodec() {
           SyncDelta.fromList(it)
         }
       }
+      132.toByte() -> {
+        return (readValue(buffer) as? List<Any?>)?.let {
+          HashResult.fromList(it)
+        }
+      }
       else -> super.readValueOfType(type, buffer)
     }
   }
@@ -244,11 +283,16 @@ private open class MessagesPigeonCodec : StandardMessageCodec() {
         stream.write(131)
         writeValue(stream, value.toList())
       }
+      is HashResult -> {
+        stream.write(132)
+        writeValue(stream, value.toList())
+      }
       else -> super.writeValue(stream, value)
     }
   }
 }
 
+
 /** Generated interface from Pigeon that represents a handler of messages from Flutter. */
 interface NativeSyncApi {
   fun shouldFullSync(): Boolean
@@ -259,7 +303,8 @@ interface NativeSyncApi {
   fun getAlbums(): List<PlatformAlbum>
   fun getAssetsCountSince(albumId: String, timestamp: Long): Long
   fun getAssetsForAlbum(albumId: String, updatedTimeCond: Long?): List<PlatformAsset>
-  fun hashPaths(paths: List<String>): List<ByteArray?>
+  fun hashAssets(assetIds: List<String>, allowNetworkAccess: Boolean, callback: (Result<List<HashResult>>) -> Unit)
+  fun cancelHashing()
 
   companion object {
     /** The codec used by NativeSyncApi. */
@@ -402,13 +447,33 @@ interface NativeSyncApi {
         }
       }
       run {
-        val channel = BasicMessageChannel<Any?>(binaryMessenger, "dev.flutter.pigeon.immich_mobile.NativeSyncApi.hashPaths$separatedMessageChannelSuffix", codec, taskQueue)
+        val channel = BasicMessageChannel<Any?>(binaryMessenger, "dev.flutter.pigeon.immich_mobile.NativeSyncApi.hashAssets$separatedMessageChannelSuffix", codec, taskQueue)
         if (api != null) {
           channel.setMessageHandler { message, reply ->
             val args = message as List<Any?>
-            val pathsArg = args[0] as List<String>
+            val assetIdsArg = args[0] as List<String>
+            val allowNetworkAccessArg = args[1] as Boolean
+            api.hashAssets(assetIdsArg, allowNetworkAccessArg) { result: Result<List<HashResult>> ->
+              val error = result.exceptionOrNull()
+              if (error != null) {
+                reply.reply(MessagesPigeonUtils.wrapError(error))
+              } else {
+                val data = result.getOrNull()
+                reply.reply(MessagesPigeonUtils.wrapResult(data))
+              }
+            }
+          }
+        } else {
+          channel.setMessageHandler(null)
+        }
+      }
+      run {
+        val channel = BasicMessageChannel<Any?>(binaryMessenger, "dev.flutter.pigeon.immich_mobile.NativeSyncApi.cancelHashing$separatedMessageChannelSuffix", codec)
+        if (api != null) {
+          channel.setMessageHandler { _, reply ->
             val wrapped: List<Any?> = try {
-              listOf(api.hashPaths(pathsArg))
+              api.cancelHashing()
+              listOf(null)
             } catch (exception: Throwable) {
               MessagesPigeonUtils.wrapError(exception)
             }

mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImplBase.kt

@@ -1,14 +1,25 @@
 package app.alextran.immich.sync
 
 import android.annotation.SuppressLint
+import android.content.ContentUris
 import android.content.Context
 import android.database.Cursor
 import android.provider.MediaStore
-import android.util.Log
+import android.util.Base64
 import androidx.core.database.getStringOrNull
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.Job
+import kotlinx.coroutines.async
+import kotlinx.coroutines.awaitAll
+import kotlinx.coroutines.ensureActive
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.sync.Semaphore
+import kotlinx.coroutines.sync.withPermit
 import java.io.File
-import java.io.FileInputStream
 import java.security.MessageDigest
+import kotlin.coroutines.cancellation.CancellationException
+import kotlin.coroutines.coroutineContext
 
 sealed class AssetResult {
   data class ValidAsset(val asset: PlatformAsset, val albumId: String) : AssetResult()
@@ -19,8 +30,12 @@ sealed class AssetResult {
 open class NativeSyncApiImplBase(context: Context) {
   private val ctx: Context = context.applicationContext
 
+  private var hashTask: Job? = null
+
   companion object {
-    private const val TAG = "NativeSyncApiImplBase"
+    private const val MAX_CONCURRENT_HASH_OPERATIONS = 16
+    private val hashSemaphore = Semaphore(MAX_CONCURRENT_HASH_OPERATIONS)
+    private const val HASHING_CANCELLED_CODE = "HASH_CANCELLED"
 
     const val MEDIA_SELECTION =
       "(${MediaStore.Files.FileColumns.MEDIA_TYPE} = ? OR ${MediaStore.Files.FileColumns.MEDIA_TYPE} = ?)"
@@ -215,23 +230,74 @@ open class NativeSyncApiImplBase(context: Context) {
       .toList()
   }
 
-  fun hashPaths(paths: List<String>): List<ByteArray?> {
-    val buffer = ByteArray(HASH_BUFFER_SIZE)
-    val digest = MessageDigest.getInstance("SHA-1")
+  fun hashAssets(
+    assetIds: List<String>,
+    // allowNetworkAccess is only used on the iOS implementation
+    @Suppress("UNUSED_PARAMETER") allowNetworkAccess: Boolean,
+    callback: (Result<List<HashResult>>) -> Unit
+  ) {
+    if (assetIds.isEmpty()) {
+      callback(Result.success(emptyList()))
+      return
+    }
 
-    return paths.map { path ->
+    hashTask?.cancel()
+    hashTask = CoroutineScope(Dispatchers.IO).launch {
       try {
-        FileInputStream(path).use { file ->
-          var bytesRead: Int
-          while (file.read(buffer).also { bytesRead = it } > 0) {
-            digest.update(buffer, 0, bytesRead)
+        val results = assetIds.map { assetId ->
+          async {
+            hashSemaphore.withPermit {
+              ensureActive()
+              hashAsset(assetId)
+            }
           }
-        }
-        digest.digest()
+        }.awaitAll()
+
+        callback(Result.success(results))
+      } catch (e: CancellationException) {
+        callback(
+          Result.failure(
+            FlutterError(
+              HASHING_CANCELLED_CODE,
+              "Hashing operation was cancelled",
+              null
+            )
+          )
+        )
       } catch (e: Exception) {
-        Log.w(TAG, "Failed to hash file $path: $e")
-        null
+        callback(Result.failure(e))
       }
     }
   }
+
+  private suspend fun hashAsset(assetId: String): HashResult {
+    return try {
+      val assetUri = ContentUris.withAppendedId(
+        MediaStore.Files.getContentUri(MediaStore.VOLUME_EXTERNAL),
+        assetId.toLong()
+      )
+
+      val digest = MessageDigest.getInstance("SHA-1")
+      ctx.contentResolver.openInputStream(assetUri)?.use { inputStream ->
+        var bytesRead: Int
+        val buffer = ByteArray(HASH_BUFFER_SIZE)
+        while (inputStream.read(buffer).also { bytesRead = it } > 0) {
+          coroutineContext.ensureActive()
+          digest.update(buffer, 0, bytesRead)
+        }
+      } ?: return HashResult(assetId, "Cannot open input stream for asset", null)
+
+      val hashString = Base64.encodeToString(digest.digest(), Base64.NO_WRAP)
+      HashResult(assetId, null, hashString)
+    } catch (e: SecurityException) {
+      HashResult(assetId, "Permission denied accessing asset: ${e.message}", null)
+    } catch (e: Exception) {
+      HashResult(assetId, "Failed to hash asset: ${e.message}", null)
+    }
+  }
+
+  fun cancelHashing() {
+    hashTask?.cancel()
+    hashTask = null
+  }
 }