memory optimization

mobile/android/app/CMakeLists.txt

@@ -8,3 +8,5 @@ project(native_buffer LANGUAGES C)
 add_library(native_buffer SHARED
     src/main/cpp/native_buffer.c
 )
+
+target_link_libraries(native_buffer jnigraphics)

mobile/android/app/src/main/cpp/native_buffer.c

@@ -1,40 +1,39 @@
 #include <jni.h>
 #include <stdlib.h>
+#include <android/bitmap.h>
 
 JNIEXPORT jlong JNICALL
-Java_app_alextran_immich_images_ThumbnailsImpl_00024Companion_allocateNative(
-        JNIEnv *env, jclass clazz, jint size) {
-    void *ptr = malloc(size);
-    return (jlong) ptr;
-}
-
-JNIEXPORT jlong JNICALL
-Java_app_alextran_immich_images_ThumbnailsImpl_allocateNative(
+Java_app_alextran_immich_images_LocalImagesImpl_allocateNative(
         JNIEnv *env, jclass clazz, jint size) {
     void *ptr = malloc(size);
     return (jlong) ptr;
 }
 
 JNIEXPORT void JNICALL
-Java_app_alextran_immich_images_ThumbnailsImpl_00024Companion_freeNative(
+Java_app_alextran_immich_images_LocalImagesImpl_freeNative(
         JNIEnv *env, jclass clazz, jlong address) {
     free((void *) address);
 }
 
+JNIEXPORT jobject JNICALL
+Java_app_alextran_immich_images_LocalImagesImpl_wrapAsBuffer(
+        JNIEnv *env, jclass clazz, jlong address, jint capacity) {
+    return (*env)->NewDirectByteBuffer(env, (void *) address, capacity);
+}
+
+JNIEXPORT jlong JNICALL
+Java_app_alextran_immich_images_RemoteImagesImpl_lockBitmapPixels(
+        JNIEnv *env, jclass clazz, jobject bitmap) {
+    void *pixels = NULL;
+    int result = AndroidBitmap_lockPixels(env, bitmap, &pixels);
+    if (result != ANDROID_BITMAP_RESULT_SUCCESS) {
+        return 0;
+    }
+    return (jlong) pixels;
+}
+
 JNIEXPORT void JNICALL
-Java_app_alextran_immich_images_ThumbnailsImpl_freeNative(
-        JNIEnv *env, jclass clazz, jlong address) {
-    free((void *) address);
-}
-
-JNIEXPORT jobject JNICALL
-Java_app_alextran_immich_images_ThumbnailsImpl_00024Companion_wrapAsBuffer(
-        JNIEnv *env, jclass clazz, jlong address, jint capacity) {
-    return (*env)->NewDirectByteBuffer(env, (void *) address, capacity);
-}
-
-JNIEXPORT jobject JNICALL
-Java_app_alextran_immich_images_ThumbnailsImpl_wrapAsBuffer(
-        JNIEnv *env, jclass clazz, jlong address, jint capacity) {
-    return (*env)->NewDirectByteBuffer(env, (void *) address, capacity);
+Java_app_alextran_immich_images_RemoteImagesImpl_unlockBitmapPixels(
+        JNIEnv *env, jclass clazz, jobject bitmap) {
+    AndroidBitmap_unlockPixels(env, bitmap);
 }

mobile/android/app/src/main/kotlin/app/alextran/immich/images/LocalImagesImpl.kt

@@ -12,9 +12,6 @@ import android.provider.MediaStore.Images
 import android.provider.MediaStore.Video
 import android.util.Size
 import androidx.annotation.RequiresApi
-import app.alextran.immich.images.LocalImagesImpl.Companion.allocateNative
-import app.alextran.immich.images.LocalImagesImpl.Companion.freeNative
-import app.alextran.immich.images.LocalImagesImpl.Companion.wrapAsBuffer
 import java.nio.ByteBuffer
 import kotlin.math.*
 import java.util.concurrent.Executors
@@ -47,9 +44,9 @@ inline fun ImageDecoder.Source.decodeBitmap(target: Size = Size(0, 0)): Bitmap {
 
 fun Bitmap.toNativeBuffer(): Map<String, Long>  {
   val size = width * height * 4
-  val pointer = allocateNative(size)
+  val pointer = LocalImagesImpl.allocateNative(size)
   try {
-    val buffer = wrapAsBuffer(pointer, size)
+    val buffer = LocalImagesImpl.wrapAsBuffer(pointer, size)
     copyPixelsToBuffer(buffer)
     recycle()
     return mapOf(
@@ -58,7 +55,7 @@ fun Bitmap.toNativeBuffer(): Map<String, Long>  {
       "height" to height.toLong()
     )
   } catch (e: Exception) {
-    freeNative(pointer)
+    LocalImagesImpl.freeNative(pointer)
     recycle()
     throw e
   }

mobile/android/app/src/main/kotlin/app/alextran/immich/images/RemoteImages.g.kt

@@ -49,6 +49,7 @@ private open class RemoteImagesPigeonCodec : StandardMessageCodec() {
 interface RemoteImageApi {
   fun requestImage(url: String, headers: Map<String, String>, requestId: Long, callback: (Result<Map<String, Long>>) -> Unit)
   fun cancelRequest(requestId: Long)
+  fun releaseImage(requestId: Long)
 
   companion object {
     /** The codec used by RemoteImageApi. */
@@ -99,6 +100,24 @@ interface RemoteImageApi {
           channel.setMessageHandler(null)
         }
       }
+      run {
+        val channel = BasicMessageChannel<Any?>(binaryMessenger, "dev.flutter.pigeon.immich_mobile.RemoteImageApi.releaseImage$separatedMessageChannelSuffix", codec)
+        if (api != null) {
+          channel.setMessageHandler { message, reply ->
+            val args = message as List<Any?>
+            val requestIdArg = args[0] as Long
+            val wrapped: List<Any?> = try {
+              api.releaseImage(requestIdArg)
+              listOf(null)
+            } catch (exception: Throwable) {
+              RemoteImagesPigeonUtils.wrapError(exception)
+            }
+            reply.reply(wrapped)
+          }
+        } else {
+          channel.setMessageHandler(null)
+        }
+      }
     }
   }
 }

mobile/android/app/src/main/kotlin/app/alextran/immich/images/RemoteImagesImpl.kt

@@ -7,7 +7,6 @@ import android.graphics.ColorSpace
 import android.graphics.ImageDecoder
 import android.os.Build
 import android.os.CancellationSignal
-import android.util.Size
 import app.alextran.immich.core.SSLConfig
 import okhttp3.Call
 import okhttp3.Callback
@@ -31,9 +30,9 @@ data class RemoteRequest(
 
 class RemoteImagesImpl(context: Context) : RemoteImageApi {
   private val requestMap = ConcurrentHashMap<Long, RemoteRequest>()
+  private val lockedBitmaps = ConcurrentHashMap<Long, Bitmap>()
 
   init {
-    System.loadLibrary("native_buffer")
     cacheDir = context.cacheDir
     client = buildClient()
   }
@@ -45,17 +44,23 @@ class RemoteImagesImpl(context: Context) : RemoteImageApi {
     private const val CACHE_SIZE_BYTES = 1024L * 1024 * 1024
 
     val CANCELLED = Result.success<Map<String, Long>>(emptyMap())
-    private val decodePool = Executors.newFixedThreadPool(
-      (Runtime.getRuntime().availableProcessors() / 2).coerceAtLeast(2)
-    )
+    private val decodePool =
+      Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() / 2 + 1)
 
     private var cacheDir: File? = null
     private var client: OkHttpClient? = null
 
     init {
+      System.loadLibrary("native_buffer")
       SSLConfig.addListener(::invalidateClient)
     }
 
+    @JvmStatic
+    external fun lockBitmapPixels(bitmap: Bitmap): Long
+
+    @JvmStatic
+    external fun unlockBitmapPixels(bitmap: Bitmap)
+
     private fun invalidateClient() {
       client?.let {
         it.dispatcher.cancelAll()
@@ -123,8 +128,20 @@ class RemoteImagesImpl(context: Context) : RemoteImageApi {
             signal.throwIfCanceled()
             val bitmap = decodeImage(bytes)
             signal.throwIfCanceled()
-            val res = bitmap.toNativeBuffer()
-            callback(Result.success(res))
+
+            val pointer = lockBitmapPixels(bitmap)
+            if (pointer == 0L) {
+              bitmap.recycle()
+              return@execute callback(Result.failure(RuntimeException("Failed to lock bitmap pixels")))
+            }
+
+            lockedBitmaps[requestId] = bitmap
+            callback(Result.success(mapOf(
+              "pointer" to pointer,
+              "width" to bitmap.width.toLong(),
+              "height" to bitmap.height.toLong(),
+              "rowBytes" to bitmap.rowBytes.toLong()
+            )))
           } catch (e: Exception) {
             val result = if (signal.isCanceled) CANCELLED else Result.failure(e)
             callback(result)
@@ -138,8 +155,14 @@ class RemoteImagesImpl(context: Context) : RemoteImageApi {
   }
 
   override fun cancelRequest(requestId: Long) {
-    val request = requestMap.remove(requestId) ?: return
-    request.cancellationSignal.cancel()
+    requestMap.remove(requestId)?.cancellationSignal?.cancel()
+    releaseImage(requestId)
+  }
+
+  override fun releaseImage(requestId: Long) {
+    val bitmap = lockedBitmaps.remove(requestId) ?: return
+    unlockBitmapPixels(bitmap)
+    bitmap.recycle()
   }
 
   private fun decodeImage(bytes: ByteArray): Bitmap {