feat(mobile): native clients (#21459)

* platform clients

* uppercase http method

* fix hot reload

* custom user agent

* init before app launch

* set defaults

* move to bootstrap

* unrelated change

* disable disk cache by default

* optimized decoding

* remove incremental

* android impl

* memory optimization

* lock approach is slower on ios

* conditional cronet

* clarify parameter

* enable disk cache

* set user agent

* flutter-side decode

* optimized http

* fixed locking

* refactor

* potential race conditions

* embedded cronet

* refactor, fix capacity handling

* fast path for known content length

* ios optimizations

* re-enable cache

* formatting

* bump concurrency

* clear cache button

* fix eviction race condition

* add extra cancellation check

* tighten dispose

* better error handling

* fix disposal

---------

Co-authored-by: Alex <alex.tran1502@gmail.com>
This commit is contained in:
Mert
2026-01-24 14:34:29 -05:00
committed by GitHub
parent 9219d559a0
commit 1803692eab
44 changed files with 1881 additions and 556 deletions
@@ -47,41 +47,41 @@ private func nilOrValue<T>(_ value: Any?) -> T? {
}
private class ThumbnailsPigeonCodecReader: FlutterStandardReader {
private class LocalImagesPigeonCodecReader: FlutterStandardReader {
}
private class ThumbnailsPigeonCodecWriter: FlutterStandardWriter {
private class LocalImagesPigeonCodecWriter: FlutterStandardWriter {
}
private class ThumbnailsPigeonCodecReaderWriter: FlutterStandardReaderWriter {
private class LocalImagesPigeonCodecReaderWriter: FlutterStandardReaderWriter {
override func reader(with data: Data) -> FlutterStandardReader {
return ThumbnailsPigeonCodecReader(data: data)
return LocalImagesPigeonCodecReader(data: data)
}
override func writer(with data: NSMutableData) -> FlutterStandardWriter {
return ThumbnailsPigeonCodecWriter(data: data)
return LocalImagesPigeonCodecWriter(data: data)
}
}
class ThumbnailsPigeonCodec: FlutterStandardMessageCodec, @unchecked Sendable {
static let shared = ThumbnailsPigeonCodec(readerWriter: ThumbnailsPigeonCodecReaderWriter())
class LocalImagesPigeonCodec: FlutterStandardMessageCodec, @unchecked Sendable {
static let shared = LocalImagesPigeonCodec(readerWriter: LocalImagesPigeonCodecReaderWriter())
}
/// Generated protocol from Pigeon that represents a handler of messages from Flutter.
protocol ThumbnailApi {
func requestImage(assetId: String, requestId: Int64, width: Int64, height: Int64, isVideo: Bool, completion: @escaping (Result<[String: Int64], Error>) -> Void)
func cancelImageRequest(requestId: Int64) throws
protocol LocalImageApi {
func requestImage(assetId: String, requestId: Int64, width: Int64, height: Int64, isVideo: Bool, completion: @escaping (Result<[String: Int64]?, Error>) -> Void)
func cancelRequest(requestId: Int64) throws
func getThumbhash(thumbhash: String, completion: @escaping (Result<[String: Int64], Error>) -> Void)
}
/// Generated setup class from Pigeon to handle messages through the `binaryMessenger`.
class ThumbnailApiSetup {
static var codec: FlutterStandardMessageCodec { ThumbnailsPigeonCodec.shared }
/// Sets up an instance of `ThumbnailApi` to handle messages through the `binaryMessenger`.
static func setUp(binaryMessenger: FlutterBinaryMessenger, api: ThumbnailApi?, messageChannelSuffix: String = "") {
class LocalImageApiSetup {
static var codec: FlutterStandardMessageCodec { LocalImagesPigeonCodec.shared }
/// Sets up an instance of `LocalImageApi` to handle messages through the `binaryMessenger`.
static func setUp(binaryMessenger: FlutterBinaryMessenger, api: LocalImageApi?, messageChannelSuffix: String = "") {
let channelSuffix = messageChannelSuffix.count > 0 ? ".\(messageChannelSuffix)" : ""
let requestImageChannel = FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.ThumbnailApi.requestImage\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec)
let requestImageChannel = FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.LocalImageApi.requestImage\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec)
if let api = api {
requestImageChannel.setMessageHandler { message, reply in
let args = message as! [Any?]
@@ -102,22 +102,22 @@ class ThumbnailApiSetup {
} else {
requestImageChannel.setMessageHandler(nil)
}
let cancelImageRequestChannel = FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.ThumbnailApi.cancelImageRequest\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec)
let cancelRequestChannel = FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.LocalImageApi.cancelRequest\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec)
if let api = api {
cancelImageRequestChannel.setMessageHandler { message, reply in
cancelRequestChannel.setMessageHandler { message, reply in
let args = message as! [Any?]
let requestIdArg = args[0] as! Int64
do {
try api.cancelImageRequest(requestId: requestIdArg)
try api.cancelRequest(requestId: requestIdArg)
reply(wrapResult(nil))
} catch {
reply(wrapError(error))
}
}
} else {
cancelImageRequestChannel.setMessageHandler(nil)
cancelRequestChannel.setMessageHandler(nil)
}
let getThumbhashChannel = FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.ThumbnailApi.getThumbhash\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec)
let getThumbhashChannel = FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.LocalImageApi.getThumbhash\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec)
if let api = api {
getThumbhashChannel.setMessageHandler { message, reply in
let args = message as! [Any?]
@@ -1,19 +1,19 @@
import CryptoKit
import Accelerate
import Flutter
import MobileCoreServices
import Photos
class Request {
class LocalImageRequest {
weak var workItem: DispatchWorkItem?
var isCancelled = false
let callback: (Result<[String: Int64], any Error>) -> Void
let callback: (Result<[String: Int64]?, any Error>) -> Void
init(callback: @escaping (Result<[String: Int64], any Error>) -> Void) {
init(callback: @escaping (Result<[String: Int64]?, any Error>) -> Void) {
self.callback = callback
}
}
class ThumbnailApiImpl: ThumbnailApi {
class LocalImageApiImpl: LocalImageApi {
private static let imageManager = PHImageManager.default()
private static let fetchOptions = {
let fetchOptions = PHFetchOptions()
@@ -36,47 +36,39 @@ class ThumbnailApiImpl: ThumbnailApi {
private static let cancelQueue = DispatchQueue(label: "thumbnail.cancellation", qos: .default)
private static let processingQueue = DispatchQueue(label: "thumbnail.processing", qos: .userInteractive, attributes: .concurrent)
private static let rgbColorSpace = CGColorSpaceCreateDeviceRGB()
private static let bitmapInfo = CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedLast.rawValue).rawValue
private static var requests = [Int64: Request]()
private static let cancelledResult = Result<[String: Int64], any Error>.success([:])
private static var rgbaFormat = vImage_CGImageFormat(
bitsPerComponent: 8,
bitsPerPixel: 32,
colorSpace: CGColorSpaceCreateDeviceRGB(),
bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedLast.rawValue),
renderingIntent: .defaultIntent
)!
private static var requests = [Int64: LocalImageRequest]()
private static let cancelledResult = Result<[String: Int64]?, any Error>.success(nil)
private static let concurrencySemaphore = DispatchSemaphore(value: ProcessInfo.processInfo.activeProcessorCount * 2)
private static let assetCache = {
let assetCache = NSCache<NSString, PHAsset>()
assetCache.countLimit = 10000
return assetCache
}()
private static let activitySemaphore = DispatchSemaphore(value: 1)
private static let willResignActiveObserver = NotificationCenter.default.addObserver(
forName: UIApplication.willResignActiveNotification,
object: nil,
queue: .main
) { _ in
processingQueue.suspend()
activitySemaphore.wait()
}
private static let didBecomeActiveObserver = NotificationCenter.default.addObserver(
forName: UIApplication.didBecomeActiveNotification,
object: nil,
queue: .main
) { _ in
processingQueue.resume()
activitySemaphore.signal()
}
func getThumbhash(thumbhash: String, completion: @escaping (Result<[String : Int64], any Error>) -> Void) {
Self.processingQueue.async {
guard let data = Data(base64Encoded: thumbhash)
else { return completion(.failure(PigeonError(code: "", message: "Invalid base64 string: \(thumbhash)", details: nil)))}
let (width, height, pointer) = thumbHashToRGBA(hash: data)
self.waitForActiveState()
completion(.success(["pointer": Int64(Int(bitPattern: pointer.baseAddress)), "width": Int64(width), "height": Int64(height)]))
completion(.success([
"pointer": Int64(Int(bitPattern: pointer.baseAddress)),
"width": Int64(width),
"height": Int64(height),
"rowBytes": Int64(width * 4)
]))
}
}
func requestImage(assetId: String, requestId: Int64, width: Int64, height: Int64, isVideo: Bool, completion: @escaping (Result<[String: Int64], any Error>) -> Void) {
let request = Request(callback: completion)
func requestImage(assetId: String, requestId: Int64, width: Int64, height: Int64, isVideo: Bool, completion: @escaping (Result<[String: Int64]?, any Error>) -> Void) {
let request = LocalImageRequest(callback: completion)
let item = DispatchWorkItem {
if request.isCancelled {
return completion(Self.cancelledResult)
@@ -93,7 +85,7 @@ class ThumbnailApiImpl: ThumbnailApi {
guard let asset = Self.requestAsset(assetId: assetId)
else {
Self.removeRequest(requestId: requestId)
Self.remove(requestId: requestId)
completion(.failure(PigeonError(code: "", message: "Could not get asset data for \(assetId)", details: nil)))
return
}
@@ -119,70 +111,54 @@ class ThumbnailApiImpl: ThumbnailApi {
guard let image = image,
let cgImage = image.cgImage else {
Self.removeRequest(requestId: requestId)
Self.remove(requestId: requestId)
return completion(.failure(PigeonError(code: "", message: "Could not get pixel data for \(assetId)", details: nil)))
}
let pointer = UnsafeMutableRawPointer.allocate(
byteCount: Int(cgImage.width) * Int(cgImage.height) * 4,
alignment: MemoryLayout<UInt8>.alignment
)
if request.isCancelled {
pointer.deallocate()
return completion(Self.cancelledResult)
}
guard let context = CGContext(
data: pointer,
width: cgImage.width,
height: cgImage.height,
bitsPerComponent: 8,
bytesPerRow: cgImage.width * 4,
space: Self.rgbColorSpace,
bitmapInfo: Self.bitmapInfo
) else {
pointer.deallocate()
Self.removeRequest(requestId: requestId)
return completion(.failure(PigeonError(code: "", message: "Could not create context for \(assetId)", details: nil)))
do {
let buffer = try vImage_Buffer(cgImage: cgImage, format: Self.rgbaFormat)
if request.isCancelled {
buffer.free()
return completion(Self.cancelledResult)
}
request.callback(.success([
"pointer": Int64(Int(bitPattern: buffer.data)),
"width": Int64(buffer.width),
"height": Int64(buffer.height),
"rowBytes": Int64(buffer.rowBytes)
]))
print("Successful response for \(requestId)")
Self.remove(requestId: requestId)
} catch {
Self.remove(requestId: requestId)
return completion(.failure(PigeonError(code: "", message: "Failed to convert image for \(assetId): \(error)", details: nil)))
}
if request.isCancelled {
pointer.deallocate()
return completion(Self.cancelledResult)
}
context.interpolationQuality = .none
context.draw(cgImage, in: CGRect(x: 0, y: 0, width: cgImage.width, height: cgImage.height))
if request.isCancelled {
pointer.deallocate()
return completion(Self.cancelledResult)
}
self.waitForActiveState()
completion(.success(["pointer": Int64(Int(bitPattern: pointer)), "width": Int64(cgImage.width), "height": Int64(cgImage.height)]))
Self.removeRequest(requestId: requestId)
}
request.workItem = item
Self.addRequest(requestId: requestId, request: request)
Self.add(requestId: requestId, request: request)
Self.processingQueue.async(execute: item)
}
func cancelImageRequest(requestId: Int64) {
Self.cancelRequest(requestId: requestId)
func cancelRequest(requestId: Int64) {
Self.cancel(requestId: requestId)
}
private static func addRequest(requestId: Int64, request: Request) -> Void {
private static func add(requestId: Int64, request: LocalImageRequest) -> Void {
requestQueue.sync { requests[requestId] = request }
}
private static func removeRequest(requestId: Int64) -> Void {
private static func remove(requestId: Int64) -> Void {
requestQueue.sync { requests[requestId] = nil }
}
private static func cancelRequest(requestId: Int64) -> Void {
private static func cancel(requestId: Int64) -> Void {
requestQueue.async {
guard let request = requests.removeValue(forKey: requestId) else { return }
request.isCancelled = true
@@ -203,9 +179,4 @@ class ThumbnailApiImpl: ThumbnailApi {
assetQueue.async { assetCache.setObject(asset, forKey: assetId as NSString) }
return asset
}
func waitForActiveState() {
Self.activitySemaphore.wait()
Self.activitySemaphore.signal()
}
}
@@ -0,0 +1,134 @@
// Autogenerated from Pigeon (v26.0.2), do not edit directly.
// See also: https://pub.dev/packages/pigeon
import Foundation
#if os(iOS)
import Flutter
#elseif os(macOS)
import FlutterMacOS
#else
#error("Unsupported platform.")
#endif
private func wrapResult(_ result: Any?) -> [Any?] {
return [result]
}
private func wrapError(_ error: Any) -> [Any?] {
if let pigeonError = error as? PigeonError {
return [
pigeonError.code,
pigeonError.message,
pigeonError.details,
]
}
if let flutterError = error as? FlutterError {
return [
flutterError.code,
flutterError.message,
flutterError.details,
]
}
return [
"\(error)",
"\(type(of: error))",
"Stacktrace: \(Thread.callStackSymbols)",
]
}
private func isNullish(_ value: Any?) -> Bool {
return value is NSNull || value == nil
}
private func nilOrValue<T>(_ value: Any?) -> T? {
if value is NSNull { return nil }
return value as! T?
}
private class RemoteImagesPigeonCodecReader: FlutterStandardReader {
}
private class RemoteImagesPigeonCodecWriter: FlutterStandardWriter {
}
private class RemoteImagesPigeonCodecReaderWriter: FlutterStandardReaderWriter {
override func reader(with data: Data) -> FlutterStandardReader {
return RemoteImagesPigeonCodecReader(data: data)
}
override func writer(with data: NSMutableData) -> FlutterStandardWriter {
return RemoteImagesPigeonCodecWriter(data: data)
}
}
class RemoteImagesPigeonCodec: FlutterStandardMessageCodec, @unchecked Sendable {
static let shared = RemoteImagesPigeonCodec(readerWriter: RemoteImagesPigeonCodecReaderWriter())
}
/// Generated protocol from Pigeon that represents a handler of messages from Flutter.
protocol RemoteImageApi {
func requestImage(url: String, headers: [String: String], requestId: Int64, completion: @escaping (Result<[String: Int64]?, Error>) -> Void)
func cancelRequest(requestId: Int64) throws
func clearCache(completion: @escaping (Result<Int64, Error>) -> Void)
}
/// Generated setup class from Pigeon to handle messages through the `binaryMessenger`.
class RemoteImageApiSetup {
static var codec: FlutterStandardMessageCodec { RemoteImagesPigeonCodec.shared }
/// Sets up an instance of `RemoteImageApi` to handle messages through the `binaryMessenger`.
static func setUp(binaryMessenger: FlutterBinaryMessenger, api: RemoteImageApi?, messageChannelSuffix: String = "") {
let channelSuffix = messageChannelSuffix.count > 0 ? ".\(messageChannelSuffix)" : ""
let requestImageChannel = FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.RemoteImageApi.requestImage\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec)
if let api = api {
requestImageChannel.setMessageHandler { message, reply in
let args = message as! [Any?]
let urlArg = args[0] as! String
let headersArg = args[1] as! [String: String]
let requestIdArg = args[2] as! Int64
api.requestImage(url: urlArg, headers: headersArg, requestId: requestIdArg) { result in
switch result {
case .success(let res):
reply(wrapResult(res))
case .failure(let error):
reply(wrapError(error))
}
}
}
} else {
requestImageChannel.setMessageHandler(nil)
}
let cancelRequestChannel = FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.RemoteImageApi.cancelRequest\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec)
if let api = api {
cancelRequestChannel.setMessageHandler { message, reply in
let args = message as! [Any?]
let requestIdArg = args[0] as! Int64
do {
try api.cancelRequest(requestId: requestIdArg)
reply(wrapResult(nil))
} catch {
reply(wrapError(error))
}
}
} else {
cancelRequestChannel.setMessageHandler(nil)
}
let clearCacheChannel = FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.RemoteImageApi.clearCache\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec)
if let api = api {
clearCacheChannel.setMessageHandler { _, reply in
api.clearCache { result in
switch result {
case .success(let res):
reply(wrapResult(res))
case .failure(let error):
reply(wrapError(error))
}
}
}
} else {
clearCacheChannel.setMessageHandler(nil)
}
}
}
@@ -0,0 +1,184 @@
import Accelerate
import Flutter
import MobileCoreServices
import Photos
class RemoteImageRequest {
weak var task: URLSessionDataTask?
let id: Int64
var isCancelled = false
var data: CFMutableData?
let completion: (Result<[String: Int64]?, any Error>) -> Void
init(id: Int64, task: URLSessionDataTask, completion: @escaping (Result<[String: Int64]?, any Error>) -> Void) {
self.id = id
self.task = task
self.data = nil
self.completion = completion
}
}
class RemoteImageApiImpl: NSObject, RemoteImageApi {
private static let delegate = RemoteImageApiDelegate()
static let session = {
let cacheDir = FileManager.default.temporaryDirectory.appendingPathComponent("thumbnails", isDirectory: true)
let config = URLSessionConfiguration.default
let version = Bundle.main.object(forInfoDictionaryKey: "CFBundleShortVersionString") as? String ?? "unknown"
config.httpAdditionalHeaders = ["User-Agent": "Immich_iOS_\(version)"]
try! FileManager.default.createDirectory(at: cacheDir, withIntermediateDirectories: true)
config.urlCache = URLCache(
memoryCapacity: 0,
diskCapacity: 1 << 30,
directory: cacheDir
)
config.httpMaximumConnectionsPerHost = 64
return URLSession(configuration: config, delegate: delegate, delegateQueue: nil)
}()
func requestImage(url: String, headers: [String : String], requestId: Int64, completion: @escaping (Result<[String : Int64]?, any Error>) -> Void) {
var urlRequest = URLRequest(url: URL(string: url)!)
for (key, value) in headers {
urlRequest.setValue(value, forHTTPHeaderField: key)
}
let task = Self.session.dataTask(with: urlRequest)
let imageRequest = RemoteImageRequest(id: requestId, task: task, completion: completion)
Self.delegate.add(taskId: task.taskIdentifier, request: imageRequest)
task.resume()
}
func cancelRequest(requestId: Int64) {
Self.delegate.cancel(requestId: requestId)
}
func clearCache(completion: @escaping (Result<Int64, any Error>) -> Void) {
Task {
let cache = Self.session.configuration.urlCache!
let cacheSize = Int64(cache.currentDiskUsage)
cache.removeAllCachedResponses()
completion(.success(cacheSize))
}
}
}
class RemoteImageApiDelegate: NSObject, URLSessionDataDelegate {
private static let requestQueue = DispatchQueue(label: "thumbnail.requests", qos: .userInitiated, attributes: .concurrent)
private static var rgbaFormat = vImage_CGImageFormat(
bitsPerComponent: 8,
bitsPerPixel: 32,
colorSpace: CGColorSpaceCreateDeviceRGB(),
bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedLast.rawValue),
renderingIntent: .perceptual
)!
private static var requestByTaskId = [Int: RemoteImageRequest]()
private static var taskIdByRequestId = [Int64: Int]()
private static let cancelledResult = Result<[String: Int64]?, any Error>.success(nil)
private static let decodeOptions = [
kCGImageSourceShouldCache: false,
kCGImageSourceShouldCacheImmediately: true,
kCGImageSourceCreateThumbnailWithTransform: true,
] as CFDictionary
func urlSession(
_ session: URLSession, dataTask: URLSessionDataTask,
didReceive response: URLResponse,
completionHandler: @escaping (URLSession.ResponseDisposition) -> Void
) {
guard let request = get(taskId: dataTask.taskIdentifier)
else {
return completionHandler(.cancel)
}
let capacity = max(Int(response.expectedContentLength), 0)
request.data = CFDataCreateMutable(nil, capacity)
completionHandler(.allow)
}
func urlSession(_ session: URLSession, dataTask: URLSessionDataTask,
didReceive data: Data) {
guard let request = get(taskId: dataTask.taskIdentifier) else { return }
data.withUnsafeBytes { bytes in
CFDataAppendBytes(request.data, bytes.bindMemory(to: UInt8.self).baseAddress, data.count)
}
}
func urlSession(_ session: URLSession, task: URLSessionTask,
didCompleteWithError error: Error?) {
guard let request = get(taskId: task.taskIdentifier) else { return }
defer { remove(taskId: task.taskIdentifier, requestId: request.id) }
if let error = error {
if request.isCancelled || (error as NSError).code == NSURLErrorCancelled {
return request.completion(Self.cancelledResult)
}
return request.completion(.failure(error))
}
if request.isCancelled {
return request.completion(Self.cancelledResult)
}
guard let data = request.data else {
return request.completion(.failure(PigeonError(code: "", message: "No data received", details: nil)))
}
guard let imageSource = CGImageSourceCreateWithData(data, nil),
let cgImage = CGImageSourceCreateThumbnailAtIndex(imageSource, 0, Self.decodeOptions) else {
return request.completion(.failure(PigeonError(code: "", message: "Failed to decode image for request", details: nil)))
}
if request.isCancelled {
return request.completion(Self.cancelledResult)
}
do {
let buffer = try vImage_Buffer(cgImage: cgImage, format: Self.rgbaFormat)
if request.isCancelled {
buffer.free()
return request.completion(Self.cancelledResult)
}
request.completion(
.success([
"pointer": Int64(Int(bitPattern: buffer.data)),
"width": Int64(buffer.width),
"height": Int64(buffer.height),
"rowBytes": Int64(buffer.rowBytes),
]))
} catch {
return request.completion(.failure(PigeonError(code: "", message: "Failed to convert image for request: \(error)", details: nil)))
}
}
@inline(__always) func get(taskId: Int) -> RemoteImageRequest? {
Self.requestQueue.sync { Self.requestByTaskId[taskId] }
}
@inline(__always) func add(taskId: Int, request: RemoteImageRequest) -> Void {
Self.requestQueue.async(flags: .barrier) {
Self.requestByTaskId[taskId] = request
Self.taskIdByRequestId[request.id] = taskId
}
}
@inline(__always) func remove(taskId: Int, requestId: Int64) -> Void {
Self.requestQueue.async(flags: .barrier) {
Self.taskIdByRequestId[requestId] = nil
Self.requestByTaskId[taskId] = nil
}
}
@inline(__always) func cancel(requestId: Int64) -> Void {
guard let request: RemoteImageRequest = (Self.requestQueue.sync {
guard let taskId = Self.taskIdByRequestId[requestId] else { return nil }
return Self.requestByTaskId[taskId]
}) else { return }
request.isCancelled = true
request.task?.cancel()
}
}