background upload plugin

add schemas

sync variants

formatting

initial implementation

use existing db, wip

move to separate folder

fix table definitions

wip

wiring it up

repository pattern
This commit is contained in:
mertalev
2025-11-22 11:09:34 -05:00
parent 41f013387f
commit 92bc22620b
19 changed files with 939 additions and 582 deletions
+62
View File
@@ -0,0 +1,62 @@
import StructuredFieldValues
struct AssetData: StructuredFieldValue {
static let structuredFieldType: StructuredFieldType = .dictionary
let deviceAssetId: String
let deviceId: String
let fileCreatedAt: String
let fileModifiedAt: String
let fileName: String
let isFavorite: Bool
let livePhotoVideoId: String?
static let boundary = "Boundary-\(UUID().uuidString)"
static let deviceAssetIdField = "--\(boundary)\r\nContent-Disposition: form-data; name=\"deviceAssetId\"\r\n\r\n"
.data(using: .utf8)!
static let deviceIdField = "\r\n--\(boundary)\r\nContent-Disposition: form-data; name=\"deviceId\"\r\n\r\n"
.data(using: .utf8)!
static let fileCreatedAtField =
"\r\n--\(boundary)\r\nContent-Disposition: form-data; name=\"fileCreatedAt\"\r\n\r\n"
.data(using: .utf8)!
static let fileModifiedAtField =
"\r\n--\(boundary)\r\nContent-Disposition: form-data; name=\"fileModifiedAt\"\r\n\r\n"
.data(using: .utf8)!
static let isFavoriteField = "\r\n--\(boundary)\r\nContent-Disposition: form-data; name=\"isFavorite\"\r\n\r\n"
.data(using: .utf8)!
static let livePhotoVideoIdField =
"\r\n--\(boundary)\r\nContent-Disposition: form-data; name=\"livePhotoVideoId\"\r\n\r\n"
.data(using: .utf8)!
static let trueData = "true".data(using: .utf8)!
static let falseData = "false".data(using: .utf8)!
static let footer = "\r\n--\(boundary)--\r\n".data(using: .utf8)!
static let contentType = "multipart/form-data; boundary=\(boundary)"
func multipart() -> (Data, Data) {
var header = Data()
header.append(Self.deviceAssetIdField)
header.append(deviceAssetId.data(using: .utf8)!)
header.append(Self.deviceIdField)
header.append(deviceId.data(using: .utf8)!)
header.append(Self.fileCreatedAtField)
header.append(fileCreatedAt.data(using: .utf8)!)
header.append(Self.fileModifiedAtField)
header.append(fileModifiedAt.data(using: .utf8)!)
header.append(Self.isFavoriteField)
header.append(isFavorite ? Self.trueData : Self.falseData)
if let livePhotoVideoId {
header.append(Self.livePhotoVideoIdField)
header.append(livePhotoVideoId.data(using: .utf8)!)
}
header.append(
"\r\n--\(Self.boundary)\r\nContent-Disposition: form-data; name=\"assetData\"; filename=\"\(fileName)\"\r\nContent-Type: application/octet-stream\r\n\r\n"
.data(using: .utf8)!
)
return (header, Self.footer)
}
}
+71 -65
View File
@@ -1,19 +1,33 @@
import SQLiteData
class UploadApiDelegate: NSObject, URLSessionDataDelegate, URLSessionTaskDelegate {
private static let stateLock = NSLock()
private static var transferStates: [Int64: NetworkTransferState] = [:]
private static var responseData: [Int64: Data] = [:]
private static let jsonDecoder = JSONDecoder()
private let stateLock = NSLock()
private var transferStates: [Int64: NetworkTransferState] = [:]
private var responseData: [Int64: Data] = [:]
private let jsonDecoder = JSONDecoder()
private let db: DatabasePool
private let statusListener: StatusEventListener
private let progressListener: ProgressEventListener
weak var downloadQueue: DownloadQueue?
weak var uploadQueue: UploadQueue?
private class NetworkTransferState {
var lastUpdateTime: Date
var totalBytesTransferred: Int64
var currentSpeed: Double?
init(db: DatabasePool, statusListener: StatusEventListener, progressListener: ProgressEventListener) {
self.db = db
init(lastUpdateTime: Date, totalBytesTransferred: Int64, currentSpeed: Double?) {
self.lastUpdateTime = lastUpdateTime
self.totalBytesTransferred = totalBytesTransferred
self.currentSpeed = currentSpeed
}
}
final class UploadApiDelegate<
TaskRepo: TaskProtocol,
StatusListener: TaskStatusListener,
ProgressListener: TaskProgressListener
>: NSObject, URLSessionDataDelegate, URLSessionTaskDelegate {
private let taskRepository: TaskRepo
private let statusListener: StatusListener
private let progressListener: ProgressListener
init(taskRepository: TaskRepo, statusListener: StatusListener, progressListener: ProgressListener) {
self.taskRepository = taskRepository
self.statusListener = statusListener
self.progressListener = progressListener
}
@@ -30,11 +44,11 @@ class UploadApiDelegate: NSObject, URLSessionDataDelegate, URLSessionTaskDelegat
let taskId = Int64(taskIdStr)
else { return }
Self.stateLock.withLock {
if var response = Self.responseData[taskId] {
stateLock.withLock {
if var response = responseData[taskId] {
response.append(data)
} else {
Self.responseData[taskId] = data
responseData[taskId] = data
}
}
}
@@ -42,8 +56,7 @@ class UploadApiDelegate: NSObject, URLSessionDataDelegate, URLSessionTaskDelegat
func urlSession(_ session: URLSession, task: URLSessionTask, didCompleteWithError error: Error?) {
Task {
defer {
downloadQueue?.startQueueProcessing()
uploadQueue?.startQueueProcessing()
NotificationCenter.default.post(name: .uploadTaskDidComplete, object: nil)
}
guard let taskDescriptionId = task.taskDescription,
@@ -53,25 +66,27 @@ class UploadApiDelegate: NSObject, URLSessionDataDelegate, URLSessionTaskDelegat
}
defer {
Self.stateLock.withLock { let _ = Self.transferStates.removeValue(forKey: taskId) }
stateLock.withLock { let _ = transferStates.removeValue(forKey: taskId) }
}
if let responseData = Self.stateLock.withLock({ Self.responseData.removeValue(forKey: taskId) }),
let httpResponse = task.response as? HTTPURLResponse
if let body = stateLock.withLock({ responseData.removeValue(forKey: taskId) }),
let response = task.response as? HTTPURLResponse
{
switch httpResponse.statusCode {
switch response.statusCode {
case 200, 201:
do {
let response = try Self.jsonDecoder.decode(UploadSuccessResponse.self, from: responseData)
let response = try jsonDecoder.decode(UploadSuccessResponse.self, from: body)
return await handleSuccess(taskId: taskId, response: response)
} catch {
return await handleFailure(taskId: taskId, code: .invalidResponse)
}
case 401: return await handleFailure(taskId: taskId, code: .unauthorized)
case 400..<500:
dPrint(
"Response \(httpResponse.statusCode): \(String(data: responseData, encoding: .utf8) ?? "No response body")"
)
dPrint("Response \(response.statusCode): \(String(data: body, encoding: .utf8) ?? "No response body")")
return await handleFailure(taskId: taskId, code: .badRequest)
case 500..<600:
dPrint("Response \(response.statusCode): \(String(data: body, encoding: .utf8) ?? "No response body")")
return await handleFailure(taskId: taskId, code: .internalServerError)
default:
break
}
@@ -111,8 +126,8 @@ class UploadApiDelegate: NSObject, URLSessionDataDelegate, URLSessionTaskDelegat
) {
guard let sessionTaskId = task.taskDescription, let taskId = Int64(sessionTaskId) else { return }
let currentTime = Date()
let state = Self.stateLock.withLock {
if let existing = Self.transferStates[taskId] {
let state = stateLock.withLock {
if let existing = transferStates[taskId] {
return existing
}
let new = NetworkTransferState(
@@ -120,7 +135,7 @@ class UploadApiDelegate: NSObject, URLSessionDataDelegate, URLSessionTaskDelegat
totalBytesTransferred: totalBytesSent,
currentSpeed: nil
)
Self.transferStates[taskId] = new
transferStates[taskId] = new
return new
}
@@ -147,30 +162,29 @@ class UploadApiDelegate: NSObject, URLSessionDataDelegate, URLSessionTaskDelegat
)
}
func urlSessionDidFinishEvents(forBackgroundURLSession session: URLSession) {
dPrint("All background events delivered for session: \(session.configuration.identifier ?? "unknown")")
DispatchQueue.main.async {
if let identifier = session.configuration.identifier,
let appDelegate = UIApplication.shared.delegate as? AppDelegate,
let completionHandler = appDelegate.completionHandler(forSession: identifier)
{
completionHandler()
}
}
}
private func handleSuccess(taskId: Int64, response: UploadSuccessResponse) async {
dPrint("Upload succeeded for task \(taskId), server ID: \(response.id)")
do {
try await db.write { conn in
let task = try UploadTask.update { $0.status = .uploadComplete }.where({ $0.id.eq(taskId) })
.returning(\.self).fetchOne(conn)
guard let task, let isLivePhoto = task.isLivePhoto, isLivePhoto, task.livePhotoVideoId == nil else { return }
try UploadTask.insert {
UploadTask.Draft(
attempts: 0,
createdAt: Date(),
filePath: nil,
isLivePhoto: true,
lastError: nil,
livePhotoVideoId: response.id,
localId: task.localId,
method: .multipart,
priority: 0.7,
retryAfter: nil,
status: .downloadPending,
)
}.execute(conn)
}
dPrint("Updated upload success status for session task \(taskId)")
try await taskRepository.markUploadSuccess(taskId: taskId, livePhotoVideoId: response.id)
statusListener.onTaskStatus(
UploadApiTaskStatus(
id: String(taskId),
filename: (try? await taskRepository.getFilename(taskId: taskId)) ?? "",
status: .uploadComplete
)
)
} catch {
dPrint(
"Failed to update upload success status for session task \(taskId): \(error.localizedDescription)"
@@ -180,10 +194,14 @@ class UploadApiDelegate: NSObject, URLSessionDataDelegate, URLSessionTaskDelegat
private func handleFailure(taskId: Int64, code: UploadErrorCode = .unknown) async {
dPrint("Upload failed for task \(taskId) with code \(code)")
try? await db.write { conn in
try UploadTask.retryOrFail(code: code, status: .uploadFailed).where { $0.id.eq(taskId) }
.execute(conn)
}
try? await taskRepository.retryOrFail(taskId: taskId, code: code, status: .uploadFailed)
statusListener.onTaskStatus(
UploadApiTaskStatus(
id: String(taskId),
filename: (try? await taskRepository.getFilename(taskId: taskId)) ?? "",
status: .uploadFailed
)
)
}
@available(iOS 17, *)
@@ -193,16 +211,4 @@ class UploadApiDelegate: NSObject, URLSessionDataDelegate, URLSessionTaskDelegat
resumeTask.taskDescription = taskDescriptionId
resumeTask.resume()
}
private class NetworkTransferState {
var lastUpdateTime: Date
var totalBytesTransferred: Int64
var currentSpeed: Double?
init(lastUpdateTime: Date, totalBytesTransferred: Int64, currentSpeed: Double?) {
self.lastUpdateTime = lastUpdateTime
self.totalBytesTransferred = totalBytesTransferred
self.currentSpeed = currentSpeed
}
}
}
+85 -131
View File
@@ -1,131 +1,105 @@
import CryptoKit
import Photos
import SQLiteData
class DownloadQueue {
private static let resourceManager = PHAssetResourceManager.default()
private static var queueProcessingTask: Task<Void, Never>?
private static var queueProcessingLock = NSLock()
private var queueProcessingTask: Task<Void, Never>?
private var queueProcessingLock = NSLock()
private let resourceManager = PHAssetResourceManager.default()
private let db: DatabasePool
private let uploadQueue: UploadQueue
private let statusListener: StatusEventListener
private let progressListener: ProgressEventListener
private final class RequestRef {
var id: PHAssetResourceDataRequestID?
var lastProgressTime = Date()
var didStall = false
}
final class DownloadQueue<
StoreRepo: StoreProtocol,
TaskRepo: TaskProtocol,
StatusListener: TaskStatusListener,
ProgressListener: TaskProgressListener
> {
private let storeRepository: StoreRepo
private let taskRepository: TaskRepo
private let statusListener: StatusListener
private let progressListener: ProgressListener
private var uploadObserver: NSObjectProtocol?
private var networkObserver: NSObjectProtocol?
init(
db: DatabasePool,
uploadQueue: UploadQueue,
statusListener: StatusEventListener,
progressListener: ProgressEventListener
storeRepository: StoreRepo,
taskRepository: TaskRepo,
statusListener: StatusListener,
progressListener: ProgressListener
) {
self.db = db
self.uploadQueue = uploadQueue
self.storeRepository = storeRepository
self.taskRepository = taskRepository
self.statusListener = statusListener
self.progressListener = progressListener
NotificationCenter.default.addObserver(forName: .networkDidConnect, object: nil, queue: nil) { [weak self] _ in
uploadObserver = NotificationCenter.default.addObserver(forName: .uploadTaskDidComplete, object: nil, queue: nil) {
[weak self] _ in
self?.startQueueProcessing()
}
networkObserver = NotificationCenter.default.addObserver(forName: .networkDidConnect, object: nil, queue: nil) {
[weak self] _ in
dPrint("Network connected")
self?.startQueueProcessing()
}
}
deinit {
uploadObserver.map(NotificationCenter.default.removeObserver(_:))
networkObserver.map(NotificationCenter.default.removeObserver(_:))
}
func enqueueAssets(localIds: [String]) async throws {
guard !localIds.isEmpty else { return dPrint("No assets to enqueue") }
defer { startQueueProcessing() }
let candidates = try await db.read { conn in
return try LocalAsset.all
.where { asset in asset.id.in(localIds) }
.select { LocalAssetCandidate.Columns(id: $0.id, type: $0.type) }
.limit { _ in UploadTaskStat.availableSlots }
.fetchAll(conn)
}
let candidates = try await taskRepository.getBackupCandidates(ids: localIds)
guard !candidates.isEmpty else { return dPrint("No candidates to enqueue") }
try await db.write { conn in
var draft = UploadTask.Draft(
attempts: 0,
createdAt: Date(),
filePath: nil,
isLivePhoto: nil,
lastError: nil,
livePhotoVideoId: nil,
localId: "",
method: .multipart,
priority: 0.5,
retryAfter: nil,
status: .downloadPending,
)
for candidate in candidates {
draft.localId = candidate.id
draft.priority = candidate.type == .image ? 0.9 : 0.8
try UploadTask.insert {
draft
} onConflict: {
($0.localId, $0.livePhotoVideoId)
}.execute(conn)
}
}
try await taskRepository.enqueue(assets: candidates, imagePriority: 0.9, videoPriority: 0.8)
dPrint("Enqueued \(candidates.count) assets for upload")
}
func startQueueProcessing() {
dPrint("Starting download queue processing")
Self.queueProcessingLock.withLock {
guard Self.queueProcessingTask == nil else { return }
Self.queueProcessingTask = Task {
queueProcessingLock.withLock {
guard queueProcessingTask == nil else { return }
queueProcessingTask = Task {
await startDownloads()
Self.queueProcessingLock.withLock { Self.queueProcessingTask = nil }
queueProcessingLock.withLock { queueProcessingTask = nil }
}
}
}
private func startDownloads() async {
dPrint("Processing download queue")
guard NetworkMonitor.shared.isConnected else {
return dPrint("Download queue paused: network disconnected")
guard await UIApplication.shared.applicationState != .background else {
return dPrint("Not processing downloads in background") // TODO: run in processing tasks
}
guard NetworkMonitor.shared.isConnected,
let backupEnabled = try? storeRepository.get(StoreKey.enableBackup), backupEnabled,
let deviceId = try? storeRepository.get(StoreKey.deviceId)
else { return dPrint("Download queue paused: missing preconditions") }
do {
let tasks: [LocalAssetDownloadData] = try await db.read({ conn in
guard let backupEnabled = try Store.get(conn, StoreKey.enableBackup), backupEnabled else { return [] }
return try UploadTask.join(LocalAsset.all) { task, asset in task.localId.eq(asset.id) }
.where { task, asset in
asset.checksum.isNot(nil) && task.status.eq(TaskStatus.downloadPending)
&& task.attempts < TaskConfig.maxRetries
&& (task.retryAfter.is(nil) || task.retryAfter.unwrapped <= Date().unixTime)
&& (task.lastError.is(nil)
|| !task.lastError.unwrapped.in([
UploadErrorCode.assetNotFound, UploadErrorCode.resourceNotFound, UploadErrorCode.invalidResource,
]))
}
.select { task, asset in
LocalAssetDownloadData.Columns(
checksum: asset.checksum,
createdAt: asset.createdAt,
livePhotoVideoId: task.livePhotoVideoId,
localId: asset.id,
taskId: task.id,
updatedAt: asset.updatedAt
)
}
.order { task, asset in (task.priority.desc(), task.createdAt) }
.limit { _, _ in UploadTaskStat.availableDownloadSlots }
.fetchAll(conn)
})
let tasks = try await taskRepository.getDownloadTasks()
if tasks.isEmpty { return dPrint("No download tasks to process") }
try await withThrowingTaskGroup(of: Void.self) { group in
var iterator = tasks.makeIterator()
for _ in 0..<min(TaskConfig.maxActiveDownloads, tasks.count) {
if let task = iterator.next() {
group.addTask { await self.downloadAndQueue(task) }
group.addTask { await self.downloadAndQueue(task, deviceId: deviceId) }
}
}
while try await group.next() != nil {
if let task = iterator.next() {
group.addTask { await self.downloadAndQueue(task) }
group.addTask { await self.downloadAndQueue(task, deviceId: deviceId) }
}
}
}
@@ -134,26 +108,21 @@ class DownloadQueue {
}
}
private func downloadAndQueue(_ task: LocalAssetDownloadData) async {
private func downloadAndQueue(_ task: LocalAssetDownloadData, deviceId: String) async {
defer { startQueueProcessing() }
dPrint("Starting download for task \(task.taskId)")
guard let asset = PHAsset.fetchAssets(withLocalIdentifiers: [task.localId], options: nil).firstObject
else {
dPrint("Asset not found")
return handleFailure(task: task, code: .assetNotFound)
return await handleFailure(task: task, code: .assetNotFound)
}
let isLivePhoto = asset.mediaSubtypes.contains(.photoLive)
let isMotion = isLivePhoto && task.livePhotoVideoId != nil
guard let resource = isMotion ? asset.getLivePhotoResource() : asset.getResource() else {
dPrint("Resource not found")
return handleFailure(task: task, code: .resourceNotFound)
}
guard let deviceId = (try? await db.read { conn in try Store.get(conn, StoreKey.deviceId) }) else {
dPrint("Device ID not found")
return handleFailure(task: task, code: .noDeviceId)
return await handleFailure(task: task, code: .resourceNotFound)
}
let fileDir = TaskConfig.originalsDir
@@ -167,17 +136,11 @@ class DownloadQueue {
)
} catch {
dPrint("Failed to create directory for download task \(task.taskId): \(error)")
return handleFailure(task: task, code: .writeFailed, filePath: filePath)
return await handleFailure(task: task, code: .writeFailed, filePath: filePath)
}
do {
try await db.write { conn in
try UploadTask.update {
$0.status = .downloadQueued
$0.isLivePhoto = isLivePhoto
$0.filePath = filePath
}.where { $0.id.eq(task.taskId) }.execute(conn)
}
try await taskRepository.markDownloadQueued(taskId: task.taskId, isLivePhoto: isLivePhoto, filePath: filePath)
} catch {
return dPrint("Failed to set file path for download task \(task.taskId): \(error)")
}
@@ -187,28 +150,21 @@ class DownloadQueue {
do {
let hash = try await download(task: task, asset: asset, resource: resource, to: filePath, deviceId: deviceId)
let status = try await db.write { conn in
if let hash { try LocalAsset.update { $0.checksum = hash }.where { $0.id.eq(task.localId) }.execute(conn) }
let status =
if let hash, try RemoteAsset.select(\.rowid).where({ $0.checksum.eq(hash) }).fetchOne(conn) != nil {
TaskStatus.uploadSkipped
} else {
TaskStatus.uploadPending
}
try UploadTask.update { $0.status = .uploadPending }.where { $0.id.eq(task.taskId) }.execute(conn)
return status
}
let status = try await taskRepository.markDownloadComplete(taskId: task.taskId, localId: task.localId, hash: hash)
statusListener.onTaskStatus(
UploadApiTaskStatus(
id: String(task.taskId),
filename: filePath.path,
filename: task.filename,
status: UploadApiStatus(rawValue: status.rawValue)!
)
)
uploadQueue.startQueueProcessing()
NotificationCenter.default.post(name: .downloadTaskDidComplete, object: nil)
} catch let error as UploadError {
dPrint("Download failed for task \(task.taskId): \(error)")
await handleFailure(task: task, code: error.code, filePath: filePath)
} catch {
dPrint("Download failed for task \(task.taskId): \(error)")
handleFailure(task: task, code: .writeFailed, filePath: filePath)
await handleFailure(task: task, code: .unknown, filePath: filePath)
}
}
@@ -240,18 +196,11 @@ class DownloadQueue {
}
try fileHandle.write(contentsOf: header)
class RequestRef {
var id: PHAssetResourceDataRequestID?
var lastProgressTime = Date()
var didStall = false
}
var lastProgressTime = Date()
nonisolated(unsafe) let progressListener = self.progressListener
let taskIdStr = String(task.taskId)
options.progressHandler = { progress in
lastProgressTime = Date()
progressListener.onTaskProgress(UploadApiTaskProgress(id: taskIdStr, progress: progress))
self.progressListener.onTaskProgress(UploadApiTaskProgress(id: taskIdStr, progress: progress))
}
let request = RequestRef()
@@ -261,7 +210,7 @@ class DownloadQueue {
request.didStall = Date().timeIntervalSince(lastProgressTime) > TaskConfig.downloadTimeoutS
if request.didStall {
if let requestId = request.id {
Self.resourceManager.cancelDataRequest(requestId)
resourceManager.cancelDataRequest(requestId)
}
break
}
@@ -271,7 +220,7 @@ class DownloadQueue {
return try await withTaskCancellationHandler {
try await withCheckedThrowingContinuation { continuation in
var hasher = task.checksum == nil && task.livePhotoVideoId == nil ? Insecure.SHA1() : nil
request.id = Self.resourceManager.requestData(
request.id = resourceManager.requestData(
for: resource,
options: options,
dataReceivedHandler: { data in
@@ -281,7 +230,7 @@ class DownloadQueue {
try fileHandle.write(contentsOf: data)
} catch {
request.id = nil
Self.resourceManager.cancelDataRequest(requestId)
resourceManager.cancelDataRequest(requestId)
}
},
completionHandler: { error in
@@ -295,7 +244,7 @@ class DownloadQueue {
case 81: .iCloudThrottled
default: .photosUnknownError
}
self.handleFailure(task: task, code: code, filePath: filePath)
continuation.resume(throwing: UploadError.iCloudError(code))
case let e as PHPhotosError:
dPrint("Photos error during download: \(e)")
let code: UploadErrorCode =
@@ -310,10 +259,10 @@ class DownloadQueue {
case .userCancelled: .cancelled
default: .photosUnknownError
}
self.handleFailure(task: task, code: code, filePath: filePath)
continuation.resume(throwing: UploadError.photosError(code))
case .some:
dPrint("Unknown error during download: \(String(describing: error))")
self.handleFailure(task: task, code: .unknown, filePath: filePath)
continuation.resume(throwing: UploadError.unknown)
case .none:
dPrint("Download completed for task \(task.taskId)")
do {
@@ -321,7 +270,7 @@ class DownloadQueue {
continuation.resume(returning: hasher.map { hasher in Data(hasher.finalize()).base64EncodedString() })
} catch {
try? FileManager.default.removeItem(at: filePath)
continuation.resume(throwing: error)
continuation.resume(throwing: UploadError.fileCreationFailed)
}
}
}
@@ -329,21 +278,26 @@ class DownloadQueue {
}
} onCancel: {
if let requestId = request.id {
Self.resourceManager.cancelDataRequest(requestId)
resourceManager.cancelDataRequest(requestId)
}
}
}
private func handleFailure(task: LocalAssetDownloadData, code: UploadErrorCode, filePath: URL? = nil) {
private func handleFailure(task: LocalAssetDownloadData, code: UploadErrorCode, filePath: URL? = nil) async {
dPrint("Handling failure for task \(task.taskId) with code \(code.rawValue)")
do {
if let filePath {
try? FileManager.default.removeItem(at: filePath)
}
try db.write { conn in
try UploadTask.retryOrFail(code: code, status: .downloadFailed).where { $0.id.eq(task.taskId) }.execute(conn)
}
try await taskRepository.retryOrFail(taskId: task.taskId, code: code, status: .downloadFailed)
statusListener.onTaskStatus(
UploadApiTaskStatus(
id: String(task.taskId),
filename: task.filename,
status: .downloadFailed
)
)
} catch {
dPrint("Failed to update download failure status for task \(task.taskId): \(error)")
}
+47 -21
View File
@@ -1,4 +1,12 @@
class StatusEventListener: StreamStatusStreamHandler {
protocol TaskProgressListener {
func onTaskProgress(_ event: UploadApiTaskProgress)
}
protocol TaskStatusListener {
func onTaskStatus(_ event: UploadApiTaskStatus)
}
final class StatusEventListener: StreamStatusStreamHandler, TaskStatusListener, @unchecked Sendable {
var eventSink: PigeonEventSink<UploadApiTaskStatus>?
override func onListen(withArguments arguments: Any?, sink: PigeonEventSink<UploadApiTaskStatus>) {
@@ -6,26 +14,7 @@ class StatusEventListener: StreamStatusStreamHandler {
}
func onTaskStatus(_ event: UploadApiTaskStatus) {
if let eventSink = eventSink {
eventSink.success(event)
}
}
func onEventsDone() {
eventSink?.endOfStream()
eventSink = nil
}
}
class ProgressEventListener: StreamProgressStreamHandler {
var eventSink: PigeonEventSink<UploadApiTaskProgress>?
override func onListen(withArguments arguments: Any?, sink: PigeonEventSink<UploadApiTaskProgress>) {
eventSink = sink
}
func onTaskProgress(_ event: UploadApiTaskProgress) {
if let eventSink = eventSink {
if let eventSink {
DispatchQueue.main.async { eventSink.success(event) }
}
}
@@ -37,3 +26,40 @@ class ProgressEventListener: StreamProgressStreamHandler {
}
}
}
final class ProgressEventListener: StreamProgressStreamHandler, TaskProgressListener, @unchecked Sendable {
var eventSink: PigeonEventSink<UploadApiTaskProgress>?
private var lastReportTimes: [String: Date] = [:]
private let lock = NSLock()
override func onListen(withArguments arguments: Any?, sink: PigeonEventSink<UploadApiTaskProgress>) {
eventSink = sink
}
func onTaskProgress(_ event: UploadApiTaskProgress) {
guard let eventSink,
lock.withLock({
let now = Date()
if let lastReport = lastReportTimes[event.id] {
guard now.timeIntervalSince(lastReport) >= TaskConfig.progressThrottleInterval else {
return false
}
}
lastReportTimes[event.id] = now
return true
})
else { return }
DispatchQueue.main.async { eventSink.success(event) }
}
func onEventsDone() {
DispatchQueue.main.async {
self.eventSink?.endOfStream()
self.eventSink = nil
self.lock.withLock {
self.lastReportTimes.removeAll()
}
}
}
}
@@ -1,6 +1,6 @@
import Network
class NetworkMonitor {
final class NetworkMonitor {
static let shared = NetworkMonitor()
private let monitor = NWPathMonitor()
private(set) var isConnected = false
@@ -17,6 +17,6 @@ class NetworkMonitor {
NotificationCenter.default.post(name: .networkDidConnect, object: nil)
}
}
monitor.start(queue: .global(qos: .utility))
monitor.start(queue: .global(qos: .default))
}
}
+73 -97
View File
@@ -1,28 +1,57 @@
import SQLiteData
import StructuredFieldValues
private var queueProcessingTask: Task<Void, Never>?
private var queueProcessingLock = NSLock()
class UploadQueue {
private static let structuredEncoder = StructuredFieldValueEncoder()
private static var queueProcessingTask: Task<Void, Never>?
private static var queueProcessingLock = NSLock()
final class UploadQueue<StoreRepo: StoreProtocol, TaskRepo: TaskProtocol, StatusListener: TaskStatusListener> {
private let storeRepository: StoreRepo
private let taskRepository: TaskRepo
private let statusListener: StatusListener
private let db: DatabasePool
private let cellularSession: URLSession
private let wifiOnlySession: URLSession
private let statusListener: StatusEventListener
private var uploadObserver: NSObjectProtocol?
private var downloadObserver: NSObjectProtocol?
private var networkObserver: NSObjectProtocol?
init(db: DatabasePool, cellularSession: URLSession, wifiOnlySession: URLSession, statusListener: StatusEventListener)
{
self.db = db
init(
storeRepository: StoreRepo,
taskRepository: TaskRepo,
statusListener: StatusListener,
cellularSession: URLSession,
wifiOnlySession: URLSession
) {
self.storeRepository = storeRepository
self.taskRepository = taskRepository
self.cellularSession = cellularSession
self.wifiOnlySession = wifiOnlySession
self.statusListener = statusListener
uploadObserver = NotificationCenter.default.addObserver(forName: .uploadTaskDidComplete, object: nil, queue: nil) {
[weak self] _ in
self?.startQueueProcessing()
}
downloadObserver = NotificationCenter.default.addObserver(
forName: .downloadTaskDidComplete,
object: nil,
queue: nil
) { [weak self] _ in
self?.startQueueProcessing()
}
networkObserver = NotificationCenter.default.addObserver(forName: .networkDidConnect, object: nil, queue: nil) {
[weak self] _ in
self?.startQueueProcessing()
}
}
deinit {
uploadObserver.map(NotificationCenter.default.removeObserver(_:))
downloadObserver.map(NotificationCenter.default.removeObserver(_:))
networkObserver.map(NotificationCenter.default.removeObserver(_:))
}
func enqueueFiles(paths: [String]) async throws {
guard !paths.isEmpty else { return dPrint("No paths to enqueue") }
guard let deviceId = (try? await db.read { conn in try Store.get(conn, StoreKey.deviceId) }) else {
guard let deviceId = try? storeRepository.get(StoreKey.deviceId) else {
throw StoreError.notFound
}
@@ -75,72 +104,36 @@ class UploadQueue {
try await group.waitForAll()
}
try await db.write { conn in
var draft = UploadTask.Draft(
attempts: 0,
createdAt: Date(),
filePath: nil,
isLivePhoto: nil,
lastError: nil,
livePhotoVideoId: nil,
localId: "",
method: .multipart,
priority: 0.5,
retryAfter: nil,
status: .downloadPending,
)
for path in paths {
draft.filePath = URL(fileURLWithPath: path, isDirectory: false)
try UploadTask.insert { draft }.execute(conn)
}
}
try await taskRepository.enqueue(files: paths)
dPrint("Enqueued \(paths.count) assets for upload")
}
func startQueueProcessing() {
dPrint("Starting upload queue processing")
Self.queueProcessingLock.withLock {
guard Self.queueProcessingTask == nil else { return }
Self.queueProcessingTask = Task {
queueProcessingLock.withLock {
guard queueProcessingTask == nil else { return }
queueProcessingTask = Task {
await startUploads()
Self.queueProcessingLock.withLock { Self.queueProcessingTask = nil }
queueProcessingLock.withLock { queueProcessingTask = nil }
}
}
}
private func startUploads() async {
dPrint("Processing download queue")
dPrint("Processing upload queue")
guard NetworkMonitor.shared.isConnected,
let backupEnabled = try? await db.read({ conn in try Store.get(conn, StoreKey.enableBackup) }),
backupEnabled
else { return dPrint("Download queue paused: network disconnected or backup disabled") }
let backupEnabled = try? storeRepository.get(StoreKey.enableBackup), backupEnabled,
let url = try? storeRepository.get(StoreKey.serverEndpoint),
let accessToken = try? storeRepository.get(StoreKey.accessToken)
else { return dPrint("Upload queue paused: missing preconditions") }
do {
let tasks: [LocalAssetUploadData] = try await db.read({ conn in
guard let backupEnabled = try Store.get(conn, StoreKey.enableBackup), backupEnabled else { return [] }
return try UploadTask.join(LocalAsset.all) { task, asset in task.localId.eq(asset.id) }
.where { task, asset in
asset.checksum.isNot(nil) && task.status.eq(TaskStatus.uploadPending)
&& task.attempts < TaskConfig.maxRetries
&& task.filePath.isNot(nil)
}
.select { task, asset in
LocalAssetUploadData.Columns(
filePath: task.filePath.unwrapped,
priority: task.priority,
taskId: task.id,
type: asset.type
)
}
.limit { task, _ in UploadTaskStat.availableUploadSlots }
.order { task, asset in (task.priority.desc(), task.createdAt) }
.fetchAll(conn)
})
let tasks = try await taskRepository.getUploadTasks()
if tasks.isEmpty { return dPrint("No upload tasks to process") }
await withTaskGroup(of: Void.self) { group in
for task in tasks {
group.addTask { await self.startUpload(multipart: task) }
group.addTask { await self.startUpload(multipart: task, url: url, accessToken: accessToken) }
}
}
} catch {
@@ -148,33 +141,18 @@ class UploadQueue {
}
}
private func startUpload(multipart task: LocalAssetUploadData) async {
private func startUpload(multipart task: LocalAssetUploadData, url: URL, accessToken: String) async {
dPrint("Uploading asset resource at \(task.filePath) of task \(task.taskId)")
defer { startQueueProcessing() }
let (url, accessToken, session): (URL, String, URLSession)
do {
(url, accessToken, session) = try await db.read { conn in
guard let url = try Store.get(conn, StoreKey.serverEndpoint),
let accessToken = try Store.get(conn, StoreKey.accessToken)
else {
throw StoreError.notFound
}
let session =
switch task.type {
case .image:
(try? Store.get(conn, StoreKey.useWifiForUploadPhotos)) ?? false ? cellularSession : wifiOnlySession
case .video:
(try? Store.get(conn, StoreKey.useWifiForUploadVideos)) ?? false ? cellularSession : wifiOnlySession
default: wifiOnlySession
}
return (url, accessToken, session)
let session =
switch task.type {
case .image:
(try? storeRepository.get(StoreKey.useWifiForUploadPhotos)) ?? false ? wifiOnlySession : cellularSession
case .video:
(try? storeRepository.get(StoreKey.useWifiForUploadVideos)) ?? false ? wifiOnlySession : cellularSession
default: wifiOnlySession
}
} catch {
dPrint("Upload failed for \(task.taskId), could not retrieve server URL or access token: \(error)")
return handleFailure(task: task, code: .noServerUrl)
}
var request = URLRequest(url: url.appendingPathComponent("/assets"))
request.httpMethod = "POST"
@@ -186,18 +164,11 @@ class UploadQueue {
sessionTask.priority = task.priority
do {
try? FileManager.default.removeItem(at: task.filePath) // upload task already copied the file
try await db.write { conn in
try UploadTask.update { row in
row.status = .uploadQueued
row.filePath = nil
}
.where { $0.id.eq(task.taskId) }
.execute(conn)
}
try await taskRepository.markUploadQueued(taskId: task.taskId)
statusListener.onTaskStatus(
UploadApiTaskStatus(
id: String(task.taskId),
filename: task.filePath.lastPathComponent,
filename: task.filename,
status: .uploadQueued,
)
)
@@ -209,11 +180,16 @@ class UploadQueue {
}
}
private func handleFailure(task: LocalAssetUploadData, code: UploadErrorCode) {
private func handleFailure(task: LocalAssetUploadData, code: UploadErrorCode) async {
do {
try db.write { conn in
try UploadTask.retryOrFail(code: code, status: .uploadFailed).where { $0.id.eq(task.taskId) }.execute(conn)
}
try await taskRepository.retryOrFail(taskId: task.taskId, code: code, status: .uploadFailed)
statusListener.onTaskStatus(
UploadApiTaskStatus(
id: String(task.taskId),
filename: task.filename,
status: .uploadFailed
)
)
} catch {
dPrint("Failed to update upload failure status for task \(task.taskId): \(error)")
}
+31 -12
View File
@@ -122,18 +122,19 @@ enum UploadApiErrorCode: Int {
case networkError = 8
case photosInternalError = 9
case photosUnknownError = 10
case noServerUrl = 11
case noDeviceId = 12
case noAccessToken = 13
case interrupted = 14
case cancelled = 15
case downloadStalled = 16
case forceQuit = 17
case outOfResources = 18
case backgroundUpdatesDisabled = 19
case uploadTimeout = 20
case iCloudRateLimit = 21
case iCloudThrottled = 22
case interrupted = 11
case cancelled = 12
case downloadStalled = 13
case forceQuit = 14
case outOfResources = 15
case backgroundUpdatesDisabled = 16
case uploadTimeout = 17
case iCloudRateLimit = 18
case iCloudThrottled = 19
case invalidResponse = 20
case badRequest = 21
case internalServerError = 22
case unauthorized = 23
}
enum UploadApiStatus: Int {
@@ -294,6 +295,7 @@ protocol UploadApi {
func cancelAll(completion: @escaping (Result<Void, Error>) -> Void)
func enqueueAssets(localIds: [String], completion: @escaping (Result<Void, Error>) -> Void)
func enqueueFiles(paths: [String], completion: @escaping (Result<Void, Error>) -> Void)
func onConfigChange(key: Int64, completion: @escaping (Result<Void, Error>) -> Void)
}
/// Generated setup class from Pigeon to handle messages through the `binaryMessenger`.
@@ -381,6 +383,23 @@ class UploadApiSetup {
} else {
enqueueFilesChannel.setMessageHandler(nil)
}
let onConfigChangeChannel = FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.UploadApi.onConfigChange\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec)
if let api = api {
onConfigChangeChannel.setMessageHandler { message, reply in
let args = message as! [Any?]
let keyArg = args[0] as! Int64
api.onConfigChange(key: keyArg) { result in
switch result {
case .success:
reply(wrapResult(nil))
case .failure(let error):
reply(wrapError(error))
}
}
}
} else {
onConfigChangeChannel.setMessageHandler(nil)
}
}
}
+54 -133
View File
@@ -1,5 +1,4 @@
import SQLiteData
import StructuredFieldValues
extension FileHandle {
static func createOrOverwrite(atPath path: String) throws -> FileHandle {
@@ -11,33 +10,41 @@ extension FileHandle {
}
}
class UploadApiImpl: ImmichPlugin, UploadApi {
private let db: DatabasePool
private let downloadQueue: DownloadQueue
private let uploadQueue: UploadQueue
final class UploadApiImpl<
StoreRepo: StoreProtocol,
TaskRepo: TaskProtocol,
StatusListener: TaskStatusListener,
ProgressListener: TaskProgressListener
>: ImmichPlugin, UploadApi {
private let storeRepository: StoreRepo
private let taskRepository: TaskRepo
private let downloadQueue: DownloadQueue<StoreRepo, TaskRepo, StatusListener, ProgressListener>
private let uploadQueue: UploadQueue<StoreRepo, TaskRepo, StatusListener>
private var isInitialized = false
private let initLock = NSLock()
private var backupTask: Task<Void, Never>?
private let backupLock = NSLock()
private let cellularSession: URLSession
private let wifiOnlySession: URLSession
init(statusListener: StatusEventListener, progressListener: ProgressEventListener) {
let dbUrl = try! FileManager.default.url(
for: .documentDirectory,
in: .userDomainMask,
appropriateFor: nil,
create: true
).appendingPathComponent("immich.sqlite")
self.db = try! DatabasePool(path: dbUrl.path)
init(
storeRepository: StoreRepo,
taskRepository: TaskRepo,
statusListener: StatusListener,
progressListener: ProgressListener
) {
self.taskRepository = taskRepository
let delegate = UploadApiDelegate(
taskRepository: taskRepository,
statusListener: statusListener,
progressListener: progressListener
)
let cellularConfig = URLSessionConfiguration.background(withIdentifier: "\(TaskConfig.sessionId).cellular")
cellularConfig.allowsCellularAccess = true
cellularConfig.waitsForConnectivity = true
let delegate = UploadApiDelegate(db: db, statusListener: statusListener, progressListener: progressListener)
self.cellularSession = URLSession(configuration: cellularConfig, delegate: delegate, delegateQueue: nil)
let wifiOnlyConfig = URLSessionConfiguration.background(withIdentifier: "\(TaskConfig.sessionId).wifi")
@@ -45,28 +52,26 @@ class UploadApiImpl: ImmichPlugin, UploadApi {
wifiOnlyConfig.waitsForConnectivity = true
self.wifiOnlySession = URLSession(configuration: wifiOnlyConfig, delegate: delegate, delegateQueue: nil)
self.storeRepository = storeRepository
self.uploadQueue = UploadQueue(
db: db,
storeRepository: storeRepository,
taskRepository: taskRepository,
statusListener: statusListener,
cellularSession: cellularSession,
wifiOnlySession: wifiOnlySession,
statusListener: statusListener
wifiOnlySession: wifiOnlySession
)
self.downloadQueue = DownloadQueue(
db: db,
uploadQueue: uploadQueue,
storeRepository: storeRepository,
taskRepository: taskRepository,
statusListener: statusListener,
progressListener: progressListener
)
delegate.downloadQueue = downloadQueue
delegate.uploadQueue = uploadQueue
}
func initialize(completion: @escaping (Result<Void, any Error>) -> Void) {
Task(priority: .high) {
do {
async let dbIds = db.read { conn in
try UploadTask.select(\.id).where { $0.status.eq(TaskStatus.uploadQueued) }.fetchAll(conn)
}
async let dbIds = taskRepository.getTaskIds(status: .uploadQueued)
async let cellularTasks = cellularSession.allTasks
async let wifiTasks = wifiOnlySession.allTasks
@@ -84,15 +89,7 @@ class UploadApiImpl: ImmichPlugin, UploadApi {
validateTasks(await cellularTasks)
validateTasks(await wifiTasks)
let orphanIds = Array(dbTaskIds)
try await db.write { conn in
try UploadTask.update {
$0.filePath = nil
$0.status = .downloadPending
}
.where { row in row.status.in([TaskStatus.downloadQueued, TaskStatus.uploadPending]) || row.id.in(orphanIds) }
.execute(conn)
}
try await taskRepository.markOrphansPending(ids: Array(dbTaskIds))
try? FileManager.default.removeItem(at: TaskConfig.originalsDir)
initLock.withLock { isInitialized = true }
@@ -139,9 +136,9 @@ class UploadApiImpl: ImmichPlugin, UploadApi {
Task {
do {
try await downloadQueue.enqueueAssets(localIds: localIds)
completion(.success(()))
self.completeWhenActive(for: completion, with: .success(()))
} catch {
completion(.failure(error))
self.completeWhenActive(for: completion, with: .failure(error))
}
}
}
@@ -150,13 +147,24 @@ class UploadApiImpl: ImmichPlugin, UploadApi {
Task {
do {
try await uploadQueue.enqueueFiles(paths: paths)
completion(.success(()))
self.completeWhenActive(for: completion, with: .success(()))
} catch {
completion(.failure(error))
self.completeWhenActive(for: completion, with: .failure(error))
}
}
}
func onConfigChange(key: Int64, completion: @escaping (Result<Void, any Error>) -> Void) {
storeRepository.invalidateCache()
Task {
if let key = StoreKey(rawValue: Int(key)), key == ._accessToken {
try? await taskRepository.resolveError(code: .unauthorized)
}
startBackup()
self.completeWhenActive(for: completion, with: .success(()))
}
}
private func cancelSessionTasks(_ tasks: [URLSessionTask]) {
dPrint("Canceling \(tasks.count) tasks")
for task in tasks {
@@ -165,107 +173,20 @@ class UploadApiImpl: ImmichPlugin, UploadApi {
}
private func _startBackup() async {
defer { downloadQueue.startQueueProcessing() }
defer {
downloadQueue.startQueueProcessing()
uploadQueue.startQueueProcessing()
}
do {
let candidates = try await db.read { conn in
return try LocalAsset.getCandidates()
.where { asset in !UploadTask.where { task in task.localId.eq(asset.id) }.exists() }
.select { LocalAssetCandidate.Columns(id: $0.id, type: $0.type) }
.limit { _ in UploadTaskStat.availableSlots }
.fetchAll(conn)
}
let candidates = try await taskRepository.getBackupCandidates()
guard !candidates.isEmpty else { return dPrint("No candidates for backup") }
try await db.write { conn in
var draft = UploadTask.Draft(
attempts: 0,
createdAt: Date(),
filePath: nil,
isLivePhoto: nil,
lastError: nil,
livePhotoVideoId: nil,
localId: "",
method: .multipart,
priority: 0.5,
retryAfter: nil,
status: .downloadPending,
)
for candidate in candidates {
draft.localId = candidate.id
draft.priority = candidate.type == .image ? 0.5 : 0.3
try UploadTask.insert {
draft
} onConflict: {
($0.localId, $0.livePhotoVideoId)
}
.execute(conn)
}
}
try await taskRepository.enqueue(assets: candidates, imagePriority: 0.5, videoPriority: 0.3)
dPrint("Backup enqueued \(candidates.count) assets for upload")
} catch {
print("Backup queue error: \(error)")
}
}
}
struct AssetData: StructuredFieldValue {
static let structuredFieldType: StructuredFieldType = .dictionary
let deviceAssetId: String
let deviceId: String
let fileCreatedAt: String
let fileModifiedAt: String
let fileName: String
let isFavorite: Bool
let livePhotoVideoId: String?
static let boundary = "Boundary-\(UUID().uuidString)"
static let deviceAssetIdField = "--\(boundary)\r\nContent-Disposition: form-data; name=\"deviceAssetId\"\r\n\r\n"
.data(using: .utf8)!
static let deviceIdField = "\r\n--\(boundary)\r\nContent-Disposition: form-data; name=\"deviceId\"\r\n\r\n"
.data(using: .utf8)!
static let fileCreatedAtField =
"\r\n--\(boundary)\r\nContent-Disposition: form-data; name=\"fileCreatedAt\"\r\n\r\n"
.data(using: .utf8)!
static let fileModifiedAtField =
"\r\n--\(boundary)\r\nContent-Disposition: form-data; name=\"fileModifiedAt\"\r\n\r\n"
.data(using: .utf8)!
static let isFavoriteField = "\r\n--\(boundary)\r\nContent-Disposition: form-data; name=\"isFavorite\"\r\n\r\n"
.data(using: .utf8)!
static let livePhotoVideoIdField =
"\r\n--\(boundary)\r\nContent-Disposition: form-data; name=\"livePhotoVideoId\"\r\n\r\n"
.data(using: .utf8)!
static let trueData = "true".data(using: .utf8)!
static let falseData = "false".data(using: .utf8)!
static let footer = "\r\n--\(boundary)--\r\n".data(using: .utf8)!
static let contentType = "multipart/form-data; boundary=\(boundary)"
func multipart() -> (Data, Data) {
var header = Data()
header.append(Self.deviceAssetIdField)
header.append(deviceAssetId.data(using: .utf8)!)
header.append(Self.deviceIdField)
header.append(deviceId.data(using: .utf8)!)
header.append(Self.fileCreatedAtField)
header.append(fileCreatedAt.data(using: .utf8)!)
header.append(Self.fileModifiedAtField)
header.append(fileModifiedAt.data(using: .utf8)!)
header.append(Self.isFavoriteField)
header.append(isFavorite ? Self.trueData : Self.falseData)
if let livePhotoVideoId {
header.append(Self.livePhotoVideoIdField)
header.append(livePhotoVideoId.data(using: .utf8)!)
}
header.append(
"\r\n--\(Self.boundary)\r\nContent-Disposition: form-data; name=\"assetData\"; filename=\"\(fileName)\"\r\nContent-Type: application/octet-stream\r\n\r\n"
.data(using: .utf8)!
)
return (header, Self.footer)
}
}