mirror of
https://github.com/immich-app/immich.git
synced 2026-05-18 03:10:24 +03:00
background upload plugin
add schemas sync variants formatting initial implementation use existing db, wip move to separate folder fix table definitions wip wiring it up repository pattern
This commit is contained in:
@@ -0,0 +1,62 @@
|
||||
import StructuredFieldValues
|
||||
|
||||
struct AssetData: StructuredFieldValue {
|
||||
static let structuredFieldType: StructuredFieldType = .dictionary
|
||||
|
||||
let deviceAssetId: String
|
||||
let deviceId: String
|
||||
let fileCreatedAt: String
|
||||
let fileModifiedAt: String
|
||||
let fileName: String
|
||||
let isFavorite: Bool
|
||||
let livePhotoVideoId: String?
|
||||
|
||||
static let boundary = "Boundary-\(UUID().uuidString)"
|
||||
static let deviceAssetIdField = "--\(boundary)\r\nContent-Disposition: form-data; name=\"deviceAssetId\"\r\n\r\n"
|
||||
.data(using: .utf8)!
|
||||
static let deviceIdField = "\r\n--\(boundary)\r\nContent-Disposition: form-data; name=\"deviceId\"\r\n\r\n"
|
||||
.data(using: .utf8)!
|
||||
static let fileCreatedAtField =
|
||||
"\r\n--\(boundary)\r\nContent-Disposition: form-data; name=\"fileCreatedAt\"\r\n\r\n"
|
||||
.data(using: .utf8)!
|
||||
static let fileModifiedAtField =
|
||||
"\r\n--\(boundary)\r\nContent-Disposition: form-data; name=\"fileModifiedAt\"\r\n\r\n"
|
||||
.data(using: .utf8)!
|
||||
static let isFavoriteField = "\r\n--\(boundary)\r\nContent-Disposition: form-data; name=\"isFavorite\"\r\n\r\n"
|
||||
.data(using: .utf8)!
|
||||
static let livePhotoVideoIdField =
|
||||
"\r\n--\(boundary)\r\nContent-Disposition: form-data; name=\"livePhotoVideoId\"\r\n\r\n"
|
||||
.data(using: .utf8)!
|
||||
static let trueData = "true".data(using: .utf8)!
|
||||
static let falseData = "false".data(using: .utf8)!
|
||||
static let footer = "\r\n--\(boundary)--\r\n".data(using: .utf8)!
|
||||
static let contentType = "multipart/form-data; boundary=\(boundary)"
|
||||
|
||||
func multipart() -> (Data, Data) {
|
||||
var header = Data()
|
||||
header.append(Self.deviceAssetIdField)
|
||||
header.append(deviceAssetId.data(using: .utf8)!)
|
||||
|
||||
header.append(Self.deviceIdField)
|
||||
header.append(deviceId.data(using: .utf8)!)
|
||||
|
||||
header.append(Self.fileCreatedAtField)
|
||||
header.append(fileCreatedAt.data(using: .utf8)!)
|
||||
|
||||
header.append(Self.fileModifiedAtField)
|
||||
header.append(fileModifiedAt.data(using: .utf8)!)
|
||||
|
||||
header.append(Self.isFavoriteField)
|
||||
header.append(isFavorite ? Self.trueData : Self.falseData)
|
||||
|
||||
if let livePhotoVideoId {
|
||||
header.append(Self.livePhotoVideoIdField)
|
||||
header.append(livePhotoVideoId.data(using: .utf8)!)
|
||||
}
|
||||
header.append(
|
||||
"\r\n--\(Self.boundary)\r\nContent-Disposition: form-data; name=\"assetData\"; filename=\"\(fileName)\"\r\nContent-Type: application/octet-stream\r\n\r\n"
|
||||
.data(using: .utf8)!
|
||||
)
|
||||
return (header, Self.footer)
|
||||
}
|
||||
}
|
||||
@@ -1,19 +1,33 @@
|
||||
import SQLiteData
|
||||
|
||||
class UploadApiDelegate: NSObject, URLSessionDataDelegate, URLSessionTaskDelegate {
|
||||
private static let stateLock = NSLock()
|
||||
private static var transferStates: [Int64: NetworkTransferState] = [:]
|
||||
private static var responseData: [Int64: Data] = [:]
|
||||
private static let jsonDecoder = JSONDecoder()
|
||||
private let stateLock = NSLock()
|
||||
private var transferStates: [Int64: NetworkTransferState] = [:]
|
||||
private var responseData: [Int64: Data] = [:]
|
||||
private let jsonDecoder = JSONDecoder()
|
||||
|
||||
private let db: DatabasePool
|
||||
private let statusListener: StatusEventListener
|
||||
private let progressListener: ProgressEventListener
|
||||
weak var downloadQueue: DownloadQueue?
|
||||
weak var uploadQueue: UploadQueue?
|
||||
private class NetworkTransferState {
|
||||
var lastUpdateTime: Date
|
||||
var totalBytesTransferred: Int64
|
||||
var currentSpeed: Double?
|
||||
|
||||
init(db: DatabasePool, statusListener: StatusEventListener, progressListener: ProgressEventListener) {
|
||||
self.db = db
|
||||
init(lastUpdateTime: Date, totalBytesTransferred: Int64, currentSpeed: Double?) {
|
||||
self.lastUpdateTime = lastUpdateTime
|
||||
self.totalBytesTransferred = totalBytesTransferred
|
||||
self.currentSpeed = currentSpeed
|
||||
}
|
||||
}
|
||||
|
||||
final class UploadApiDelegate<
|
||||
TaskRepo: TaskProtocol,
|
||||
StatusListener: TaskStatusListener,
|
||||
ProgressListener: TaskProgressListener
|
||||
>: NSObject, URLSessionDataDelegate, URLSessionTaskDelegate {
|
||||
private let taskRepository: TaskRepo
|
||||
private let statusListener: StatusListener
|
||||
private let progressListener: ProgressListener
|
||||
|
||||
init(taskRepository: TaskRepo, statusListener: StatusListener, progressListener: ProgressListener) {
|
||||
self.taskRepository = taskRepository
|
||||
self.statusListener = statusListener
|
||||
self.progressListener = progressListener
|
||||
}
|
||||
@@ -30,11 +44,11 @@ class UploadApiDelegate: NSObject, URLSessionDataDelegate, URLSessionTaskDelegat
|
||||
let taskId = Int64(taskIdStr)
|
||||
else { return }
|
||||
|
||||
Self.stateLock.withLock {
|
||||
if var response = Self.responseData[taskId] {
|
||||
stateLock.withLock {
|
||||
if var response = responseData[taskId] {
|
||||
response.append(data)
|
||||
} else {
|
||||
Self.responseData[taskId] = data
|
||||
responseData[taskId] = data
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -42,8 +56,7 @@ class UploadApiDelegate: NSObject, URLSessionDataDelegate, URLSessionTaskDelegat
|
||||
func urlSession(_ session: URLSession, task: URLSessionTask, didCompleteWithError error: Error?) {
|
||||
Task {
|
||||
defer {
|
||||
downloadQueue?.startQueueProcessing()
|
||||
uploadQueue?.startQueueProcessing()
|
||||
NotificationCenter.default.post(name: .uploadTaskDidComplete, object: nil)
|
||||
}
|
||||
|
||||
guard let taskDescriptionId = task.taskDescription,
|
||||
@@ -53,25 +66,27 @@ class UploadApiDelegate: NSObject, URLSessionDataDelegate, URLSessionTaskDelegat
|
||||
}
|
||||
|
||||
defer {
|
||||
Self.stateLock.withLock { let _ = Self.transferStates.removeValue(forKey: taskId) }
|
||||
stateLock.withLock { let _ = transferStates.removeValue(forKey: taskId) }
|
||||
}
|
||||
|
||||
if let responseData = Self.stateLock.withLock({ Self.responseData.removeValue(forKey: taskId) }),
|
||||
let httpResponse = task.response as? HTTPURLResponse
|
||||
if let body = stateLock.withLock({ responseData.removeValue(forKey: taskId) }),
|
||||
let response = task.response as? HTTPURLResponse
|
||||
{
|
||||
switch httpResponse.statusCode {
|
||||
switch response.statusCode {
|
||||
case 200, 201:
|
||||
do {
|
||||
let response = try Self.jsonDecoder.decode(UploadSuccessResponse.self, from: responseData)
|
||||
let response = try jsonDecoder.decode(UploadSuccessResponse.self, from: body)
|
||||
return await handleSuccess(taskId: taskId, response: response)
|
||||
} catch {
|
||||
return await handleFailure(taskId: taskId, code: .invalidResponse)
|
||||
}
|
||||
case 401: return await handleFailure(taskId: taskId, code: .unauthorized)
|
||||
case 400..<500:
|
||||
dPrint(
|
||||
"Response \(httpResponse.statusCode): \(String(data: responseData, encoding: .utf8) ?? "No response body")"
|
||||
)
|
||||
dPrint("Response \(response.statusCode): \(String(data: body, encoding: .utf8) ?? "No response body")")
|
||||
return await handleFailure(taskId: taskId, code: .badRequest)
|
||||
case 500..<600:
|
||||
dPrint("Response \(response.statusCode): \(String(data: body, encoding: .utf8) ?? "No response body")")
|
||||
return await handleFailure(taskId: taskId, code: .internalServerError)
|
||||
default:
|
||||
break
|
||||
}
|
||||
@@ -111,8 +126,8 @@ class UploadApiDelegate: NSObject, URLSessionDataDelegate, URLSessionTaskDelegat
|
||||
) {
|
||||
guard let sessionTaskId = task.taskDescription, let taskId = Int64(sessionTaskId) else { return }
|
||||
let currentTime = Date()
|
||||
let state = Self.stateLock.withLock {
|
||||
if let existing = Self.transferStates[taskId] {
|
||||
let state = stateLock.withLock {
|
||||
if let existing = transferStates[taskId] {
|
||||
return existing
|
||||
}
|
||||
let new = NetworkTransferState(
|
||||
@@ -120,7 +135,7 @@ class UploadApiDelegate: NSObject, URLSessionDataDelegate, URLSessionTaskDelegat
|
||||
totalBytesTransferred: totalBytesSent,
|
||||
currentSpeed: nil
|
||||
)
|
||||
Self.transferStates[taskId] = new
|
||||
transferStates[taskId] = new
|
||||
return new
|
||||
}
|
||||
|
||||
@@ -147,30 +162,29 @@ class UploadApiDelegate: NSObject, URLSessionDataDelegate, URLSessionTaskDelegat
|
||||
)
|
||||
}
|
||||
|
||||
func urlSessionDidFinishEvents(forBackgroundURLSession session: URLSession) {
|
||||
dPrint("All background events delivered for session: \(session.configuration.identifier ?? "unknown")")
|
||||
DispatchQueue.main.async {
|
||||
if let identifier = session.configuration.identifier,
|
||||
let appDelegate = UIApplication.shared.delegate as? AppDelegate,
|
||||
let completionHandler = appDelegate.completionHandler(forSession: identifier)
|
||||
{
|
||||
completionHandler()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func handleSuccess(taskId: Int64, response: UploadSuccessResponse) async {
|
||||
dPrint("Upload succeeded for task \(taskId), server ID: \(response.id)")
|
||||
do {
|
||||
try await db.write { conn in
|
||||
let task = try UploadTask.update { $0.status = .uploadComplete }.where({ $0.id.eq(taskId) })
|
||||
.returning(\.self).fetchOne(conn)
|
||||
guard let task, let isLivePhoto = task.isLivePhoto, isLivePhoto, task.livePhotoVideoId == nil else { return }
|
||||
try UploadTask.insert {
|
||||
UploadTask.Draft(
|
||||
attempts: 0,
|
||||
createdAt: Date(),
|
||||
filePath: nil,
|
||||
isLivePhoto: true,
|
||||
lastError: nil,
|
||||
livePhotoVideoId: response.id,
|
||||
localId: task.localId,
|
||||
method: .multipart,
|
||||
priority: 0.7,
|
||||
retryAfter: nil,
|
||||
status: .downloadPending,
|
||||
)
|
||||
}.execute(conn)
|
||||
}
|
||||
dPrint("Updated upload success status for session task \(taskId)")
|
||||
try await taskRepository.markUploadSuccess(taskId: taskId, livePhotoVideoId: response.id)
|
||||
statusListener.onTaskStatus(
|
||||
UploadApiTaskStatus(
|
||||
id: String(taskId),
|
||||
filename: (try? await taskRepository.getFilename(taskId: taskId)) ?? "",
|
||||
status: .uploadComplete
|
||||
)
|
||||
)
|
||||
} catch {
|
||||
dPrint(
|
||||
"Failed to update upload success status for session task \(taskId): \(error.localizedDescription)"
|
||||
@@ -180,10 +194,14 @@ class UploadApiDelegate: NSObject, URLSessionDataDelegate, URLSessionTaskDelegat
|
||||
|
||||
private func handleFailure(taskId: Int64, code: UploadErrorCode = .unknown) async {
|
||||
dPrint("Upload failed for task \(taskId) with code \(code)")
|
||||
try? await db.write { conn in
|
||||
try UploadTask.retryOrFail(code: code, status: .uploadFailed).where { $0.id.eq(taskId) }
|
||||
.execute(conn)
|
||||
}
|
||||
try? await taskRepository.retryOrFail(taskId: taskId, code: code, status: .uploadFailed)
|
||||
statusListener.onTaskStatus(
|
||||
UploadApiTaskStatus(
|
||||
id: String(taskId),
|
||||
filename: (try? await taskRepository.getFilename(taskId: taskId)) ?? "",
|
||||
status: .uploadFailed
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
@available(iOS 17, *)
|
||||
@@ -193,16 +211,4 @@ class UploadApiDelegate: NSObject, URLSessionDataDelegate, URLSessionTaskDelegat
|
||||
resumeTask.taskDescription = taskDescriptionId
|
||||
resumeTask.resume()
|
||||
}
|
||||
|
||||
private class NetworkTransferState {
|
||||
var lastUpdateTime: Date
|
||||
var totalBytesTransferred: Int64
|
||||
var currentSpeed: Double?
|
||||
|
||||
init(lastUpdateTime: Date, totalBytesTransferred: Int64, currentSpeed: Double?) {
|
||||
self.lastUpdateTime = lastUpdateTime
|
||||
self.totalBytesTransferred = totalBytesTransferred
|
||||
self.currentSpeed = currentSpeed
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,131 +1,105 @@
|
||||
import CryptoKit
|
||||
import Photos
|
||||
import SQLiteData
|
||||
|
||||
class DownloadQueue {
|
||||
private static let resourceManager = PHAssetResourceManager.default()
|
||||
private static var queueProcessingTask: Task<Void, Never>?
|
||||
private static var queueProcessingLock = NSLock()
|
||||
private var queueProcessingTask: Task<Void, Never>?
|
||||
private var queueProcessingLock = NSLock()
|
||||
private let resourceManager = PHAssetResourceManager.default()
|
||||
|
||||
private let db: DatabasePool
|
||||
private let uploadQueue: UploadQueue
|
||||
private let statusListener: StatusEventListener
|
||||
private let progressListener: ProgressEventListener
|
||||
private final class RequestRef {
|
||||
var id: PHAssetResourceDataRequestID?
|
||||
var lastProgressTime = Date()
|
||||
var didStall = false
|
||||
}
|
||||
|
||||
final class DownloadQueue<
|
||||
StoreRepo: StoreProtocol,
|
||||
TaskRepo: TaskProtocol,
|
||||
StatusListener: TaskStatusListener,
|
||||
ProgressListener: TaskProgressListener
|
||||
> {
|
||||
private let storeRepository: StoreRepo
|
||||
private let taskRepository: TaskRepo
|
||||
private let statusListener: StatusListener
|
||||
private let progressListener: ProgressListener
|
||||
private var uploadObserver: NSObjectProtocol?
|
||||
private var networkObserver: NSObjectProtocol?
|
||||
|
||||
init(
|
||||
db: DatabasePool,
|
||||
uploadQueue: UploadQueue,
|
||||
statusListener: StatusEventListener,
|
||||
progressListener: ProgressEventListener
|
||||
storeRepository: StoreRepo,
|
||||
taskRepository: TaskRepo,
|
||||
statusListener: StatusListener,
|
||||
progressListener: ProgressListener
|
||||
) {
|
||||
self.db = db
|
||||
self.uploadQueue = uploadQueue
|
||||
self.storeRepository = storeRepository
|
||||
self.taskRepository = taskRepository
|
||||
self.statusListener = statusListener
|
||||
self.progressListener = progressListener
|
||||
NotificationCenter.default.addObserver(forName: .networkDidConnect, object: nil, queue: nil) { [weak self] _ in
|
||||
uploadObserver = NotificationCenter.default.addObserver(forName: .uploadTaskDidComplete, object: nil, queue: nil) {
|
||||
[weak self] _ in
|
||||
self?.startQueueProcessing()
|
||||
}
|
||||
networkObserver = NotificationCenter.default.addObserver(forName: .networkDidConnect, object: nil, queue: nil) {
|
||||
[weak self] _ in
|
||||
dPrint("Network connected")
|
||||
self?.startQueueProcessing()
|
||||
}
|
||||
}
|
||||
|
||||
deinit {
|
||||
uploadObserver.map(NotificationCenter.default.removeObserver(_:))
|
||||
networkObserver.map(NotificationCenter.default.removeObserver(_:))
|
||||
}
|
||||
|
||||
func enqueueAssets(localIds: [String]) async throws {
|
||||
guard !localIds.isEmpty else { return dPrint("No assets to enqueue") }
|
||||
|
||||
defer { startQueueProcessing() }
|
||||
let candidates = try await db.read { conn in
|
||||
return try LocalAsset.all
|
||||
.where { asset in asset.id.in(localIds) }
|
||||
.select { LocalAssetCandidate.Columns(id: $0.id, type: $0.type) }
|
||||
.limit { _ in UploadTaskStat.availableSlots }
|
||||
.fetchAll(conn)
|
||||
}
|
||||
let candidates = try await taskRepository.getBackupCandidates(ids: localIds)
|
||||
|
||||
guard !candidates.isEmpty else { return dPrint("No candidates to enqueue") }
|
||||
|
||||
try await db.write { conn in
|
||||
var draft = UploadTask.Draft(
|
||||
attempts: 0,
|
||||
createdAt: Date(),
|
||||
filePath: nil,
|
||||
isLivePhoto: nil,
|
||||
lastError: nil,
|
||||
livePhotoVideoId: nil,
|
||||
localId: "",
|
||||
method: .multipart,
|
||||
priority: 0.5,
|
||||
retryAfter: nil,
|
||||
status: .downloadPending,
|
||||
)
|
||||
for candidate in candidates {
|
||||
draft.localId = candidate.id
|
||||
draft.priority = candidate.type == .image ? 0.9 : 0.8
|
||||
try UploadTask.insert {
|
||||
draft
|
||||
} onConflict: {
|
||||
($0.localId, $0.livePhotoVideoId)
|
||||
}.execute(conn)
|
||||
}
|
||||
}
|
||||
try await taskRepository.enqueue(assets: candidates, imagePriority: 0.9, videoPriority: 0.8)
|
||||
dPrint("Enqueued \(candidates.count) assets for upload")
|
||||
}
|
||||
|
||||
func startQueueProcessing() {
|
||||
dPrint("Starting download queue processing")
|
||||
Self.queueProcessingLock.withLock {
|
||||
guard Self.queueProcessingTask == nil else { return }
|
||||
Self.queueProcessingTask = Task {
|
||||
queueProcessingLock.withLock {
|
||||
guard queueProcessingTask == nil else { return }
|
||||
queueProcessingTask = Task {
|
||||
await startDownloads()
|
||||
Self.queueProcessingLock.withLock { Self.queueProcessingTask = nil }
|
||||
queueProcessingLock.withLock { queueProcessingTask = nil }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func startDownloads() async {
|
||||
dPrint("Processing download queue")
|
||||
guard NetworkMonitor.shared.isConnected else {
|
||||
return dPrint("Download queue paused: network disconnected")
|
||||
|
||||
guard await UIApplication.shared.applicationState != .background else {
|
||||
return dPrint("Not processing downloads in background") // TODO: run in processing tasks
|
||||
}
|
||||
|
||||
guard NetworkMonitor.shared.isConnected,
|
||||
let backupEnabled = try? storeRepository.get(StoreKey.enableBackup), backupEnabled,
|
||||
let deviceId = try? storeRepository.get(StoreKey.deviceId)
|
||||
else { return dPrint("Download queue paused: missing preconditions") }
|
||||
|
||||
do {
|
||||
let tasks: [LocalAssetDownloadData] = try await db.read({ conn in
|
||||
guard let backupEnabled = try Store.get(conn, StoreKey.enableBackup), backupEnabled else { return [] }
|
||||
return try UploadTask.join(LocalAsset.all) { task, asset in task.localId.eq(asset.id) }
|
||||
.where { task, asset in
|
||||
asset.checksum.isNot(nil) && task.status.eq(TaskStatus.downloadPending)
|
||||
&& task.attempts < TaskConfig.maxRetries
|
||||
&& (task.retryAfter.is(nil) || task.retryAfter.unwrapped <= Date().unixTime)
|
||||
&& (task.lastError.is(nil)
|
||||
|| !task.lastError.unwrapped.in([
|
||||
UploadErrorCode.assetNotFound, UploadErrorCode.resourceNotFound, UploadErrorCode.invalidResource,
|
||||
]))
|
||||
}
|
||||
.select { task, asset in
|
||||
LocalAssetDownloadData.Columns(
|
||||
checksum: asset.checksum,
|
||||
createdAt: asset.createdAt,
|
||||
livePhotoVideoId: task.livePhotoVideoId,
|
||||
localId: asset.id,
|
||||
taskId: task.id,
|
||||
updatedAt: asset.updatedAt
|
||||
)
|
||||
}
|
||||
.order { task, asset in (task.priority.desc(), task.createdAt) }
|
||||
.limit { _, _ in UploadTaskStat.availableDownloadSlots }
|
||||
.fetchAll(conn)
|
||||
})
|
||||
let tasks = try await taskRepository.getDownloadTasks()
|
||||
if tasks.isEmpty { return dPrint("No download tasks to process") }
|
||||
|
||||
try await withThrowingTaskGroup(of: Void.self) { group in
|
||||
var iterator = tasks.makeIterator()
|
||||
for _ in 0..<min(TaskConfig.maxActiveDownloads, tasks.count) {
|
||||
if let task = iterator.next() {
|
||||
group.addTask { await self.downloadAndQueue(task) }
|
||||
group.addTask { await self.downloadAndQueue(task, deviceId: deviceId) }
|
||||
}
|
||||
}
|
||||
|
||||
while try await group.next() != nil {
|
||||
if let task = iterator.next() {
|
||||
group.addTask { await self.downloadAndQueue(task) }
|
||||
group.addTask { await self.downloadAndQueue(task, deviceId: deviceId) }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -134,26 +108,21 @@ class DownloadQueue {
|
||||
}
|
||||
}
|
||||
|
||||
private func downloadAndQueue(_ task: LocalAssetDownloadData) async {
|
||||
private func downloadAndQueue(_ task: LocalAssetDownloadData, deviceId: String) async {
|
||||
defer { startQueueProcessing() }
|
||||
dPrint("Starting download for task \(task.taskId)")
|
||||
|
||||
guard let asset = PHAsset.fetchAssets(withLocalIdentifiers: [task.localId], options: nil).firstObject
|
||||
else {
|
||||
dPrint("Asset not found")
|
||||
return handleFailure(task: task, code: .assetNotFound)
|
||||
return await handleFailure(task: task, code: .assetNotFound)
|
||||
}
|
||||
|
||||
let isLivePhoto = asset.mediaSubtypes.contains(.photoLive)
|
||||
let isMotion = isLivePhoto && task.livePhotoVideoId != nil
|
||||
guard let resource = isMotion ? asset.getLivePhotoResource() : asset.getResource() else {
|
||||
dPrint("Resource not found")
|
||||
return handleFailure(task: task, code: .resourceNotFound)
|
||||
}
|
||||
|
||||
guard let deviceId = (try? await db.read { conn in try Store.get(conn, StoreKey.deviceId) }) else {
|
||||
dPrint("Device ID not found")
|
||||
return handleFailure(task: task, code: .noDeviceId)
|
||||
return await handleFailure(task: task, code: .resourceNotFound)
|
||||
}
|
||||
|
||||
let fileDir = TaskConfig.originalsDir
|
||||
@@ -167,17 +136,11 @@ class DownloadQueue {
|
||||
)
|
||||
} catch {
|
||||
dPrint("Failed to create directory for download task \(task.taskId): \(error)")
|
||||
return handleFailure(task: task, code: .writeFailed, filePath: filePath)
|
||||
return await handleFailure(task: task, code: .writeFailed, filePath: filePath)
|
||||
}
|
||||
|
||||
do {
|
||||
try await db.write { conn in
|
||||
try UploadTask.update {
|
||||
$0.status = .downloadQueued
|
||||
$0.isLivePhoto = isLivePhoto
|
||||
$0.filePath = filePath
|
||||
}.where { $0.id.eq(task.taskId) }.execute(conn)
|
||||
}
|
||||
try await taskRepository.markDownloadQueued(taskId: task.taskId, isLivePhoto: isLivePhoto, filePath: filePath)
|
||||
} catch {
|
||||
return dPrint("Failed to set file path for download task \(task.taskId): \(error)")
|
||||
}
|
||||
@@ -187,28 +150,21 @@ class DownloadQueue {
|
||||
|
||||
do {
|
||||
let hash = try await download(task: task, asset: asset, resource: resource, to: filePath, deviceId: deviceId)
|
||||
let status = try await db.write { conn in
|
||||
if let hash { try LocalAsset.update { $0.checksum = hash }.where { $0.id.eq(task.localId) }.execute(conn) }
|
||||
let status =
|
||||
if let hash, try RemoteAsset.select(\.rowid).where({ $0.checksum.eq(hash) }).fetchOne(conn) != nil {
|
||||
TaskStatus.uploadSkipped
|
||||
} else {
|
||||
TaskStatus.uploadPending
|
||||
}
|
||||
try UploadTask.update { $0.status = .uploadPending }.where { $0.id.eq(task.taskId) }.execute(conn)
|
||||
return status
|
||||
}
|
||||
let status = try await taskRepository.markDownloadComplete(taskId: task.taskId, localId: task.localId, hash: hash)
|
||||
statusListener.onTaskStatus(
|
||||
UploadApiTaskStatus(
|
||||
id: String(task.taskId),
|
||||
filename: filePath.path,
|
||||
filename: task.filename,
|
||||
status: UploadApiStatus(rawValue: status.rawValue)!
|
||||
)
|
||||
)
|
||||
uploadQueue.startQueueProcessing()
|
||||
NotificationCenter.default.post(name: .downloadTaskDidComplete, object: nil)
|
||||
} catch let error as UploadError {
|
||||
dPrint("Download failed for task \(task.taskId): \(error)")
|
||||
await handleFailure(task: task, code: error.code, filePath: filePath)
|
||||
} catch {
|
||||
dPrint("Download failed for task \(task.taskId): \(error)")
|
||||
handleFailure(task: task, code: .writeFailed, filePath: filePath)
|
||||
await handleFailure(task: task, code: .unknown, filePath: filePath)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -240,18 +196,11 @@ class DownloadQueue {
|
||||
}
|
||||
try fileHandle.write(contentsOf: header)
|
||||
|
||||
class RequestRef {
|
||||
var id: PHAssetResourceDataRequestID?
|
||||
var lastProgressTime = Date()
|
||||
var didStall = false
|
||||
}
|
||||
|
||||
var lastProgressTime = Date()
|
||||
nonisolated(unsafe) let progressListener = self.progressListener
|
||||
let taskIdStr = String(task.taskId)
|
||||
options.progressHandler = { progress in
|
||||
lastProgressTime = Date()
|
||||
progressListener.onTaskProgress(UploadApiTaskProgress(id: taskIdStr, progress: progress))
|
||||
self.progressListener.onTaskProgress(UploadApiTaskProgress(id: taskIdStr, progress: progress))
|
||||
}
|
||||
|
||||
let request = RequestRef()
|
||||
@@ -261,7 +210,7 @@ class DownloadQueue {
|
||||
request.didStall = Date().timeIntervalSince(lastProgressTime) > TaskConfig.downloadTimeoutS
|
||||
if request.didStall {
|
||||
if let requestId = request.id {
|
||||
Self.resourceManager.cancelDataRequest(requestId)
|
||||
resourceManager.cancelDataRequest(requestId)
|
||||
}
|
||||
break
|
||||
}
|
||||
@@ -271,7 +220,7 @@ class DownloadQueue {
|
||||
return try await withTaskCancellationHandler {
|
||||
try await withCheckedThrowingContinuation { continuation in
|
||||
var hasher = task.checksum == nil && task.livePhotoVideoId == nil ? Insecure.SHA1() : nil
|
||||
request.id = Self.resourceManager.requestData(
|
||||
request.id = resourceManager.requestData(
|
||||
for: resource,
|
||||
options: options,
|
||||
dataReceivedHandler: { data in
|
||||
@@ -281,7 +230,7 @@ class DownloadQueue {
|
||||
try fileHandle.write(contentsOf: data)
|
||||
} catch {
|
||||
request.id = nil
|
||||
Self.resourceManager.cancelDataRequest(requestId)
|
||||
resourceManager.cancelDataRequest(requestId)
|
||||
}
|
||||
},
|
||||
completionHandler: { error in
|
||||
@@ -295,7 +244,7 @@ class DownloadQueue {
|
||||
case 81: .iCloudThrottled
|
||||
default: .photosUnknownError
|
||||
}
|
||||
self.handleFailure(task: task, code: code, filePath: filePath)
|
||||
continuation.resume(throwing: UploadError.iCloudError(code))
|
||||
case let e as PHPhotosError:
|
||||
dPrint("Photos error during download: \(e)")
|
||||
let code: UploadErrorCode =
|
||||
@@ -310,10 +259,10 @@ class DownloadQueue {
|
||||
case .userCancelled: .cancelled
|
||||
default: .photosUnknownError
|
||||
}
|
||||
self.handleFailure(task: task, code: code, filePath: filePath)
|
||||
continuation.resume(throwing: UploadError.photosError(code))
|
||||
case .some:
|
||||
dPrint("Unknown error during download: \(String(describing: error))")
|
||||
self.handleFailure(task: task, code: .unknown, filePath: filePath)
|
||||
continuation.resume(throwing: UploadError.unknown)
|
||||
case .none:
|
||||
dPrint("Download completed for task \(task.taskId)")
|
||||
do {
|
||||
@@ -321,7 +270,7 @@ class DownloadQueue {
|
||||
continuation.resume(returning: hasher.map { hasher in Data(hasher.finalize()).base64EncodedString() })
|
||||
} catch {
|
||||
try? FileManager.default.removeItem(at: filePath)
|
||||
continuation.resume(throwing: error)
|
||||
continuation.resume(throwing: UploadError.fileCreationFailed)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -329,21 +278,26 @@ class DownloadQueue {
|
||||
}
|
||||
} onCancel: {
|
||||
if let requestId = request.id {
|
||||
Self.resourceManager.cancelDataRequest(requestId)
|
||||
resourceManager.cancelDataRequest(requestId)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func handleFailure(task: LocalAssetDownloadData, code: UploadErrorCode, filePath: URL? = nil) {
|
||||
private func handleFailure(task: LocalAssetDownloadData, code: UploadErrorCode, filePath: URL? = nil) async {
|
||||
dPrint("Handling failure for task \(task.taskId) with code \(code.rawValue)")
|
||||
do {
|
||||
if let filePath {
|
||||
try? FileManager.default.removeItem(at: filePath)
|
||||
}
|
||||
|
||||
try db.write { conn in
|
||||
try UploadTask.retryOrFail(code: code, status: .downloadFailed).where { $0.id.eq(task.taskId) }.execute(conn)
|
||||
}
|
||||
try await taskRepository.retryOrFail(taskId: task.taskId, code: code, status: .downloadFailed)
|
||||
statusListener.onTaskStatus(
|
||||
UploadApiTaskStatus(
|
||||
id: String(task.taskId),
|
||||
filename: task.filename,
|
||||
status: .downloadFailed
|
||||
)
|
||||
)
|
||||
} catch {
|
||||
dPrint("Failed to update download failure status for task \(task.taskId): \(error)")
|
||||
}
|
||||
|
||||
@@ -1,4 +1,12 @@
|
||||
class StatusEventListener: StreamStatusStreamHandler {
|
||||
protocol TaskProgressListener {
|
||||
func onTaskProgress(_ event: UploadApiTaskProgress)
|
||||
}
|
||||
|
||||
protocol TaskStatusListener {
|
||||
func onTaskStatus(_ event: UploadApiTaskStatus)
|
||||
}
|
||||
|
||||
final class StatusEventListener: StreamStatusStreamHandler, TaskStatusListener, @unchecked Sendable {
|
||||
var eventSink: PigeonEventSink<UploadApiTaskStatus>?
|
||||
|
||||
override func onListen(withArguments arguments: Any?, sink: PigeonEventSink<UploadApiTaskStatus>) {
|
||||
@@ -6,26 +14,7 @@ class StatusEventListener: StreamStatusStreamHandler {
|
||||
}
|
||||
|
||||
func onTaskStatus(_ event: UploadApiTaskStatus) {
|
||||
if let eventSink = eventSink {
|
||||
eventSink.success(event)
|
||||
}
|
||||
}
|
||||
|
||||
func onEventsDone() {
|
||||
eventSink?.endOfStream()
|
||||
eventSink = nil
|
||||
}
|
||||
}
|
||||
|
||||
class ProgressEventListener: StreamProgressStreamHandler {
|
||||
var eventSink: PigeonEventSink<UploadApiTaskProgress>?
|
||||
|
||||
override func onListen(withArguments arguments: Any?, sink: PigeonEventSink<UploadApiTaskProgress>) {
|
||||
eventSink = sink
|
||||
}
|
||||
|
||||
func onTaskProgress(_ event: UploadApiTaskProgress) {
|
||||
if let eventSink = eventSink {
|
||||
if let eventSink {
|
||||
DispatchQueue.main.async { eventSink.success(event) }
|
||||
}
|
||||
}
|
||||
@@ -37,3 +26,40 @@ class ProgressEventListener: StreamProgressStreamHandler {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
final class ProgressEventListener: StreamProgressStreamHandler, TaskProgressListener, @unchecked Sendable {
|
||||
var eventSink: PigeonEventSink<UploadApiTaskProgress>?
|
||||
private var lastReportTimes: [String: Date] = [:]
|
||||
private let lock = NSLock()
|
||||
|
||||
override func onListen(withArguments arguments: Any?, sink: PigeonEventSink<UploadApiTaskProgress>) {
|
||||
eventSink = sink
|
||||
}
|
||||
|
||||
func onTaskProgress(_ event: UploadApiTaskProgress) {
|
||||
guard let eventSink,
|
||||
lock.withLock({
|
||||
let now = Date()
|
||||
if let lastReport = lastReportTimes[event.id] {
|
||||
guard now.timeIntervalSince(lastReport) >= TaskConfig.progressThrottleInterval else {
|
||||
return false
|
||||
}
|
||||
}
|
||||
lastReportTimes[event.id] = now
|
||||
return true
|
||||
})
|
||||
else { return }
|
||||
|
||||
DispatchQueue.main.async { eventSink.success(event) }
|
||||
}
|
||||
|
||||
func onEventsDone() {
|
||||
DispatchQueue.main.async {
|
||||
self.eventSink?.endOfStream()
|
||||
self.eventSink = nil
|
||||
self.lock.withLock {
|
||||
self.lastReportTimes.removeAll()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import Network
|
||||
|
||||
class NetworkMonitor {
|
||||
final class NetworkMonitor {
|
||||
static let shared = NetworkMonitor()
|
||||
private let monitor = NWPathMonitor()
|
||||
private(set) var isConnected = false
|
||||
@@ -17,6 +17,6 @@ class NetworkMonitor {
|
||||
NotificationCenter.default.post(name: .networkDidConnect, object: nil)
|
||||
}
|
||||
}
|
||||
monitor.start(queue: .global(qos: .utility))
|
||||
monitor.start(queue: .global(qos: .default))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,28 +1,57 @@
|
||||
import SQLiteData
|
||||
import StructuredFieldValues
|
||||
private var queueProcessingTask: Task<Void, Never>?
|
||||
private var queueProcessingLock = NSLock()
|
||||
|
||||
class UploadQueue {
|
||||
private static let structuredEncoder = StructuredFieldValueEncoder()
|
||||
private static var queueProcessingTask: Task<Void, Never>?
|
||||
private static var queueProcessingLock = NSLock()
|
||||
final class UploadQueue<StoreRepo: StoreProtocol, TaskRepo: TaskProtocol, StatusListener: TaskStatusListener> {
|
||||
private let storeRepository: StoreRepo
|
||||
private let taskRepository: TaskRepo
|
||||
private let statusListener: StatusListener
|
||||
|
||||
private let db: DatabasePool
|
||||
private let cellularSession: URLSession
|
||||
private let wifiOnlySession: URLSession
|
||||
private let statusListener: StatusEventListener
|
||||
private var uploadObserver: NSObjectProtocol?
|
||||
private var downloadObserver: NSObjectProtocol?
|
||||
private var networkObserver: NSObjectProtocol?
|
||||
|
||||
init(db: DatabasePool, cellularSession: URLSession, wifiOnlySession: URLSession, statusListener: StatusEventListener)
|
||||
{
|
||||
self.db = db
|
||||
init(
|
||||
storeRepository: StoreRepo,
|
||||
taskRepository: TaskRepo,
|
||||
statusListener: StatusListener,
|
||||
cellularSession: URLSession,
|
||||
wifiOnlySession: URLSession
|
||||
) {
|
||||
self.storeRepository = storeRepository
|
||||
self.taskRepository = taskRepository
|
||||
self.cellularSession = cellularSession
|
||||
self.wifiOnlySession = wifiOnlySession
|
||||
self.statusListener = statusListener
|
||||
|
||||
uploadObserver = NotificationCenter.default.addObserver(forName: .uploadTaskDidComplete, object: nil, queue: nil) {
|
||||
[weak self] _ in
|
||||
self?.startQueueProcessing()
|
||||
}
|
||||
downloadObserver = NotificationCenter.default.addObserver(
|
||||
forName: .downloadTaskDidComplete,
|
||||
object: nil,
|
||||
queue: nil
|
||||
) { [weak self] _ in
|
||||
self?.startQueueProcessing()
|
||||
}
|
||||
networkObserver = NotificationCenter.default.addObserver(forName: .networkDidConnect, object: nil, queue: nil) {
|
||||
[weak self] _ in
|
||||
self?.startQueueProcessing()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
deinit {
|
||||
uploadObserver.map(NotificationCenter.default.removeObserver(_:))
|
||||
downloadObserver.map(NotificationCenter.default.removeObserver(_:))
|
||||
networkObserver.map(NotificationCenter.default.removeObserver(_:))
|
||||
}
|
||||
|
||||
func enqueueFiles(paths: [String]) async throws {
|
||||
guard !paths.isEmpty else { return dPrint("No paths to enqueue") }
|
||||
|
||||
guard let deviceId = (try? await db.read { conn in try Store.get(conn, StoreKey.deviceId) }) else {
|
||||
guard let deviceId = try? storeRepository.get(StoreKey.deviceId) else {
|
||||
throw StoreError.notFound
|
||||
}
|
||||
|
||||
@@ -75,72 +104,36 @@ class UploadQueue {
|
||||
try await group.waitForAll()
|
||||
}
|
||||
|
||||
try await db.write { conn in
|
||||
var draft = UploadTask.Draft(
|
||||
attempts: 0,
|
||||
createdAt: Date(),
|
||||
filePath: nil,
|
||||
isLivePhoto: nil,
|
||||
lastError: nil,
|
||||
livePhotoVideoId: nil,
|
||||
localId: "",
|
||||
method: .multipart,
|
||||
priority: 0.5,
|
||||
retryAfter: nil,
|
||||
status: .downloadPending,
|
||||
)
|
||||
for path in paths {
|
||||
draft.filePath = URL(fileURLWithPath: path, isDirectory: false)
|
||||
try UploadTask.insert { draft }.execute(conn)
|
||||
}
|
||||
}
|
||||
try await taskRepository.enqueue(files: paths)
|
||||
dPrint("Enqueued \(paths.count) assets for upload")
|
||||
}
|
||||
|
||||
func startQueueProcessing() {
|
||||
dPrint("Starting upload queue processing")
|
||||
Self.queueProcessingLock.withLock {
|
||||
guard Self.queueProcessingTask == nil else { return }
|
||||
Self.queueProcessingTask = Task {
|
||||
queueProcessingLock.withLock {
|
||||
guard queueProcessingTask == nil else { return }
|
||||
queueProcessingTask = Task {
|
||||
await startUploads()
|
||||
Self.queueProcessingLock.withLock { Self.queueProcessingTask = nil }
|
||||
queueProcessingLock.withLock { queueProcessingTask = nil }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func startUploads() async {
|
||||
dPrint("Processing download queue")
|
||||
dPrint("Processing upload queue")
|
||||
guard NetworkMonitor.shared.isConnected,
|
||||
let backupEnabled = try? await db.read({ conn in try Store.get(conn, StoreKey.enableBackup) }),
|
||||
backupEnabled
|
||||
else { return dPrint("Download queue paused: network disconnected or backup disabled") }
|
||||
let backupEnabled = try? storeRepository.get(StoreKey.enableBackup), backupEnabled,
|
||||
let url = try? storeRepository.get(StoreKey.serverEndpoint),
|
||||
let accessToken = try? storeRepository.get(StoreKey.accessToken)
|
||||
else { return dPrint("Upload queue paused: missing preconditions") }
|
||||
|
||||
do {
|
||||
let tasks: [LocalAssetUploadData] = try await db.read({ conn in
|
||||
guard let backupEnabled = try Store.get(conn, StoreKey.enableBackup), backupEnabled else { return [] }
|
||||
return try UploadTask.join(LocalAsset.all) { task, asset in task.localId.eq(asset.id) }
|
||||
.where { task, asset in
|
||||
asset.checksum.isNot(nil) && task.status.eq(TaskStatus.uploadPending)
|
||||
&& task.attempts < TaskConfig.maxRetries
|
||||
&& task.filePath.isNot(nil)
|
||||
}
|
||||
.select { task, asset in
|
||||
LocalAssetUploadData.Columns(
|
||||
filePath: task.filePath.unwrapped,
|
||||
priority: task.priority,
|
||||
taskId: task.id,
|
||||
type: asset.type
|
||||
)
|
||||
}
|
||||
.limit { task, _ in UploadTaskStat.availableUploadSlots }
|
||||
.order { task, asset in (task.priority.desc(), task.createdAt) }
|
||||
.fetchAll(conn)
|
||||
})
|
||||
let tasks = try await taskRepository.getUploadTasks()
|
||||
if tasks.isEmpty { return dPrint("No upload tasks to process") }
|
||||
|
||||
await withTaskGroup(of: Void.self) { group in
|
||||
for task in tasks {
|
||||
group.addTask { await self.startUpload(multipart: task) }
|
||||
group.addTask { await self.startUpload(multipart: task, url: url, accessToken: accessToken) }
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
@@ -148,33 +141,18 @@ class UploadQueue {
|
||||
}
|
||||
}
|
||||
|
||||
private func startUpload(multipart task: LocalAssetUploadData) async {
|
||||
private func startUpload(multipart task: LocalAssetUploadData, url: URL, accessToken: String) async {
|
||||
dPrint("Uploading asset resource at \(task.filePath) of task \(task.taskId)")
|
||||
defer { startQueueProcessing() }
|
||||
|
||||
let (url, accessToken, session): (URL, String, URLSession)
|
||||
do {
|
||||
(url, accessToken, session) = try await db.read { conn in
|
||||
guard let url = try Store.get(conn, StoreKey.serverEndpoint),
|
||||
let accessToken = try Store.get(conn, StoreKey.accessToken)
|
||||
else {
|
||||
throw StoreError.notFound
|
||||
}
|
||||
|
||||
let session =
|
||||
switch task.type {
|
||||
case .image:
|
||||
(try? Store.get(conn, StoreKey.useWifiForUploadPhotos)) ?? false ? cellularSession : wifiOnlySession
|
||||
case .video:
|
||||
(try? Store.get(conn, StoreKey.useWifiForUploadVideos)) ?? false ? cellularSession : wifiOnlySession
|
||||
default: wifiOnlySession
|
||||
}
|
||||
return (url, accessToken, session)
|
||||
let session =
|
||||
switch task.type {
|
||||
case .image:
|
||||
(try? storeRepository.get(StoreKey.useWifiForUploadPhotos)) ?? false ? wifiOnlySession : cellularSession
|
||||
case .video:
|
||||
(try? storeRepository.get(StoreKey.useWifiForUploadVideos)) ?? false ? wifiOnlySession : cellularSession
|
||||
default: wifiOnlySession
|
||||
}
|
||||
} catch {
|
||||
dPrint("Upload failed for \(task.taskId), could not retrieve server URL or access token: \(error)")
|
||||
return handleFailure(task: task, code: .noServerUrl)
|
||||
}
|
||||
|
||||
var request = URLRequest(url: url.appendingPathComponent("/assets"))
|
||||
request.httpMethod = "POST"
|
||||
@@ -186,18 +164,11 @@ class UploadQueue {
|
||||
sessionTask.priority = task.priority
|
||||
do {
|
||||
try? FileManager.default.removeItem(at: task.filePath) // upload task already copied the file
|
||||
try await db.write { conn in
|
||||
try UploadTask.update { row in
|
||||
row.status = .uploadQueued
|
||||
row.filePath = nil
|
||||
}
|
||||
.where { $0.id.eq(task.taskId) }
|
||||
.execute(conn)
|
||||
}
|
||||
try await taskRepository.markUploadQueued(taskId: task.taskId)
|
||||
statusListener.onTaskStatus(
|
||||
UploadApiTaskStatus(
|
||||
id: String(task.taskId),
|
||||
filename: task.filePath.lastPathComponent,
|
||||
filename: task.filename,
|
||||
status: .uploadQueued,
|
||||
)
|
||||
)
|
||||
@@ -209,11 +180,16 @@ class UploadQueue {
|
||||
}
|
||||
}
|
||||
|
||||
private func handleFailure(task: LocalAssetUploadData, code: UploadErrorCode) {
|
||||
private func handleFailure(task: LocalAssetUploadData, code: UploadErrorCode) async {
|
||||
do {
|
||||
try db.write { conn in
|
||||
try UploadTask.retryOrFail(code: code, status: .uploadFailed).where { $0.id.eq(task.taskId) }.execute(conn)
|
||||
}
|
||||
try await taskRepository.retryOrFail(taskId: task.taskId, code: code, status: .uploadFailed)
|
||||
statusListener.onTaskStatus(
|
||||
UploadApiTaskStatus(
|
||||
id: String(task.taskId),
|
||||
filename: task.filename,
|
||||
status: .uploadFailed
|
||||
)
|
||||
)
|
||||
} catch {
|
||||
dPrint("Failed to update upload failure status for task \(task.taskId): \(error)")
|
||||
}
|
||||
|
||||
@@ -122,18 +122,19 @@ enum UploadApiErrorCode: Int {
|
||||
case networkError = 8
|
||||
case photosInternalError = 9
|
||||
case photosUnknownError = 10
|
||||
case noServerUrl = 11
|
||||
case noDeviceId = 12
|
||||
case noAccessToken = 13
|
||||
case interrupted = 14
|
||||
case cancelled = 15
|
||||
case downloadStalled = 16
|
||||
case forceQuit = 17
|
||||
case outOfResources = 18
|
||||
case backgroundUpdatesDisabled = 19
|
||||
case uploadTimeout = 20
|
||||
case iCloudRateLimit = 21
|
||||
case iCloudThrottled = 22
|
||||
case interrupted = 11
|
||||
case cancelled = 12
|
||||
case downloadStalled = 13
|
||||
case forceQuit = 14
|
||||
case outOfResources = 15
|
||||
case backgroundUpdatesDisabled = 16
|
||||
case uploadTimeout = 17
|
||||
case iCloudRateLimit = 18
|
||||
case iCloudThrottled = 19
|
||||
case invalidResponse = 20
|
||||
case badRequest = 21
|
||||
case internalServerError = 22
|
||||
case unauthorized = 23
|
||||
}
|
||||
|
||||
enum UploadApiStatus: Int {
|
||||
@@ -294,6 +295,7 @@ protocol UploadApi {
|
||||
func cancelAll(completion: @escaping (Result<Void, Error>) -> Void)
|
||||
func enqueueAssets(localIds: [String], completion: @escaping (Result<Void, Error>) -> Void)
|
||||
func enqueueFiles(paths: [String], completion: @escaping (Result<Void, Error>) -> Void)
|
||||
func onConfigChange(key: Int64, completion: @escaping (Result<Void, Error>) -> Void)
|
||||
}
|
||||
|
||||
/// Generated setup class from Pigeon to handle messages through the `binaryMessenger`.
|
||||
@@ -381,6 +383,23 @@ class UploadApiSetup {
|
||||
} else {
|
||||
enqueueFilesChannel.setMessageHandler(nil)
|
||||
}
|
||||
let onConfigChangeChannel = FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.UploadApi.onConfigChange\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec)
|
||||
if let api = api {
|
||||
onConfigChangeChannel.setMessageHandler { message, reply in
|
||||
let args = message as! [Any?]
|
||||
let keyArg = args[0] as! Int64
|
||||
api.onConfigChange(key: keyArg) { result in
|
||||
switch result {
|
||||
case .success:
|
||||
reply(wrapResult(nil))
|
||||
case .failure(let error):
|
||||
reply(wrapError(error))
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
onConfigChangeChannel.setMessageHandler(nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import SQLiteData
|
||||
import StructuredFieldValues
|
||||
|
||||
extension FileHandle {
|
||||
static func createOrOverwrite(atPath path: String) throws -> FileHandle {
|
||||
@@ -11,33 +10,41 @@ extension FileHandle {
|
||||
}
|
||||
}
|
||||
|
||||
class UploadApiImpl: ImmichPlugin, UploadApi {
|
||||
private let db: DatabasePool
|
||||
private let downloadQueue: DownloadQueue
|
||||
private let uploadQueue: UploadQueue
|
||||
final class UploadApiImpl<
|
||||
StoreRepo: StoreProtocol,
|
||||
TaskRepo: TaskProtocol,
|
||||
StatusListener: TaskStatusListener,
|
||||
ProgressListener: TaskProgressListener
|
||||
>: ImmichPlugin, UploadApi {
|
||||
private let storeRepository: StoreRepo
|
||||
private let taskRepository: TaskRepo
|
||||
private let downloadQueue: DownloadQueue<StoreRepo, TaskRepo, StatusListener, ProgressListener>
|
||||
private let uploadQueue: UploadQueue<StoreRepo, TaskRepo, StatusListener>
|
||||
|
||||
private var isInitialized = false
|
||||
private let initLock = NSLock()
|
||||
|
||||
private var backupTask: Task<Void, Never>?
|
||||
private let backupLock = NSLock()
|
||||
|
||||
private let cellularSession: URLSession
|
||||
private let wifiOnlySession: URLSession
|
||||
|
||||
init(statusListener: StatusEventListener, progressListener: ProgressEventListener) {
|
||||
let dbUrl = try! FileManager.default.url(
|
||||
for: .documentDirectory,
|
||||
in: .userDomainMask,
|
||||
appropriateFor: nil,
|
||||
create: true
|
||||
).appendingPathComponent("immich.sqlite")
|
||||
|
||||
self.db = try! DatabasePool(path: dbUrl.path)
|
||||
init(
|
||||
storeRepository: StoreRepo,
|
||||
taskRepository: TaskRepo,
|
||||
statusListener: StatusListener,
|
||||
progressListener: ProgressListener
|
||||
) {
|
||||
self.taskRepository = taskRepository
|
||||
let delegate = UploadApiDelegate(
|
||||
taskRepository: taskRepository,
|
||||
statusListener: statusListener,
|
||||
progressListener: progressListener
|
||||
)
|
||||
let cellularConfig = URLSessionConfiguration.background(withIdentifier: "\(TaskConfig.sessionId).cellular")
|
||||
cellularConfig.allowsCellularAccess = true
|
||||
cellularConfig.waitsForConnectivity = true
|
||||
let delegate = UploadApiDelegate(db: db, statusListener: statusListener, progressListener: progressListener)
|
||||
|
||||
self.cellularSession = URLSession(configuration: cellularConfig, delegate: delegate, delegateQueue: nil)
|
||||
|
||||
let wifiOnlyConfig = URLSessionConfiguration.background(withIdentifier: "\(TaskConfig.sessionId).wifi")
|
||||
@@ -45,28 +52,26 @@ class UploadApiImpl: ImmichPlugin, UploadApi {
|
||||
wifiOnlyConfig.waitsForConnectivity = true
|
||||
self.wifiOnlySession = URLSession(configuration: wifiOnlyConfig, delegate: delegate, delegateQueue: nil)
|
||||
|
||||
self.storeRepository = storeRepository
|
||||
self.uploadQueue = UploadQueue(
|
||||
db: db,
|
||||
storeRepository: storeRepository,
|
||||
taskRepository: taskRepository,
|
||||
statusListener: statusListener,
|
||||
cellularSession: cellularSession,
|
||||
wifiOnlySession: wifiOnlySession,
|
||||
statusListener: statusListener
|
||||
wifiOnlySession: wifiOnlySession
|
||||
)
|
||||
self.downloadQueue = DownloadQueue(
|
||||
db: db,
|
||||
uploadQueue: uploadQueue,
|
||||
storeRepository: storeRepository,
|
||||
taskRepository: taskRepository,
|
||||
statusListener: statusListener,
|
||||
progressListener: progressListener
|
||||
)
|
||||
delegate.downloadQueue = downloadQueue
|
||||
delegate.uploadQueue = uploadQueue
|
||||
}
|
||||
|
||||
func initialize(completion: @escaping (Result<Void, any Error>) -> Void) {
|
||||
Task(priority: .high) {
|
||||
do {
|
||||
async let dbIds = db.read { conn in
|
||||
try UploadTask.select(\.id).where { $0.status.eq(TaskStatus.uploadQueued) }.fetchAll(conn)
|
||||
}
|
||||
async let dbIds = taskRepository.getTaskIds(status: .uploadQueued)
|
||||
async let cellularTasks = cellularSession.allTasks
|
||||
async let wifiTasks = wifiOnlySession.allTasks
|
||||
|
||||
@@ -84,15 +89,7 @@ class UploadApiImpl: ImmichPlugin, UploadApi {
|
||||
validateTasks(await cellularTasks)
|
||||
validateTasks(await wifiTasks)
|
||||
|
||||
let orphanIds = Array(dbTaskIds)
|
||||
try await db.write { conn in
|
||||
try UploadTask.update {
|
||||
$0.filePath = nil
|
||||
$0.status = .downloadPending
|
||||
}
|
||||
.where { row in row.status.in([TaskStatus.downloadQueued, TaskStatus.uploadPending]) || row.id.in(orphanIds) }
|
||||
.execute(conn)
|
||||
}
|
||||
try await taskRepository.markOrphansPending(ids: Array(dbTaskIds))
|
||||
|
||||
try? FileManager.default.removeItem(at: TaskConfig.originalsDir)
|
||||
initLock.withLock { isInitialized = true }
|
||||
@@ -139,9 +136,9 @@ class UploadApiImpl: ImmichPlugin, UploadApi {
|
||||
Task {
|
||||
do {
|
||||
try await downloadQueue.enqueueAssets(localIds: localIds)
|
||||
completion(.success(()))
|
||||
self.completeWhenActive(for: completion, with: .success(()))
|
||||
} catch {
|
||||
completion(.failure(error))
|
||||
self.completeWhenActive(for: completion, with: .failure(error))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -150,13 +147,24 @@ class UploadApiImpl: ImmichPlugin, UploadApi {
|
||||
Task {
|
||||
do {
|
||||
try await uploadQueue.enqueueFiles(paths: paths)
|
||||
completion(.success(()))
|
||||
self.completeWhenActive(for: completion, with: .success(()))
|
||||
} catch {
|
||||
completion(.failure(error))
|
||||
self.completeWhenActive(for: completion, with: .failure(error))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func onConfigChange(key: Int64, completion: @escaping (Result<Void, any Error>) -> Void) {
|
||||
storeRepository.invalidateCache()
|
||||
Task {
|
||||
if let key = StoreKey(rawValue: Int(key)), key == ._accessToken {
|
||||
try? await taskRepository.resolveError(code: .unauthorized)
|
||||
}
|
||||
startBackup()
|
||||
self.completeWhenActive(for: completion, with: .success(()))
|
||||
}
|
||||
}
|
||||
|
||||
private func cancelSessionTasks(_ tasks: [URLSessionTask]) {
|
||||
dPrint("Canceling \(tasks.count) tasks")
|
||||
for task in tasks {
|
||||
@@ -165,107 +173,20 @@ class UploadApiImpl: ImmichPlugin, UploadApi {
|
||||
}
|
||||
|
||||
private func _startBackup() async {
|
||||
defer { downloadQueue.startQueueProcessing() }
|
||||
defer {
|
||||
downloadQueue.startQueueProcessing()
|
||||
uploadQueue.startQueueProcessing()
|
||||
}
|
||||
|
||||
do {
|
||||
let candidates = try await db.read { conn in
|
||||
return try LocalAsset.getCandidates()
|
||||
.where { asset in !UploadTask.where { task in task.localId.eq(asset.id) }.exists() }
|
||||
.select { LocalAssetCandidate.Columns(id: $0.id, type: $0.type) }
|
||||
.limit { _ in UploadTaskStat.availableSlots }
|
||||
.fetchAll(conn)
|
||||
}
|
||||
let candidates = try await taskRepository.getBackupCandidates()
|
||||
|
||||
guard !candidates.isEmpty else { return dPrint("No candidates for backup") }
|
||||
|
||||
try await db.write { conn in
|
||||
var draft = UploadTask.Draft(
|
||||
attempts: 0,
|
||||
createdAt: Date(),
|
||||
filePath: nil,
|
||||
isLivePhoto: nil,
|
||||
lastError: nil,
|
||||
livePhotoVideoId: nil,
|
||||
localId: "",
|
||||
method: .multipart,
|
||||
priority: 0.5,
|
||||
retryAfter: nil,
|
||||
status: .downloadPending,
|
||||
)
|
||||
for candidate in candidates {
|
||||
draft.localId = candidate.id
|
||||
draft.priority = candidate.type == .image ? 0.5 : 0.3
|
||||
try UploadTask.insert {
|
||||
draft
|
||||
} onConflict: {
|
||||
($0.localId, $0.livePhotoVideoId)
|
||||
}
|
||||
.execute(conn)
|
||||
}
|
||||
}
|
||||
try await taskRepository.enqueue(assets: candidates, imagePriority: 0.5, videoPriority: 0.3)
|
||||
dPrint("Backup enqueued \(candidates.count) assets for upload")
|
||||
} catch {
|
||||
print("Backup queue error: \(error)")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct AssetData: StructuredFieldValue {
|
||||
static let structuredFieldType: StructuredFieldType = .dictionary
|
||||
|
||||
let deviceAssetId: String
|
||||
let deviceId: String
|
||||
let fileCreatedAt: String
|
||||
let fileModifiedAt: String
|
||||
let fileName: String
|
||||
let isFavorite: Bool
|
||||
let livePhotoVideoId: String?
|
||||
|
||||
static let boundary = "Boundary-\(UUID().uuidString)"
|
||||
static let deviceAssetIdField = "--\(boundary)\r\nContent-Disposition: form-data; name=\"deviceAssetId\"\r\n\r\n"
|
||||
.data(using: .utf8)!
|
||||
static let deviceIdField = "\r\n--\(boundary)\r\nContent-Disposition: form-data; name=\"deviceId\"\r\n\r\n"
|
||||
.data(using: .utf8)!
|
||||
static let fileCreatedAtField =
|
||||
"\r\n--\(boundary)\r\nContent-Disposition: form-data; name=\"fileCreatedAt\"\r\n\r\n"
|
||||
.data(using: .utf8)!
|
||||
static let fileModifiedAtField =
|
||||
"\r\n--\(boundary)\r\nContent-Disposition: form-data; name=\"fileModifiedAt\"\r\n\r\n"
|
||||
.data(using: .utf8)!
|
||||
static let isFavoriteField = "\r\n--\(boundary)\r\nContent-Disposition: form-data; name=\"isFavorite\"\r\n\r\n"
|
||||
.data(using: .utf8)!
|
||||
static let livePhotoVideoIdField =
|
||||
"\r\n--\(boundary)\r\nContent-Disposition: form-data; name=\"livePhotoVideoId\"\r\n\r\n"
|
||||
.data(using: .utf8)!
|
||||
static let trueData = "true".data(using: .utf8)!
|
||||
static let falseData = "false".data(using: .utf8)!
|
||||
static let footer = "\r\n--\(boundary)--\r\n".data(using: .utf8)!
|
||||
static let contentType = "multipart/form-data; boundary=\(boundary)"
|
||||
|
||||
func multipart() -> (Data, Data) {
|
||||
var header = Data()
|
||||
header.append(Self.deviceAssetIdField)
|
||||
header.append(deviceAssetId.data(using: .utf8)!)
|
||||
|
||||
header.append(Self.deviceIdField)
|
||||
header.append(deviceId.data(using: .utf8)!)
|
||||
|
||||
header.append(Self.fileCreatedAtField)
|
||||
header.append(fileCreatedAt.data(using: .utf8)!)
|
||||
|
||||
header.append(Self.fileModifiedAtField)
|
||||
header.append(fileModifiedAt.data(using: .utf8)!)
|
||||
|
||||
header.append(Self.isFavoriteField)
|
||||
header.append(isFavorite ? Self.trueData : Self.falseData)
|
||||
|
||||
if let livePhotoVideoId {
|
||||
header.append(Self.livePhotoVideoIdField)
|
||||
header.append(livePhotoVideoId.data(using: .utf8)!)
|
||||
}
|
||||
header.append(
|
||||
"\r\n--\(Self.boundary)\r\nContent-Disposition: form-data; name=\"assetData\"; filename=\"\(fileName)\"\r\nContent-Type: application/octet-stream\r\n\r\n"
|
||||
.data(using: .utf8)!
|
||||
)
|
||||
return (header, Self.footer)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user