mirror of
https://github.com/immich-app/immich.git
synced 2026-05-18 03:10:24 +03:00
background upload plugin
add schemas sync variants formatting initial implementation use existing db, wip move to separate folder fix table definitions wip wiring it up
This commit is contained in:
mertalev
@@ -0,0 +1,208 @@
|
||||
import SQLiteData
|
||||
|
||||
class UploadApiDelegate: NSObject, URLSessionDataDelegate, URLSessionTaskDelegate {
|
||||
private static let stateLock = NSLock()
|
||||
private static var transferStates: [Int64: NetworkTransferState] = [:]
|
||||
private static var responseData: [Int64: Data] = [:]
|
||||
private static let jsonDecoder = JSONDecoder()
|
||||
|
||||
private let db: DatabasePool
|
||||
private let statusListener: StatusEventListener
|
||||
private let progressListener: ProgressEventListener
|
||||
weak var downloadQueue: DownloadQueue?
|
||||
weak var uploadQueue: UploadQueue?
|
||||
|
||||
init(db: DatabasePool, statusListener: StatusEventListener, progressListener: ProgressEventListener) {
|
||||
self.db = db
|
||||
self.statusListener = statusListener
|
||||
self.progressListener = progressListener
|
||||
}
|
||||
|
||||
static func reset() {
|
||||
stateLock.withLock {
|
||||
transferStates.removeAll()
|
||||
responseData.removeAll()
|
||||
}
|
||||
}
|
||||
|
||||
func urlSession(_ session: URLSession, dataTask: URLSessionDataTask, didReceive data: Data) {
|
||||
guard let taskIdStr = dataTask.taskDescription,
|
||||
let taskId = Int64(taskIdStr)
|
||||
else { return }
|
||||
|
||||
Self.stateLock.withLock {
|
||||
if var response = Self.responseData[taskId] {
|
||||
response.append(data)
|
||||
} else {
|
||||
Self.responseData[taskId] = data
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func urlSession(_ session: URLSession, task: URLSessionTask, didCompleteWithError error: Error?) {
|
||||
Task {
|
||||
defer {
|
||||
downloadQueue?.startQueueProcessing()
|
||||
uploadQueue?.startQueueProcessing()
|
||||
}
|
||||
|
||||
guard let taskDescriptionId = task.taskDescription,
|
||||
let taskId = Int64(taskDescriptionId)
|
||||
else {
|
||||
return dPrint("Unexpected: task without session ID completed")
|
||||
}
|
||||
|
||||
defer {
|
||||
Self.stateLock.withLock { let _ = Self.transferStates.removeValue(forKey: taskId) }
|
||||
}
|
||||
|
||||
if let responseData = Self.stateLock.withLock({ Self.responseData.removeValue(forKey: taskId) }),
|
||||
let httpResponse = task.response as? HTTPURLResponse
|
||||
{
|
||||
switch httpResponse.statusCode {
|
||||
case 200, 201:
|
||||
do {
|
||||
let response = try Self.jsonDecoder.decode(UploadSuccessResponse.self, from: responseData)
|
||||
return await handleSuccess(taskId: taskId, response: response)
|
||||
} catch {
|
||||
return await handleFailure(taskId: taskId, code: .invalidResponse)
|
||||
}
|
||||
case 400..<500:
|
||||
dPrint(
|
||||
"Response \(httpResponse.statusCode): \(String(data: responseData, encoding: .utf8) ?? "No response body")"
|
||||
)
|
||||
return await handleFailure(taskId: taskId, code: .badRequest)
|
||||
default:
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
guard let urlError = error as? URLError else {
|
||||
return await handleFailure(taskId: taskId)
|
||||
}
|
||||
|
||||
if #available(iOS 17, *), let resumeData = urlError.uploadTaskResumeData {
|
||||
return await handleFailure(taskDescriptionId: taskDescriptionId, session: session, resumeData: resumeData)
|
||||
}
|
||||
|
||||
let code: UploadErrorCode =
|
||||
switch urlError.backgroundTaskCancelledReason {
|
||||
case .backgroundUpdatesDisabled: .backgroundUpdatesDisabled
|
||||
case .insufficientSystemResources: .outOfResources
|
||||
case .userForceQuitApplication: .forceQuit
|
||||
default:
|
||||
switch urlError.code {
|
||||
case .networkConnectionLost, .notConnectedToInternet: .networkError
|
||||
case .timedOut: .uploadTimeout
|
||||
case .resourceUnavailable, .fileDoesNotExist: .fileNotFound
|
||||
default: .unknown
|
||||
}
|
||||
}
|
||||
await handleFailure(taskId: taskId, code: code)
|
||||
}
|
||||
}
|
||||
|
||||
func urlSession(
|
||||
_ session: URLSession,
|
||||
task: URLSessionTask,
|
||||
didSendBodyData bytesSent: Int64,
|
||||
totalBytesSent: Int64,
|
||||
totalBytesExpectedToSend: Int64
|
||||
) {
|
||||
guard let sessionTaskId = task.taskDescription, let taskId = Int64(sessionTaskId) else { return }
|
||||
let currentTime = Date()
|
||||
let state = Self.stateLock.withLock {
|
||||
if let existing = Self.transferStates[taskId] {
|
||||
return existing
|
||||
}
|
||||
let new = NetworkTransferState(
|
||||
lastUpdateTime: currentTime,
|
||||
totalBytesTransferred: totalBytesSent,
|
||||
currentSpeed: nil
|
||||
)
|
||||
Self.transferStates[taskId] = new
|
||||
return new
|
||||
}
|
||||
|
||||
let timeDelta = currentTime.timeIntervalSince(state.lastUpdateTime)
|
||||
guard timeDelta > 0 else { return }
|
||||
|
||||
let bytesDelta = totalBytesSent - state.totalBytesTransferred
|
||||
let instantSpeed = Double(bytesDelta) / timeDelta
|
||||
let currentSpeed =
|
||||
if let previousSpeed = state.currentSpeed {
|
||||
TaskConfig.transferSpeedAlpha * instantSpeed + (1 - TaskConfig.transferSpeedAlpha) * previousSpeed
|
||||
} else {
|
||||
instantSpeed
|
||||
}
|
||||
state.currentSpeed = currentSpeed
|
||||
state.lastUpdateTime = currentTime
|
||||
state.totalBytesTransferred = totalBytesSent
|
||||
self.progressListener.onTaskProgress(
|
||||
UploadApiTaskProgress(
|
||||
id: sessionTaskId,
|
||||
progress: Double(totalBytesSent) / Double(totalBytesExpectedToSend),
|
||||
speed: currentSpeed
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
private func handleSuccess(taskId: Int64, response: UploadSuccessResponse) async {
|
||||
dPrint("Upload succeeded for task \(taskId), server ID: \(response.id)")
|
||||
do {
|
||||
try await db.write { conn in
|
||||
let task = try UploadTask.update { $0.status = .uploadComplete }.where({ $0.id.eq(taskId) })
|
||||
.returning(\.self).fetchOne(conn)
|
||||
guard let task, let isLivePhoto = task.isLivePhoto, isLivePhoto, task.livePhotoVideoId == nil else { return }
|
||||
try UploadTask.insert {
|
||||
UploadTask.Draft(
|
||||
attempts: 0,
|
||||
createdAt: Date(),
|
||||
filePath: nil,
|
||||
isLivePhoto: true,
|
||||
lastError: nil,
|
||||
livePhotoVideoId: response.id,
|
||||
localId: task.localId,
|
||||
method: .multipart,
|
||||
priority: 0.7,
|
||||
retryAfter: nil,
|
||||
status: .downloadPending,
|
||||
)
|
||||
}.execute(conn)
|
||||
}
|
||||
dPrint("Updated upload success status for session task \(taskId)")
|
||||
} catch {
|
||||
dPrint(
|
||||
"Failed to update upload success status for session task \(taskId): \(error.localizedDescription)"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
private func handleFailure(taskId: Int64, code: UploadErrorCode = .unknown) async {
|
||||
dPrint("Upload failed for task \(taskId) with code \(code)")
|
||||
try? await db.write { conn in
|
||||
try UploadTask.retryOrFail(code: code, status: .uploadFailed).where { $0.id.eq(taskId) }
|
||||
.execute(conn)
|
||||
}
|
||||
}
|
||||
|
||||
@available(iOS 17, *)
|
||||
private func handleFailure(taskDescriptionId: String, session: URLSession, resumeData: Data) async {
|
||||
dPrint("Resuming upload for task \(taskDescriptionId)")
|
||||
let resumeTask = session.uploadTask(withResumeData: resumeData)
|
||||
resumeTask.taskDescription = taskDescriptionId
|
||||
resumeTask.resume()
|
||||
}
|
||||
|
||||
private class NetworkTransferState {
|
||||
var lastUpdateTime: Date
|
||||
var totalBytesTransferred: Int64
|
||||
var currentSpeed: Double?
|
||||
|
||||
init(lastUpdateTime: Date, totalBytesTransferred: Int64, currentSpeed: Double?) {
|
||||
self.lastUpdateTime = lastUpdateTime
|
||||
self.totalBytesTransferred = totalBytesTransferred
|
||||
self.currentSpeed = currentSpeed
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,351 @@
|
||||
import CryptoKit
|
||||
import Photos
|
||||
import SQLiteData
|
||||
|
||||
class DownloadQueue {
|
||||
private static let resourceManager = PHAssetResourceManager.default()
|
||||
private static var queueProcessingTask: Task<Void, Never>?
|
||||
private static var queueProcessingLock = NSLock()
|
||||
|
||||
private let db: DatabasePool
|
||||
private let uploadQueue: UploadQueue
|
||||
private let statusListener: StatusEventListener
|
||||
private let progressListener: ProgressEventListener
|
||||
|
||||
init(
|
||||
db: DatabasePool,
|
||||
uploadQueue: UploadQueue,
|
||||
statusListener: StatusEventListener,
|
||||
progressListener: ProgressEventListener
|
||||
) {
|
||||
self.db = db
|
||||
self.uploadQueue = uploadQueue
|
||||
self.statusListener = statusListener
|
||||
self.progressListener = progressListener
|
||||
NotificationCenter.default.addObserver(forName: .networkDidConnect, object: nil, queue: nil) { [weak self] _ in
|
||||
dPrint("Network connected")
|
||||
self?.startQueueProcessing()
|
||||
}
|
||||
}
|
||||
|
||||
func enqueueAssets(localIds: [String]) async throws {
|
||||
guard !localIds.isEmpty else { return dPrint("No assets to enqueue") }
|
||||
|
||||
defer { startQueueProcessing() }
|
||||
let candidates = try await db.read { conn in
|
||||
return try LocalAsset.all
|
||||
.where { asset in asset.id.in(localIds) }
|
||||
.select { LocalAssetCandidate.Columns(id: $0.id, type: $0.type) }
|
||||
.limit { _ in UploadTaskStat.availableSlots }
|
||||
.fetchAll(conn)
|
||||
}
|
||||
|
||||
guard !candidates.isEmpty else { return dPrint("No candidates to enqueue") }
|
||||
|
||||
try await db.write { conn in
|
||||
var draft = UploadTask.Draft(
|
||||
attempts: 0,
|
||||
createdAt: Date(),
|
||||
filePath: nil,
|
||||
isLivePhoto: nil,
|
||||
lastError: nil,
|
||||
livePhotoVideoId: nil,
|
||||
localId: "",
|
||||
method: .multipart,
|
||||
priority: 0.5,
|
||||
retryAfter: nil,
|
||||
status: .downloadPending,
|
||||
)
|
||||
for candidate in candidates {
|
||||
draft.localId = candidate.id
|
||||
draft.priority = candidate.type == .image ? 0.9 : 0.8
|
||||
try UploadTask.insert {
|
||||
draft
|
||||
} onConflict: {
|
||||
($0.localId, $0.livePhotoVideoId)
|
||||
}.execute(conn)
|
||||
}
|
||||
}
|
||||
dPrint("Enqueued \(candidates.count) assets for upload")
|
||||
}
|
||||
|
||||
func startQueueProcessing() {
|
||||
dPrint("Starting download queue processing")
|
||||
Self.queueProcessingLock.withLock {
|
||||
guard Self.queueProcessingTask == nil else { return }
|
||||
Self.queueProcessingTask = Task {
|
||||
await startDownloads()
|
||||
Self.queueProcessingLock.withLock { Self.queueProcessingTask = nil }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func startDownloads() async {
|
||||
dPrint("Processing download queue")
|
||||
guard NetworkMonitor.shared.isConnected else {
|
||||
return dPrint("Download queue paused: network disconnected")
|
||||
}
|
||||
|
||||
do {
|
||||
let tasks: [LocalAssetDownloadData] = try await db.read({ conn in
|
||||
guard let backupEnabled = try Store.get(conn, StoreKey.enableBackup), backupEnabled else { return [] }
|
||||
return try UploadTask.join(LocalAsset.all) { task, asset in task.localId.eq(asset.id) }
|
||||
.where { task, asset in
|
||||
asset.checksum.isNot(nil) && task.status.eq(TaskStatus.downloadPending)
|
||||
&& task.attempts < TaskConfig.maxRetries
|
||||
&& (task.retryAfter.is(nil) || task.retryAfter.unwrapped <= Date().unixTime)
|
||||
&& (task.lastError.is(nil)
|
||||
|| !task.lastError.unwrapped.in([
|
||||
UploadErrorCode.assetNotFound, UploadErrorCode.resourceNotFound, UploadErrorCode.invalidResource,
|
||||
]))
|
||||
}
|
||||
.select { task, asset in
|
||||
LocalAssetDownloadData.Columns(
|
||||
checksum: asset.checksum,
|
||||
createdAt: asset.createdAt,
|
||||
livePhotoVideoId: task.livePhotoVideoId,
|
||||
localId: asset.id,
|
||||
taskId: task.id,
|
||||
updatedAt: asset.updatedAt
|
||||
)
|
||||
}
|
||||
.order { task, asset in (task.priority.desc(), task.createdAt) }
|
||||
.limit { _, _ in UploadTaskStat.availableDownloadSlots }
|
||||
.fetchAll(conn)
|
||||
})
|
||||
if tasks.isEmpty { return dPrint("No download tasks to process") }
|
||||
|
||||
try await withThrowingTaskGroup(of: Void.self) { group in
|
||||
var iterator = tasks.makeIterator()
|
||||
for _ in 0..<min(TaskConfig.maxActiveDownloads, tasks.count) {
|
||||
if let task = iterator.next() {
|
||||
group.addTask { await self.downloadAndQueue(task) }
|
||||
}
|
||||
}
|
||||
|
||||
while try await group.next() != nil {
|
||||
if let task = iterator.next() {
|
||||
group.addTask { await self.downloadAndQueue(task) }
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
dPrint("Download queue error: \(error)")
|
||||
}
|
||||
}
|
||||
|
||||
private func downloadAndQueue(_ task: LocalAssetDownloadData) async {
|
||||
defer { startQueueProcessing() }
|
||||
dPrint("Starting download for task \(task.taskId)")
|
||||
|
||||
guard let asset = PHAsset.fetchAssets(withLocalIdentifiers: [task.localId], options: nil).firstObject
|
||||
else {
|
||||
dPrint("Asset not found")
|
||||
return handleFailure(task: task, code: .assetNotFound)
|
||||
}
|
||||
|
||||
let isLivePhoto = asset.mediaSubtypes.contains(.photoLive)
|
||||
let isMotion = isLivePhoto && task.livePhotoVideoId != nil
|
||||
guard let resource = isMotion ? asset.getLivePhotoResource() : asset.getResource() else {
|
||||
dPrint("Resource not found")
|
||||
return handleFailure(task: task, code: .resourceNotFound)
|
||||
}
|
||||
|
||||
guard let deviceId = (try? await db.read { conn in try Store.get(conn, StoreKey.deviceId) }) else {
|
||||
dPrint("Device ID not found")
|
||||
return handleFailure(task: task, code: .noDeviceId)
|
||||
}
|
||||
|
||||
let fileDir = TaskConfig.originalsDir
|
||||
let fileName = "\(resource.assetLocalIdentifier.replacingOccurrences(of: "/", with: "_"))_\(resource.type.rawValue)"
|
||||
let filePath = fileDir.appendingPathComponent(fileName)
|
||||
do {
|
||||
try FileManager.default.createDirectory(
|
||||
at: fileDir,
|
||||
withIntermediateDirectories: true,
|
||||
attributes: nil
|
||||
)
|
||||
} catch {
|
||||
dPrint("Failed to create directory for download task \(task.taskId): \(error)")
|
||||
return handleFailure(task: task, code: .writeFailed, filePath: filePath)
|
||||
}
|
||||
|
||||
do {
|
||||
try await db.write { conn in
|
||||
try UploadTask.update {
|
||||
$0.status = .downloadQueued
|
||||
$0.isLivePhoto = isLivePhoto
|
||||
$0.filePath = filePath
|
||||
}.where { $0.id.eq(task.taskId) }.execute(conn)
|
||||
}
|
||||
} catch {
|
||||
return dPrint("Failed to set file path for download task \(task.taskId): \(error)")
|
||||
}
|
||||
statusListener.onTaskStatus(
|
||||
UploadApiTaskStatus(id: String(task.taskId), filename: filePath.path, status: .downloadQueued)
|
||||
)
|
||||
|
||||
do {
|
||||
let hash = try await download(task: task, asset: asset, resource: resource, to: filePath, deviceId: deviceId)
|
||||
let status = try await db.write { conn in
|
||||
if let hash { try LocalAsset.update { $0.checksum = hash }.where { $0.id.eq(task.localId) }.execute(conn) }
|
||||
let status =
|
||||
if let hash, try RemoteAsset.select(\.rowid).where({ $0.checksum.eq(hash) }).fetchOne(conn) != nil {
|
||||
TaskStatus.uploadSkipped
|
||||
} else {
|
||||
TaskStatus.uploadPending
|
||||
}
|
||||
try UploadTask.update { $0.status = .uploadPending }.where { $0.id.eq(task.taskId) }.execute(conn)
|
||||
return status
|
||||
}
|
||||
statusListener.onTaskStatus(
|
||||
UploadApiTaskStatus(
|
||||
id: String(task.taskId),
|
||||
filename: filePath.path,
|
||||
status: UploadApiStatus(rawValue: status.rawValue)!
|
||||
)
|
||||
)
|
||||
uploadQueue.startQueueProcessing()
|
||||
} catch {
|
||||
dPrint("Download failed for task \(task.taskId): \(error)")
|
||||
handleFailure(task: task, code: .writeFailed, filePath: filePath)
|
||||
}
|
||||
}
|
||||
|
||||
func download(
|
||||
task: LocalAssetDownloadData,
|
||||
asset: PHAsset,
|
||||
resource: PHAssetResource,
|
||||
to filePath: URL,
|
||||
deviceId: String
|
||||
) async throws
|
||||
-> String?
|
||||
{
|
||||
dPrint("Downloading asset resource \(resource.assetLocalIdentifier) of type \(resource.type.rawValue)")
|
||||
let options = PHAssetResourceRequestOptions()
|
||||
options.isNetworkAccessAllowed = true
|
||||
let (header, footer) = AssetData(
|
||||
deviceAssetId: task.localId,
|
||||
deviceId: deviceId,
|
||||
fileCreatedAt: task.createdAt,
|
||||
fileModifiedAt: task.updatedAt,
|
||||
fileName: resource.originalFilename,
|
||||
isFavorite: asset.isFavorite,
|
||||
livePhotoVideoId: nil
|
||||
).multipart()
|
||||
|
||||
guard let fileHandle = try? FileHandle.createOrOverwrite(atPath: filePath.path) else {
|
||||
dPrint("Failed to open file handle for download task \(task.taskId), path: \(filePath.path)")
|
||||
throw UploadError.fileCreationFailed
|
||||
}
|
||||
try fileHandle.write(contentsOf: header)
|
||||
|
||||
class RequestRef {
|
||||
var id: PHAssetResourceDataRequestID?
|
||||
var lastProgressTime = Date()
|
||||
var didStall = false
|
||||
}
|
||||
|
||||
var lastProgressTime = Date()
|
||||
nonisolated(unsafe) let progressListener = self.progressListener
|
||||
let taskIdStr = String(task.taskId)
|
||||
options.progressHandler = { progress in
|
||||
lastProgressTime = Date()
|
||||
progressListener.onTaskProgress(UploadApiTaskProgress(id: taskIdStr, progress: progress))
|
||||
}
|
||||
|
||||
let request = RequestRef()
|
||||
let timeoutTask = Task {
|
||||
while !Task.isCancelled {
|
||||
try? await Task.sleep(nanoseconds: TaskConfig.downloadCheckIntervalNs)
|
||||
request.didStall = Date().timeIntervalSince(lastProgressTime) > TaskConfig.downloadTimeoutS
|
||||
if request.didStall {
|
||||
if let requestId = request.id {
|
||||
Self.resourceManager.cancelDataRequest(requestId)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return try await withTaskCancellationHandler {
|
||||
try await withCheckedThrowingContinuation { continuation in
|
||||
var hasher = task.checksum == nil && task.livePhotoVideoId == nil ? Insecure.SHA1() : nil
|
||||
request.id = Self.resourceManager.requestData(
|
||||
for: resource,
|
||||
options: options,
|
||||
dataReceivedHandler: { data in
|
||||
guard let requestId = request.id else { return }
|
||||
do {
|
||||
hasher?.update(data: data)
|
||||
try fileHandle.write(contentsOf: data)
|
||||
} catch {
|
||||
request.id = nil
|
||||
Self.resourceManager.cancelDataRequest(requestId)
|
||||
}
|
||||
},
|
||||
completionHandler: { error in
|
||||
timeoutTask.cancel()
|
||||
switch error {
|
||||
case let e as NSError where e.domain == "CloudPhotoLibraryErrorDomain":
|
||||
dPrint("iCloud error during download: \(e)")
|
||||
let code: UploadErrorCode =
|
||||
switch e.code {
|
||||
case 1005: .iCloudRateLimit
|
||||
case 81: .iCloudThrottled
|
||||
default: .photosUnknownError
|
||||
}
|
||||
self.handleFailure(task: task, code: code, filePath: filePath)
|
||||
case let e as PHPhotosError:
|
||||
dPrint("Photos error during download: \(e)")
|
||||
let code: UploadErrorCode =
|
||||
switch e.code {
|
||||
case .notEnoughSpace: .notEnoughSpace
|
||||
case .missingResource: .resourceNotFound
|
||||
case .networkError: .networkError
|
||||
case .internalError: .photosInternalError
|
||||
case .invalidResource: .invalidResource
|
||||
case .operationInterrupted: .interrupted
|
||||
case .userCancelled where request.didStall: .downloadStalled
|
||||
case .userCancelled: .cancelled
|
||||
default: .photosUnknownError
|
||||
}
|
||||
self.handleFailure(task: task, code: code, filePath: filePath)
|
||||
case .some:
|
||||
dPrint("Unknown error during download: \(String(describing: error))")
|
||||
self.handleFailure(task: task, code: .unknown, filePath: filePath)
|
||||
case .none:
|
||||
dPrint("Download completed for task \(task.taskId)")
|
||||
do {
|
||||
try fileHandle.write(contentsOf: footer)
|
||||
continuation.resume(returning: hasher.map { hasher in Data(hasher.finalize()).base64EncodedString() })
|
||||
} catch {
|
||||
try? FileManager.default.removeItem(at: filePath)
|
||||
continuation.resume(throwing: error)
|
||||
}
|
||||
}
|
||||
}
|
||||
)
|
||||
}
|
||||
} onCancel: {
|
||||
if let requestId = request.id {
|
||||
Self.resourceManager.cancelDataRequest(requestId)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func handleFailure(task: LocalAssetDownloadData, code: UploadErrorCode, filePath: URL? = nil) {
|
||||
dPrint("Handling failure for task \(task.taskId) with code \(code.rawValue)")
|
||||
do {
|
||||
if let filePath {
|
||||
try? FileManager.default.removeItem(at: filePath)
|
||||
}
|
||||
|
||||
try db.write { conn in
|
||||
try UploadTask.retryOrFail(code: code, status: .downloadFailed).where { $0.id.eq(task.taskId) }.execute(conn)
|
||||
}
|
||||
} catch {
|
||||
dPrint("Failed to update download failure status for task \(task.taskId): \(error)")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
class StatusEventListener: StreamStatusStreamHandler {
|
||||
var eventSink: PigeonEventSink<UploadApiTaskStatus>?
|
||||
|
||||
override func onListen(withArguments arguments: Any?, sink: PigeonEventSink<UploadApiTaskStatus>) {
|
||||
eventSink = sink
|
||||
}
|
||||
|
||||
func onTaskStatus(_ event: UploadApiTaskStatus) {
|
||||
if let eventSink = eventSink {
|
||||
eventSink.success(event)
|
||||
}
|
||||
}
|
||||
|
||||
func onEventsDone() {
|
||||
eventSink?.endOfStream()
|
||||
eventSink = nil
|
||||
}
|
||||
}
|
||||
|
||||
class ProgressEventListener: StreamProgressStreamHandler {
|
||||
var eventSink: PigeonEventSink<UploadApiTaskProgress>?
|
||||
|
||||
override func onListen(withArguments arguments: Any?, sink: PigeonEventSink<UploadApiTaskProgress>) {
|
||||
eventSink = sink
|
||||
}
|
||||
|
||||
func onTaskProgress(_ event: UploadApiTaskProgress) {
|
||||
if let eventSink = eventSink {
|
||||
DispatchQueue.main.async { eventSink.success(event) }
|
||||
}
|
||||
}
|
||||
|
||||
func onEventsDone() {
|
||||
DispatchQueue.main.async {
|
||||
self.eventSink?.endOfStream()
|
||||
self.eventSink = nil
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
import Network
|
||||
|
||||
class NetworkMonitor {
|
||||
static let shared = NetworkMonitor()
|
||||
private let monitor = NWPathMonitor()
|
||||
private(set) var isConnected = false
|
||||
private(set) var isExpensive = false
|
||||
|
||||
private init() {
|
||||
monitor.pathUpdateHandler = { [weak self] path in
|
||||
guard let self else { return }
|
||||
let wasConnected = self.isConnected
|
||||
self.isConnected = path.status == .satisfied
|
||||
self.isExpensive = path.isExpensive
|
||||
|
||||
if !wasConnected && self.isConnected {
|
||||
NotificationCenter.default.post(name: .networkDidConnect, object: nil)
|
||||
}
|
||||
}
|
||||
monitor.start(queue: .global(qos: .utility))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,221 @@
|
||||
import SQLiteData
|
||||
import StructuredFieldValues
|
||||
|
||||
class UploadQueue {
|
||||
private static let structuredEncoder = StructuredFieldValueEncoder()
|
||||
private static var queueProcessingTask: Task<Void, Never>?
|
||||
private static var queueProcessingLock = NSLock()
|
||||
|
||||
private let db: DatabasePool
|
||||
private let cellularSession: URLSession
|
||||
private let wifiOnlySession: URLSession
|
||||
private let statusListener: StatusEventListener
|
||||
|
||||
init(db: DatabasePool, cellularSession: URLSession, wifiOnlySession: URLSession, statusListener: StatusEventListener)
|
||||
{
|
||||
self.db = db
|
||||
self.cellularSession = cellularSession
|
||||
self.wifiOnlySession = wifiOnlySession
|
||||
self.statusListener = statusListener
|
||||
}
|
||||
|
||||
func enqueueFiles(paths: [String]) async throws {
|
||||
guard !paths.isEmpty else { return dPrint("No paths to enqueue") }
|
||||
|
||||
guard let deviceId = (try? await db.read { conn in try Store.get(conn, StoreKey.deviceId) }) else {
|
||||
throw StoreError.notFound
|
||||
}
|
||||
|
||||
defer { startQueueProcessing() }
|
||||
|
||||
try await withThrowingTaskGroup(of: Void.self, returning: Void.self) { group in
|
||||
let date = Date()
|
||||
try FileManager.default.createDirectory(
|
||||
at: TaskConfig.originalsDir,
|
||||
withIntermediateDirectories: true,
|
||||
attributes: nil
|
||||
)
|
||||
|
||||
for path in paths {
|
||||
group.addTask {
|
||||
let inputURL = URL(fileURLWithPath: path, isDirectory: false)
|
||||
let outputURL = TaskConfig.originalsDir.appendingPathComponent(UUID().uuidString)
|
||||
let resources = try inputURL.resourceValues(forKeys: [.creationDateKey, .contentModificationDateKey])
|
||||
|
||||
let formatter = ISO8601DateFormatter()
|
||||
let (header, footer) = AssetData(
|
||||
deviceAssetId: "",
|
||||
deviceId: deviceId,
|
||||
fileCreatedAt: formatter.string(from: resources.creationDate ?? date),
|
||||
fileModifiedAt: formatter.string(from: resources.contentModificationDate ?? date),
|
||||
fileName: resources.name ?? inputURL.lastPathComponent,
|
||||
isFavorite: false,
|
||||
livePhotoVideoId: nil
|
||||
).multipart()
|
||||
|
||||
do {
|
||||
let writeHandle = try FileHandle.createOrOverwrite(atPath: outputURL.path)
|
||||
try writeHandle.write(contentsOf: header)
|
||||
let readHandle = try FileHandle(forReadingFrom: inputURL)
|
||||
|
||||
let bufferSize = 1024 * 1024
|
||||
while true {
|
||||
let data = try readHandle.read(upToCount: bufferSize)
|
||||
guard let data = data, !data.isEmpty else { break }
|
||||
try writeHandle.write(contentsOf: data)
|
||||
}
|
||||
try writeHandle.write(contentsOf: footer)
|
||||
} catch {
|
||||
try? FileManager.default.removeItem(at: outputURL)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try await group.waitForAll()
|
||||
}
|
||||
|
||||
try await db.write { conn in
|
||||
var draft = UploadTask.Draft(
|
||||
attempts: 0,
|
||||
createdAt: Date(),
|
||||
filePath: nil,
|
||||
isLivePhoto: nil,
|
||||
lastError: nil,
|
||||
livePhotoVideoId: nil,
|
||||
localId: "",
|
||||
method: .multipart,
|
||||
priority: 0.5,
|
||||
retryAfter: nil,
|
||||
status: .downloadPending,
|
||||
)
|
||||
for path in paths {
|
||||
draft.filePath = URL(fileURLWithPath: path, isDirectory: false)
|
||||
try UploadTask.insert { draft }.execute(conn)
|
||||
}
|
||||
}
|
||||
dPrint("Enqueued \(paths.count) assets for upload")
|
||||
}
|
||||
|
||||
func startQueueProcessing() {
|
||||
dPrint("Starting upload queue processing")
|
||||
Self.queueProcessingLock.withLock {
|
||||
guard Self.queueProcessingTask == nil else { return }
|
||||
Self.queueProcessingTask = Task {
|
||||
await startUploads()
|
||||
Self.queueProcessingLock.withLock { Self.queueProcessingTask = nil }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func startUploads() async {
|
||||
dPrint("Processing download queue")
|
||||
guard NetworkMonitor.shared.isConnected,
|
||||
let backupEnabled = try? await db.read({ conn in try Store.get(conn, StoreKey.enableBackup) }),
|
||||
backupEnabled
|
||||
else { return dPrint("Download queue paused: network disconnected or backup disabled") }
|
||||
|
||||
do {
|
||||
let tasks: [LocalAssetUploadData] = try await db.read({ conn in
|
||||
guard let backupEnabled = try Store.get(conn, StoreKey.enableBackup), backupEnabled else { return [] }
|
||||
return try UploadTask.join(LocalAsset.all) { task, asset in task.localId.eq(asset.id) }
|
||||
.where { task, asset in
|
||||
asset.checksum.isNot(nil) && task.status.eq(TaskStatus.uploadPending)
|
||||
&& task.attempts < TaskConfig.maxRetries
|
||||
&& task.filePath.isNot(nil)
|
||||
}
|
||||
.select { task, asset in
|
||||
LocalAssetUploadData.Columns(
|
||||
filePath: task.filePath.unwrapped,
|
||||
priority: task.priority,
|
||||
taskId: task.id,
|
||||
type: asset.type
|
||||
)
|
||||
}
|
||||
.limit { task, _ in UploadTaskStat.availableUploadSlots }
|
||||
.order { task, asset in (task.priority.desc(), task.createdAt) }
|
||||
.fetchAll(conn)
|
||||
})
|
||||
if tasks.isEmpty { return dPrint("No upload tasks to process") }
|
||||
|
||||
await withTaskGroup(of: Void.self) { group in
|
||||
for task in tasks {
|
||||
group.addTask { await self.startUpload(multipart: task) }
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
dPrint("Upload queue error: \(error)")
|
||||
}
|
||||
}
|
||||
|
||||
private func startUpload(multipart task: LocalAssetUploadData) async {
|
||||
dPrint("Uploading asset resource at \(task.filePath) of task \(task.taskId)")
|
||||
defer { startQueueProcessing() }
|
||||
|
||||
let (url, accessToken, session): (URL, String, URLSession)
|
||||
do {
|
||||
(url, accessToken, session) = try await db.read { conn in
|
||||
guard let url = try Store.get(conn, StoreKey.serverEndpoint),
|
||||
let accessToken = try Store.get(conn, StoreKey.accessToken)
|
||||
else {
|
||||
throw StoreError.notFound
|
||||
}
|
||||
|
||||
let session =
|
||||
switch task.type {
|
||||
case .image:
|
||||
(try? Store.get(conn, StoreKey.useWifiForUploadPhotos)) ?? false ? cellularSession : wifiOnlySession
|
||||
case .video:
|
||||
(try? Store.get(conn, StoreKey.useWifiForUploadVideos)) ?? false ? cellularSession : wifiOnlySession
|
||||
default: wifiOnlySession
|
||||
}
|
||||
return (url, accessToken, session)
|
||||
}
|
||||
} catch {
|
||||
dPrint("Upload failed for \(task.taskId), could not retrieve server URL or access token: \(error)")
|
||||
return handleFailure(task: task, code: .noServerUrl)
|
||||
}
|
||||
|
||||
var request = URLRequest(url: url.appendingPathComponent("/assets"))
|
||||
request.httpMethod = "POST"
|
||||
request.setValue(accessToken, forHTTPHeaderField: UploadHeaders.userToken.rawValue)
|
||||
request.setValue(AssetData.contentType, forHTTPHeaderField: "Content-Type")
|
||||
|
||||
let sessionTask = session.uploadTask(with: request, fromFile: task.filePath)
|
||||
sessionTask.taskDescription = String(task.taskId)
|
||||
sessionTask.priority = task.priority
|
||||
do {
|
||||
try? FileManager.default.removeItem(at: task.filePath) // upload task already copied the file
|
||||
try await db.write { conn in
|
||||
try UploadTask.update { row in
|
||||
row.status = .uploadQueued
|
||||
row.filePath = nil
|
||||
}
|
||||
.where { $0.id.eq(task.taskId) }
|
||||
.execute(conn)
|
||||
}
|
||||
statusListener.onTaskStatus(
|
||||
UploadApiTaskStatus(
|
||||
id: String(task.taskId),
|
||||
filename: task.filePath.lastPathComponent,
|
||||
status: .uploadQueued,
|
||||
)
|
||||
)
|
||||
|
||||
sessionTask.resume()
|
||||
dPrint("Upload started for task \(task.taskId) using \(session == wifiOnlySession ? "WiFi" : "Cellular") session")
|
||||
} catch {
|
||||
dPrint("Upload failed for \(task.taskId), could not update queue status: \(error.localizedDescription)")
|
||||
}
|
||||
}
|
||||
|
||||
private func handleFailure(task: LocalAssetUploadData, code: UploadErrorCode) {
|
||||
do {
|
||||
try db.write { conn in
|
||||
try UploadTask.retryOrFail(code: code, status: .uploadFailed).where { $0.id.eq(task.taskId) }.execute(conn)
|
||||
}
|
||||
} catch {
|
||||
dPrint("Failed to update upload failure status for task \(task.taskId): \(error)")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,463 @@
|
||||
// Autogenerated from Pigeon (v26.0.2), do not edit directly.
|
||||
// See also: https://pub.dev/packages/pigeon
|
||||
|
||||
import Foundation
|
||||
|
||||
#if os(iOS)
|
||||
import Flutter
|
||||
#elseif os(macOS)
|
||||
import FlutterMacOS
|
||||
#else
|
||||
#error("Unsupported platform.")
|
||||
#endif
|
||||
|
||||
private func wrapResult(_ result: Any?) -> [Any?] {
|
||||
return [result]
|
||||
}
|
||||
|
||||
private func wrapError(_ error: Any) -> [Any?] {
|
||||
if let pigeonError = error as? PigeonError {
|
||||
return [
|
||||
pigeonError.code,
|
||||
pigeonError.message,
|
||||
pigeonError.details,
|
||||
]
|
||||
}
|
||||
if let flutterError = error as? FlutterError {
|
||||
return [
|
||||
flutterError.code,
|
||||
flutterError.message,
|
||||
flutterError.details,
|
||||
]
|
||||
}
|
||||
return [
|
||||
"\(error)",
|
||||
"\(type(of: error))",
|
||||
"Stacktrace: \(Thread.callStackSymbols)",
|
||||
]
|
||||
}
|
||||
|
||||
private func isNullish(_ value: Any?) -> Bool {
|
||||
return value is NSNull || value == nil
|
||||
}
|
||||
|
||||
private func nilOrValue<T>(_ value: Any?) -> T? {
|
||||
if value is NSNull { return nil }
|
||||
return value as! T?
|
||||
}
|
||||
|
||||
func deepEqualsUploadTask(_ lhs: Any?, _ rhs: Any?) -> Bool {
|
||||
let cleanLhs = nilOrValue(lhs) as Any?
|
||||
let cleanRhs = nilOrValue(rhs) as Any?
|
||||
switch (cleanLhs, cleanRhs) {
|
||||
case (nil, nil):
|
||||
return true
|
||||
|
||||
case (nil, _), (_, nil):
|
||||
return false
|
||||
|
||||
case is (Void, Void):
|
||||
return true
|
||||
|
||||
case let (cleanLhsHashable, cleanRhsHashable) as (AnyHashable, AnyHashable):
|
||||
return cleanLhsHashable == cleanRhsHashable
|
||||
|
||||
case let (cleanLhsArray, cleanRhsArray) as ([Any?], [Any?]):
|
||||
guard cleanLhsArray.count == cleanRhsArray.count else { return false }
|
||||
for (index, element) in cleanLhsArray.enumerated() {
|
||||
if !deepEqualsUploadTask(element, cleanRhsArray[index]) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
|
||||
case let (cleanLhsDictionary, cleanRhsDictionary) as ([AnyHashable: Any?], [AnyHashable: Any?]):
|
||||
guard cleanLhsDictionary.count == cleanRhsDictionary.count else { return false }
|
||||
for (key, cleanLhsValue) in cleanLhsDictionary {
|
||||
guard cleanRhsDictionary.index(forKey: key) != nil else { return false }
|
||||
if !deepEqualsUploadTask(cleanLhsValue, cleanRhsDictionary[key]!) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
|
||||
default:
|
||||
// Any other type shouldn't be able to be used with pigeon. File an issue if you find this to be untrue.
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func deepHashUploadTask(value: Any?, hasher: inout Hasher) {
|
||||
if let valueList = value as? [AnyHashable] {
|
||||
for item in valueList { deepHashUploadTask(value: item, hasher: &hasher) }
|
||||
return
|
||||
}
|
||||
|
||||
if let valueDict = value as? [AnyHashable: AnyHashable] {
|
||||
for key in valueDict.keys {
|
||||
hasher.combine(key)
|
||||
deepHashUploadTask(value: valueDict[key]!, hasher: &hasher)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if let hashableValue = value as? AnyHashable {
|
||||
hasher.combine(hashableValue.hashValue)
|
||||
}
|
||||
|
||||
return hasher.combine(String(describing: value))
|
||||
}
|
||||
|
||||
|
||||
|
||||
enum UploadApiErrorCode: Int {
|
||||
case unknown = 0
|
||||
case assetNotFound = 1
|
||||
case fileNotFound = 2
|
||||
case resourceNotFound = 3
|
||||
case invalidResource = 4
|
||||
case encodingFailed = 5
|
||||
case writeFailed = 6
|
||||
case notEnoughSpace = 7
|
||||
case networkError = 8
|
||||
case photosInternalError = 9
|
||||
case photosUnknownError = 10
|
||||
case noServerUrl = 11
|
||||
case noDeviceId = 12
|
||||
case noAccessToken = 13
|
||||
case interrupted = 14
|
||||
case cancelled = 15
|
||||
case downloadStalled = 16
|
||||
case forceQuit = 17
|
||||
case outOfResources = 18
|
||||
case backgroundUpdatesDisabled = 19
|
||||
case uploadTimeout = 20
|
||||
case iCloudRateLimit = 21
|
||||
case iCloudThrottled = 22
|
||||
}
|
||||
|
||||
enum UploadApiStatus: Int {
|
||||
case downloadPending = 0
|
||||
case downloadQueued = 1
|
||||
case downloadFailed = 2
|
||||
case uploadPending = 3
|
||||
case uploadQueued = 4
|
||||
case uploadFailed = 5
|
||||
case uploadComplete = 6
|
||||
case uploadSkipped = 7
|
||||
}
|
||||
|
||||
/// Generated class from Pigeon that represents data sent in messages.
|
||||
struct UploadApiTaskStatus: Hashable {
|
||||
var id: String
|
||||
var filename: String
|
||||
var status: UploadApiStatus
|
||||
var errorCode: UploadApiErrorCode? = nil
|
||||
var httpStatusCode: Int64? = nil
|
||||
|
||||
|
||||
// swift-format-ignore: AlwaysUseLowerCamelCase
|
||||
static func fromList(_ pigeonVar_list: [Any?]) -> UploadApiTaskStatus? {
|
||||
let id = pigeonVar_list[0] as! String
|
||||
let filename = pigeonVar_list[1] as! String
|
||||
let status = pigeonVar_list[2] as! UploadApiStatus
|
||||
let errorCode: UploadApiErrorCode? = nilOrValue(pigeonVar_list[3])
|
||||
let httpStatusCode: Int64? = nilOrValue(pigeonVar_list[4])
|
||||
|
||||
return UploadApiTaskStatus(
|
||||
id: id,
|
||||
filename: filename,
|
||||
status: status,
|
||||
errorCode: errorCode,
|
||||
httpStatusCode: httpStatusCode
|
||||
)
|
||||
}
|
||||
func toList() -> [Any?] {
|
||||
return [
|
||||
id,
|
||||
filename,
|
||||
status,
|
||||
errorCode,
|
||||
httpStatusCode,
|
||||
]
|
||||
}
|
||||
static func == (lhs: UploadApiTaskStatus, rhs: UploadApiTaskStatus) -> Bool {
|
||||
return deepEqualsUploadTask(lhs.toList(), rhs.toList()) }
|
||||
func hash(into hasher: inout Hasher) {
|
||||
deepHashUploadTask(value: toList(), hasher: &hasher)
|
||||
}
|
||||
}
|
||||
|
||||
/// Generated class from Pigeon that represents data sent in messages.
|
||||
struct UploadApiTaskProgress: Hashable {
|
||||
var id: String
|
||||
var progress: Double
|
||||
var speed: Double? = nil
|
||||
var totalBytes: Int64? = nil
|
||||
|
||||
|
||||
// swift-format-ignore: AlwaysUseLowerCamelCase
|
||||
static func fromList(_ pigeonVar_list: [Any?]) -> UploadApiTaskProgress? {
|
||||
let id = pigeonVar_list[0] as! String
|
||||
let progress = pigeonVar_list[1] as! Double
|
||||
let speed: Double? = nilOrValue(pigeonVar_list[2])
|
||||
let totalBytes: Int64? = nilOrValue(pigeonVar_list[3])
|
||||
|
||||
return UploadApiTaskProgress(
|
||||
id: id,
|
||||
progress: progress,
|
||||
speed: speed,
|
||||
totalBytes: totalBytes
|
||||
)
|
||||
}
|
||||
func toList() -> [Any?] {
|
||||
return [
|
||||
id,
|
||||
progress,
|
||||
speed,
|
||||
totalBytes,
|
||||
]
|
||||
}
|
||||
static func == (lhs: UploadApiTaskProgress, rhs: UploadApiTaskProgress) -> Bool {
|
||||
return deepEqualsUploadTask(lhs.toList(), rhs.toList()) }
|
||||
func hash(into hasher: inout Hasher) {
|
||||
deepHashUploadTask(value: toList(), hasher: &hasher)
|
||||
}
|
||||
}
|
||||
|
||||
private class UploadTaskPigeonCodecReader: FlutterStandardReader {
|
||||
override func readValue(ofType type: UInt8) -> Any? {
|
||||
switch type {
|
||||
case 129:
|
||||
let enumResultAsInt: Int? = nilOrValue(self.readValue() as! Int?)
|
||||
if let enumResultAsInt = enumResultAsInt {
|
||||
return UploadApiErrorCode(rawValue: enumResultAsInt)
|
||||
}
|
||||
return nil
|
||||
case 130:
|
||||
let enumResultAsInt: Int? = nilOrValue(self.readValue() as! Int?)
|
||||
if let enumResultAsInt = enumResultAsInt {
|
||||
return UploadApiStatus(rawValue: enumResultAsInt)
|
||||
}
|
||||
return nil
|
||||
case 131:
|
||||
return UploadApiTaskStatus.fromList(self.readValue() as! [Any?])
|
||||
case 132:
|
||||
return UploadApiTaskProgress.fromList(self.readValue() as! [Any?])
|
||||
default:
|
||||
return super.readValue(ofType: type)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private class UploadTaskPigeonCodecWriter: FlutterStandardWriter {
|
||||
override func writeValue(_ value: Any) {
|
||||
if let value = value as? UploadApiErrorCode {
|
||||
super.writeByte(129)
|
||||
super.writeValue(value.rawValue)
|
||||
} else if let value = value as? UploadApiStatus {
|
||||
super.writeByte(130)
|
||||
super.writeValue(value.rawValue)
|
||||
} else if let value = value as? UploadApiTaskStatus {
|
||||
super.writeByte(131)
|
||||
super.writeValue(value.toList())
|
||||
} else if let value = value as? UploadApiTaskProgress {
|
||||
super.writeByte(132)
|
||||
super.writeValue(value.toList())
|
||||
} else {
|
||||
super.writeValue(value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private class UploadTaskPigeonCodecReaderWriter: FlutterStandardReaderWriter {
|
||||
override func reader(with data: Data) -> FlutterStandardReader {
|
||||
return UploadTaskPigeonCodecReader(data: data)
|
||||
}
|
||||
|
||||
override func writer(with data: NSMutableData) -> FlutterStandardWriter {
|
||||
return UploadTaskPigeonCodecWriter(data: data)
|
||||
}
|
||||
}
|
||||
|
||||
class UploadTaskPigeonCodec: FlutterStandardMessageCodec, @unchecked Sendable {
|
||||
static let shared = UploadTaskPigeonCodec(readerWriter: UploadTaskPigeonCodecReaderWriter())
|
||||
}
|
||||
|
||||
var uploadTaskPigeonMethodCodec = FlutterStandardMethodCodec(readerWriter: UploadTaskPigeonCodecReaderWriter());
|
||||
|
||||
|
||||
/// Generated protocol from Pigeon that represents a handler of messages from Flutter.
|
||||
protocol UploadApi {
|
||||
func initialize(completion: @escaping (Result<Void, Error>) -> Void)
|
||||
func refresh(completion: @escaping (Result<Void, Error>) -> Void)
|
||||
func cancelAll(completion: @escaping (Result<Void, Error>) -> Void)
|
||||
func enqueueAssets(localIds: [String], completion: @escaping (Result<Void, Error>) -> Void)
|
||||
func enqueueFiles(paths: [String], completion: @escaping (Result<Void, Error>) -> Void)
|
||||
}
|
||||
|
||||
/// Generated setup class from Pigeon to handle messages through the `binaryMessenger`.
|
||||
class UploadApiSetup {
|
||||
static var codec: FlutterStandardMessageCodec { UploadTaskPigeonCodec.shared }
|
||||
/// Sets up an instance of `UploadApi` to handle messages through the `binaryMessenger`.
|
||||
static func setUp(binaryMessenger: FlutterBinaryMessenger, api: UploadApi?, messageChannelSuffix: String = "") {
|
||||
let channelSuffix = messageChannelSuffix.count > 0 ? ".\(messageChannelSuffix)" : ""
|
||||
let initializeChannel = FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.UploadApi.initialize\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec)
|
||||
if let api = api {
|
||||
initializeChannel.setMessageHandler { _, reply in
|
||||
api.initialize { result in
|
||||
switch result {
|
||||
case .success:
|
||||
reply(wrapResult(nil))
|
||||
case .failure(let error):
|
||||
reply(wrapError(error))
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
initializeChannel.setMessageHandler(nil)
|
||||
}
|
||||
let refreshChannel = FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.UploadApi.refresh\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec)
|
||||
if let api = api {
|
||||
refreshChannel.setMessageHandler { _, reply in
|
||||
api.refresh { result in
|
||||
switch result {
|
||||
case .success:
|
||||
reply(wrapResult(nil))
|
||||
case .failure(let error):
|
||||
reply(wrapError(error))
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
refreshChannel.setMessageHandler(nil)
|
||||
}
|
||||
let cancelAllChannel = FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.UploadApi.cancelAll\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec)
|
||||
if let api = api {
|
||||
cancelAllChannel.setMessageHandler { _, reply in
|
||||
api.cancelAll { result in
|
||||
switch result {
|
||||
case .success:
|
||||
reply(wrapResult(nil))
|
||||
case .failure(let error):
|
||||
reply(wrapError(error))
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
cancelAllChannel.setMessageHandler(nil)
|
||||
}
|
||||
let enqueueAssetsChannel = FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.UploadApi.enqueueAssets\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec)
|
||||
if let api = api {
|
||||
enqueueAssetsChannel.setMessageHandler { message, reply in
|
||||
let args = message as! [Any?]
|
||||
let localIdsArg = args[0] as! [String]
|
||||
api.enqueueAssets(localIds: localIdsArg) { result in
|
||||
switch result {
|
||||
case .success:
|
||||
reply(wrapResult(nil))
|
||||
case .failure(let error):
|
||||
reply(wrapError(error))
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
enqueueAssetsChannel.setMessageHandler(nil)
|
||||
}
|
||||
let enqueueFilesChannel = FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.UploadApi.enqueueFiles\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec)
|
||||
if let api = api {
|
||||
enqueueFilesChannel.setMessageHandler { message, reply in
|
||||
let args = message as! [Any?]
|
||||
let pathsArg = args[0] as! [String]
|
||||
api.enqueueFiles(paths: pathsArg) { result in
|
||||
switch result {
|
||||
case .success:
|
||||
reply(wrapResult(nil))
|
||||
case .failure(let error):
|
||||
reply(wrapError(error))
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
enqueueFilesChannel.setMessageHandler(nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private class PigeonStreamHandler<ReturnType>: NSObject, FlutterStreamHandler {
|
||||
private let wrapper: PigeonEventChannelWrapper<ReturnType>
|
||||
private var pigeonSink: PigeonEventSink<ReturnType>? = nil
|
||||
|
||||
init(wrapper: PigeonEventChannelWrapper<ReturnType>) {
|
||||
self.wrapper = wrapper
|
||||
}
|
||||
|
||||
func onListen(withArguments arguments: Any?, eventSink events: @escaping FlutterEventSink)
|
||||
-> FlutterError?
|
||||
{
|
||||
pigeonSink = PigeonEventSink<ReturnType>(events)
|
||||
wrapper.onListen(withArguments: arguments, sink: pigeonSink!)
|
||||
return nil
|
||||
}
|
||||
|
||||
func onCancel(withArguments arguments: Any?) -> FlutterError? {
|
||||
pigeonSink = nil
|
||||
wrapper.onCancel(withArguments: arguments)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
class PigeonEventChannelWrapper<ReturnType> {
|
||||
func onListen(withArguments arguments: Any?, sink: PigeonEventSink<ReturnType>) {}
|
||||
func onCancel(withArguments arguments: Any?) {}
|
||||
}
|
||||
|
||||
class PigeonEventSink<ReturnType> {
|
||||
private let sink: FlutterEventSink
|
||||
|
||||
init(_ sink: @escaping FlutterEventSink) {
|
||||
self.sink = sink
|
||||
}
|
||||
|
||||
func success(_ value: ReturnType) {
|
||||
sink(value)
|
||||
}
|
||||
|
||||
func error(code: String, message: String?, details: Any?) {
|
||||
sink(FlutterError(code: code, message: message, details: details))
|
||||
}
|
||||
|
||||
func endOfStream() {
|
||||
sink(FlutterEndOfEventStream)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
class StreamStatusStreamHandler: PigeonEventChannelWrapper<UploadApiTaskStatus> {
|
||||
static func register(with messenger: FlutterBinaryMessenger,
|
||||
instanceName: String = "",
|
||||
streamHandler: StreamStatusStreamHandler) {
|
||||
var channelName = "dev.flutter.pigeon.immich_mobile.UploadFlutterApi.streamStatus"
|
||||
if !instanceName.isEmpty {
|
||||
channelName += ".\(instanceName)"
|
||||
}
|
||||
let internalStreamHandler = PigeonStreamHandler<UploadApiTaskStatus>(wrapper: streamHandler)
|
||||
let channel = FlutterEventChannel(name: channelName, binaryMessenger: messenger, codec: uploadTaskPigeonMethodCodec)
|
||||
channel.setStreamHandler(internalStreamHandler)
|
||||
}
|
||||
}
|
||||
|
||||
class StreamProgressStreamHandler: PigeonEventChannelWrapper<UploadApiTaskProgress> {
|
||||
static func register(with messenger: FlutterBinaryMessenger,
|
||||
instanceName: String = "",
|
||||
streamHandler: StreamProgressStreamHandler) {
|
||||
var channelName = "dev.flutter.pigeon.immich_mobile.UploadFlutterApi.streamProgress"
|
||||
if !instanceName.isEmpty {
|
||||
channelName += ".\(instanceName)"
|
||||
}
|
||||
let internalStreamHandler = PigeonStreamHandler<UploadApiTaskProgress>(wrapper: streamHandler)
|
||||
let channel = FlutterEventChannel(name: channelName, binaryMessenger: messenger, codec: uploadTaskPigeonMethodCodec)
|
||||
channel.setStreamHandler(internalStreamHandler)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,271 @@
|
||||
import SQLiteData
|
||||
import StructuredFieldValues
|
||||
|
||||
extension FileHandle {
|
||||
static func createOrOverwrite(atPath path: String) throws -> FileHandle {
|
||||
let fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0o644)
|
||||
guard fd >= 0 else {
|
||||
throw NSError(domain: NSPOSIXErrorDomain, code: Int(errno))
|
||||
}
|
||||
return FileHandle(fileDescriptor: fd, closeOnDealloc: true)
|
||||
}
|
||||
}
|
||||
|
||||
class UploadApiImpl: ImmichPlugin, UploadApi {
|
||||
private let db: DatabasePool
|
||||
private let downloadQueue: DownloadQueue
|
||||
private let uploadQueue: UploadQueue
|
||||
|
||||
private var isInitialized = false
|
||||
private let initLock = NSLock()
|
||||
|
||||
private var backupTask: Task<Void, Never>?
|
||||
private let backupLock = NSLock()
|
||||
|
||||
private let cellularSession: URLSession
|
||||
private let wifiOnlySession: URLSession
|
||||
|
||||
init(statusListener: StatusEventListener, progressListener: ProgressEventListener) {
|
||||
let dbUrl = try! FileManager.default.url(
|
||||
for: .documentDirectory,
|
||||
in: .userDomainMask,
|
||||
appropriateFor: nil,
|
||||
create: true
|
||||
).appendingPathComponent("immich.sqlite")
|
||||
|
||||
self.db = try! DatabasePool(path: dbUrl.path)
|
||||
let cellularConfig = URLSessionConfiguration.background(withIdentifier: "\(TaskConfig.sessionId).cellular")
|
||||
cellularConfig.allowsCellularAccess = true
|
||||
cellularConfig.waitsForConnectivity = true
|
||||
let delegate = UploadApiDelegate(db: db, statusListener: statusListener, progressListener: progressListener)
|
||||
self.cellularSession = URLSession(configuration: cellularConfig, delegate: delegate, delegateQueue: nil)
|
||||
|
||||
let wifiOnlyConfig = URLSessionConfiguration.background(withIdentifier: "\(TaskConfig.sessionId).wifi")
|
||||
wifiOnlyConfig.allowsCellularAccess = false
|
||||
wifiOnlyConfig.waitsForConnectivity = true
|
||||
self.wifiOnlySession = URLSession(configuration: wifiOnlyConfig, delegate: delegate, delegateQueue: nil)
|
||||
|
||||
self.uploadQueue = UploadQueue(
|
||||
db: db,
|
||||
cellularSession: cellularSession,
|
||||
wifiOnlySession: wifiOnlySession,
|
||||
statusListener: statusListener
|
||||
)
|
||||
self.downloadQueue = DownloadQueue(
|
||||
db: db,
|
||||
uploadQueue: uploadQueue,
|
||||
statusListener: statusListener,
|
||||
progressListener: progressListener
|
||||
)
|
||||
delegate.downloadQueue = downloadQueue
|
||||
delegate.uploadQueue = uploadQueue
|
||||
}
|
||||
|
||||
func initialize(completion: @escaping (Result<Void, any Error>) -> Void) {
|
||||
Task(priority: .high) {
|
||||
do {
|
||||
async let dbIds = db.read { conn in
|
||||
try UploadTask.select(\.id).where { $0.status.eq(TaskStatus.uploadQueued) }.fetchAll(conn)
|
||||
}
|
||||
async let cellularTasks = cellularSession.allTasks
|
||||
async let wifiTasks = wifiOnlySession.allTasks
|
||||
|
||||
var dbTaskIds = Set(try await dbIds)
|
||||
func validateTasks(_ tasks: [URLSessionTask]) {
|
||||
for task in tasks {
|
||||
if let taskIdStr = task.taskDescription, let taskId = Int64(taskIdStr), task.state != .canceling {
|
||||
dbTaskIds.remove(taskId)
|
||||
} else {
|
||||
task.cancel()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
validateTasks(await cellularTasks)
|
||||
validateTasks(await wifiTasks)
|
||||
|
||||
let orphanIds = Array(dbTaskIds)
|
||||
try await db.write { conn in
|
||||
try UploadTask.update {
|
||||
$0.filePath = nil
|
||||
$0.status = .downloadPending
|
||||
}
|
||||
.where { row in row.status.in([TaskStatus.downloadQueued, TaskStatus.uploadPending]) || row.id.in(orphanIds) }
|
||||
.execute(conn)
|
||||
}
|
||||
|
||||
try? FileManager.default.removeItem(at: TaskConfig.originalsDir)
|
||||
initLock.withLock { isInitialized = true }
|
||||
startBackup()
|
||||
self.completeWhenActive(for: completion, with: .success(()))
|
||||
} catch {
|
||||
self.completeWhenActive(for: completion, with: .failure(error))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func refresh(completion: @escaping (Result<Void, any Error>) -> Void) {
|
||||
Task {
|
||||
startBackup()
|
||||
self.completeWhenActive(for: completion, with: .success(()))
|
||||
}
|
||||
}
|
||||
|
||||
func startBackup() {
|
||||
dPrint("Starting backup task")
|
||||
guard (initLock.withLock { isInitialized }) else { return dPrint("Not initialized, skipping backup") }
|
||||
|
||||
backupLock.withLock {
|
||||
guard backupTask == nil else { return dPrint("Backup task already running") }
|
||||
backupTask = Task {
|
||||
await _startBackup()
|
||||
backupLock.withLock { backupTask = nil }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func cancelAll(completion: @escaping (Result<Void, any Error>) -> Void) {
|
||||
Task {
|
||||
async let cellularTasks = cellularSession.allTasks
|
||||
async let wifiTasks = wifiOnlySession.allTasks
|
||||
|
||||
cancelSessionTasks(await cellularTasks)
|
||||
cancelSessionTasks(await wifiTasks)
|
||||
self.completeWhenActive(for: completion, with: .success(()))
|
||||
}
|
||||
}
|
||||
|
||||
func enqueueAssets(localIds: [String], completion: @escaping (Result<Void, any Error>) -> Void) {
|
||||
Task {
|
||||
do {
|
||||
try await downloadQueue.enqueueAssets(localIds: localIds)
|
||||
completion(.success(()))
|
||||
} catch {
|
||||
completion(.failure(error))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func enqueueFiles(paths: [String], completion: @escaping (Result<Void, any Error>) -> Void) {
|
||||
Task {
|
||||
do {
|
||||
try await uploadQueue.enqueueFiles(paths: paths)
|
||||
completion(.success(()))
|
||||
} catch {
|
||||
completion(.failure(error))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func cancelSessionTasks(_ tasks: [URLSessionTask]) {
|
||||
dPrint("Canceling \(tasks.count) tasks")
|
||||
for task in tasks {
|
||||
task.cancel()
|
||||
}
|
||||
}
|
||||
|
||||
private func _startBackup() async {
|
||||
defer { downloadQueue.startQueueProcessing() }
|
||||
do {
|
||||
let candidates = try await db.read { conn in
|
||||
return try LocalAsset.getCandidates()
|
||||
.where { asset in !UploadTask.where { task in task.localId.eq(asset.id) }.exists() }
|
||||
.select { LocalAssetCandidate.Columns(id: $0.id, type: $0.type) }
|
||||
.limit { _ in UploadTaskStat.availableSlots }
|
||||
.fetchAll(conn)
|
||||
}
|
||||
|
||||
guard !candidates.isEmpty else { return dPrint("No candidates for backup") }
|
||||
|
||||
try await db.write { conn in
|
||||
var draft = UploadTask.Draft(
|
||||
attempts: 0,
|
||||
createdAt: Date(),
|
||||
filePath: nil,
|
||||
isLivePhoto: nil,
|
||||
lastError: nil,
|
||||
livePhotoVideoId: nil,
|
||||
localId: "",
|
||||
method: .multipart,
|
||||
priority: 0.5,
|
||||
retryAfter: nil,
|
||||
status: .downloadPending,
|
||||
)
|
||||
for candidate in candidates {
|
||||
draft.localId = candidate.id
|
||||
draft.priority = candidate.type == .image ? 0.5 : 0.3
|
||||
try UploadTask.insert {
|
||||
draft
|
||||
} onConflict: {
|
||||
($0.localId, $0.livePhotoVideoId)
|
||||
}
|
||||
.execute(conn)
|
||||
}
|
||||
}
|
||||
dPrint("Backup enqueued \(candidates.count) assets for upload")
|
||||
} catch {
|
||||
print("Backup queue error: \(error)")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct AssetData: StructuredFieldValue {
|
||||
static let structuredFieldType: StructuredFieldType = .dictionary
|
||||
|
||||
let deviceAssetId: String
|
||||
let deviceId: String
|
||||
let fileCreatedAt: String
|
||||
let fileModifiedAt: String
|
||||
let fileName: String
|
||||
let isFavorite: Bool
|
||||
let livePhotoVideoId: String?
|
||||
|
||||
static let boundary = "Boundary-\(UUID().uuidString)"
|
||||
static let deviceAssetIdField = "--\(boundary)\r\nContent-Disposition: form-data; name=\"deviceAssetId\"\r\n\r\n"
|
||||
.data(using: .utf8)!
|
||||
static let deviceIdField = "\r\n--\(boundary)\r\nContent-Disposition: form-data; name=\"deviceId\"\r\n\r\n"
|
||||
.data(using: .utf8)!
|
||||
static let fileCreatedAtField =
|
||||
"\r\n--\(boundary)\r\nContent-Disposition: form-data; name=\"fileCreatedAt\"\r\n\r\n"
|
||||
.data(using: .utf8)!
|
||||
static let fileModifiedAtField =
|
||||
"\r\n--\(boundary)\r\nContent-Disposition: form-data; name=\"fileModifiedAt\"\r\n\r\n"
|
||||
.data(using: .utf8)!
|
||||
static let isFavoriteField = "\r\n--\(boundary)\r\nContent-Disposition: form-data; name=\"isFavorite\"\r\n\r\n"
|
||||
.data(using: .utf8)!
|
||||
static let livePhotoVideoIdField =
|
||||
"\r\n--\(boundary)\r\nContent-Disposition: form-data; name=\"livePhotoVideoId\"\r\n\r\n"
|
||||
.data(using: .utf8)!
|
||||
static let trueData = "true".data(using: .utf8)!
|
||||
static let falseData = "false".data(using: .utf8)!
|
||||
static let footer = "\r\n--\(boundary)--\r\n".data(using: .utf8)!
|
||||
static let contentType = "multipart/form-data; boundary=\(boundary)"
|
||||
|
||||
func multipart() -> (Data, Data) {
|
||||
var header = Data()
|
||||
header.append(Self.deviceAssetIdField)
|
||||
header.append(deviceAssetId.data(using: .utf8)!)
|
||||
|
||||
header.append(Self.deviceIdField)
|
||||
header.append(deviceId.data(using: .utf8)!)
|
||||
|
||||
header.append(Self.fileCreatedAtField)
|
||||
header.append(fileCreatedAt.data(using: .utf8)!)
|
||||
|
||||
header.append(Self.fileModifiedAtField)
|
||||
header.append(fileModifiedAt.data(using: .utf8)!)
|
||||
|
||||
header.append(Self.isFavoriteField)
|
||||
header.append(isFavorite ? Self.trueData : Self.falseData)
|
||||
|
||||
if let livePhotoVideoId {
|
||||
header.append(Self.livePhotoVideoIdField)
|
||||
header.append(livePhotoVideoId.data(using: .utf8)!)
|
||||
}
|
||||
header.append(
|
||||
"\r\n--\(Self.boundary)\r\nContent-Disposition: form-data; name=\"assetData\"; filename=\"\(fileName)\"\r\nContent-Type: application/octet-stream\r\n\r\n"
|
||||
.data(using: .utf8)!
|
||||
)
|
||||
return (header, Self.footer)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user