mirror of
https://github.com/immich-app/immich.git
synced 2026-05-18 03:10:24 +03:00
background upload plugin
add schemas sync variants formatting initial implementation use existing db, wip move to separate folder fix table definitions wip wiring it up repository pattern
This commit is contained in:
mertalev
@@ -0,0 +1,203 @@
|
||||
import SQLiteData
|
||||
|
||||
protocol StoreProtocol {
|
||||
func get<T: StoreConvertible<Int>>(_ key: StoreKey.Typed<T>) throws -> T?
|
||||
func get<T: StoreConvertible<String>>(_ key: StoreKey.Typed<T>) throws -> T?
|
||||
func set<T: StoreConvertible<Int>>(_ key: StoreKey.Typed<T>, value: T) throws
|
||||
func set<T: StoreConvertible<String>>(_ key: StoreKey.Typed<T>, value: T) throws
|
||||
func invalidateCache()
|
||||
}
|
||||
|
||||
protocol StoreConvertible<StorageType> {
|
||||
associatedtype StorageType
|
||||
static var cacheKeyPath: ReferenceWritableKeyPath<StoreCache, [StoreKey: Self]> { get }
|
||||
static func fromValue(_ value: StorageType) throws(StoreError) -> Self
|
||||
static func toValue(_ value: Self) throws(StoreError) -> StorageType
|
||||
}
|
||||
|
||||
final class StoreRepository: StoreProtocol {
|
||||
private let db: DatabasePool
|
||||
private static let cache = StoreCache()
|
||||
private static var lock = os_unfair_lock()
|
||||
|
||||
init(db: DatabasePool) {
|
||||
self.db = db
|
||||
}
|
||||
|
||||
func get<T: StoreConvertible<Int>>(_ key: StoreKey.Typed<T>) throws -> T? {
|
||||
os_unfair_lock_lock(&Self.lock)
|
||||
defer { os_unfair_lock_unlock(&Self.lock) }
|
||||
let cached = Self.cache.get(key)
|
||||
if _fastPath(cached != nil) { return cached! }
|
||||
return try db.read { conn in
|
||||
let query = Store.select(\.intValue).where { $0.id.eq(key.rawValue) }
|
||||
if let value = try query.fetchOne(conn) ?? nil {
|
||||
let converted = try T.fromValue(value)
|
||||
Self.cache.set(key, value: converted)
|
||||
return converted
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func get<T: StoreConvertible<String>>(_ key: StoreKey.Typed<T>) throws -> T? {
|
||||
os_unfair_lock_lock(&Self.lock)
|
||||
defer { os_unfair_lock_unlock(&Self.lock) }
|
||||
let cached = Self.cache.get(key)
|
||||
if _fastPath(cached != nil) { return cached! }
|
||||
return try db.read { conn in
|
||||
let query = Store.select(\.stringValue).where { $0.id.eq(key.rawValue) }
|
||||
if let value = try query.fetchOne(conn) ?? nil {
|
||||
let converted = try T.fromValue(value)
|
||||
Self.cache.set(key, value: converted)
|
||||
return converted
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func set<T: StoreConvertible<Int>>(_ key: StoreKey.Typed<T>, value: T) throws {
|
||||
os_unfair_lock_lock(&Self.lock)
|
||||
defer { os_unfair_lock_unlock(&Self.lock) }
|
||||
let converted = try T.toValue(value)
|
||||
try db.write { conn in
|
||||
try Store.upsert { Store(id: key.rawValue, stringValue: nil, intValue: converted) }.execute(conn)
|
||||
}
|
||||
Self.cache.set(key, value: value)
|
||||
}
|
||||
|
||||
func set<T: StoreConvertible<String>>(_ key: StoreKey.Typed<T>, value: T) throws {
|
||||
os_unfair_lock_lock(&Self.lock)
|
||||
defer { os_unfair_lock_unlock(&Self.lock) }
|
||||
let converted = try T.toValue(value)
|
||||
try db.write { conn in
|
||||
try Store.upsert { Store(id: key.rawValue, stringValue: converted, intValue: nil) }.execute(conn)
|
||||
}
|
||||
Self.cache.set(key, value: value)
|
||||
}
|
||||
|
||||
func invalidateCache() {
|
||||
Self.cache.reset()
|
||||
}
|
||||
}
|
||||
|
||||
enum StoreError: Error {
|
||||
case invalidJSON(String)
|
||||
case invalidURL(String)
|
||||
case encodingFailed
|
||||
case notFound
|
||||
}
|
||||
|
||||
extension StoreConvertible {
|
||||
fileprivate static func get(_ cache: StoreCache, key: StoreKey) -> Self? {
|
||||
return cache[keyPath: cacheKeyPath][key]
|
||||
}
|
||||
|
||||
fileprivate static func set(_ cache: StoreCache, key: StoreKey, value: Self?) {
|
||||
cache[keyPath: cacheKeyPath][key] = value
|
||||
}
|
||||
|
||||
fileprivate static func reset(_ cache: StoreCache) {
|
||||
cache.reset()
|
||||
}
|
||||
}
|
||||
|
||||
final class StoreCache {
|
||||
fileprivate var intCache: [StoreKey: Int] = [:]
|
||||
fileprivate var boolCache: [StoreKey: Bool] = [:]
|
||||
fileprivate var dateCache: [StoreKey: Date] = [:]
|
||||
fileprivate var stringCache: [StoreKey: String] = [:]
|
||||
fileprivate var urlCache: [StoreKey: URL] = [:]
|
||||
fileprivate var endpointArrayCache: [StoreKey: [Endpoint]] = [:]
|
||||
fileprivate var stringDictCache: [StoreKey: [String: String]] = [:]
|
||||
|
||||
func get<T: StoreConvertible>(_ key: StoreKey.Typed<T>) -> T? {
|
||||
return T.get(self, key: key.rawValue)
|
||||
}
|
||||
|
||||
func set<T: StoreConvertible>(_ key: StoreKey.Typed<T>, value: T?) {
|
||||
return T.set(self, key: key.rawValue, value: value)
|
||||
}
|
||||
|
||||
func reset() {
|
||||
intCache.removeAll(keepingCapacity: true)
|
||||
boolCache.removeAll(keepingCapacity: true)
|
||||
dateCache.removeAll(keepingCapacity: true)
|
||||
stringCache.removeAll(keepingCapacity: true)
|
||||
urlCache.removeAll(keepingCapacity: true)
|
||||
endpointArrayCache.removeAll(keepingCapacity: true)
|
||||
stringDictCache.removeAll(keepingCapacity: true)
|
||||
}
|
||||
}
|
||||
|
||||
extension Int: StoreConvertible {
|
||||
static let cacheKeyPath = \StoreCache.intCache
|
||||
static func fromValue(_ value: Int) -> Int { value }
|
||||
static func toValue(_ value: Int) -> Int { value }
|
||||
}
|
||||
|
||||
extension Bool: StoreConvertible {
|
||||
static let cacheKeyPath = \StoreCache.boolCache
|
||||
static func fromValue(_ value: Int) -> Bool { value == 1 }
|
||||
static func toValue(_ value: Bool) -> Int { value ? 1 : 0 }
|
||||
}
|
||||
|
||||
extension Date: StoreConvertible {
|
||||
static let cacheKeyPath = \StoreCache.dateCache
|
||||
static func fromValue(_ value: Int) -> Date { Date(timeIntervalSince1970: TimeInterval(value) / 1000) }
|
||||
static func toValue(_ value: Date) -> Int { Int(value.timeIntervalSince1970 * 1000) }
|
||||
}
|
||||
|
||||
extension String: StoreConvertible {
|
||||
static let cacheKeyPath = \StoreCache.stringCache
|
||||
static func fromValue(_ value: String) -> String { value }
|
||||
static func toValue(_ value: String) -> String { value }
|
||||
}
|
||||
|
||||
extension URL: StoreConvertible {
|
||||
static let cacheKeyPath = \StoreCache.urlCache
|
||||
static func fromValue(_ value: String) throws(StoreError) -> URL {
|
||||
guard let url = URL(string: value) else {
|
||||
throw StoreError.invalidURL(value)
|
||||
}
|
||||
return url
|
||||
}
|
||||
static func toValue(_ value: URL) -> String { value.absoluteString }
|
||||
}
|
||||
|
||||
extension StoreConvertible<String> where Self: Codable {
|
||||
static var jsonDecoder: JSONDecoder { JSONDecoder() }
|
||||
static var jsonEncoder: JSONEncoder { JSONEncoder() }
|
||||
|
||||
static func fromValue(_ value: String) throws(StoreError) -> Self {
|
||||
do {
|
||||
return try jsonDecoder.decode(Self.self, from: Data(value.utf8))
|
||||
} catch {
|
||||
throw StoreError.invalidJSON(value)
|
||||
}
|
||||
}
|
||||
|
||||
static func toValue(_ value: Self) throws(StoreError) -> String {
|
||||
let encoded: Data
|
||||
do {
|
||||
encoded = try jsonEncoder.encode(value)
|
||||
} catch {
|
||||
throw StoreError.encodingFailed
|
||||
}
|
||||
|
||||
guard let string = String(data: encoded, encoding: .utf8) else {
|
||||
throw StoreError.encodingFailed
|
||||
}
|
||||
return string
|
||||
}
|
||||
}
|
||||
|
||||
extension Array: StoreConvertible where Element == Endpoint {
|
||||
static let cacheKeyPath = \StoreCache.endpointArrayCache
|
||||
typealias StorageType = String
|
||||
}
|
||||
|
||||
extension Dictionary: StoreConvertible where Key == String, Value == String {
|
||||
static let cacheKeyPath = \StoreCache.stringDictCache
|
||||
typealias StorageType = String
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
import SQLiteData
|
||||
|
||||
protocol TaskProtocol {
|
||||
func getTaskIds(status: TaskStatus) async throws -> [Int64]
|
||||
func getBackupCandidates() async throws -> [LocalAssetCandidate]
|
||||
func getBackupCandidates(ids: [String]) async throws -> [LocalAssetCandidate]
|
||||
func getDownloadTasks() async throws -> [LocalAssetDownloadData]
|
||||
func getUploadTasks() async throws -> [LocalAssetUploadData]
|
||||
func markOrphansPending(ids: [Int64]) async throws
|
||||
func markDownloadQueued(taskId: Int64, isLivePhoto: Bool, filePath: URL) async throws
|
||||
func markUploadQueued(taskId: Int64) async throws
|
||||
func markDownloadComplete(taskId: Int64, localId: String, hash: String?) async throws -> TaskStatus
|
||||
func markUploadSuccess(taskId: Int64, livePhotoVideoId: String?) async throws
|
||||
func retryOrFail(taskId: Int64, code: UploadErrorCode, status: TaskStatus) async throws
|
||||
func enqueue(assets: [LocalAssetCandidate], imagePriority: Float, videoPriority: Float) async throws
|
||||
func enqueue(files: [String]) async throws
|
||||
func resolveError(code: UploadErrorCode) async throws
|
||||
func getFilename(taskId: Int64) async throws -> String?
|
||||
}
|
||||
|
||||
final class TaskRepository: TaskProtocol {
|
||||
private let db: DatabasePool
|
||||
|
||||
init(db: DatabasePool) {
|
||||
self.db = db
|
||||
}
|
||||
|
||||
func getTaskIds(status: TaskStatus) async throws -> [Int64] {
|
||||
return try await db.read { conn in
|
||||
try UploadTask.select(\.id).where { $0.status.eq(status) }.fetchAll(conn)
|
||||
}
|
||||
}
|
||||
|
||||
func getBackupCandidates() async throws -> [LocalAssetCandidate] {
|
||||
return try await db.read { conn in
|
||||
return try LocalAsset.backupCandidates.fetchAll(conn)
|
||||
}
|
||||
}
|
||||
|
||||
func getBackupCandidates(ids: [String]) async throws -> [LocalAssetCandidate] {
|
||||
return try await db.read { conn in
|
||||
return try LocalAsset.backupCandidates.where { $0.id.in(ids) }.fetchAll(conn)
|
||||
}
|
||||
}
|
||||
|
||||
func getDownloadTasks() async throws -> [LocalAssetDownloadData] {
|
||||
return try await db.read({ conn in
|
||||
return try UploadTask.join(LocalAsset.all) { task, asset in task.localId.eq(asset.id) }
|
||||
.where { task, _ in task.canRetry && task.noFatalError && LocalAsset.withChecksum.exists() }
|
||||
.select { task, asset in
|
||||
LocalAssetDownloadData.Columns(
|
||||
checksum: asset.checksum,
|
||||
createdAt: asset.createdAt,
|
||||
filename: asset.name,
|
||||
livePhotoVideoId: task.livePhotoVideoId,
|
||||
localId: asset.id,
|
||||
taskId: task.id,
|
||||
updatedAt: asset.updatedAt
|
||||
)
|
||||
}
|
||||
.order { task, asset in (task.priority.desc(), task.createdAt) }
|
||||
.limit { _, _ in UploadTaskStat.availableDownloadSlots }
|
||||
.fetchAll(conn)
|
||||
})
|
||||
}
|
||||
|
||||
func getUploadTasks() async throws -> [LocalAssetUploadData] {
|
||||
return try await db.read({ conn in
|
||||
return try UploadTask.join(LocalAsset.all) { task, asset in task.localId.eq(asset.id) }
|
||||
.where { task, _ in task.canRetry && task.noFatalError && LocalAsset.withChecksum.exists() }
|
||||
.select { task, asset in
|
||||
LocalAssetUploadData.Columns(
|
||||
filename: asset.name,
|
||||
filePath: task.filePath.unwrapped,
|
||||
priority: task.priority,
|
||||
taskId: task.id,
|
||||
type: asset.type
|
||||
)
|
||||
}
|
||||
.order { task, asset in (task.priority.desc(), task.createdAt) }
|
||||
.limit { task, _ in UploadTaskStat.availableUploadSlots }
|
||||
.fetchAll(conn)
|
||||
})
|
||||
}
|
||||
|
||||
func markOrphansPending(ids: [Int64]) async throws {
|
||||
try await db.write { conn in
|
||||
try UploadTask.update {
|
||||
$0.filePath = nil
|
||||
$0.status = .downloadPending
|
||||
}
|
||||
.where { row in row.status.in([TaskStatus.downloadQueued, TaskStatus.uploadPending]) || row.id.in(ids) }
|
||||
.execute(conn)
|
||||
}
|
||||
}
|
||||
|
||||
func markDownloadQueued(taskId: Int64, isLivePhoto: Bool, filePath: URL) async throws {
|
||||
try await db.write { conn in
|
||||
try UploadTask.update {
|
||||
$0.status = .downloadQueued
|
||||
$0.isLivePhoto = isLivePhoto
|
||||
$0.filePath = filePath
|
||||
}
|
||||
.where { $0.id.eq(taskId) }.execute(conn)
|
||||
}
|
||||
}
|
||||
|
||||
func markUploadQueued(taskId: Int64) async throws {
|
||||
try await db.write { conn in
|
||||
try UploadTask.update { row in
|
||||
row.status = .uploadQueued
|
||||
row.filePath = nil
|
||||
}
|
||||
.where { $0.id.eq(taskId) }.execute(conn)
|
||||
}
|
||||
}
|
||||
|
||||
func markDownloadComplete(taskId: Int64, localId: String, hash: String?) async throws -> TaskStatus {
|
||||
return try await db.write { conn in
|
||||
if let hash {
|
||||
try LocalAsset.update { $0.checksum = hash }.where { $0.id.eq(localId) }.execute(conn)
|
||||
}
|
||||
let status =
|
||||
if let hash, try RemoteAsset.select(\.rowid).where({ $0.checksum.eq(hash) }).fetchOne(conn) != nil {
|
||||
TaskStatus.uploadSkipped
|
||||
} else {
|
||||
TaskStatus.uploadPending
|
||||
}
|
||||
try UploadTask.update { $0.status = status }.where { $0.id.eq(taskId) }.execute(conn)
|
||||
return status
|
||||
}
|
||||
}
|
||||
|
||||
func markUploadSuccess(taskId: Int64, livePhotoVideoId: String?) async throws {
|
||||
try await db.write { conn in
|
||||
let task =
|
||||
try UploadTask
|
||||
.update { $0.status = .uploadComplete }
|
||||
.where { $0.id.eq(taskId) }
|
||||
.returning(\.self)
|
||||
.fetchOne(conn)
|
||||
guard let task, let localId = task.localId, let isLivePhoto = task.isLivePhoto, isLivePhoto,
|
||||
task.livePhotoVideoId == nil
|
||||
else { return }
|
||||
try UploadTask.insert {
|
||||
UploadTask.Draft(
|
||||
attempts: 0,
|
||||
createdAt: Date(),
|
||||
filePath: nil,
|
||||
isLivePhoto: true,
|
||||
lastError: nil,
|
||||
livePhotoVideoId: livePhotoVideoId,
|
||||
localId: localId,
|
||||
method: .multipart,
|
||||
priority: 0.7,
|
||||
retryAfter: nil,
|
||||
status: .downloadPending,
|
||||
)
|
||||
}.execute(conn)
|
||||
}
|
||||
}
|
||||
|
||||
func retryOrFail(taskId: Int64, code: UploadErrorCode, status: TaskStatus) async throws {
|
||||
try await db.write { conn in
|
||||
try UploadTask.update { row in
|
||||
let retryOffset =
|
||||
switch code {
|
||||
case .iCloudThrottled, .iCloudRateLimit, .notEnoughSpace: 3000
|
||||
default: 0
|
||||
}
|
||||
row.status = Case()
|
||||
.when(row.localId.is(nil) && row.attempts.lte(TaskConfig.maxRetries), then: TaskStatus.uploadPending)
|
||||
.when(row.attempts.lte(TaskConfig.maxRetries), then: TaskStatus.downloadPending)
|
||||
.else(status)
|
||||
row.attempts += 1
|
||||
row.lastError = code
|
||||
row.retryAfter = #sql("unixepoch('now') + (\(4 << row.attempts)) + \(retryOffset)")
|
||||
}
|
||||
.where { $0.id.eq(taskId) }.execute(conn)
|
||||
}
|
||||
}
|
||||
|
||||
func enqueue(assets: [LocalAssetCandidate], imagePriority: Float, videoPriority: Float) async throws {
|
||||
try await db.write { conn in
|
||||
var draft = draftStub
|
||||
for candidate in assets {
|
||||
draft.localId = candidate.id
|
||||
draft.priority = candidate.type == .image ? imagePriority : videoPriority
|
||||
try UploadTask.insert {
|
||||
draft
|
||||
} onConflict: {
|
||||
($0.localId, $0.livePhotoVideoId)
|
||||
}
|
||||
.execute(conn)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func enqueue(files: [String]) async throws {
|
||||
try await db.write { conn in
|
||||
var draft = draftStub
|
||||
draft.priority = 1.0
|
||||
draft.status = .uploadPending
|
||||
for file in files {
|
||||
draft.filePath = URL(fileURLWithPath: file, isDirectory: false)
|
||||
try UploadTask.insert { draft }.execute(conn)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func resolveError(code: UploadErrorCode) async throws {
|
||||
try await db.write { conn in
|
||||
try UploadTask.update { $0.lastError = nil }.where { $0.lastError.unwrapped.eq(code) }.execute(conn)
|
||||
}
|
||||
}
|
||||
|
||||
func getFilename(taskId: Int64) async throws -> String? {
|
||||
try await db.read { conn in
|
||||
try UploadTask.join(LocalAsset.all) { task, asset in task.localId.eq(asset.id) }.select(\.1.name).fetchOne(conn)
|
||||
}
|
||||
}
|
||||
|
||||
private var draftStub: UploadTask.Draft {
|
||||
.init(
|
||||
attempts: 0,
|
||||
createdAt: Date(),
|
||||
filePath: nil,
|
||||
isLivePhoto: nil,
|
||||
lastError: nil,
|
||||
livePhotoVideoId: nil,
|
||||
localId: nil,
|
||||
method: .multipart,
|
||||
priority: 0.5,
|
||||
retryAfter: nil,
|
||||
status: .downloadPending,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
extension UploadTask.TableColumns {
|
||||
var noFatalError: some QueryExpression<Bool> { lastError.is(nil) || !lastError.unwrapped.in(UploadErrorCode.fatal) }
|
||||
var canRetry: some QueryExpression<Bool> {
|
||||
attempts.lte(TaskConfig.maxRetries) && (retryAfter.is(nil) || retryAfter.unwrapped <= Date().unixTime)
|
||||
}
|
||||
}
|
||||
|
||||
extension LocalAlbum {
|
||||
static let selected = Self.where { $0.backupSelection.eq(BackupSelection.selected) }
|
||||
static let excluded = Self.where { $0.backupSelection.eq(BackupSelection.excluded) }
|
||||
}
|
||||
|
||||
extension LocalAlbumAsset {
|
||||
static let selected = Self.where {
|
||||
$0.id.assetId.eq(LocalAsset.columns.id) && $0.id.albumId.in(LocalAlbum.selected.select(\.id))
|
||||
}
|
||||
static let excluded = Self.where {
|
||||
$0.id.assetId.eq(LocalAsset.columns.id) && $0.id.albumId.in(LocalAlbum.excluded.select(\.id))
|
||||
}
|
||||
}
|
||||
|
||||
extension RemoteAsset {
|
||||
static let currentUser = Self.where { _ in
|
||||
ownerId.eq(Store.select(\.stringValue).where { $0.id.eq(StoreKey.currentUser.rawValue) }.unwrapped)
|
||||
}
|
||||
}
|
||||
|
||||
extension LocalAsset {
|
||||
static let withChecksum = Self.where { $0.checksum.isNot(nil) }
|
||||
static let shouldBackup = Self.where { _ in LocalAlbumAsset.selected.exists() && !LocalAlbumAsset.excluded.exists() }
|
||||
static let notBackedUp = Self.where { local in
|
||||
!RemoteAsset.currentUser.where { remote in local.checksum.eq(remote.checksum) }.exists()
|
||||
}
|
||||
static let backupCandidates = Self
|
||||
.shouldBackup
|
||||
.notBackedUp
|
||||
.where { local in !UploadTask.where { $0.localId.eq(local.id) }.exists() }
|
||||
.select { LocalAssetCandidate.Columns(id: $0.id, type: $0.type) }
|
||||
.limit { _ in UploadTaskStat.availableSlots }
|
||||
}
|
||||
Reference in New Issue
Block a user