background upload plugin

add schemas

sync variants

formatting

initial implementation

use existing db, wip

move to separate folder

fix table definitions

wip

wiring it up
This commit is contained in:
mertalev
2025-11-09 20:09:08 -05:00
parent 1e1c4ac9d2
commit 41f013387f
67 changed files with 17482 additions and 1260 deletions
+105 -10
View File
@@ -1,12 +1,22 @@
import SQLiteData
struct Endpoint: Codable {
let url: URL
let status: Status
extension Notification.Name {
static let networkDidConnect = Notification.Name("networkDidConnect")
}
enum Status: String, Codable {
case loading, valid, error, unknown
}
enum TaskConfig {
static let maxActiveDownloads = 3
static let maxPendingDownloads = 50
static let maxPendingUploads = 50
static let maxRetries = 10
static let sessionId = "app.mertalev.immich.upload"
static let downloadCheckIntervalNs: UInt64 = 30_000_000_000 // 30 seconds
static let downloadTimeoutS = TimeInterval(60)
static let transferSpeedAlpha = 0.4
static let originalsDir = FileManager.default.temporaryDirectory.appendingPathComponent(
"originals",
isDirectory: true
)
}
enum StoreKey: Int, CaseIterable, QueryBindable {
@@ -47,8 +57,6 @@ enum StoreKey: Int, CaseIterable, QueryBindable {
static let deviceId = Typed<String>(rawValue: ._deviceId)
case _accessToken = 11
static let accessToken = Typed<String>(rawValue: ._accessToken)
case _serverEndpoint = 12
static let serverEndpoint = Typed<String>(rawValue: ._serverEndpoint)
case _sslClientCertData = 15
static let sslClientCertData = Typed<String>(rawValue: ._sslClientCertData)
case _sslClientPasswd = 16
@@ -67,10 +75,12 @@ enum StoreKey: Int, CaseIterable, QueryBindable {
static let externalEndpointList = Typed<[Endpoint]>(rawValue: ._externalEndpointList)
// MARK: - URL
case _localEndpoint = 134
static let localEndpoint = Typed<URL>(rawValue: ._localEndpoint)
case _serverUrl = 10
static let serverUrl = Typed<URL>(rawValue: ._serverUrl)
case _serverEndpoint = 12
static let serverEndpoint = Typed<URL>(rawValue: ._serverEndpoint)
case _localEndpoint = 134
static let localEndpoint = Typed<URL>(rawValue: ._localEndpoint)
// MARK: - Date
case _backupFailedSince = 5
@@ -160,6 +170,17 @@ enum StoreKey: Int, CaseIterable, QueryBindable {
}
}
enum UploadHeaders: String {
case reprDigest = "Repr-Digest"
case userToken = "X-Immich-User-Token"
case assetData = "X-Immich-Asset-Data"
}
enum TaskStatus: Int, QueryBindable {
case downloadPending, downloadQueued, downloadFailed, uploadPending, uploadQueued, uploadFailed, uploadComplete,
uploadSkipped
}
enum BackupSelection: Int, QueryBindable {
case selected, none, excluded
}
@@ -175,3 +196,77 @@ enum AlbumUserRole: Int, QueryBindable {
enum MemoryType: Int, QueryBindable {
case onThisDay
}
enum AssetVisibility: Int, QueryBindable {
case timeline, hidden, archive, locked
}
enum SourceType: String, QueryBindable {
case machineLearning = "machine-learning"
case exif, manual
}
enum UploadMethod: Int, QueryBindable {
case multipart, resumable
}
enum UploadError: Error {
case fileCreationFailed
case iCloudError(UploadErrorCode)
case photosError(UploadErrorCode)
}
enum UploadErrorCode: Int, QueryBindable {
case unknown
case assetNotFound
case fileNotFound
case resourceNotFound
case invalidResource
case encodingFailed
case writeFailed
case notEnoughSpace
case networkError
case photosInternalError
case photosUnknownError
case noServerUrl
case noDeviceId
case noAccessToken
case interrupted
case cancelled
case downloadStalled
case forceQuit
case outOfResources
case backgroundUpdatesDisabled
case uploadTimeout
case iCloudRateLimit
case iCloudThrottled
case invalidResponse
case badRequest
case internalServerError
}
enum AssetType: Int, QueryBindable {
case other, image, video, audio
}
enum AssetMediaStatus: String, Codable {
case created, replaced, duplicate
}
struct Endpoint: Codable {
let url: URL
let status: Status
enum Status: String, Codable {
case loading, valid, error, unknown
}
}
struct UploadSuccessResponse: Codable {
let status: AssetMediaStatus
let id: String
}
struct UploadErrorResponse: Codable {
let message: String
}
+69 -55
View File
@@ -4,35 +4,75 @@ enum StoreError: Error {
case invalidJSON(String)
case invalidURL(String)
case encodingFailed
case notFound
}
protocol StoreConvertible {
static var cacheKeyPath: ReferenceWritableKeyPath<StoreCache, [StoreKey: Self]> { get }
associatedtype StorageType
static func fromValue(_ value: StorageType) throws(StoreError) -> Self
static func toValue(_ value: Self) throws(StoreError) -> StorageType
}
extension StoreConvertible {
static func get(_ cache: StoreCache, key: StoreKey) -> Self? {
os_unfair_lock_lock(&cache.lock)
defer { os_unfair_lock_unlock(&cache.lock) }
return cache[keyPath: cacheKeyPath][key]
}
static func set(_ cache: StoreCache, key: StoreKey, value: Self?) {
os_unfair_lock_lock(&cache.lock)
defer { os_unfair_lock_unlock(&cache.lock) }
cache[keyPath: cacheKeyPath][key] = value
}
}
final class StoreCache {
fileprivate var lock = os_unfair_lock()
fileprivate var intCache: [StoreKey: Int] = [:]
fileprivate var boolCache: [StoreKey: Bool] = [:]
fileprivate var dateCache: [StoreKey: Date] = [:]
fileprivate var stringCache: [StoreKey: String] = [:]
fileprivate var urlCache: [StoreKey: URL] = [:]
fileprivate var endpointArrayCache: [StoreKey: [Endpoint]] = [:]
fileprivate var stringDictCache: [StoreKey: [String: String]] = [:]
func get<T: StoreConvertible>(_ key: StoreKey.Typed<T>) -> T? {
T.get(self, key: key.rawValue)
}
func set<T: StoreConvertible>(_ key: StoreKey.Typed<T>, value: T?) {
T.set(self, key: key.rawValue, value: value)
}
}
extension Int: StoreConvertible {
static let cacheKeyPath = \StoreCache.intCache
static func fromValue(_ value: Int) -> Int { value }
static func toValue(_ value: Int) -> Int { value }
}
extension Bool: StoreConvertible {
static let cacheKeyPath = \StoreCache.boolCache
static func fromValue(_ value: Int) -> Bool { value == 1 }
static func toValue(_ value: Bool) -> Int { value ? 1 : 0 }
}
extension Date: StoreConvertible {
static let cacheKeyPath = \StoreCache.dateCache
static func fromValue(_ value: Int) -> Date { Date(timeIntervalSince1970: TimeInterval(value) / 1000) }
static func toValue(_ value: Date) -> Int { Int(value.timeIntervalSince1970 * 1000) }
}
extension String: StoreConvertible {
static let cacheKeyPath = \StoreCache.stringCache
static func fromValue(_ value: String) -> String { value }
static func toValue(_ value: String) -> String { value }
}
extension URL: StoreConvertible {
static let cacheKeyPath = \StoreCache.urlCache
static func fromValue(_ value: String) throws(StoreError) -> URL {
guard let url = URL(string: value) else {
throw StoreError.invalidURL(value)
@@ -69,78 +109,52 @@ extension StoreConvertible where Self: Codable, StorageType == String {
}
}
extension Array: StoreConvertible where Element: Codable {
extension Array: StoreConvertible where Element == Endpoint {
static let cacheKeyPath = \StoreCache.endpointArrayCache
typealias StorageType = String
}
extension Dictionary: StoreConvertible where Key == String, Value: Codable {
extension Dictionary: StoreConvertible where Key == String, Value == String {
static let cacheKeyPath = \StoreCache.stringDictCache
typealias StorageType = String
}
class StoreRepository {
private let db: DatabasePool
init(db: DatabasePool) {
self.db = db
}
func get<T: StoreConvertible>(_ key: StoreKey.Typed<T>) throws -> T? where T.StorageType == Int {
extension Store {
static let cache = StoreCache()
static func get<T: StoreConvertible>(_ conn: Database, _ key: StoreKey.Typed<T>) throws -> T?
where T.StorageType == Int {
if let cached = cache.get(key) { return cached }
let query = Store.select(\.intValue).where { $0.id.eq(key.rawValue) }
if let value = try db.read({ conn in try query.fetchOne(conn) }) ?? nil {
return try T.fromValue(value)
if let value = try query.fetchOne(conn) ?? nil {
let converted = try T.fromValue(value)
cache.set(key, value: converted)
}
return nil
}
func get<T: StoreConvertible>(_ key: StoreKey.Typed<T>) throws -> T? where T.StorageType == String {
static func get<T: StoreConvertible>(_ conn: Database, _ key: StoreKey.Typed<T>) throws -> T?
where T.StorageType == String {
if let cached = cache.get(key) { return cached }
let query = Store.select(\.stringValue).where { $0.id.eq(key.rawValue) }
if let value = try db.read({ conn in try query.fetchOne(conn) }) ?? nil {
return try T.fromValue(value)
if let value = try query.fetchOne(conn) ?? nil {
let converted = try T.fromValue(value)
cache.set(key, value: converted)
}
return nil
}
func get<T: StoreConvertible>(_ key: StoreKey.Typed<T>) async throws -> T? where T.StorageType == Int {
let query = Store.select(\.intValue).where { $0.id.eq(key.rawValue) }
if let value = try await db.read({ conn in try query.fetchOne(conn) }) ?? nil {
return try T.fromValue(value)
}
return nil
static func set<T: StoreConvertible>(_ conn: Database, _ key: StoreKey.Typed<T>, value: T) throws
where T.StorageType == Int {
let converted = try T.toValue(value)
try Store.upsert { Store(id: key.rawValue, stringValue: nil, intValue: converted) }.execute(conn)
cache.set(key, value: value)
}
func get<T: StoreConvertible>(_ key: StoreKey.Typed<T>) async throws -> T? where T.StorageType == String {
let query = Store.select(\.stringValue).where { $0.id.eq(key.rawValue) }
if let value = try await db.read({ conn in try query.fetchOne(conn) }) ?? nil {
return try T.fromValue(value)
}
return nil
}
func set<T: StoreConvertible>(_ key: StoreKey.Typed<T>, value: T) throws where T.StorageType == Int {
let value = try T.toValue(value)
try db.write { conn in
try Store.upsert { Store(id: key.rawValue, stringValue: nil, intValue: value) }.execute(conn)
}
}
func set<T: StoreConvertible>(_ key: StoreKey.Typed<T>, value: T) throws where T.StorageType == String {
let value = try T.toValue(value)
try db.write { conn in
try Store.upsert { Store(id: key.rawValue, stringValue: value, intValue: nil) }.execute(conn)
}
}
func set<T: StoreConvertible>(_ key: StoreKey.Typed<T>, value: T) async throws where T.StorageType == Int {
let value = try T.toValue(value)
try await db.write { conn in
try Store.upsert { Store(id: key.rawValue, stringValue: nil, intValue: value) }.execute(conn)
}
}
func set<T: StoreConvertible>(_ key: StoreKey.Typed<T>, value: T) async throws where T.StorageType == String {
let value = try T.toValue(value)
try await db.write { conn in
try Store.upsert { Store(id: key.rawValue, stringValue: value, intValue: nil) }.execute(conn)
}
static func set<T: StoreConvertible>(_ conn: Database, _ key: StoreKey.Typed<T>, value: T) throws
where T.StorageType == String {
let converted = try T.toValue(value)
try Store.upsert { Store(id: key.rawValue, stringValue: converted, intValue: nil) }.execute(conn)
cache.set(key, value: value)
}
}
+243 -33
View File
@@ -1,70 +1,160 @@
import GRDB
import SQLiteData
extension QueryExpression where QueryValue: _OptionalProtocol {
// asserts column result cannot be nil
var unwrapped: SQLQueryExpression<QueryValue.Wrapped> {
SQLQueryExpression(self.queryFragment, as: QueryValue.Wrapped.self)
}
}
extension Date {
var unixTime: Date.UnixTimeRepresentation {
return Date.UnixTimeRepresentation(queryOutput: self)
}
}
@Table("asset_face_entity")
struct AssetFace {
struct AssetFace: Identifiable {
let id: String
let assetId: String
let personId: String?
@Column("asset_id")
let assetId: RemoteAsset.ID
@Column("person_id")
let personId: Person.ID?
@Column("image_width")
let imageWidth: Int
@Column("image_height")
let imageHeight: Int
@Column("bounding_box_x1")
let boundingBoxX1: Int
@Column("bounding_box_y1")
let boundingBoxY1: Int
@Column("bounding_box_x2")
let boundingBoxX2: Int
@Column("bounding_box_y2")
let boundingBoxY2: Int
let sourceType: String
@Column("source_type")
let sourceType: SourceType
}
@Table("auth_user_entity")
struct AuthUser {
struct AuthUser: Identifiable {
let id: String
let name: String
let email: String
@Column("is_admin")
let isAdmin: Bool
@Column("has_profile_image")
let hasProfileImage: Bool
@Column("profile_changed_at")
let profileChangedAt: Date
@Column("avatar_color")
let avatarColor: AvatarColor
@Column("quota_size_in_bytes")
let quotaSizeInBytes: Int
@Column("quota_usage_in_bytes")
let quotaUsageInBytes: Int
@Column("pin_code")
let pinCode: String?
}
@Table("local_album_entity")
struct LocalAlbum {
struct LocalAlbum: Identifiable {
let id: String
@Column("backup_selection")
let backupSelection: BackupSelection
let linkedRemoteAlbumId: String?
@Column("linked_remote_album_id")
let linkedRemoteAlbumId: RemoteAlbum.ID?
@Column("marker")
let marker_: Bool?
let name: String
@Column("is_ios_shared_album")
let isIosSharedAlbum: Bool
@Column("updated_at")
let updatedAt: Date
}
extension LocalAlbum {
static let selected = Self.where { $0.backupSelection.eq(BackupSelection.selected) }
static let excluded = Self.where { $0.backupSelection.eq(BackupSelection.excluded) }
}
@Table("local_album_asset_entity")
struct LocalAlbumAsset {
let id: ID
@Column("marker")
let marker_: String?
@Selection
struct ID {
@Column("asset_id")
let assetId: String
@Column("album_id")
let albumId: String
}
}
extension LocalAlbumAsset {
static let selected = Self.where {
$0.id.assetId.eq(LocalAsset.columns.id) && $0.id.albumId.in(LocalAlbum.selected.select(\.id))
}
static let excluded = Self.where {
$0.id.assetId.eq(LocalAsset.columns.id) && $0.id.albumId.in(LocalAlbum.excluded.select(\.id))
}
}
@Table("local_asset_entity")
struct LocalAsset {
struct LocalAsset: Identifiable {
let id: String
let checksum: String?
let createdAt: Date
@Column("created_at")
let createdAt: String
@Column("duration_in_seconds")
let durationInSeconds: Int?
let height: Int?
@Column("is_favorite")
let isFavorite: Bool
let name: String
let orientation: String
let type: Int
let updatedAt: Date
let type: AssetType
@Column("updated_at")
let updatedAt: String
let width: Int?
static func getCandidates() -> Where<LocalAsset> {
return Self.where { local in
LocalAlbumAsset.selected.exists()
&& !LocalAlbumAsset.excluded.exists()
&& !RemoteAsset.where {
local.checksum.eq($0.checksum)
&& $0.ownerId.eq(Store.select(\.stringValue).where { $0.id.eq(StoreKey.currentUser.rawValue) }.unwrapped)
}.exists()
&& !UploadTask.where { $0.localId.eq(local.id) }.exists()
}
}
}
@Selection
struct LocalAssetCandidate {
let id: LocalAsset.ID
let type: AssetType
}
@Selection
struct LocalAssetDownloadData {
let checksum: String?
let createdAt: String
let livePhotoVideoId: RemoteAsset.ID?
let localId: LocalAsset.ID
let taskId: UploadTask.ID
let updatedAt: String
}
@Selection
struct LocalAssetUploadData {
let filePath: URL
let priority: Float
let taskId: UploadTask.ID
let type: AssetType
}
@Table("memory_asset_entity")
@@ -73,63 +163,89 @@ struct MemoryAsset {
@Selection
struct ID {
@Column("asset_id")
let assetId: String
@Column("album_id")
let albumId: String
}
}
@Table("memory_entity")
struct Memory {
struct Memory: Identifiable {
let id: String
@Column("created_at")
let createdAt: Date
@Column("updated_at")
let updatedAt: Date
@Column("deleted_at")
let deletedAt: Date?
let ownerId: String
@Column("owner_id")
let ownerId: User.ID
let type: MemoryType
let data: String
@Column("is_saved")
let isSaved: Bool
@Column("memory_at")
let memoryAt: Date
@Column("seen_at")
let seenAt: Date?
@Column("show_at")
let showAt: Date?
@Column("hide_at")
let hideAt: Date?
}
@Table("partner_entity")
struct Partner {
let id: ID
@Column("in_timeline")
let inTimeline: Bool
@Selection
struct ID {
@Column("shared_by_id")
let sharedById: String
@Column("shared_with_id")
let sharedWithId: String
}
}
@Table("person_entity")
struct Person {
struct Person: Identifiable {
let id: String
@Column("created_at")
let createdAt: Date
@Column("updated_at")
let updatedAt: Date
@Column("owner_id")
let ownerId: String
let name: String
let faceAssetId: String?
@Column("face_asset_id")
let faceAssetId: AssetFace.ID?
@Column("is_favorite")
let isFavorite: Bool
@Column("is_hidden")
let isHidden: Bool
let color: String?
@Column("birth_date")
let birthDate: Date?
}
@Table("remote_album_entity")
struct RemoteAlbum {
struct RemoteAlbum: Identifiable {
let id: String
@Column("created_at")
let createdAt: Date
let description: String?
@Column("is_activity_enabled")
let isActivityEnabled: Bool
let name: String
let order: Int
@Column("owner_id")
let ownerId: String
let thumbnailAssetId: String?
@Column("thumbnail_asset_id")
let thumbnailAssetId: RemoteAsset.ID?
@Column("updated_at")
let updatedAt: Date
}
@@ -139,7 +255,9 @@ struct RemoteAlbumAsset {
@Selection
struct ID {
@Column("asset_id")
let assetId: String
@Column("album_id")
let albumId: String
}
}
@@ -151,40 +269,55 @@ struct RemoteAlbumUser {
@Selection
struct ID {
@Column("album_id")
let albumId: String
@Column("user_id")
let userId: String
}
}
@Table("remote_asset_entity")
struct RemoteAsset {
struct RemoteAsset: Identifiable {
let id: String
let checksum: String?
let checksum: String
@Column("is_favorite")
let isFavorite: Bool
@Column("deleted_at")
let deletedAt: Date?
let isFavorite: Int
let libraryId: String?
let livePhotoVideoId: String?
@Column("owner_id")
let ownerId: User.ID
@Column("local_date_time")
let localDateTime: Date?
let orientation: String
let ownerId: String
let stackId: String?
let visibility: Int
@Column("thumb_hash")
let thumbHash: String?
@Column("library_id")
let libraryId: String?
@Column("live_photo_video_id")
let livePhotoVideoId: String?
@Column("stack_id")
let stackId: Stack.ID?
let visibility: AssetVisibility
}
@Table("remote_exif_entity")
struct RemoteExif {
@Column(primaryKey: true)
let assetId: String
@Column("asset_id", primaryKey: true)
let assetId: RemoteAsset.ID
let city: String?
let state: String?
let country: String?
@Column("date_time_original")
let dateTimeOriginal: Date?
let description: String?
let height: Int?
let width: Int?
@Column("exposure_time")
let exposureTime: String?
@Column("f_number")
let fNumber: Double?
@Column("file_size")
let fileSize: Int?
@Column("focal_length")
let focalLength: Double?
let latitude: Double?
let longitude: Double?
@@ -193,34 +326,110 @@ struct RemoteExif {
let model: String?
let lens: String?
let orientation: String?
@Column("time_zone")
let timeZone: String?
let rating: Int?
@Column("projection_type")
let projectionType: String?
}
@Table("stack_entity")
struct Stack {
struct Stack: Identifiable {
let id: String
@Column("created_at")
let createdAt: Date
@Column("updated_at")
let updatedAt: Date
let ownerId: String
@Column("owner_id")
let ownerId: User.ID
@Column("primary_asset_id")
let primaryAssetId: String
}
@Table("store_entity")
struct Store {
struct Store: Identifiable {
let id: StoreKey
@Column("string_value")
let stringValue: String?
@Column("int_value")
let intValue: Int?
}
@Table("upload_tasks")
struct UploadTask: Identifiable {
let id: Int64
let attempts: Int
@Column("created_at", as: Date.UnixTimeRepresentation.self)
let createdAt: Date
@Column("file_path")
var filePath: URL?
@Column("is_live_photo")
let isLivePhoto: Bool?
@Column("last_error")
let lastError: UploadErrorCode?
@Column("live_photo_video_id")
let livePhotoVideoId: RemoteAsset.ID?
@Column("local_id")
var localId: LocalAsset.ID?
let method: UploadMethod
var priority: Float
@Column("retry_after", as: Date?.UnixTimeRepresentation.self)
let retryAfter: Date?
let status: TaskStatus
static func retryOrFail(code: UploadErrorCode, status: TaskStatus) -> Update<UploadTask, ()> {
return Self.update { row in
row.status = Case().when(row.attempts.lte(TaskConfig.maxRetries), then: TaskStatus.downloadPending).else(status)
row.attempts += 1
row.lastError = code
row.retryAfter = #sql("unixepoch('now') + (\(4 << row.attempts))")
}
}
}
@Table("upload_task_stats")
struct UploadTaskStat {
@Column("pending_downloads")
let pendingDownloads: Int
@Column("pending_uploads")
let pendingUploads: Int
@Column("queued_downloads")
let queuedDownloads: Int
@Column("queued_uploads")
let queuedUploads: Int
@Column("failed_downloads")
let failedDownloads: Int
@Column("failed_uploads")
let failedUploads: Int
@Column("completed_uploads")
let completedUploads: Int
@Column("skipped_uploads")
let skippedUploads: Int
static let availableDownloadSlots = Self.select {
TaskConfig.maxPendingDownloads - ($0.pendingDownloads + $0.queuedDownloads)
}
static let availableUploadSlots = Self.select {
TaskConfig.maxPendingUploads - ($0.pendingUploads + $0.queuedUploads)
}
static let availableSlots = Self.select {
TaskConfig.maxPendingUploads + TaskConfig.maxPendingDownloads
- ($0.pendingDownloads + $0.queuedDownloads + $0.pendingUploads + $0.queuedUploads)
}
}
@Table("user_entity")
struct User {
struct User: Identifiable {
let id: String
let name: String
let email: String
@Column("has_profile_image")
let hasProfileImage: Bool
@Column("profile_changed_at")
let profileChangedAt: Date
@Column("avatar_color")
let avatarColor: AvatarColor
}
@@ -231,6 +440,7 @@ struct UserMetadata {
@Selection
struct ID {
@Column("user_id")
let userId: String
let key: Date
}