mirror of
https://github.com/immich-app/immich.git
synced 2026-05-18 03:10:24 +03:00
feat: opt-in sync of deletes and restores from web to Android (beta timeline) (#20473)
* feature(mobile, beta, Android): handle remote asset trash/restore events and rescan media - Handle move to trash and restore from trash for remote assets on Android - Trigger MediaScannerConnection to rescan affected media files * feature(mobile, beta, Android): fix rescan * fix imports * fix checking conditions * refactor naming * fix line breaks * refactor code rollback changes in BackgroundServicePlugin * refactor code (use separate TrashService) * refactor code * parallelize restoreFromTrash calls with Future.wait format trash.provider.dart * try to re-format trash.provider.dart * re-format trash.provider.dart * rename TrashService to TrashSyncService to avoid duplicated names revert changes in original trash.provider.dart * refactor code (minor nitpicks) * process restoreFromTrash sequentially instead of Future.wait * group local assets by checksum before moving to trash delete LocalAssetEntity records when moved to trash refactor code * fix format * use checksum for asset restoration refactro code * fix format * sync trash only for backup-selected assets * feat(db): add local_trashed_asset table and integrate with restoration flow - Add new `local_trashed_asset` table to store metadata of trashed assets - Save trashed asset info into `local_trashed_asset` before deletion - Use `local_trashed_asset` as source for asset restoration - Implement file restoration by `mediaId` * resolve merge conflicts * fix index creating on migration * rework trashed assets handling - add new table trashed_local_asset - mirror trashed assets data in trashed_local_asset. - compute checksums for assets trashed out-of-app. - restore assets present in trashed_local_asset and non-trashed in remote_asset. - simplify moving-to-trash logic based on remote_asset events. * resolve merge conflicts use updated approach for calculating checksums * use CurrentPlatform instead _platform fix mocks * revert redundant changes * Include trashed items in getMediaChanges Process trashed items delta during incremental sync * fix merge conflicts * fix format * trashed_local_asset table mirror of local_asset table structure trashed_local_asset<->local_asset transfer data on move to trash or restore refactor code * refactor and format code * refactor TrashedAsset model fix missed data transfering * refactor code remove unused model * fix label * fix merge conflicts * optimize, refactor code remove redundant code and checking getTrashedAssetsForAlbum for iOS tests for hash trashed assets * format code * fix migration fix tests * fix generated file * reuse exist checksums on trash data update handle restoration errors fix import * format code * sync_stream.service depend on repos refactor assets restoration update dependencies in tests * remove trashed asset model remove trash_sync.service refactor DriftTrashedLocalAssetRepository, LocalSyncService * rework fetching trashed assets data on native side optimize handling trashed assets in local sync service refactor code * update NativeSyncApi on iOS side remove unused code * optimize sync trashed assets call in full sync mode refactor code * fix format * remove albumIds from getTrashedAssets params fix upsert in trashed local asset repo refactor code * fix getTrashedAssets params * fix(trash-sync): clean up NativeSyncApiImplBase and correct applyDelta * refactor(trash-sync): optimize performance and fix minor issues * refactor(trash-sync): add missed index * feat(trash-sync): remove sinceLastCheckpoint param from getTrashedAssets * fix(trash-sync): fix target table * fix(trash-sync): remove unused extension * fix(trash-sync): remove unused code * fix(trash-sync): refactor code * fix(trash-sync): reformat file * fix(trash_sync): refactor code * fix(trash_sync): improve moving to trash * refactor(trash_sync): integrate MANAGE_MEDIA permission request into login flow and advanced settings * refactor(trash_sync): add additional checking for experimental trash sync flag and MANAGE_MEDIA permission. * refactor(trash_sync): resolve merge conflicts * refactor(trash_sync): fix format * resolve merge conflicts add await for alert dialog add missed request * refactor(trash_sync): rework MANAGE_MEDIA info widget show rationale text in permission request alert dialog refactor setting getter * fix(trash_sync): restore missing text values * fix(trash_sync): format file * fix(trash_sync): check backup enabled and remove remote asset existence check * fix(trash_sync): remove checking backup enabled test(trash_sync): cover sync-stream trash/restore paths and dedupe mocks * test(trash_sync): cover trash/restore flows for local_sync_service * chore(e2e): restore test-assets submodule pointer --------- Co-authored-by: Peter Ombodi <peter.ombodi@gmail.com> Co-authored-by: Alex <alex.tran1502@gmail.com>
This commit is contained in:
Peter Ombodi
@@ -2,8 +2,10 @@ import 'package:flutter/services.dart';
|
||||
import 'package:immich_mobile/constants/constants.dart';
|
||||
import 'package:immich_mobile/domain/models/album/local_album.model.dart';
|
||||
import 'package:immich_mobile/domain/models/asset/base_asset.model.dart';
|
||||
import 'package:immich_mobile/extensions/platform_extensions.dart';
|
||||
import 'package:immich_mobile/infrastructure/repositories/local_album.repository.dart';
|
||||
import 'package:immich_mobile/infrastructure/repositories/local_asset.repository.dart';
|
||||
import 'package:immich_mobile/infrastructure/repositories/trashed_local_asset.repository.dart';
|
||||
import 'package:immich_mobile/platform/native_sync_api.g.dart';
|
||||
import 'package:logging/logging.dart';
|
||||
|
||||
@@ -13,6 +15,7 @@ class HashService {
|
||||
final int _batchSize;
|
||||
final DriftLocalAlbumRepository _localAlbumRepository;
|
||||
final DriftLocalAssetRepository _localAssetRepository;
|
||||
final DriftTrashedLocalAssetRepository _trashedLocalAssetRepository;
|
||||
final NativeSyncApi _nativeSyncApi;
|
||||
final bool Function()? _cancelChecker;
|
||||
final _log = Logger('HashService');
|
||||
@@ -20,11 +23,13 @@ class HashService {
|
||||
HashService({
|
||||
required DriftLocalAlbumRepository localAlbumRepository,
|
||||
required DriftLocalAssetRepository localAssetRepository,
|
||||
required DriftTrashedLocalAssetRepository trashedLocalAssetRepository,
|
||||
required NativeSyncApi nativeSyncApi,
|
||||
bool Function()? cancelChecker,
|
||||
int? batchSize,
|
||||
}) : _localAlbumRepository = localAlbumRepository,
|
||||
_localAssetRepository = localAssetRepository,
|
||||
_trashedLocalAssetRepository = trashedLocalAssetRepository,
|
||||
_cancelChecker = cancelChecker,
|
||||
_nativeSyncApi = nativeSyncApi,
|
||||
_batchSize = batchSize ?? kBatchHashFileLimit;
|
||||
@@ -49,6 +54,14 @@ class HashService {
|
||||
await _hashAssets(album, assetsToHash);
|
||||
}
|
||||
}
|
||||
if (CurrentPlatform.isAndroid && localAlbums.isNotEmpty) {
|
||||
final backupAlbumIds = localAlbums.map((e) => e.id);
|
||||
final trashedToHash = await _trashedLocalAssetRepository.getAssetsToHash(backupAlbumIds);
|
||||
if (trashedToHash.isNotEmpty) {
|
||||
final pseudoAlbum = LocalAlbum(id: '-pseudoAlbum', name: 'Trash', updatedAt: DateTime.now());
|
||||
await _hashAssets(pseudoAlbum, trashedToHash, isTrashed: true);
|
||||
}
|
||||
}
|
||||
} on PlatformException catch (e) {
|
||||
if (e.code == _kHashCancelledCode) {
|
||||
_log.warning("Hashing cancelled by platform");
|
||||
@@ -65,7 +78,7 @@ class HashService {
|
||||
/// Processes a list of [LocalAsset]s, storing their hash and updating the assets in the DB
|
||||
/// with hash for those that were successfully hashed. Hashes are looked up in a table
|
||||
/// [LocalAssetHashEntity] by local id. Only missing entries are newly hashed and added to the DB.
|
||||
Future<void> _hashAssets(LocalAlbum album, List<LocalAsset> assetsToHash) async {
|
||||
Future<void> _hashAssets(LocalAlbum album, List<LocalAsset> assetsToHash, {bool isTrashed = false}) async {
|
||||
final toHash = <String, LocalAsset>{};
|
||||
|
||||
for (final asset in assetsToHash) {
|
||||
@@ -76,16 +89,16 @@ class HashService {
|
||||
|
||||
toHash[asset.id] = asset;
|
||||
if (toHash.length == _batchSize) {
|
||||
await _processBatch(album, toHash);
|
||||
await _processBatch(album, toHash, isTrashed);
|
||||
toHash.clear();
|
||||
}
|
||||
}
|
||||
|
||||
await _processBatch(album, toHash);
|
||||
await _processBatch(album, toHash, isTrashed);
|
||||
}
|
||||
|
||||
/// Processes a batch of assets.
|
||||
Future<void> _processBatch(LocalAlbum album, Map<String, LocalAsset> toHash) async {
|
||||
Future<void> _processBatch(LocalAlbum album, Map<String, LocalAsset> toHash, bool isTrashed) async {
|
||||
if (toHash.isEmpty) {
|
||||
return;
|
||||
}
|
||||
@@ -120,7 +133,10 @@ class HashService {
|
||||
}
|
||||
|
||||
_log.fine("Hashed ${hashed.length}/${toHash.length} assets");
|
||||
|
||||
await _localAssetRepository.updateHashes(hashed);
|
||||
if (isTrashed) {
|
||||
await _trashedLocalAssetRepository.updateHashes(hashed);
|
||||
} else {
|
||||
await _localAssetRepository.updateHashes(hashed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,9 +4,14 @@ import 'package:collection/collection.dart';
|
||||
import 'package:flutter/foundation.dart';
|
||||
import 'package:immich_mobile/domain/models/album/local_album.model.dart';
|
||||
import 'package:immich_mobile/domain/models/asset/base_asset.model.dart';
|
||||
import 'package:immich_mobile/domain/models/store.model.dart';
|
||||
import 'package:immich_mobile/entities/store.entity.dart';
|
||||
import 'package:immich_mobile/extensions/platform_extensions.dart';
|
||||
import 'package:immich_mobile/infrastructure/repositories/local_album.repository.dart';
|
||||
import 'package:immich_mobile/infrastructure/repositories/storage.repository.dart';
|
||||
import 'package:immich_mobile/infrastructure/repositories/trashed_local_asset.repository.dart';
|
||||
import 'package:immich_mobile/platform/native_sync_api.g.dart';
|
||||
import 'package:immich_mobile/repositories/local_files_manager.repository.dart';
|
||||
import 'package:immich_mobile/utils/datetime_helpers.dart';
|
||||
import 'package:immich_mobile/utils/diff.dart';
|
||||
import 'package:logging/logging.dart';
|
||||
@@ -14,15 +19,34 @@ import 'package:logging/logging.dart';
|
||||
class LocalSyncService {
|
||||
final DriftLocalAlbumRepository _localAlbumRepository;
|
||||
final NativeSyncApi _nativeSyncApi;
|
||||
final DriftTrashedLocalAssetRepository _trashedLocalAssetRepository;
|
||||
final LocalFilesManagerRepository _localFilesManager;
|
||||
final StorageRepository _storageRepository;
|
||||
final Logger _log = Logger("DeviceSyncService");
|
||||
|
||||
LocalSyncService({required DriftLocalAlbumRepository localAlbumRepository, required NativeSyncApi nativeSyncApi})
|
||||
: _localAlbumRepository = localAlbumRepository,
|
||||
_nativeSyncApi = nativeSyncApi;
|
||||
LocalSyncService({
|
||||
required DriftLocalAlbumRepository localAlbumRepository,
|
||||
required DriftTrashedLocalAssetRepository trashedLocalAssetRepository,
|
||||
required LocalFilesManagerRepository localFilesManager,
|
||||
required StorageRepository storageRepository,
|
||||
required NativeSyncApi nativeSyncApi,
|
||||
}) : _localAlbumRepository = localAlbumRepository,
|
||||
_trashedLocalAssetRepository = trashedLocalAssetRepository,
|
||||
_localFilesManager = localFilesManager,
|
||||
_storageRepository = storageRepository,
|
||||
_nativeSyncApi = nativeSyncApi;
|
||||
|
||||
Future<void> sync({bool full = false}) async {
|
||||
final Stopwatch stopwatch = Stopwatch()..start();
|
||||
try {
|
||||
if (CurrentPlatform.isAndroid && Store.get(StoreKey.manageLocalMediaAndroid, false)) {
|
||||
final hasPermission = await _localFilesManager.hasManageMediaPermission();
|
||||
if (hasPermission) {
|
||||
await _syncTrashedAssets();
|
||||
} else {
|
||||
_log.warning("syncTrashedAssets cannot proceed because MANAGE_MEDIA permission is missing");
|
||||
}
|
||||
}
|
||||
if (full || await _nativeSyncApi.shouldFullSync()) {
|
||||
_log.fine("Full sync request from ${full ? "user" : "native"}");
|
||||
return await fullSync();
|
||||
@@ -69,7 +93,6 @@ class LocalSyncService {
|
||||
await updateAlbum(dbAlbum, album);
|
||||
}
|
||||
}
|
||||
|
||||
await _nativeSyncApi.checkpointSync();
|
||||
} catch (e, s) {
|
||||
_log.severe("Error performing device sync", e, s);
|
||||
@@ -273,6 +296,48 @@ class LocalSyncService {
|
||||
bool _albumsEqual(LocalAlbum a, LocalAlbum b) {
|
||||
return a.name == b.name && a.assetCount == b.assetCount && a.updatedAt.isAtSameMomentAs(b.updatedAt);
|
||||
}
|
||||
|
||||
Future<void> _syncTrashedAssets() async {
|
||||
final trashedAssetMap = await _nativeSyncApi.getTrashedAssets();
|
||||
await processTrashedAssets(trashedAssetMap);
|
||||
}
|
||||
|
||||
@visibleForTesting
|
||||
Future<void> processTrashedAssets(Map<String, List<PlatformAsset>> trashedAssetMap) async {
|
||||
if (trashedAssetMap.isEmpty) {
|
||||
_log.info("syncTrashedAssets, No trashed assets found");
|
||||
}
|
||||
final trashedAssets = trashedAssetMap.cast<String, List<Object?>>().entries.expand(
|
||||
(entry) => entry.value.cast<PlatformAsset>().toTrashedAssets(entry.key),
|
||||
);
|
||||
|
||||
_log.fine("syncTrashedAssets, trashedAssets: ${trashedAssets.map((e) => e.asset.id)}");
|
||||
await _trashedLocalAssetRepository.processTrashSnapshot(trashedAssets);
|
||||
|
||||
final assetsToRestore = await _trashedLocalAssetRepository.getToRestore();
|
||||
if (assetsToRestore.isNotEmpty) {
|
||||
final restoredIds = await _localFilesManager.restoreAssetsFromTrash(assetsToRestore);
|
||||
await _trashedLocalAssetRepository.applyRestoredAssets(restoredIds);
|
||||
} else {
|
||||
_log.info("syncTrashedAssets, No remote assets found for restoration");
|
||||
}
|
||||
|
||||
final localAssetsToTrash = await _trashedLocalAssetRepository.getToTrash();
|
||||
if (localAssetsToTrash.isNotEmpty) {
|
||||
final mediaUrls = await Future.wait(
|
||||
localAssetsToTrash.values
|
||||
.expand((e) => e)
|
||||
.map((localAsset) => _storageRepository.getAssetEntityForAsset(localAsset).then((e) => e?.getMediaUrl())),
|
||||
);
|
||||
_log.info("Moving to trash ${mediaUrls.join(", ")} assets");
|
||||
final result = await _localFilesManager.moveToTrash(mediaUrls.nonNulls.toList());
|
||||
if (result) {
|
||||
await _trashedLocalAssetRepository.trashLocalAsset(localAssetsToTrash);
|
||||
}
|
||||
} else {
|
||||
_log.info("syncTrashedAssets, No assets found in backup-enabled albums for move to trash");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
extension on Iterable<PlatformAlbum> {
|
||||
@@ -290,20 +355,26 @@ extension on Iterable<PlatformAlbum> {
|
||||
|
||||
extension on Iterable<PlatformAsset> {
|
||||
List<LocalAsset> toLocalAssets() {
|
||||
return map(
|
||||
(e) => LocalAsset(
|
||||
id: e.id,
|
||||
name: e.name,
|
||||
checksum: null,
|
||||
type: AssetType.values.elementAtOrNull(e.type) ?? AssetType.other,
|
||||
createdAt: tryFromSecondsSinceEpoch(e.createdAt, isUtc: true) ?? DateTime.timestamp(),
|
||||
updatedAt: tryFromSecondsSinceEpoch(e.updatedAt, isUtc: true) ?? DateTime.timestamp(),
|
||||
width: e.width,
|
||||
height: e.height,
|
||||
durationInSeconds: e.durationInSeconds,
|
||||
orientation: e.orientation,
|
||||
isFavorite: e.isFavorite,
|
||||
),
|
||||
).toList();
|
||||
return map((e) => e.toLocalAsset()).toList();
|
||||
}
|
||||
|
||||
Iterable<TrashedAsset> toTrashedAssets(String albumId) {
|
||||
return map((e) => (albumId: albumId, asset: e.toLocalAsset()));
|
||||
}
|
||||
}
|
||||
|
||||
extension on PlatformAsset {
|
||||
LocalAsset toLocalAsset() => LocalAsset(
|
||||
id: id,
|
||||
name: name,
|
||||
checksum: null,
|
||||
type: AssetType.values.elementAtOrNull(type) ?? AssetType.other,
|
||||
createdAt: tryFromSecondsSinceEpoch(createdAt, isUtc: true) ?? DateTime.timestamp(),
|
||||
updatedAt: tryFromSecondsSinceEpoch(createdAt, isUtc: true) ?? DateTime.timestamp(),
|
||||
width: width,
|
||||
height: height,
|
||||
durationInSeconds: durationInSeconds,
|
||||
isFavorite: isFavorite,
|
||||
orientation: orientation,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,8 +1,15 @@
|
||||
import 'dart:async';
|
||||
|
||||
import 'package:immich_mobile/domain/models/store.model.dart';
|
||||
import 'package:immich_mobile/domain/models/sync_event.model.dart';
|
||||
import 'package:immich_mobile/entities/store.entity.dart';
|
||||
import 'package:immich_mobile/extensions/platform_extensions.dart';
|
||||
import 'package:immich_mobile/infrastructure/repositories/local_asset.repository.dart';
|
||||
import 'package:immich_mobile/infrastructure/repositories/storage.repository.dart';
|
||||
import 'package:immich_mobile/infrastructure/repositories/sync_api.repository.dart';
|
||||
import 'package:immich_mobile/infrastructure/repositories/sync_stream.repository.dart';
|
||||
import 'package:immich_mobile/infrastructure/repositories/trashed_local_asset.repository.dart';
|
||||
import 'package:immich_mobile/repositories/local_files_manager.repository.dart';
|
||||
import 'package:logging/logging.dart';
|
||||
import 'package:openapi/api.dart';
|
||||
|
||||
@@ -11,14 +18,26 @@ class SyncStreamService {
|
||||
|
||||
final SyncApiRepository _syncApiRepository;
|
||||
final SyncStreamRepository _syncStreamRepository;
|
||||
final DriftLocalAssetRepository _localAssetRepository;
|
||||
final DriftTrashedLocalAssetRepository _trashedLocalAssetRepository;
|
||||
final LocalFilesManagerRepository _localFilesManager;
|
||||
final StorageRepository _storageRepository;
|
||||
final bool Function()? _cancelChecker;
|
||||
|
||||
SyncStreamService({
|
||||
required SyncApiRepository syncApiRepository,
|
||||
required SyncStreamRepository syncStreamRepository,
|
||||
required DriftLocalAssetRepository localAssetRepository,
|
||||
required DriftTrashedLocalAssetRepository trashedLocalAssetRepository,
|
||||
required LocalFilesManagerRepository localFilesManager,
|
||||
required StorageRepository storageRepository,
|
||||
bool Function()? cancelChecker,
|
||||
}) : _syncApiRepository = syncApiRepository,
|
||||
_syncStreamRepository = syncStreamRepository,
|
||||
_localAssetRepository = localAssetRepository,
|
||||
_trashedLocalAssetRepository = trashedLocalAssetRepository,
|
||||
_localFilesManager = localFilesManager,
|
||||
_storageRepository = storageRepository,
|
||||
_cancelChecker = cancelChecker;
|
||||
|
||||
bool get isCancelled => _cancelChecker?.call() ?? false;
|
||||
@@ -83,7 +102,18 @@ class SyncStreamService {
|
||||
case SyncEntityType.partnerDeleteV1:
|
||||
return _syncStreamRepository.deletePartnerV1(data.cast());
|
||||
case SyncEntityType.assetV1:
|
||||
return _syncStreamRepository.updateAssetsV1(data.cast());
|
||||
final remoteSyncAssets = data.cast<SyncAssetV1>();
|
||||
await _syncStreamRepository.updateAssetsV1(remoteSyncAssets);
|
||||
if (CurrentPlatform.isAndroid && Store.get(StoreKey.manageLocalMediaAndroid, false)) {
|
||||
final hasPermission = await _localFilesManager.hasManageMediaPermission();
|
||||
if (hasPermission) {
|
||||
await _handleRemoteTrashed(remoteSyncAssets.where((e) => e.deletedAt != null).map((e) => e.checksum));
|
||||
await _applyRemoteRestoreToLocal();
|
||||
} else {
|
||||
_logger.warning("sync Trashed Assets cannot proceed because MANAGE_MEDIA permission is missing");
|
||||
}
|
||||
}
|
||||
return;
|
||||
case SyncEntityType.assetDeleteV1:
|
||||
return _syncStreamRepository.deleteAssetsV1(data.cast());
|
||||
case SyncEntityType.assetExifV1:
|
||||
@@ -212,4 +242,36 @@ class SyncStreamService {
|
||||
_logger.severe("Error processing AssetUploadReadyV1 websocket batch events", error, stackTrace);
|
||||
}
|
||||
}
|
||||
|
||||
Future<void> _handleRemoteTrashed(Iterable<String> checksums) async {
|
||||
if (checksums.isEmpty) {
|
||||
return Future.value();
|
||||
} else {
|
||||
final localAssetsToTrash = await _localAssetRepository.getAssetsFromBackupAlbums(checksums);
|
||||
if (localAssetsToTrash.isNotEmpty) {
|
||||
final mediaUrls = await Future.wait(
|
||||
localAssetsToTrash.values
|
||||
.expand((e) => e)
|
||||
.map((localAsset) => _storageRepository.getAssetEntityForAsset(localAsset).then((e) => e?.getMediaUrl())),
|
||||
);
|
||||
_logger.info("Moving to trash ${mediaUrls.join(", ")} assets");
|
||||
final result = await _localFilesManager.moveToTrash(mediaUrls.nonNulls.toList());
|
||||
if (result) {
|
||||
await _trashedLocalAssetRepository.trashLocalAsset(localAssetsToTrash);
|
||||
}
|
||||
} else {
|
||||
_logger.info("No assets found in backup-enabled albums for assets: $checksums");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Future<void> _applyRemoteRestoreToLocal() async {
|
||||
final assetsToRestore = await _trashedLocalAssetRepository.getToRestore();
|
||||
if (assetsToRestore.isNotEmpty) {
|
||||
final restoredIds = await _localFilesManager.restoreAssetsFromTrash(assetsToRestore);
|
||||
await _trashedLocalAssetRepository.applyRestoredAssets(restoredIds);
|
||||
} else {
|
||||
_logger.info("No remote assets found for restoration");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user