From 55acafc1ab361e512bf231711a75752daba3a56b Mon Sep 17 00:00:00 2001 From: Thibault Deckers Date: Wed, 10 Feb 2021 11:32:14 +0900 Subject: [PATCH] #45 collection: find entries with obsolete paths --- .../deckers/thibault/aves/MainActivity.kt | 1 + .../aves/channel/calls/ImageFileHandler.kt | 55 +++++++---------- .../aves/channel/calls/MediaStoreHandler.kt | 43 +++++++++++++ .../model/provider/MediaStoreImageProvider.kt | 60 +++++++++++++------ lib/model/source/media_store_source.dart | 19 ++++-- lib/services/image_file_service.dart | 37 ++---------- lib/services/media_store_service.dart | 47 +++++++++++++++ 7 files changed, 175 insertions(+), 87 deletions(-) create mode 100644 android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MediaStoreHandler.kt create mode 100644 lib/services/media_store_service.dart diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/MainActivity.kt b/android/app/src/main/kotlin/deckers/thibault/aves/MainActivity.kt index a6742b22c..98e308d7f 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/MainActivity.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/MainActivity.kt @@ -33,6 +33,7 @@ class MainActivity : FlutterActivity() { MethodChannel(messenger, AppAdapterHandler.CHANNEL).setMethodCallHandler(AppAdapterHandler(this)) MethodChannel(messenger, AppShortcutHandler.CHANNEL).setMethodCallHandler(AppShortcutHandler(this)) MethodChannel(messenger, ImageFileHandler.CHANNEL).setMethodCallHandler(ImageFileHandler(this)) + MethodChannel(messenger, MediaStoreHandler.CHANNEL).setMethodCallHandler(MediaStoreHandler(this)) MethodChannel(messenger, MetadataHandler.CHANNEL).setMethodCallHandler(MetadataHandler(this)) MethodChannel(messenger, StorageHandler.CHANNEL).setMethodCallHandler(StorageHandler(this)) MethodChannel(messenger, DebugHandler.CHANNEL).setMethodCallHandler(DebugHandler(this)) diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/ImageFileHandler.kt b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/ImageFileHandler.kt index a9c831565..5af8a45f3 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/ImageFileHandler.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/ImageFileHandler.kt @@ -14,7 +14,6 @@ import deckers.thibault.aves.model.ExifOrientationOp import deckers.thibault.aves.model.FieldMap import deckers.thibault.aves.model.provider.ImageProvider.ImageOpCallback import deckers.thibault.aves.model.provider.ImageProviderFactory.getProvider -import deckers.thibault.aves.model.provider.MediaStoreImageProvider import deckers.thibault.aves.utils.MimeTypes import io.flutter.plugin.common.MethodCall import io.flutter.plugin.common.MethodChannel @@ -31,25 +30,35 @@ class ImageFileHandler(private val activity: Activity) : MethodCallHandler { override fun onMethodCall(call: MethodCall, result: MethodChannel.Result) { when (call.method) { - "getObsoleteEntries" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::getObsoleteEntries) } "getEntry" -> GlobalScope.launch(Dispatchers.IO) { safesus(call, result, ::getEntry) } "getThumbnail" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::getThumbnail) } "getRegion" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::getRegion) } - "clearSizedThumbnailDiskCache" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::clearSizedThumbnailDiskCache) } "rename" -> GlobalScope.launch(Dispatchers.IO) { safesus(call, result, ::rename) } "rotate" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::rotate) } "flip" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::flip) } + "clearSizedThumbnailDiskCache" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::clearSizedThumbnailDiskCache) } else -> result.notImplemented() } } - private fun getObsoleteEntries(call: MethodCall, result: MethodChannel.Result) { - val known = call.argument>("knownContentIds") - if (known == null) { - result.error("getObsoleteEntries-args", "failed because of missing arguments", null) + private suspend fun getEntry(call: MethodCall, result: MethodChannel.Result) { + val mimeType = call.argument("mimeType") // MIME type is optional + val uri = call.argument("uri")?.let { Uri.parse(it) } + if (uri == null) { + result.error("getEntry-args", "failed because of missing arguments", null) return } - result.success(MediaStoreImageProvider().getObsoleteContentIds(activity, known)) + + val provider = getProvider(uri) + if (provider == null) { + result.error("getEntry-provider", "failed to find provider for uri=$uri", null) + return + } + + provider.fetchSingle(activity, uri, mimeType, object : ImageOpCallback { + override fun onSuccess(fields: FieldMap) = result.success(fields) + override fun onFailure(throwable: Throwable) = result.error("getEntry-failure", "failed to get entry for uri=$uri", throwable.message) + }) } private fun getThumbnail(call: MethodCall, result: MethodChannel.Result) { @@ -122,31 +131,6 @@ class ImageFileHandler(private val activity: Activity) : MethodCallHandler { } } - private suspend fun getEntry(call: MethodCall, result: MethodChannel.Result) { - val mimeType = call.argument("mimeType") // MIME type is optional - val uri = call.argument("uri")?.let { Uri.parse(it) } - if (uri == null) { - result.error("getEntry-args", "failed because of missing arguments", null) - return - } - - val provider = getProvider(uri) - if (provider == null) { - result.error("getEntry-provider", "failed to find provider for uri=$uri", null) - return - } - - provider.fetchSingle(activity, uri, mimeType, object : ImageOpCallback { - override fun onSuccess(fields: FieldMap) = result.success(fields) - override fun onFailure(throwable: Throwable) = result.error("getEntry-failure", "failed to get entry for uri=$uri", throwable.message) - }) - } - - private fun clearSizedThumbnailDiskCache(@Suppress("UNUSED_PARAMETER") call: MethodCall, result: MethodChannel.Result) { - Glide.get(activity).clearDiskCache() - result.success(null) - } - private suspend fun rename(call: MethodCall, result: MethodChannel.Result) { val entryMap = call.argument("entry") val newName = call.argument("newName") @@ -217,6 +201,11 @@ class ImageFileHandler(private val activity: Activity) : MethodCallHandler { }) } + private fun clearSizedThumbnailDiskCache(@Suppress("UNUSED_PARAMETER") call: MethodCall, result: MethodChannel.Result) { + Glide.get(activity).clearDiskCache() + result.success(null) + } + companion object { const val CHANNEL = "deckers.thibault/aves/image" } diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MediaStoreHandler.kt b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MediaStoreHandler.kt new file mode 100644 index 000000000..9ac2992a0 --- /dev/null +++ b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MediaStoreHandler.kt @@ -0,0 +1,43 @@ +package deckers.thibault.aves.channel.calls + +import android.app.Activity +import deckers.thibault.aves.channel.calls.Coresult.Companion.safe +import deckers.thibault.aves.model.provider.MediaStoreImageProvider +import io.flutter.plugin.common.MethodCall +import io.flutter.plugin.common.MethodChannel +import io.flutter.plugin.common.MethodChannel.MethodCallHandler +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.GlobalScope +import kotlinx.coroutines.launch + +class MediaStoreHandler(private val activity: Activity) : MethodCallHandler { + override fun onMethodCall(call: MethodCall, result: MethodChannel.Result) { + when (call.method) { + "checkObsoleteContentIds" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::checkObsoleteContentIds) } + "checkObsoletePaths" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::checkObsoletePaths) } + else -> result.notImplemented() + } + } + + private fun checkObsoleteContentIds(call: MethodCall, result: MethodChannel.Result) { + val knownContentIds = call.argument>("knownContentIds") + if (knownContentIds == null) { + result.error("checkObsoleteContentIds-args", "failed because of missing arguments", null) + return + } + result.success(MediaStoreImageProvider().checkObsoleteContentIds(activity, knownContentIds)) + } + + private fun checkObsoletePaths(call: MethodCall, result: MethodChannel.Result) { + val knownPathById = call.argument>("knownPathById") + if (knownPathById == null) { + result.error("checkObsoletePaths-args", "failed because of missing arguments", null) + return + } + result.success(MediaStoreImageProvider().checkObsoletePaths(activity, knownPathById)) + } + + companion object { + const val CHANNEL = "deckers.thibault/aves/mediastore" + } +} \ No newline at end of file diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/model/provider/MediaStoreImageProvider.kt b/android/app/src/main/kotlin/deckers/thibault/aves/model/provider/MediaStoreImageProvider.kt index b2386f961..78e65b524 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/model/provider/MediaStoreImageProvider.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/model/provider/MediaStoreImageProvider.kt @@ -23,6 +23,7 @@ import deckers.thibault.aves.utils.UriUtils.tryParseId import kotlinx.coroutines.delay import java.io.File import java.util.* +import kotlin.collections.ArrayList class MediaStoreImageProvider : ImageProvider() { suspend fun fetchAll(context: Context, knownEntries: Map, handleNewEntry: NewEntryHandler) { @@ -59,30 +60,53 @@ class MediaStoreImageProvider : ImageProvider() { callback.onFailure(Exception("failed to fetch entry at uri=$uri")) } - fun getObsoleteContentIds(context: Context, knownContentIds: List): List { - val current = arrayListOf().apply { - addAll(getContentIdList(context, IMAGE_CONTENT_URI)) - addAll(getContentIdList(context, VIDEO_CONTENT_URI)) + fun checkObsoleteContentIds(context: Context, knownContentIds: List): List { + val foundContentIds = ArrayList() + fun check(context: Context, contentUri: Uri) { + val projection = arrayOf(MediaStore.MediaColumns._ID) + try { + val cursor = context.contentResolver.query(contentUri, projection, null, null, null) + if (cursor != null) { + val idColumn = cursor.getColumnIndexOrThrow(MediaStore.MediaColumns._ID) + while (cursor.moveToNext()) { + foundContentIds.add(cursor.getInt(idColumn)) + } + cursor.close() + } + } catch (e: Exception) { + Log.e(LOG_TAG, "failed to get content IDs for contentUri=$contentUri", e) + } } - return knownContentIds.filter { id: Int -> !current.contains(id) }.toList() + check(context, IMAGE_CONTENT_URI) + check(context, VIDEO_CONTENT_URI) + return knownContentIds.filter { id: Int -> !foundContentIds.contains(id) }.toList() } - private fun getContentIdList(context: Context, contentUri: Uri): List { - val foundContentIds = ArrayList() - val projection = arrayOf(MediaStore.MediaColumns._ID) - try { - val cursor = context.contentResolver.query(contentUri, projection, null, null, null) - if (cursor != null) { - val idColumn = cursor.getColumnIndexOrThrow(MediaStore.MediaColumns._ID) - while (cursor.moveToNext()) { - foundContentIds.add(cursor.getInt(idColumn)) + fun checkObsoletePaths(context: Context, knownPathById: Map): List { + val obsoleteIds = ArrayList() + fun check(context: Context, contentUri: Uri) { + val projection = arrayOf(MediaStore.MediaColumns._ID, MediaColumns.PATH) + try { + val cursor = context.contentResolver.query(contentUri, projection, null, null, null) + if (cursor != null) { + val idColumn = cursor.getColumnIndexOrThrow(MediaStore.MediaColumns._ID) + val pathColumn = cursor.getColumnIndexOrThrow(MediaColumns.PATH) + while (cursor.moveToNext()) { + val id = cursor.getInt(idColumn) + val path = cursor.getString(pathColumn) + if (knownPathById.containsKey(id) && knownPathById[id] != path) { + obsoleteIds.add(id) + } + } + cursor.close() } - cursor.close() + } catch (e: Exception) { + Log.e(LOG_TAG, "failed to get content IDs for contentUri=$contentUri", e) } - } catch (e: Exception) { - Log.e(LOG_TAG, "failed to get content IDs for contentUri=$contentUri", e) } - return foundContentIds + check(context, IMAGE_CONTENT_URI) + check(context, VIDEO_CONTENT_URI) + return obsoleteIds } private suspend fun fetchFrom( diff --git a/lib/model/source/media_store_source.dart b/lib/model/source/media_store_source.dart index 8b098132b..103d82ec3 100644 --- a/lib/model/source/media_store_source.dart +++ b/lib/model/source/media_store_source.dart @@ -7,6 +7,7 @@ import 'package:aves/model/metadata_db.dart'; import 'package:aves/model/settings/settings.dart'; import 'package:aves/model/source/collection_source.dart'; import 'package:aves/services/image_file_service.dart'; +import 'package:aves/services/media_store_service.dart'; import 'package:aves/utils/android_file_utils.dart'; import 'package:aves/utils/math_utils.dart'; import 'package:firebase_analytics/firebase_analytics.dart'; @@ -49,8 +50,8 @@ class MediaStoreSource extends CollectionSource { clearEntries(); final oldEntries = await metadataDb.loadEntries(); // 400ms for 5500 entries - final knownEntryMap = Map.fromEntries(oldEntries.map((entry) => MapEntry(entry.contentId, entry.dateModifiedSecs))); - final obsoleteContentIds = (await ImageFileService.getObsoleteEntries(knownEntryMap.keys.toList())).toSet(); + final knownDateById = Map.fromEntries(oldEntries.map((entry) => MapEntry(entry.contentId, entry.dateModifiedSecs))); + final obsoleteContentIds = (await MediaStoreService.checkObsoleteContentIds(knownDateById.keys.toList())).toSet(); oldEntries.removeWhere((entry) => obsoleteContentIds.contains(entry.contentId)); // show known entries @@ -62,6 +63,13 @@ class MediaStoreSource extends CollectionSource { // clean up obsolete entries metadataDb.removeIds(obsoleteContentIds, updateFavourites: true); + // verify paths because some apps move files without updating their `last modified date` + final knownPathById = Map.fromEntries(allEntries.map((entry) => MapEntry(entry.contentId, entry.path))); + final movedContentIds = (await MediaStoreService.checkObsoletePaths(knownPathById)).toSet(); + movedContentIds.forEach((contentId) { + knownDateById[contentId] = 0; + }); + // fetch new entries // refresh after the first 10 entries, then after 100 more, then every 1000 entries var refreshCount = 10; @@ -73,7 +81,7 @@ class MediaStoreSource extends CollectionSource { pendingNewEntries.clear(); } - ImageFileService.getEntries(knownEntryMap).listen( + MediaStoreService.getEntries(knownDateById).listen( (entry) { pendingNewEntries.add(entry); if (pendingNewEntries.length >= refreshCount) { @@ -124,7 +132,7 @@ class MediaStoreSource extends CollectionSource { }).where((kv) => kv != null)); // clean up obsolete entries - final obsoleteContentIds = (await ImageFileService.getObsoleteEntries(uriByContentId.keys.toList())).toSet(); + final obsoleteContentIds = (await MediaStoreService.checkObsoleteContentIds(uriByContentId.keys.toList())).toSet(); final obsoleteUris = obsoleteContentIds.map((contentId) => uriByContentId[contentId]).toSet(); removeEntries(obsoleteUris); obsoleteContentIds.forEach(uriByContentId.remove); @@ -138,7 +146,8 @@ class MediaStoreSource extends CollectionSource { final sourceEntry = await ImageFileService.getEntry(uri, null); if (sourceEntry != null) { final existingEntry = allEntries.firstWhere((entry) => entry.contentId == contentId, orElse: () => null); - if (existingEntry == null || sourceEntry.dateModifiedSecs > existingEntry.dateModifiedSecs) { + // compare paths because some apps move files without updating their `last modified date` + if (existingEntry == null || sourceEntry.dateModifiedSecs > existingEntry.dateModifiedSecs || sourceEntry.path != existingEntry.path) { final volume = androidFileUtils.getStorageVolume(sourceEntry.path); if (volume != null) { newEntries.add(sourceEntry); diff --git a/lib/services/image_file_service.dart b/lib/services/image_file_service.dart index 75ad7eda1..20b75ecee 100644 --- a/lib/services/image_file_service.dart +++ b/lib/services/image_file_service.dart @@ -13,9 +13,8 @@ import 'package:streams_channel/streams_channel.dart'; class ImageFileService { static const platform = MethodChannel('deckers.thibault/aves/image'); - static final StreamsChannel mediaStoreChannel = StreamsChannel('deckers.thibault/aves/mediastorestream'); - static final StreamsChannel byteChannel = StreamsChannel('deckers.thibault/aves/imagebytestream'); - static final StreamsChannel opChannel = StreamsChannel('deckers.thibault/aves/imageopstream'); + static final StreamsChannel _byteStreamChannel = StreamsChannel('deckers.thibault/aves/imagebytestream'); + static final StreamsChannel _opStreamChannel = StreamsChannel('deckers.thibault/aves/imageopstream'); static const double thumbnailDefaultSize = 64.0; static Map _toPlatformEntryMap(AvesEntry entry) { @@ -32,30 +31,6 @@ class ImageFileService { }; } - // knownEntries: map of contentId -> dateModifiedSecs - static Stream getEntries(Map knownEntries) { - try { - return mediaStoreChannel.receiveBroadcastStream({ - 'knownEntries': knownEntries, - }).map((event) => AvesEntry.fromMap(event)); - } on PlatformException catch (e) { - debugPrint('getEntries failed with code=${e.code}, exception=${e.message}, details=${e.details}'); - return Stream.error(e); - } - } - - static Future> getObsoleteEntries(List knownContentIds) async { - try { - final result = await platform.invokeMethod('getObsoleteEntries', { - 'knownContentIds': knownContentIds, - }); - return (result as List).cast(); - } on PlatformException catch (e) { - debugPrint('getObsoleteEntries failed with code=${e.code}, exception=${e.message}, details=${e.details}'); - } - return []; - } - static Future getEntry(String uri, String mimeType) async { try { final result = await platform.invokeMethod('getEntry', { @@ -97,7 +72,7 @@ class ImageFileService { final completer = Completer.sync(); final sink = _OutputBuffer(); var bytesReceived = 0; - byteChannel.receiveBroadcastStream({ + _byteStreamChannel.receiveBroadcastStream({ 'uri': uri, 'mimeType': mimeType, 'rotationDegrees': rotationDegrees ?? 0, @@ -225,7 +200,7 @@ class ImageFileService { static Stream delete(Iterable entries) { try { - return opChannel.receiveBroadcastStream({ + return _opStreamChannel.receiveBroadcastStream({ 'op': 'delete', 'entries': entries.map(_toPlatformEntryMap).toList(), }).map((event) => ImageOpEvent.fromMap(event)); @@ -241,7 +216,7 @@ class ImageFileService { @required String destinationAlbum, }) { try { - return opChannel.receiveBroadcastStream({ + return _opStreamChannel.receiveBroadcastStream({ 'op': 'move', 'entries': entries.map(_toPlatformEntryMap).toList(), 'copy': copy, @@ -259,7 +234,7 @@ class ImageFileService { @required String destinationAlbum, }) { try { - return opChannel.receiveBroadcastStream({ + return _opStreamChannel.receiveBroadcastStream({ 'op': 'export', 'entries': entries.map(_toPlatformEntryMap).toList(), 'mimeType': mimeType, diff --git a/lib/services/media_store_service.dart b/lib/services/media_store_service.dart new file mode 100644 index 000000000..43f380358 --- /dev/null +++ b/lib/services/media_store_service.dart @@ -0,0 +1,47 @@ +import 'dart:async'; + +import 'package:aves/model/entry.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter/widgets.dart'; +import 'package:streams_channel/streams_channel.dart'; + +class MediaStoreService { + static const platform = MethodChannel('deckers.thibault/aves/mediastore'); + static final StreamsChannel _streamChannel = StreamsChannel('deckers.thibault/aves/mediastorestream'); + + static Future> checkObsoleteContentIds(List knownContentIds) async { + try { + final result = await platform.invokeMethod('checkObsoleteContentIds', { + 'knownContentIds': knownContentIds, + }); + return (result as List).cast(); + } on PlatformException catch (e) { + debugPrint('checkObsoleteContentIds failed with code=${e.code}, exception=${e.message}, details=${e.details}'); + } + return []; + } + + static Future> checkObsoletePaths(Map knownPathById) async { + try { + final result = await platform.invokeMethod('checkObsoletePaths', { + 'knownPathById': knownPathById, + }); + return (result as List).cast(); + } on PlatformException catch (e) { + debugPrint('checkObsoletePaths failed with code=${e.code}, exception=${e.message}, details=${e.details}'); + } + return []; + } + + // knownEntries: map of contentId -> dateModifiedSecs + static Stream getEntries(Map knownEntries) { + try { + return _streamChannel.receiveBroadcastStream({ + 'knownEntries': knownEntries, + }).map((event) => AvesEntry.fromMap(event)); + } on PlatformException catch (e) { + debugPrint('getEntries failed with code=${e.code}, exception=${e.message}, details=${e.details}'); + return Stream.error(e); + } + } +}