#45 collection: find entries with obsolete paths

android/app/src/main/kotlin/deckers/thibault/aves/MainActivity.kt

@@ -33,6 +33,7 @@ class MainActivity : FlutterActivity() {
         MethodChannel(messenger, AppAdapterHandler.CHANNEL).setMethodCallHandler(AppAdapterHandler(this))
         MethodChannel(messenger, AppShortcutHandler.CHANNEL).setMethodCallHandler(AppShortcutHandler(this))
         MethodChannel(messenger, ImageFileHandler.CHANNEL).setMethodCallHandler(ImageFileHandler(this))
+        MethodChannel(messenger, MediaStoreHandler.CHANNEL).setMethodCallHandler(MediaStoreHandler(this))
         MethodChannel(messenger, MetadataHandler.CHANNEL).setMethodCallHandler(MetadataHandler(this))
         MethodChannel(messenger, StorageHandler.CHANNEL).setMethodCallHandler(StorageHandler(this))
         MethodChannel(messenger, DebugHandler.CHANNEL).setMethodCallHandler(DebugHandler(this))

android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/ImageFileHandler.kt

@@ -14,7 +14,6 @@ import deckers.thibault.aves.model.ExifOrientationOp
 import deckers.thibault.aves.model.FieldMap
 import deckers.thibault.aves.model.provider.ImageProvider.ImageOpCallback
 import deckers.thibault.aves.model.provider.ImageProviderFactory.getProvider
-import deckers.thibault.aves.model.provider.MediaStoreImageProvider
 import deckers.thibault.aves.utils.MimeTypes
 import io.flutter.plugin.common.MethodCall
 import io.flutter.plugin.common.MethodChannel
@@ -31,25 +30,35 @@ class ImageFileHandler(private val activity: Activity) : MethodCallHandler {
 
     override fun onMethodCall(call: MethodCall, result: MethodChannel.Result) {
         when (call.method) {
-            "getObsoleteEntries" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::getObsoleteEntries) }
             "getEntry" -> GlobalScope.launch(Dispatchers.IO) { safesus(call, result, ::getEntry) }
             "getThumbnail" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::getThumbnail) }
             "getRegion" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::getRegion) }
-            "clearSizedThumbnailDiskCache" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::clearSizedThumbnailDiskCache) }
             "rename" -> GlobalScope.launch(Dispatchers.IO) { safesus(call, result, ::rename) }
             "rotate" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::rotate) }
             "flip" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::flip) }
+            "clearSizedThumbnailDiskCache" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::clearSizedThumbnailDiskCache) }
             else -> result.notImplemented()
         }
     }
 
-    private fun getObsoleteEntries(call: MethodCall, result: MethodChannel.Result) {
+    private suspend fun getEntry(call: MethodCall, result: MethodChannel.Result) {
-        val known = call.argument<List<Int>>("knownContentIds")
+        val mimeType = call.argument<String>("mimeType") // MIME type is optional
-        if (known == null) {
+        val uri = call.argument<String>("uri")?.let { Uri.parse(it) }
-            result.error("getObsoleteEntries-args", "failed because of missing arguments", null)
+        if (uri == null) {
+            result.error("getEntry-args", "failed because of missing arguments", null)
             return
         }
-        result.success(MediaStoreImageProvider().getObsoleteContentIds(activity, known))
+
+        val provider = getProvider(uri)
+        if (provider == null) {
+            result.error("getEntry-provider", "failed to find provider for uri=$uri", null)
+            return
+        }
+
+        provider.fetchSingle(activity, uri, mimeType, object : ImageOpCallback {
+            override fun onSuccess(fields: FieldMap) = result.success(fields)
+            override fun onFailure(throwable: Throwable) = result.error("getEntry-failure", "failed to get entry for uri=$uri", throwable.message)
+        })
     }
 
     private fun getThumbnail(call: MethodCall, result: MethodChannel.Result) {
@@ -122,31 +131,6 @@ class ImageFileHandler(private val activity: Activity) : MethodCallHandler {
         }
     }
 
-    private suspend fun getEntry(call: MethodCall, result: MethodChannel.Result) {
-        val mimeType = call.argument<String>("mimeType") // MIME type is optional
-        val uri = call.argument<String>("uri")?.let { Uri.parse(it) }
-        if (uri == null) {
-            result.error("getEntry-args", "failed because of missing arguments", null)
-            return
-        }
-
-        val provider = getProvider(uri)
-        if (provider == null) {
-            result.error("getEntry-provider", "failed to find provider for uri=$uri", null)
-            return
-        }
-
-        provider.fetchSingle(activity, uri, mimeType, object : ImageOpCallback {
-            override fun onSuccess(fields: FieldMap) = result.success(fields)
-            override fun onFailure(throwable: Throwable) = result.error("getEntry-failure", "failed to get entry for uri=$uri", throwable.message)
-        })
-    }
-
-    private fun clearSizedThumbnailDiskCache(@Suppress("UNUSED_PARAMETER") call: MethodCall, result: MethodChannel.Result) {
-        Glide.get(activity).clearDiskCache()
-        result.success(null)
-    }
-
     private suspend fun rename(call: MethodCall, result: MethodChannel.Result) {
         val entryMap = call.argument<FieldMap>("entry")
         val newName = call.argument<String>("newName")
@@ -217,6 +201,11 @@ class ImageFileHandler(private val activity: Activity) : MethodCallHandler {
         })
     }
 
+    private fun clearSizedThumbnailDiskCache(@Suppress("UNUSED_PARAMETER") call: MethodCall, result: MethodChannel.Result) {
+        Glide.get(activity).clearDiskCache()
+        result.success(null)
+    }
+
     companion object {
         const val CHANNEL = "deckers.thibault/aves/image"
     }

android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MediaStoreHandler.kt

@@ -0,0 +1,43 @@
+package deckers.thibault.aves.channel.calls
+
+import android.app.Activity
+import deckers.thibault.aves.channel.calls.Coresult.Companion.safe
+import deckers.thibault.aves.model.provider.MediaStoreImageProvider
+import io.flutter.plugin.common.MethodCall
+import io.flutter.plugin.common.MethodChannel
+import io.flutter.plugin.common.MethodChannel.MethodCallHandler
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.GlobalScope
+import kotlinx.coroutines.launch
+
+class MediaStoreHandler(private val activity: Activity) : MethodCallHandler {
+    override fun onMethodCall(call: MethodCall, result: MethodChannel.Result) {
+        when (call.method) {
+            "checkObsoleteContentIds" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::checkObsoleteContentIds) }
+            "checkObsoletePaths" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::checkObsoletePaths) }
+            else -> result.notImplemented()
+        }
+    }
+
+    private fun checkObsoleteContentIds(call: MethodCall, result: MethodChannel.Result) {
+        val knownContentIds = call.argument<List<Int>>("knownContentIds")
+        if (knownContentIds == null) {
+            result.error("checkObsoleteContentIds-args", "failed because of missing arguments", null)
+            return
+        }
+        result.success(MediaStoreImageProvider().checkObsoleteContentIds(activity, knownContentIds))
+    }
+
+    private fun checkObsoletePaths(call: MethodCall, result: MethodChannel.Result) {
+        val knownPathById = call.argument<Map<Int, String>>("knownPathById")
+        if (knownPathById == null) {
+            result.error("checkObsoletePaths-args", "failed because of missing arguments", null)
+            return
+        }
+        result.success(MediaStoreImageProvider().checkObsoletePaths(activity, knownPathById))
+    }
+
+    companion object {
+        const val CHANNEL = "deckers.thibault/aves/mediastore"
+    }
+}

android/app/src/main/kotlin/deckers/thibault/aves/model/provider/MediaStoreImageProvider.kt

@@ -23,6 +23,7 @@ import deckers.thibault.aves.utils.UriUtils.tryParseId
 import kotlinx.coroutines.delay
 import java.io.File
 import java.util.*
+import kotlin.collections.ArrayList
 
 class MediaStoreImageProvider : ImageProvider() {
     suspend fun fetchAll(context: Context, knownEntries: Map<Int, Int?>, handleNewEntry: NewEntryHandler) {
@@ -59,16 +60,9 @@ class MediaStoreImageProvider : ImageProvider() {
         callback.onFailure(Exception("failed to fetch entry at uri=$uri"))
     }
 
-    fun getObsoleteContentIds(context: Context, knownContentIds: List<Int>): List<Int> {
+    fun checkObsoleteContentIds(context: Context, knownContentIds: List<Int>): List<Int> {
-        val current = arrayListOf<Int>().apply {
-            addAll(getContentIdList(context, IMAGE_CONTENT_URI))
-            addAll(getContentIdList(context, VIDEO_CONTENT_URI))
-        }
-        return knownContentIds.filter { id: Int -> !current.contains(id) }.toList()
-    }
-
-    private fun getContentIdList(context: Context, contentUri: Uri): List<Int> {
         val foundContentIds = ArrayList<Int>()
+        fun check(context: Context, contentUri: Uri) {
             val projection = arrayOf(MediaStore.MediaColumns._ID)
             try {
                 val cursor = context.contentResolver.query(contentUri, projection, null, null, null)
@@ -82,7 +76,37 @@ class MediaStoreImageProvider : ImageProvider() {
             } catch (e: Exception) {
                 Log.e(LOG_TAG, "failed to get content IDs for contentUri=$contentUri", e)
             }
-        return foundContentIds
+        }
+        check(context, IMAGE_CONTENT_URI)
+        check(context, VIDEO_CONTENT_URI)
+        return knownContentIds.filter { id: Int -> !foundContentIds.contains(id) }.toList()
+    }
+
+    fun checkObsoletePaths(context: Context, knownPathById: Map<Int, String>): List<Int> {
+        val obsoleteIds = ArrayList<Int>()
+        fun check(context: Context, contentUri: Uri) {
+            val projection = arrayOf(MediaStore.MediaColumns._ID, MediaColumns.PATH)
+            try {
+                val cursor = context.contentResolver.query(contentUri, projection, null, null, null)
+                if (cursor != null) {
+                    val idColumn = cursor.getColumnIndexOrThrow(MediaStore.MediaColumns._ID)
+                    val pathColumn = cursor.getColumnIndexOrThrow(MediaColumns.PATH)
+                    while (cursor.moveToNext()) {
+                        val id = cursor.getInt(idColumn)
+                        val path = cursor.getString(pathColumn)
+                        if (knownPathById.containsKey(id) && knownPathById[id] != path) {
+                            obsoleteIds.add(id)
+                        }
+                    }
+                    cursor.close()
+                }
+            } catch (e: Exception) {
+                Log.e(LOG_TAG, "failed to get content IDs for contentUri=$contentUri", e)
+            }
+        }
+        check(context, IMAGE_CONTENT_URI)
+        check(context, VIDEO_CONTENT_URI)
+        return obsoleteIds
     }
 
     private suspend fun fetchFrom(

lib/model/source/media_store_source.dart

@@ -7,6 +7,7 @@ import 'package:aves/model/metadata_db.dart';
 import 'package:aves/model/settings/settings.dart';
 import 'package:aves/model/source/collection_source.dart';
 import 'package:aves/services/image_file_service.dart';
+import 'package:aves/services/media_store_service.dart';
 import 'package:aves/utils/android_file_utils.dart';
 import 'package:aves/utils/math_utils.dart';
 import 'package:firebase_analytics/firebase_analytics.dart';
@@ -49,8 +50,8 @@ class MediaStoreSource extends CollectionSource {
     clearEntries();
 
     final oldEntries = await metadataDb.loadEntries(); // 400ms for 5500 entries
-    final knownEntryMap = Map.fromEntries(oldEntries.map((entry) => MapEntry(entry.contentId, entry.dateModifiedSecs)));
+    final knownDateById = Map.fromEntries(oldEntries.map((entry) => MapEntry(entry.contentId, entry.dateModifiedSecs)));
-    final obsoleteContentIds = (await ImageFileService.getObsoleteEntries(knownEntryMap.keys.toList())).toSet();
+    final obsoleteContentIds = (await MediaStoreService.checkObsoleteContentIds(knownDateById.keys.toList())).toSet();
     oldEntries.removeWhere((entry) => obsoleteContentIds.contains(entry.contentId));
 
     // show known entries
@@ -62,6 +63,13 @@ class MediaStoreSource extends CollectionSource {
     // clean up obsolete entries
     metadataDb.removeIds(obsoleteContentIds, updateFavourites: true);
 
+    // verify paths because some apps move files without updating their `last modified date`
+    final knownPathById = Map.fromEntries(allEntries.map((entry) => MapEntry(entry.contentId, entry.path)));
+    final movedContentIds = (await MediaStoreService.checkObsoletePaths(knownPathById)).toSet();
+    movedContentIds.forEach((contentId) {
+      knownDateById[contentId] = 0;
+    });
+
     // fetch new entries
     // refresh after the first 10 entries, then after 100 more, then every 1000 entries
     var refreshCount = 10;
@@ -73,7 +81,7 @@ class MediaStoreSource extends CollectionSource {
       pendingNewEntries.clear();
     }
 
-    ImageFileService.getEntries(knownEntryMap).listen(
+    MediaStoreService.getEntries(knownDateById).listen(
       (entry) {
         pendingNewEntries.add(entry);
         if (pendingNewEntries.length >= refreshCount) {
@@ -124,7 +132,7 @@ class MediaStoreSource extends CollectionSource {
     }).where((kv) => kv != null));
 
     // clean up obsolete entries
-    final obsoleteContentIds = (await ImageFileService.getObsoleteEntries(uriByContentId.keys.toList())).toSet();
+    final obsoleteContentIds = (await MediaStoreService.checkObsoleteContentIds(uriByContentId.keys.toList())).toSet();
     final obsoleteUris = obsoleteContentIds.map((contentId) => uriByContentId[contentId]).toSet();
     removeEntries(obsoleteUris);
     obsoleteContentIds.forEach(uriByContentId.remove);
@@ -138,7 +146,8 @@ class MediaStoreSource extends CollectionSource {
       final sourceEntry = await ImageFileService.getEntry(uri, null);
       if (sourceEntry != null) {
         final existingEntry = allEntries.firstWhere((entry) => entry.contentId == contentId, orElse: () => null);
-        if (existingEntry == null || sourceEntry.dateModifiedSecs > existingEntry.dateModifiedSecs) {
+        // compare paths because some apps move files without updating their `last modified date`
+        if (existingEntry == null || sourceEntry.dateModifiedSecs > existingEntry.dateModifiedSecs || sourceEntry.path != existingEntry.path) {
           final volume = androidFileUtils.getStorageVolume(sourceEntry.path);
           if (volume != null) {
             newEntries.add(sourceEntry);

lib/services/image_file_service.dart

@@ -13,9 +13,8 @@ import 'package:streams_channel/streams_channel.dart';
 
 class ImageFileService {
   static const platform = MethodChannel('deckers.thibault/aves/image');
-  static final StreamsChannel mediaStoreChannel = StreamsChannel('deckers.thibault/aves/mediastorestream');
+  static final StreamsChannel _byteStreamChannel = StreamsChannel('deckers.thibault/aves/imagebytestream');
-  static final StreamsChannel byteChannel = StreamsChannel('deckers.thibault/aves/imagebytestream');
+  static final StreamsChannel _opStreamChannel = StreamsChannel('deckers.thibault/aves/imageopstream');
-  static final StreamsChannel opChannel = StreamsChannel('deckers.thibault/aves/imageopstream');
   static const double thumbnailDefaultSize = 64.0;
 
   static Map<String, dynamic> _toPlatformEntryMap(AvesEntry entry) {
@@ -32,30 +31,6 @@ class ImageFileService {
     };
   }
 
-  // knownEntries: map of contentId -> dateModifiedSecs
-  static Stream<AvesEntry> getEntries(Map<int, int> knownEntries) {
-    try {
-      return mediaStoreChannel.receiveBroadcastStream(<String, dynamic>{
-        'knownEntries': knownEntries,
-      }).map((event) => AvesEntry.fromMap(event));
-    } on PlatformException catch (e) {
-      debugPrint('getEntries failed with code=${e.code}, exception=${e.message}, details=${e.details}');
-      return Stream.error(e);
-    }
-  }
-
-  static Future<List<int>> getObsoleteEntries(List<int> knownContentIds) async {
-    try {
-      final result = await platform.invokeMethod('getObsoleteEntries', <String, dynamic>{
-        'knownContentIds': knownContentIds,
-      });
-      return (result as List).cast<int>();
-    } on PlatformException catch (e) {
-      debugPrint('getObsoleteEntries failed with code=${e.code}, exception=${e.message}, details=${e.details}');
-    }
-    return [];
-  }
-
   static Future<AvesEntry> getEntry(String uri, String mimeType) async {
     try {
       final result = await platform.invokeMethod('getEntry', <String, dynamic>{
@@ -97,7 +72,7 @@ class ImageFileService {
       final completer = Completer<Uint8List>.sync();
       final sink = _OutputBuffer();
       var bytesReceived = 0;
-      byteChannel.receiveBroadcastStream(<String, dynamic>{
+      _byteStreamChannel.receiveBroadcastStream(<String, dynamic>{
         'uri': uri,
         'mimeType': mimeType,
         'rotationDegrees': rotationDegrees ?? 0,
@@ -225,7 +200,7 @@ class ImageFileService {
 
   static Stream<ImageOpEvent> delete(Iterable<AvesEntry> entries) {
     try {
-      return opChannel.receiveBroadcastStream(<String, dynamic>{
+      return _opStreamChannel.receiveBroadcastStream(<String, dynamic>{
         'op': 'delete',
         'entries': entries.map(_toPlatformEntryMap).toList(),
       }).map((event) => ImageOpEvent.fromMap(event));
@@ -241,7 +216,7 @@ class ImageFileService {
     @required String destinationAlbum,
   }) {
     try {
-      return opChannel.receiveBroadcastStream(<String, dynamic>{
+      return _opStreamChannel.receiveBroadcastStream(<String, dynamic>{
         'op': 'move',
         'entries': entries.map(_toPlatformEntryMap).toList(),
         'copy': copy,
@@ -259,7 +234,7 @@ class ImageFileService {
     @required String destinationAlbum,
   }) {
     try {
-      return opChannel.receiveBroadcastStream(<String, dynamic>{
+      return _opStreamChannel.receiveBroadcastStream(<String, dynamic>{
         'op': 'export',
         'entries': entries.map(_toPlatformEntryMap).toList(),
         'mimeType': mimeType,

lib/services/media_store_service.dart

@@ -0,0 +1,47 @@
+import 'dart:async';
+
+import 'package:aves/model/entry.dart';
+import 'package:flutter/services.dart';
+import 'package:flutter/widgets.dart';
+import 'package:streams_channel/streams_channel.dart';
+
+class MediaStoreService {
+  static const platform = MethodChannel('deckers.thibault/aves/mediastore');
+  static final StreamsChannel _streamChannel = StreamsChannel('deckers.thibault/aves/mediastorestream');
+
+  static Future<List<int>> checkObsoleteContentIds(List<int> knownContentIds) async {
+    try {
+      final result = await platform.invokeMethod('checkObsoleteContentIds', <String, dynamic>{
+        'knownContentIds': knownContentIds,
+      });
+      return (result as List).cast<int>();
+    } on PlatformException catch (e) {
+      debugPrint('checkObsoleteContentIds failed with code=${e.code}, exception=${e.message}, details=${e.details}');
+    }
+    return [];
+  }
+
+  static Future<List<int>> checkObsoletePaths(Map<int, String> knownPathById) async {
+    try {
+      final result = await platform.invokeMethod('checkObsoletePaths', <String, dynamic>{
+        'knownPathById': knownPathById,
+      });
+      return (result as List).cast<int>();
+    } on PlatformException catch (e) {
+      debugPrint('checkObsoletePaths failed with code=${e.code}, exception=${e.message}, details=${e.details}');
+    }
+    return [];
+  }
+
+  // knownEntries: map of contentId -> dateModifiedSecs
+  static Stream<AvesEntry> getEntries(Map<int, int> knownEntries) {
+    try {
+      return _streamChannel.receiveBroadcastStream(<String, dynamic>{
+        'knownEntries': knownEntries,
+      }).map((event) => AvesEntry.fromMap(event));
+    } on PlatformException catch (e) {
+      debugPrint('getEntries failed with code=${e.code}, exception=${e.message}, details=${e.details}');
+      return Stream.error(e);
+    }
+  }
+}