save known entries in sqlite and only fetch from mediastore new/modified entries

android/app/src/main/java/deckers/thibault/aves/MainActivity.java

@@ -48,7 +48,7 @@ public class MainActivity extends FlutterActivity {
 
         new StreamsChannel(messenger, ImageByteStreamHandler.CHANNEL).setStreamHandlerFactory(args -> new ImageByteStreamHandler(this, args));
         new StreamsChannel(messenger, ImageOpStreamHandler.CHANNEL).setStreamHandlerFactory(args -> new ImageOpStreamHandler(this, args));
-        new StreamsChannel(messenger, MediaStoreStreamHandler.CHANNEL).setStreamHandlerFactory(args -> new MediaStoreStreamHandler(this));
+        new StreamsChannel(messenger, MediaStoreStreamHandler.CHANNEL).setStreamHandlerFactory(args -> new MediaStoreStreamHandler(this, args));
         new StreamsChannel(messenger, StorageAccessStreamHandler.CHANNEL).setStreamHandlerFactory(args -> new StorageAccessStreamHandler(this, args));
 
         new MethodChannel(messenger, VIEWER_CHANNEL).setMethodCallHandler(

android/app/src/main/java/deckers/thibault/aves/channel/calls/ImageFileHandler.java

@@ -9,11 +9,13 @@ import androidx.annotation.NonNull;
 
 import com.bumptech.glide.Glide;
 
+import java.util.List;
 import java.util.Map;
 
 import deckers.thibault.aves.model.ImageEntry;
 import deckers.thibault.aves.model.provider.ImageProvider;
 import deckers.thibault.aves.model.provider.ImageProviderFactory;
+import deckers.thibault.aves.model.provider.MediaStoreImageProvider;
 import io.flutter.plugin.common.MethodCall;
 import io.flutter.plugin.common.MethodChannel;
 
@@ -37,6 +39,9 @@ public class ImageFileHandler implements MethodChannel.MethodCallHandler {
     @Override
     public void onMethodCall(@NonNull MethodCall call, @NonNull MethodChannel.Result result) {
         switch (call.method) {
+            case "getObsoleteEntries":
+                new Thread(() -> getObsoleteEntries(call, new MethodResultWrapper(result))).start();
+                break;
             case "getImageEntry":
                 new Thread(() -> getImageEntry(call, new MethodResultWrapper(result))).start();
                 break;
@@ -79,6 +84,16 @@ public class ImageFileHandler implements MethodChannel.MethodCallHandler {
         new ImageDecodeTask(activity).execute(new ImageDecodeTask.Params(entry, width, height, defaultSize, result));
     }
 
+    private void getObsoleteEntries(@NonNull MethodCall call, @NonNull MethodChannel.Result result) {
+        List<Integer> known = call.argument("knownContentIds");
+        if (known == null) {
+            result.error("getObsoleteEntries-args", "failed because of missing arguments", null);
+            return;
+        }
+        List<Integer> obsolete = new MediaStoreImageProvider().getObsoleteContentIds(activity, known);
+        result.success(obsolete);
+    }
+
     private void getImageEntry(@NonNull MethodCall call, @NonNull MethodChannel.Result result) {
         String uriString = call.argument("uri");
         String mimeType = call.argument("mimeType");

android/app/src/main/java/deckers/thibault/aves/channel/streams/ImageOpStreamHandler.java

@@ -94,7 +94,7 @@ public class ImageOpStreamHandler implements EventChannel.StreamHandler {
         String destinationDir = (String) argMap.get("destinationPath");
         if (copy == null || destinationDir == null) return;
 
-        ArrayList<ImageEntry> entries = entryMapList.stream().map(ImageEntry::new).collect(Collectors.toCollection(ArrayList::new));
+        List<ImageEntry> entries = entryMapList.stream().map(ImageEntry::new).collect(Collectors.toList());
         provider.moveMultiple(activity, copy, destinationDir, entries, new ImageProvider.ImageOpCallback() {
             @Override
             public void onSuccess(Map<String, Object> fields) {

android/app/src/main/java/deckers/thibault/aves/channel/streams/MediaStoreStreamHandler.java

@@ -15,9 +15,15 @@ public class MediaStoreStreamHandler implements EventChannel.StreamHandler {
     private Activity activity;
     private EventChannel.EventSink eventSink;
     private Handler handler;
+    private Map<Integer, Integer> knownEntries;
 
-    public MediaStoreStreamHandler(Activity activity) {
+    @SuppressWarnings("unchecked")
+    public MediaStoreStreamHandler(Activity activity, Object arguments) {
         this.activity = activity;
+        if (arguments instanceof Map) {
+            Map<String, Object> argMap = (Map<String, Object>) arguments;
+            this.knownEntries = (Map<Integer, Integer>) argMap.get("knownEntries");
+        }
     }
 
     @Override
@@ -41,7 +47,7 @@ public class MediaStoreStreamHandler implements EventChannel.StreamHandler {
     }
 
     void fetchAll() {
-        new MediaStoreImageProvider().fetchAll(activity, this::success); // 350ms
+        new MediaStoreImageProvider().fetchAll(activity, knownEntries, this::success); // 350ms
         endOfStream();
     }
 }

android/app/src/main/java/deckers/thibault/aves/model/provider/ImageProvider.java

@@ -33,8 +33,8 @@ import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import deckers.thibault.aves.model.ImageEntry;
@@ -65,7 +65,7 @@ public abstract class ImageProvider {
         return Futures.immediateFailedFuture(new UnsupportedOperationException());
     }
 
-    public void moveMultiple(final Activity activity, Boolean copy, String destinationDir, ArrayList<ImageEntry> entries, @NonNull ImageOpCallback callback) {
+    public void moveMultiple(final Activity activity, Boolean copy, String destinationDir, List<ImageEntry> entries, @NonNull ImageOpCallback callback) {
         callback.onFailure(new UnsupportedOperationException());
     }
 

android/app/src/main/java/deckers/thibault/aves/model/provider/MediaStoreImageProvider.java

@@ -25,8 +25,10 @@ import java.io.File;
 import java.io.FileNotFoundException;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import deckers.thibault.aves.model.ImageEntry;
@@ -68,16 +70,13 @@ public class MediaStoreImageProvider extends ImageProvider {
                     MediaStore.Video.Media.ORIENTATION,
             } : new String[0]).flatMap(Stream::of).toArray(String[]::new);
 
-    public void fetchAll(Activity activity, NewEntryHandler newEntryHandler) {
+    public void fetchAll(Context context, Map<Integer, Integer> knownEntries, NewEntryHandler newEntryHandler) {
-        String orderBy;
+        NewEntryChecker isModified = (contentId, dateModifiedSecs) -> {
-        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
+            final Integer knownDate = knownEntries.get(contentId);
-            orderBy = MediaStore.MediaColumns.DATE_TAKEN + " DESC";
+            return knownDate == null || knownDate < dateModifiedSecs;
-        } else {
+        };
-            orderBy = MediaStore.MediaColumns.DATE_MODIFIED + " DESC";
+        fetchFrom(context, isModified, newEntryHandler, MediaStore.Images.Media.EXTERNAL_CONTENT_URI, IMAGE_PROJECTION);
-        }
+        fetchFrom(context, isModified, newEntryHandler, MediaStore.Video.Media.EXTERNAL_CONTENT_URI, VIDEO_PROJECTION);
-
-        fetchFrom(activity, newEntryHandler, MediaStore.Images.Media.EXTERNAL_CONTENT_URI, IMAGE_PROJECTION, orderBy);
-        fetchFrom(activity, newEntryHandler, MediaStore.Video.Media.EXTERNAL_CONTENT_URI, VIDEO_PROJECTION, orderBy);
     }
 
     @Override
@@ -88,23 +87,48 @@ public class MediaStoreImageProvider extends ImageProvider {
             entry.put("uri", uri.toString());
             callback.onSuccess(entry);
         };
+        NewEntryChecker alwaysValid = (contentId, dateModifiedSecs) -> true;
         if (mimeType.startsWith(MimeTypes.IMAGE)) {
             Uri contentUri = ContentUris.withAppendedId(MediaStore.Images.Media.EXTERNAL_CONTENT_URI, id);
-            entryCount = fetchFrom(context, onSuccess, contentUri, IMAGE_PROJECTION, null);
+            entryCount = fetchFrom(context, alwaysValid, onSuccess, contentUri, IMAGE_PROJECTION);
         } else if (mimeType.startsWith(MimeTypes.VIDEO)) {
             Uri contentUri = ContentUris.withAppendedId(MediaStore.Video.Media.EXTERNAL_CONTENT_URI, id);
-            entryCount = fetchFrom(context, onSuccess, contentUri, VIDEO_PROJECTION, null);
+            entryCount = fetchFrom(context, alwaysValid, onSuccess, contentUri, VIDEO_PROJECTION);
         }
         if (entryCount == 0) {
             callback.onFailure(new Exception("failed to fetch entry at uri=" + uri));
         }
     }
 
-    @SuppressLint("InlinedApi")
+    public List<Integer> getObsoleteContentIds(Context context, List<Integer> knownContentIds) {
-    private int fetchFrom(final Context context, NewEntryHandler newEntryHandler, final Uri contentUri, String[] projection, String orderBy) {
+        final ArrayList<Integer> current = new ArrayList<>();
-        int entryCount = 0;
+        current.addAll(getContentIdList(context, MediaStore.Images.Media.EXTERNAL_CONTENT_URI));
+        current.addAll(getContentIdList(context, MediaStore.Video.Media.EXTERNAL_CONTENT_URI));
+        return knownContentIds.stream().filter(id -> !current.contains(id)).collect(Collectors.toList());
+    }
 
+    private List<Integer> getContentIdList(Context context, Uri contentUri) {
+        final ArrayList<Integer> foundContentIds = new ArrayList<>();
+        try {
+            Cursor cursor = context.getContentResolver().query(contentUri, new String[]{MediaStore.MediaColumns._ID}, null, null, null);
+            if (cursor != null) {
+                int idColumn = cursor.getColumnIndexOrThrow(MediaStore.MediaColumns._ID);
+                while (cursor.moveToNext()) {
+                    foundContentIds.add(cursor.getInt(idColumn));
+                }
+                cursor.close();
+            }
+        } catch (Exception e) {
+            Log.e(LOG_TAG, "failed to get content IDs for contentUri=" + contentUri, e);
+        }
+        return foundContentIds;
+    }
+
+    @SuppressLint("InlinedApi")
+    private int fetchFrom(final Context context, NewEntryChecker newEntryChecker, NewEntryHandler newEntryHandler, final Uri contentUri, String[] projection) {
+        int newEntryCount = 0;
         final boolean needDuration = projection == VIDEO_PROJECTION;
+        final String orderBy = MediaStore.MediaColumns.DATE_MODIFIED + " DESC";
 
         try {
             Cursor cursor = context.getContentResolver().query(contentUri, projection, null, null, orderBy);
@@ -127,49 +151,52 @@ public class MediaStoreImageProvider extends ImageProvider {
                 int durationColumn = cursor.getColumnIndex(MediaStore.MediaColumns.DURATION);
 
                 while (cursor.moveToNext()) {
-                    final long contentId = cursor.getLong(idColumn);
+                    final int contentId = cursor.getInt(idColumn);
-                    // this is fine if `contentUri` does not already contain the ID
+                    final int dateModifiedSecs = cursor.getInt(dateModifiedColumn);
-                    final Uri itemUri = ContentUris.withAppendedId(contentUri, contentId);
+                    if (newEntryChecker.where(contentId, dateModifiedSecs)) {
-                    final String path = cursor.getString(pathColumn);
+                        // this is fine if `contentUri` does not already contain the ID
-                    final String mimeType = cursor.getString(mimeTypeColumn);
+                        final Uri itemUri = ContentUris.withAppendedId(contentUri, contentId);
-                    int width = cursor.getInt(widthColumn);
+                        final String path = cursor.getString(pathColumn);
-                    int height = cursor.getInt(heightColumn);
+                        final String mimeType = cursor.getString(mimeTypeColumn);
-                    long durationMillis = durationColumn != -1 ? cursor.getLong(durationColumn) : 0;
+                        int width = cursor.getInt(widthColumn);
+                        int height = cursor.getInt(heightColumn);
+                        final long durationMillis = durationColumn != -1 ? cursor.getLong(durationColumn) : 0;
 
-                    Map<String, Object> entryMap = new HashMap<String, Object>() {{
+                        Map<String, Object> entryMap = new HashMap<String, Object>() {{
-                        put("uri", itemUri.toString());
+                            put("uri", itemUri.toString());
-                        put("path", path);
+                            put("path", path);
-                        put("mimeType", mimeType);
+                            put("mimeType", mimeType);
-                        put("orientationDegrees", orientationColumn != -1 ? cursor.getInt(orientationColumn) : 0);
+                            put("orientationDegrees", orientationColumn != -1 ? cursor.getInt(orientationColumn) : 0);
-                        put("sizeBytes", cursor.getLong(sizeColumn));
+                            put("sizeBytes", cursor.getLong(sizeColumn));
-                        put("title", cursor.getString(titleColumn));
+                            put("title", cursor.getString(titleColumn));
-                        put("dateModifiedSecs", cursor.getLong(dateModifiedColumn));
+                            put("dateModifiedSecs", dateModifiedSecs);
-                        put("sourceDateTakenMillis", cursor.getLong(dateTakenColumn));
+                            put("sourceDateTakenMillis", cursor.getLong(dateTakenColumn));
-                        // only for map export
+                            // only for map export
-                        put("contentId", contentId);
+                            put("contentId", contentId);
-                    }};
+                        }};
-                    entryMap.put("width", width);
+                        entryMap.put("width", width);
-                    entryMap.put("height", height);
+                        entryMap.put("height", height);
-                    entryMap.put("durationMillis", durationMillis);
+                        entryMap.put("durationMillis", durationMillis);
 
-                    if (((width <= 0 || height <= 0) && needSize(mimeType)) || (durationMillis == 0 && needDuration)) {
+                        if (((width <= 0 || height <= 0) && needSize(mimeType)) || (durationMillis == 0 && needDuration)) {
-                        // some images are incorrectly registered in the Media Store,
+                            // some images are incorrectly registered in the Media Store,
-                        // they are valid but miss some attributes, such as width, height, orientation
+                            // they are valid but miss some attributes, such as width, height, orientation
-                        ImageEntry entry = new ImageEntry(entryMap).fillPreCatalogMetadata(context);
+                            ImageEntry entry = new ImageEntry(entryMap).fillPreCatalogMetadata(context);
-                        entryMap = entry.toMap();
+                            entryMap = entry.toMap();
-                        width = entry.width != null ? entry.width : 0;
+                            width = entry.width != null ? entry.width : 0;
-                        height = entry.height != null ? entry.height : 0;
+                            height = entry.height != null ? entry.height : 0;
-                    }
+                        }
-
+
-                    if ((width <= 0 || height <= 0) && needSize(mimeType)) {
+                        if ((width <= 0 || height <= 0) && needSize(mimeType)) {
-                        // this is probably not a real image, like "/storage/emulated/0", so we skip it
+                            // this is probably not a real image, like "/storage/emulated/0", so we skip it
-                        Log.w(LOG_TAG, "failed to get size for uri=" + itemUri + ", path=" + path + ", mimeType=" + mimeType);
+                            Log.w(LOG_TAG, "failed to get size for uri=" + itemUri + ", path=" + path + ", mimeType=" + mimeType);
-                    } else {
+                        } else {
-                        newEntryHandler.handleEntry(entryMap);
+                            newEntryHandler.handleEntry(entryMap);
-                        if (entryCount % 30 == 0) {
+                            if (newEntryCount % 30 == 0) {
-                            Thread.sleep(10);
+                                Thread.sleep(10);
+                            }
+                            newEntryCount++;
                         }
-                        entryCount++;
                     }
                 }
                 cursor.close();
@@ -177,7 +204,7 @@ public class MediaStoreImageProvider extends ImageProvider {
         } catch (Exception e) {
             Log.e(LOG_TAG, "failed to get entries", e);
         }
-        return entryCount;
+        return newEntryCount;
     }
 
     private boolean needSize(String mimeType) {
@@ -231,7 +258,7 @@ public class MediaStoreImageProvider extends ImageProvider {
     }
 
     @Override
-    public void moveMultiple(final Activity activity, Boolean copy, String destinationDir, ArrayList<ImageEntry> entries, ImageOpCallback callback) {
+    public void moveMultiple(final Activity activity, Boolean copy, String destinationDir, List<ImageEntry> entries, ImageOpCallback callback) {
         String volumeName = "external";
         StorageManager sm = activity.getSystemService(StorageManager.class);
         if (sm != null) {
@@ -323,4 +350,8 @@ public class MediaStoreImageProvider extends ImageProvider {
     public interface NewEntryHandler {
         void handleEntry(Map<String, Object> entry);
     }
+
+    public interface NewEntryChecker {
+        boolean where(int contentId, int dateModifiedSecs);
+    }
 }

lib/model/metadata_db.dart

@@ -1,5 +1,6 @@
 import 'dart:io';
 
+import 'package:aves/model/image_entry.dart';
 import 'package:aves/model/image_metadata.dart';
 import 'package:flutter/foundation.dart';
 import 'package:path/path.dart';
@@ -12,6 +13,7 @@ class MetadataDb {
 
   Future<String> get path async => join(await getDatabasesPath(), 'metadata.db');
 
+  static const entryTable = 'entry';
   static const dateTakenTable = 'dateTaken';
   static const metadataTable = 'metadata';
   static const addressTable = 'address';
@@ -24,6 +26,20 @@ class MetadataDb {
     _database = openDatabase(
       await path,
       onCreate: (db, version) async {
+        await db.execute('CREATE TABLE $entryTable('
+            'contentId INTEGER PRIMARY KEY'
+            ', uri TEXT'
+            ', path TEXT'
+            ', mimeType TEXT'
+            ', width INTEGER'
+            ', height INTEGER'
+            ', orientationDegrees INTEGER'
+            ', sizeBytes INTEGER'
+            ', title TEXT'
+            ', dateModifiedSecs INTEGER'
+            ', sourceDateTakenMillis INTEGER'
+            ', durationMillis INTEGER'
+            ')');
         await db.execute('CREATE TABLE $dateTakenTable('
             'contentId INTEGER PRIMARY KEY'
             ', dateMillis INTEGER'
@@ -67,6 +83,62 @@ class MetadataDb {
     await init();
   }
 
+  void removeIds(List<int> contentIds) async {
+    if (contentIds == null || contentIds.isEmpty) return;
+
+    final stopwatch = Stopwatch()..start();
+    final db = await _database;
+    // using array in `whereArgs` and using it with `where contentId IN ?` is a pain, so we prefer `batch` instead
+    final batch = db.batch();
+    const where = 'contentId = ?';
+    contentIds.forEach((id) {
+      final whereArgs = [id];
+      batch.delete(entryTable, where: where, whereArgs: whereArgs);
+      batch.delete(dateTakenTable, where: where, whereArgs: whereArgs);
+      batch.delete(metadataTable, where: where, whereArgs: whereArgs);
+      batch.delete(addressTable, where: where, whereArgs: whereArgs);
+      batch.delete(favouriteTable, where: where, whereArgs: whereArgs);
+    });
+    await batch.commit(noResult: true);
+    debugPrint('$runtimeType removeIds complete in ${stopwatch.elapsed.inMilliseconds}ms for ${contentIds.length} entries');
+  }
+
+  // entries
+
+  Future<void> clearEntries() async {
+    final db = await _database;
+    final count = await db.delete(entryTable, where: '1');
+    debugPrint('$runtimeType clearEntries deleted $count entries');
+  }
+
+  Future<List<ImageEntry>> loadEntries() async {
+    final stopwatch = Stopwatch()..start();
+    final db = await _database;
+    final maps = await db.query(entryTable);
+    final entries = maps.map((map) => ImageEntry.fromMap(map)).toList();
+    debugPrint('$runtimeType loadEntries complete in ${stopwatch.elapsed.inMilliseconds}ms for ${entries.length} entries');
+    return entries;
+  }
+
+  Future<void> saveEntries(Iterable<ImageEntry> entries) async {
+    if (entries == null || entries.isEmpty) return;
+    final stopwatch = Stopwatch()..start();
+    final db = await _database;
+    final batch = db.batch();
+    entries.forEach((entry) => _batchInsertEntry(batch, entry));
+    await batch.commit(noResult: true);
+    debugPrint('$runtimeType saveEntries complete in ${stopwatch.elapsed.inMilliseconds}ms for ${entries.length} entries');
+  }
+
+  void _batchInsertEntry(Batch batch, ImageEntry entry) {
+    if (entry == null) return;
+    batch.insert(
+      entryTable,
+      entry.toMap(),
+      conflictAlgorithm: ConflictAlgorithm.replace,
+    );
+  }
+
   // date taken
 
   Future<void> clearDates() async {
@@ -229,12 +301,10 @@ class MetadataDb {
     final ids = favouriteRows.where((row) => row != null).map((row) => row.contentId);
     if (ids.isEmpty) return;
 
-    // using array in `whereArgs` and using it with `where contentId IN ?` is a pain, so we prefer `batch` instead
-//    final stopwatch = Stopwatch()..start();
     final db = await _database;
+    // using array in `whereArgs` and using it with `where contentId IN ?` is a pain, so we prefer `batch` instead
     final batch = db.batch();
     ids.forEach((id) => batch.delete(favouriteTable, where: 'contentId = ?', whereArgs: [id]));
     await batch.commit(noResult: true);
-//    debugPrint('$runtimeType removeFavourites complete in ${stopwatch.elapsed.inMilliseconds}ms for ${favouriteRows.length} entries');
   }
 }

lib/model/source/collection_source.dart

@@ -40,10 +40,14 @@ class CollectionSource with SourceBase, AlbumMixin, LocationMixin, TagMixin {
   Future<void> loadDates() async {
     final stopwatch = Stopwatch()..start();
     _savedDates = List.unmodifiable(await metadataDb.loadDates());
-    debugPrint('$runtimeType loadDates complete in ${stopwatch.elapsed.inMilliseconds}ms for ${_savedDates.length} saved entries');
+    debugPrint('$runtimeType loadDates complete in ${stopwatch.elapsed.inMilliseconds}ms for ${_savedDates.length} entries');
   }
 
   void addAll(Iterable<ImageEntry> entries) {
+    if (_rawEntries.isNotEmpty) {
+      final newContentIds = entries.map((entry) => entry.contentId).toList();
+      _rawEntries.removeWhere((entry) => newContentIds.contains(entry.contentId));
+    }
     entries.forEach((entry) {
       final contentId = entry.contentId;
       entry.catalogDateMillis = _savedDates.firstWhere((metadata) => metadata.contentId == contentId, orElse: () => null)?.dateMillis;

lib/model/source/location.dart

@@ -6,6 +6,8 @@ import 'package:collection/collection.dart';
 import 'package:flutter/foundation.dart';
 
 mixin LocationMixin on SourceBase {
+  static const _commitCountThreshold = 50;
+
   List<String> sortedCountries = List.unmodifiable([]);
   List<String> sortedPlaces = List.unmodifiable([]);
 
@@ -16,19 +18,21 @@ mixin LocationMixin on SourceBase {
       final contentId = entry.contentId;
       entry.addressDetails = saved.firstWhere((address) => address.contentId == contentId, orElse: () => null);
     });
-    debugPrint('$runtimeType loadAddresses complete in ${stopwatch.elapsed.inMilliseconds}ms for ${saved.length} saved entries');
+    debugPrint('$runtimeType loadAddresses complete in ${stopwatch.elapsed.inMilliseconds}ms for ${saved.length} entries');
     onAddressMetadataChanged();
   }
 
   Future<void> locateEntries() async {
-    final stopwatch = Stopwatch()..start();
+//    final stopwatch = Stopwatch()..start();
     final unlocatedEntries = rawEntries.where((entry) => entry.hasGps && !entry.isLocated).toList();
+    if (unlocatedEntries.isEmpty) return;
+
     final newAddresses = <AddressDetails>[];
     await Future.forEach<ImageEntry>(unlocatedEntries, (entry) async {
       await entry.locate();
       if (entry.isLocated) {
         newAddresses.add(entry.addressDetails);
-        if (newAddresses.length >= 50) {
+        if (newAddresses.length >= _commitCountThreshold) {
           await metadataDb.saveAddresses(List.unmodifiable(newAddresses));
           newAddresses.clear();
         }
@@ -36,7 +40,7 @@ mixin LocationMixin on SourceBase {
     });
     await metadataDb.saveAddresses(List.unmodifiable(newAddresses));
     onAddressMetadataChanged();
-    debugPrint('$runtimeType locateEntries complete in ${stopwatch.elapsed.inMilliseconds}ms');
+//    debugPrint('$runtimeType locateEntries complete in ${stopwatch.elapsed.inSeconds}s');
   }
 
   void onAddressMetadataChanged() {

lib/model/source/tag.dart

@@ -6,6 +6,8 @@ import 'package:collection/collection.dart';
 import 'package:flutter/foundation.dart';
 
 mixin TagMixin on SourceBase {
+  static const _commitCountThreshold = 300;
+
   List<String> sortedTags = List.unmodifiable([]);
 
   Future<void> loadCatalogMetadata() async {
@@ -15,12 +17,12 @@ mixin TagMixin on SourceBase {
       final contentId = entry.contentId;
       entry.catalogMetadata = saved.firstWhere((metadata) => metadata.contentId == contentId, orElse: () => null);
     });
-    debugPrint('$runtimeType loadCatalogMetadata complete in ${stopwatch.elapsed.inMilliseconds}ms for ${saved.length} saved entries');
+    debugPrint('$runtimeType loadCatalogMetadata complete in ${stopwatch.elapsed.inMilliseconds}ms for ${saved.length} entries');
     onCatalogMetadataChanged();
   }
 
   Future<void> catalogEntries() async {
-    final stopwatch = Stopwatch()..start();
+//    final stopwatch = Stopwatch()..start();
     final uncataloguedEntries = rawEntries.where((entry) => !entry.isCatalogued).toList();
     if (uncataloguedEntries.isEmpty) return;
 
@@ -29,7 +31,7 @@ mixin TagMixin on SourceBase {
       await entry.catalog();
       if (entry.isCatalogued) {
         newMetadata.add(entry.catalogMetadata);
-        if (newMetadata.length >= 500) {
+        if (newMetadata.length >= _commitCountThreshold) {
           await metadataDb.saveMetadata(List.unmodifiable(newMetadata));
           newMetadata.clear();
         }
@@ -37,7 +39,7 @@ mixin TagMixin on SourceBase {
     });
     await metadataDb.saveMetadata(List.unmodifiable(newMetadata));
     onCatalogMetadataChanged();
-    debugPrint('$runtimeType catalogEntries complete in ${stopwatch.elapsed.inSeconds}s with ${newMetadata.length} new entries');
+//    debugPrint('$runtimeType catalogEntries complete in ${stopwatch.elapsed.inSeconds}s');
   }
 
   void onCatalogMetadataChanged() {

lib/services/image_file_service.dart

@@ -16,15 +16,30 @@ class ImageFileService {
   static final StreamsChannel opChannel = StreamsChannel('deckers.thibault/aves/imageopstream');
   static const double thumbnailDefaultSize = 64.0;
 
-  static Stream<ImageEntry> getImageEntries() {
+  // knownEntries: map of contentId -> dateModifiedSecs
+  static Stream<ImageEntry> getImageEntries(Map<int, int> knownEntries) {
     try {
-      return mediaStoreChannel.receiveBroadcastStream().map((event) => ImageEntry.fromMap(event));
+      return mediaStoreChannel.receiveBroadcastStream(<String, dynamic>{
+        'knownEntries': knownEntries,
+      }).map((event) => ImageEntry.fromMap(event));
     } on PlatformException catch (e) {
       debugPrint('getImageEntries failed with code=${e.code}, exception=${e.message}, details=${e.details}');
       return Stream.error(e);
     }
   }
 
+  static Future<List> getObsoleteEntries(List<int> knownContentIds) async {
+    try {
+      final result = await platform.invokeMethod('getObsoleteEntries', <String, dynamic>{
+        'knownContentIds': knownContentIds,
+      });
+      return (result as List).cast<int>();
+    } on PlatformException catch (e) {
+      debugPrint('getObsoleteEntries failed with code=${e.code}, exception=${e.message}, details=${e.details}');
+    }
+    return [];
+  }
+
   static Future<ImageEntry> getImageEntry(String uri, String mimeType) async {
     debugPrint('getImageEntry for uri=$uri, mimeType=$mimeType');
     try {

lib/widgets/album/grid/list_section_layout.dart

@@ -34,7 +34,7 @@ class SectionedListLayoutProvider extends StatelessWidget {
   }
 
   SectionedListLayout _updateLayouts(BuildContext context) {
-    debugPrint('$runtimeType _updateLayouts entries=${collection.entryCount} columnCount=$columnCount tileExtent=$tileExtent');
+//    debugPrint('$runtimeType _updateLayouts entries=${collection.entryCount} columnCount=$columnCount tileExtent=$tileExtent');
     final sectionLayouts = <SectionLayout>[];
     final showHeaders = collection.showHeaders;
     final source = collection.source;

lib/widgets/common/data_providers/media_store_collection_provider.dart

@@ -13,9 +13,9 @@ class MediaStoreSource extends CollectionSource {
   Future<void> init() async {
     final stopwatch = Stopwatch()..start();
     stateNotifier.value = SourceState.loading;
-    await metadataDb.init(); // <20ms
+    await metadataDb.init();
     await favourites.init();
-    final currentTimeZone = await FlutterNativeTimezone.getLocalTimezone(); // <20ms
+    final currentTimeZone = await FlutterNativeTimezone.getLocalTimezone();
     final catalogTimeZone = settings.catalogTimeZone;
     if (currentTimeZone != catalogTimeZone) {
       // clear catalog metadata to get correct date/times when moving to a different time zone
@@ -29,35 +29,50 @@ class MediaStoreSource extends CollectionSource {
   }
 
   Future<void> refresh() async {
+    debugPrint('$runtimeType refresh start');
     final stopwatch = Stopwatch()..start();
     stateNotifier.value = SourceState.loading;
 
+    final oldEntries = await metadataDb.loadEntries(); // 400ms for 5500 entries
+    final knownEntryMap = Map.fromEntries(oldEntries.map((entry) => MapEntry(entry.contentId, entry.dateModifiedSecs)));
+    final obsoleteEntries = await ImageFileService.getObsoleteEntries(knownEntryMap.keys.toList());
+    oldEntries.removeWhere((entry) => obsoleteEntries.contains(entry.contentId));
+
+    // show known entries
+    addAll(oldEntries);
+    await loadCatalogMetadata(); // 600ms for 5500 entries
+    await loadAddresses(); // 200ms for 3000 entries
+    debugPrint('$runtimeType refresh loaded ${oldEntries.length} known entries, elapsed=${stopwatch.elapsed}');
+
+    // clean up obsolete entries
+    metadataDb.removeIds(obsoleteEntries);
+
+    // fetch new entries
     var refreshCount = 10;
     const refreshCountMax = 1000;
-    final allEntries = <ImageEntry>[];
+    final allNewEntries = <ImageEntry>[], pendingNewEntries = <ImageEntry>[];
-
+    final addPendingEntries = () {
-    // TODO split image fetch AND/OR cache fetch across sessions
+      allNewEntries.addAll(pendingNewEntries);
-    ImageFileService.getImageEntries().listen(
+      addAll(pendingNewEntries);
+      pendingNewEntries.clear();
+    };
+    ImageFileService.getImageEntries(knownEntryMap).listen(
       (entry) {
-        allEntries.add(entry);
+        pendingNewEntries.add(entry);
-        if (allEntries.length >= refreshCount) {
+        if (pendingNewEntries.length >= refreshCount) {
           refreshCount = min(refreshCount * 10, refreshCountMax);
-          addAll(allEntries);
+          addPendingEntries();
-          allEntries.clear();
-//          debugPrint('$runtimeType streamed ${entries.length} entries at ${stopwatch.elapsed.inMilliseconds}ms');
         }
       },
       onDone: () async {
-        debugPrint('$runtimeType stream done, elapsed=${stopwatch.elapsed}');
+        addPendingEntries();
-        addAll(allEntries);
+        debugPrint('$runtimeType refresh loaded ${allNewEntries.length} new entries, elapsed=${stopwatch.elapsed}');
-        // TODO reduce setup time until here
+        await metadataDb.saveEntries(allNewEntries); // 700ms for 5500 entries
-        updateAlbums(); // <50ms
+        updateAlbums();
         stateNotifier.value = SourceState.cataloguing;
-        await loadCatalogMetadata(); // 400ms for 5400 entries
+        await catalogEntries();
-        await catalogEntries(); // <50ms
         stateNotifier.value = SourceState.locating;
-        await loadAddresses(); // 350ms
+        await locateEntries();
-        await locateEntries(); // <50ms
         stateNotifier.value = SourceState.ready;
         debugPrint('$runtimeType refresh done, elapsed=${stopwatch.elapsed}');
       },

lib/widgets/debug_page.dart

@@ -1,11 +1,11 @@
 import 'dart:collection';
 
-import 'package:aves/model/source/collection_source.dart';
 import 'package:aves/model/favourite_repo.dart';
 import 'package:aves/model/image_entry.dart';
 import 'package:aves/model/image_metadata.dart';
 import 'package:aves/model/metadata_db.dart';
 import 'package:aves/model/settings.dart';
+import 'package:aves/model/source/collection_source.dart';
 import 'package:aves/services/android_app_service.dart';
 import 'package:aves/services/android_file_service.dart';
 import 'package:aves/services/image_file_service.dart';
@@ -30,6 +30,7 @@ class DebugPage extends StatefulWidget {
 
 class DebugPageState extends State<DebugPage> {
   Future<int> _dbFileSizeLoader;
+  Future<List<ImageEntry>> _dbEntryLoader;
   Future<List<DateMetadata>> _dbDateLoader;
   Future<List<CatalogMetadata>> _dbMetadataLoader;
   Future<List<AddressDetails>> _dbAddressLoader;
@@ -169,9 +170,28 @@ class DebugPageState extends State<DebugPage> {
             );
           },
         ),
+        FutureBuilder(
+          future: _dbEntryLoader,
+          builder: (context, AsyncSnapshot<List> snapshot) {
+            if (snapshot.hasError) return Text(snapshot.error.toString());
+            if (snapshot.connectionState != ConnectionState.done) return const SizedBox.shrink();
+            return Row(
+              children: [
+                Expanded(
+                  child: Text('DB entry rows: ${snapshot.data.length}'),
+                ),
+                const SizedBox(width: 8),
+                RaisedButton(
+                  onPressed: () => metadataDb.clearEntries().then((_) => _startDbReport()),
+                  child: const Text('Clear'),
+                ),
+              ],
+            );
+          },
+        ),
         FutureBuilder(
           future: _dbDateLoader,
-          builder: (context, AsyncSnapshot<List<DateMetadata>> snapshot) {
+          builder: (context, AsyncSnapshot<List> snapshot) {
             if (snapshot.hasError) return Text(snapshot.error.toString());
             if (snapshot.connectionState != ConnectionState.done) return const SizedBox.shrink();
             return Row(
@@ -190,7 +210,7 @@ class DebugPageState extends State<DebugPage> {
         ),
         FutureBuilder(
           future: _dbMetadataLoader,
-          builder: (context, AsyncSnapshot<List<CatalogMetadata>> snapshot) {
+          builder: (context, AsyncSnapshot<List> snapshot) {
             if (snapshot.hasError) return Text(snapshot.error.toString());
             if (snapshot.connectionState != ConnectionState.done) return const SizedBox.shrink();
             return Row(
@@ -209,7 +229,7 @@ class DebugPageState extends State<DebugPage> {
         ),
         FutureBuilder(
           future: _dbAddressLoader,
-          builder: (context, AsyncSnapshot<List<AddressDetails>> snapshot) {
+          builder: (context, AsyncSnapshot<List> snapshot) {
             if (snapshot.hasError) return Text(snapshot.error.toString());
             if (snapshot.connectionState != ConnectionState.done) return const SizedBox.shrink();
             return Row(
@@ -228,7 +248,7 @@ class DebugPageState extends State<DebugPage> {
         ),
         FutureBuilder(
           future: _dbFavouritesLoader,
-          builder: (context, AsyncSnapshot<List<FavouriteRow>> snapshot) {
+          builder: (context, AsyncSnapshot<List> snapshot) {
             if (snapshot.hasError) return Text(snapshot.error.toString());
             if (snapshot.connectionState != ConnectionState.done) return const SizedBox.shrink();
             return Row(
@@ -327,6 +347,7 @@ class DebugPageState extends State<DebugPage> {
 
   void _startDbReport() {
     _dbFileSizeLoader = metadataDb.dbFileSize();
+    _dbEntryLoader = metadataDb.loadEntries();
     _dbDateLoader = metadataDb.loadDates();
     _dbMetadataLoader = metadataDb.loadMetadataEntries();
     _dbAddressLoader = metadataDb.loadAddresses();

lib/widgets/fullscreen/fullscreen_body.dart

@@ -1,9 +1,9 @@
 import 'dart:io';
 import 'dart:math';
 
-import 'package:aves/model/source/collection_lens.dart';
 import 'package:aves/model/filters/filters.dart';
 import 'package:aves/model/image_entry.dart';
+import 'package:aves/model/source/collection_lens.dart';
 import 'package:aves/utils/change_notifier.dart';
 import 'package:aves/utils/durations.dart';
 import 'package:aves/widgets/album/collection_page.dart';
@@ -451,6 +451,7 @@ class _FullscreenVerticalPageViewState extends State<FullscreenVerticalPageView>
   void _unregisterWidget(FullscreenVerticalPageView widget) {
     widget.verticalPager.removeListener(_onVerticalPageControllerChanged);
     widget.entryNotifier.removeListener(_onEntryChanged);
+    _oldEntry?.imageChangeNotifier?.removeListener(_onImageChanged);
   }
 
   @override
@@ -517,7 +518,12 @@ class _FullscreenVerticalPageViewState extends State<FullscreenVerticalPageView>
   // when the entry image itself changed (e.g. after rotation)
   void _onImageChanged() async {
     await UriImage(uri: entry.uri, mimeType: entry.mimeType).evict();
-    // TODO TLAD also evict `ThumbnailProvider` with specified extents
+    // evict low quality thumbnail (without specified extents)
+    await ThumbnailProvider(entry: entry).evict();
+    // evict higher quality thumbnails (with powers of 2 from 32 to 1024 as specified extents)
+    final extents = List.generate(6, (index) => pow(2, index + 5).toDouble());
+    await Future.forEach(extents, (extent) => ThumbnailProvider(entry: entry, extent: extent).evict());
+
     await ThumbnailProvider(entry: entry).evict();
     if (entry.path != null) await FileImage(File(entry.path)).evict();
     // rebuild to refresh the Image inside ImagePage

lib/widgets/home_page.dart

@@ -36,20 +36,16 @@ class _HomePageState extends State<HomePage> {
   }
 
   Future<void> _setup() async {
-    debugPrint('$runtimeType _setup');
-
-    // TODO reduce permission check time
     final permissions = await [
       Permission.storage,
       // to access media with unredacted metadata with scoped storage (Android 10+)
       Permission.accessMediaLocation,
-    ].request(); // 350ms
+    ].request();
     if (permissions[Permission.storage] != PermissionStatus.granted) {
       unawaited(SystemNavigator.pop());
       return;
     }
 
-    // TODO notify when icons are ready for drawer and section header refresh
     await androidFileUtils.init(); // 170ms
 
     final intentData = await ViewerService.getIntentData();
@@ -97,7 +93,6 @@ class _HomePageState extends State<HomePage> {
         builder: (context, AsyncSnapshot<void> snapshot) {
           if (snapshot.hasError) return const Icon(AIcons.error);
           if (snapshot.connectionState != ConnectionState.done) return const SizedBox.shrink();
-          debugPrint('$runtimeType app setup future complete');
           if (AvesApp.mode == AppMode.view) {
             return SingleFullscreenPage(entry: _viewerEntry);
           }