diff --git a/lib/model/db/db_sqflite.dart b/lib/model/db/db_sqflite.dart index 50ebd29f4..77bc690be 100644 --- a/lib/model/db/db_sqflite.dart +++ b/lib/model/db/db_sqflite.dart @@ -112,7 +112,7 @@ class SqfliteLocalMediaDb implements LocalMediaDb { version: 11, ); - final maxIdRows = await _db.rawQuery('SELECT max(id) AS maxId FROM $entryTable'); + final maxIdRows = await _db.rawQuery('SELECT MAX(id) AS maxId FROM $entryTable'); _lastId = (maxIdRows.firstOrNull?['maxId'] as int?) ?? 0; } @@ -252,27 +252,20 @@ class SqfliteLocalMediaDb implements LocalMediaDb { if (entries != null) { where += ' AND contentId IN (${entries.map((v) => v.contentId).join(',')})'; } - final rows = await _db.query( - entryTable, - where: where, - whereArgs: [origin, 0], - groupBy: 'contentId', - having: 'COUNT(id) > 1', + final rows = await _db.rawQuery( + 'SELECT *, MAX(id) AS id' + ' FROM $entryTable' + ' WHERE $where' + ' GROUP BY contentId' + ' HAVING COUNT(id) > 1', + [origin, 0], ); final duplicates = rows.map(AvesEntry.fromMap).toSet(); - if (duplicates.isEmpty) { - return {}; - } - - debugPrint('Found duplicates=$duplicates'); - if (entries != null) { - // return duplicates among the provided entries - final duplicateIds = duplicates.map((v) => v.id).toSet(); - return entries.where((v) => duplicateIds.contains(v.id)).toSet(); - } else { - // return latest duplicates for each content ID - return duplicates.groupFoldBy((v) => v.contentId, (prev, v) => prev != null && prev.id > v.id ? prev : v).values.toSet(); + if (duplicates.isNotEmpty) { + debugPrint('Found duplicates=$duplicates'); } + // return most recent duplicate for each duplicated content ID + return duplicates; } // date taken diff --git a/lib/model/source/media_store_source.dart b/lib/model/source/media_store_source.dart index df927f831..6aca12916 100644 --- a/lib/model/source/media_store_source.dart +++ b/lib/model/source/media_store_source.dart @@ -1,5 +1,4 @@ import 'dart:async'; -import 'dart:math'; import 'package:aves/model/covers.dart'; import 'package:aves/model/entry/entry.dart'; @@ -159,26 +158,16 @@ class MediaStoreSource extends CollectionSource { }); // items to add to the collection - final pendingNewEntries = {}; + final newEntries = {}; // recover untracked trash items debugPrint('$runtimeType refresh ${stopwatch.elapsed} recover untracked entries'); if (directory == null) { - pendingNewEntries.addAll(await recoverUntrackedTrashItems()); + newEntries.addAll(await recoverUntrackedTrashItems()); } // fetch new & modified entries debugPrint('$runtimeType refresh ${stopwatch.elapsed} fetch new entries'); - // refresh after the first 10 entries, then after 100 more, then every 1000 entries - var refreshCount = 10; - const refreshCountMax = 1000; - final allNewEntries = {}; - void addPendingEntries() { - allNewEntries.addAll(pendingNewEntries); - addEntries(pendingNewEntries); - pendingNewEntries.clear(); - } - mediaStoreService.getEntries(_safeMode, knownDateByContentId, directory: directory).listen( (entry) { // when discovering modified entry with known content ID, @@ -187,25 +176,27 @@ class MediaStoreSource extends CollectionSource { final existingEntry = knownContentIds.contains(contentId) ? knownLiveEntries.firstWhereOrNull((entry) => entry.contentId == contentId) : null; entry.id = existingEntry?.id ?? localMediaDb.nextId; - pendingNewEntries.add(entry); - if (pendingNewEntries.length >= refreshCount) { - refreshCount = min(refreshCount * 10, refreshCountMax); - addPendingEntries(); - } + newEntries.add(entry); }, onDone: () async { - addPendingEntries(); - - if (allNewEntries.isNotEmpty) { + if (newEntries.isNotEmpty) { debugPrint('$runtimeType refresh ${stopwatch.elapsed} save new entries'); - await localMediaDb.insertEntries(allNewEntries); + await localMediaDb.insertEntries(newEntries); - // TODO TLAD [971] check duplicates - final duplicates = await localMediaDb.searchLiveDuplicates(EntryOrigins.mediaStoreContent, allNewEntries); + // TODO TLAD find duplication cause + final duplicates = await localMediaDb.searchLiveDuplicates(EntryOrigins.mediaStoreContent, newEntries); if (duplicates.isNotEmpty) { unawaited(reportService.recordError(Exception('Loading entries yielded duplicates=${duplicates.join(', ')}'), StackTrace.current)); + // post-error cleanup + await localMediaDb.removeIds(duplicates.map((v) => v.id).toSet()); + for (final duplicate in duplicates) { + final duplicateId = duplicate.id; + newEntries.removeWhere((v) => duplicateId == v.id); + } } + addEntries(newEntries); + // new entries include existing entries with obsolete paths // so directories may be added, but also removed or simply have their content summary changed invalidateAlbumFilterSummary(); @@ -230,7 +221,7 @@ class MediaStoreSource extends CollectionSource { notifyAlbumsChanged(); debugPrint('$runtimeType refresh ${stopwatch.elapsed} done'); - unawaited(reportService.log('Source refresh complete in ${stopwatch.elapsed.inSeconds}s for ${knownEntries.length} known, ${allNewEntries.length} new, ${removedEntries.length} removed')); + unawaited(reportService.log('Source refresh complete in ${stopwatch.elapsed.inSeconds}s for ${knownEntries.length} known, ${newEntries.length} new, ${removedEntries.length} removed')); }, onError: (error) => debugPrint('$runtimeType stream error=$error'), ); @@ -248,7 +239,7 @@ class MediaStoreSource extends CollectionSource { state = SourceState.loading; debugPrint('$runtimeType refreshUris ${changedUris.length} uris'); - final uriByContentId = Map.fromEntries(changedUris.map((uri) { + final changedUriByContentId = Map.fromEntries(changedUris.map((uri) { final pathSegments = Uri.parse(uri).pathSegments; // e.g. URI `content://media/` has no path segment if (pathSegments.isEmpty) return null; @@ -259,16 +250,16 @@ class MediaStoreSource extends CollectionSource { }).whereNotNull()); // clean up obsolete entries - final obsoleteContentIds = (await mediaStoreService.checkObsoleteContentIds(uriByContentId.keys.toList())).toSet(); - final obsoleteUris = obsoleteContentIds.map((contentId) => uriByContentId[contentId]).whereNotNull().toSet(); + final obsoleteContentIds = (await mediaStoreService.checkObsoleteContentIds(changedUriByContentId.keys.toList())).toSet(); + final obsoleteUris = obsoleteContentIds.map((contentId) => changedUriByContentId[contentId]).whereNotNull().toSet(); await removeEntries(obsoleteUris, includeTrash: false); - obsoleteContentIds.forEach(uriByContentId.remove); + obsoleteContentIds.forEach(changedUriByContentId.remove); // fetch new entries final tempUris = {}; final newEntries = {}, entriesToRefresh = {}; final existingDirectories = {}; - for (final kv in uriByContentId.entries) { + for (final kv in changedUriByContentId.entries) { final contentId = kv.key; final uri = kv.value; final sourceEntry = await mediaFetchService.getEntry(uri, null); @@ -309,15 +300,22 @@ class MediaStoreSource extends CollectionSource { state = SourceState.ready; if (newEntries.isNotEmpty) { - addEntries(newEntries); await localMediaDb.insertEntries(newEntries); - // TODO TLAD [971] check duplicates + // TODO TLAD find duplication cause final duplicates = await localMediaDb.searchLiveDuplicates(EntryOrigins.mediaStoreContent, newEntries); if (duplicates.isNotEmpty) { unawaited(reportService.recordError(Exception('Refreshing entries yielded duplicates=${duplicates.join(', ')}'), StackTrace.current)); + // post-error cleanup + await localMediaDb.removeIds(duplicates.map((v) => v.id).toSet()); + for (final duplicate in duplicates) { + final duplicateId = duplicate.id; + newEntries.removeWhere((v) => duplicateId == v.id); + tempUris.add(duplicate.uri); + } } + addEntries(newEntries); await analyze(analysisController, entries: newEntries); } diff --git a/lib/widgets/collection/collection_grid.dart b/lib/widgets/collection/collection_grid.dart index e7f2ecfd3..76b14adbc 100644 --- a/lib/widgets/collection/collection_grid.dart +++ b/lib/widgets/collection/collection_grid.dart @@ -20,6 +20,7 @@ import 'package:aves/widgets/collection/draggable_thumb_label.dart'; import 'package:aves/widgets/collection/grid/list_details_theme.dart'; import 'package:aves/widgets/collection/grid/section_layout.dart'; import 'package:aves/widgets/collection/grid/tile.dart'; +import 'package:aves/widgets/common/action_mixins/feedback.dart'; import 'package:aves/widgets/common/basic/draggable_scrollbar/scrollbar.dart'; import 'package:aves/widgets/common/basic/insets.dart'; import 'package:aves/widgets/common/behaviour/routes.dart'; @@ -587,7 +588,13 @@ class _CollectionScrollViewState extends State<_CollectionScrollView> with Widge valueListenable: collection.source.stateNotifier, builder: (context, sourceState, child) { if (sourceState == SourceState.loading) { - return const SizedBox(); + return EmptyContent( + text: context.l10n.sourceStateLoading, + bottom: const Padding( + padding: EdgeInsets.only(top: 16), + child: ReportProgressIndicator(), + ), + ); } return FutureBuilder( diff --git a/lib/widgets/common/action_mixins/feedback.dart b/lib/widgets/common/action_mixins/feedback.dart index a8ac02206..48a0a8cf5 100644 --- a/lib/widgets/common/action_mixins/feedback.dart +++ b/lib/widgets/common/action_mixins/feedback.dart @@ -166,6 +166,9 @@ class ReportOverlay extends StatefulWidget { final VoidCallback? onCancel; final void Function(Set processed) onDone; + static const double diameter = 160.0; + static const double strokeWidth = 8.0; + const ReportOverlay({ super.key, required this.opStream, @@ -186,8 +189,6 @@ class _ReportOverlayState extends State> with SingleTickerPr Stream get opStream => widget.opStream; static const double fontSize = 18.0; - static const double diameter = 160.0; - static const double strokeWidth = 8.0; @override void initState() { @@ -222,6 +223,8 @@ class _ReportOverlayState extends State> with SingleTickerPr @override Widget build(BuildContext context) { + const diameter = ReportOverlay.diameter; + const strokeWidth = ReportOverlay.strokeWidth; final percentFormatter = NumberFormat.percentPattern(context.locale); final theme = Theme.of(context); @@ -249,16 +252,7 @@ class _ReportOverlayState extends State> with SingleTickerPr shape: BoxShape.circle, ), ), - if (animate) - Container( - width: diameter, - height: diameter, - padding: const EdgeInsets.all(strokeWidth / 2), - child: CircularProgressIndicator( - color: progressColor.withOpacity(.1), - strokeWidth: strokeWidth, - ), - ), + if (animate) const ReportProgressIndicator(opacity: .1), CircularPercentIndicator( percent: percent, lineWidth: strokeWidth, @@ -301,6 +295,32 @@ class _ReportOverlayState extends State> with SingleTickerPr } } +class ReportProgressIndicator extends StatelessWidget { + final double opacity; + + const ReportProgressIndicator({ + super.key, + this.opacity = 1, + }); + + @override + Widget build(BuildContext context) { + const diameter = ReportOverlay.diameter; + const strokeWidth = ReportOverlay.strokeWidth; + final progressColor = Theme.of(context).colorScheme.primary; + + return Container( + width: diameter, + height: diameter, + padding: const EdgeInsets.all(strokeWidth / 2), + child: CircularProgressIndicator( + color: progressColor.withOpacity(opacity), + strokeWidth: strokeWidth, + ), + ); + } +} + class _FeedbackMessage extends StatefulWidget { final FeedbackType type; final String message;