From 7aee4c6db3e33b1d0e4d651a2688afad563d9c89 Mon Sep 17 00:00:00 2001 From: Thibault Deckers Date: Tue, 24 Aug 2021 19:08:25 +0900 Subject: [PATCH] improved start-up loading time --- lib/model/metadata.dart | 24 --------------------- lib/model/metadata_db.dart | 17 ++++++--------- lib/model/source/collection_source.dart | 27 +++++++++++++++++------- lib/model/source/location.dart | 6 ++---- lib/model/source/media_store_source.dart | 10 ++++----- lib/model/source/tag.dart | 6 ++---- lib/widgets/debug/app_debug_page.dart | 8 +++++++ lib/widgets/debug/database.dart | 4 ++-- lib/widgets/viewer/debug/db.dart | 8 +++---- test/fake/metadata_db.dart | 2 +- 10 files changed, 50 insertions(+), 62 deletions(-) diff --git a/lib/model/metadata.dart b/lib/model/metadata.dart index 50e76bd00..73c57b90e 100644 --- a/lib/model/metadata.dart +++ b/lib/model/metadata.dart @@ -3,30 +3,6 @@ import 'package:flutter/foundation.dart'; import 'package:flutter/widgets.dart'; import 'package:intl/intl.dart'; -class DateMetadata { - final int? contentId, dateMillis; - - DateMetadata({ - this.contentId, - this.dateMillis, - }); - - factory DateMetadata.fromMap(Map map) { - return DateMetadata( - contentId: map['contentId'], - dateMillis: map['dateMillis'] ?? 0, - ); - } - - Map toMap() => { - 'contentId': contentId, - 'dateMillis': dateMillis, - }; - - @override - String toString() => '$runtimeType#${shortHash(this)}{contentId=$contentId, dateMillis=$dateMillis}'; -} - class CatalogMetadata { final int? contentId, dateMillis; final bool isAnimated, isGeotiff, is360, isMultiPage; diff --git a/lib/model/metadata_db.dart b/lib/model/metadata_db.dart index 9393603e2..a46558835 100644 --- a/lib/model/metadata_db.dart +++ b/lib/model/metadata_db.dart @@ -36,7 +36,7 @@ abstract class MetadataDb { Future clearDates(); - Future> loadDates(); + Future> loadDates(); // catalog metadata @@ -260,12 +260,10 @@ class SqfliteMetadataDb implements MetadataDb { } @override - Future> loadDates() async { -// final stopwatch = Stopwatch()..start(); + Future> loadDates() async { final db = await _database; final maps = await db.query(dateTakenTable); - final metadataEntries = maps.map((map) => DateMetadata.fromMap(map)).toList(); -// debugPrint('$runtimeType loadDates complete in ${stopwatch.elapsed.inMilliseconds}ms for ${metadataEntries.length} entries'); + final metadataEntries = Map.fromEntries(maps.map((map) => MapEntry(map['contentId'] as int, (map['dateMillis'] ?? 0) as int))); return metadataEntries; } @@ -280,11 +278,9 @@ class SqfliteMetadataDb implements MetadataDb { @override Future> loadMetadataEntries() async { -// final stopwatch = Stopwatch()..start(); final db = await _database; final maps = await db.query(metadataTable); final metadataEntries = maps.map((map) => CatalogMetadata.fromMap(map)).toList(); -// debugPrint('$runtimeType loadMetadataEntries complete in ${stopwatch.elapsed.inMilliseconds}ms for ${metadataEntries.length} entries'); return metadataEntries; } @@ -318,7 +314,10 @@ class SqfliteMetadataDb implements MetadataDb { if (metadata.dateMillis != 0) { batch.insert( dateTakenTable, - DateMetadata(contentId: metadata.contentId, dateMillis: metadata.dateMillis).toMap(), + { + 'contentId': metadata.contentId, + 'dateMillis': metadata.dateMillis, + }, conflictAlgorithm: ConflictAlgorithm.replace, ); } @@ -340,11 +339,9 @@ class SqfliteMetadataDb implements MetadataDb { @override Future> loadAddresses() async { -// final stopwatch = Stopwatch()..start(); final db = await _database; final maps = await db.query(addressTable); final addresses = maps.map((map) => AddressDetails.fromMap(map)).toList(); -// debugPrint('$runtimeType loadAddresses complete in ${stopwatch.elapsed.inMilliseconds}ms for ${addresses.length} entries'); return addresses; } diff --git a/lib/model/source/collection_source.dart b/lib/model/source/collection_source.dart index f91df8aeb..d99d9e652 100644 --- a/lib/model/source/collection_source.dart +++ b/lib/model/source/collection_source.dart @@ -7,7 +7,6 @@ import 'package:aves/model/filters/album.dart'; import 'package:aves/model/filters/filters.dart'; import 'package:aves/model/filters/location.dart'; import 'package:aves/model/filters/tag.dart'; -import 'package:aves/model/metadata.dart'; import 'package:aves/model/settings/settings.dart'; import 'package:aves/model/source/album.dart'; import 'package:aves/model/source/enums.dart'; @@ -22,6 +21,8 @@ import 'package:flutter/foundation.dart'; mixin SourceBase { EventBus get eventBus; + Map get entryById; + Set get visibleEntries; List get sortedEntriesByDate; @@ -41,6 +42,11 @@ abstract class CollectionSource with SourceBase, AlbumMixin, LocationMixin, TagM @override EventBus get eventBus => _eventBus; + final Map _entryById = {}; + + @override + Map get entryById => Map.unmodifiable(_entryById); + final Set _rawEntries = {}; Set get allEntries => Set.unmodifiable(_rawEntries); @@ -61,11 +67,11 @@ abstract class CollectionSource with SourceBase, AlbumMixin, LocationMixin, TagM return _sortedEntriesByDate!; } - late List _savedDates; + late Map _savedDates; Future loadDates() async { final stopwatch = Stopwatch()..start(); - _savedDates = List.unmodifiable(await metadataDb.loadDates()); + _savedDates = Map.unmodifiable(await metadataDb.loadDates()); debugPrint('$runtimeType loadDates complete in ${stopwatch.elapsed.inMilliseconds}ms for ${_savedDates.length} entries'); } @@ -84,14 +90,16 @@ abstract class CollectionSource with SourceBase, AlbumMixin, LocationMixin, TagM void addEntries(Set entries) { if (entries.isEmpty) return; + + final newIdMapEntries = Map.fromEntries(entries.map((v) => MapEntry(v.contentId, v))); if (_rawEntries.isNotEmpty) { - final newContentIds = entries.map((entry) => entry.contentId).toSet(); + final newContentIds = newIdMapEntries.keys.toSet(); _rawEntries.removeWhere((entry) => newContentIds.contains(entry.contentId)); } - entries.forEach((entry) { - final contentId = entry.contentId; - entry.catalogDateMillis = _savedDates.firstWhereOrNull((metadata) => metadata.contentId == contentId)?.dateMillis; - }); + + entries.forEach((entry) => entry.catalogDateMillis = _savedDates[entry.contentId]); + + _entryById.addAll(newIdMapEntries); _rawEntries.addAll(entries); _invalidate(entries); @@ -104,6 +112,8 @@ abstract class CollectionSource with SourceBase, AlbumMixin, LocationMixin, TagM final entries = _rawEntries.where((entry) => uris.contains(entry.uri)).toSet(); await favourites.remove(entries); await covers.removeEntries(entries); + + entries.forEach((v) => _entryById.remove(v.contentId)); _rawEntries.removeAll(entries); _invalidate(entries); @@ -114,6 +124,7 @@ abstract class CollectionSource with SourceBase, AlbumMixin, LocationMixin, TagM } void clearEntries() { + _entryById.clear(); _rawEntries.clear(); _invalidate(); diff --git a/lib/model/source/location.dart b/lib/model/source/location.dart index 114ea713e..05ec1b788 100644 --- a/lib/model/source/location.dart +++ b/lib/model/source/location.dart @@ -20,10 +20,8 @@ mixin LocationMixin on SourceBase { Future loadAddresses() async { final stopwatch = Stopwatch()..start(); final saved = await metadataDb.loadAddresses(); - visibleEntries.forEach((entry) { - final contentId = entry.contentId; - entry.addressDetails = saved.firstWhereOrNull((address) => address.contentId == contentId); - }); + final idMap = entryById; + saved.forEach((metadata) => idMap[metadata.contentId]?.addressDetails = metadata); debugPrint('$runtimeType loadAddresses complete in ${stopwatch.elapsed.inMilliseconds}ms for ${saved.length} entries'); onAddressMetadataChanged(); } diff --git a/lib/model/source/media_store_source.dart b/lib/model/source/media_store_source.dart index 6be48b411..0fcf68337 100644 --- a/lib/model/source/media_store_source.dart +++ b/lib/model/source/media_store_source.dart @@ -36,7 +36,7 @@ class MediaStoreSource extends CollectionSource { settings.catalogTimeZone = currentTimeZone; } } - await loadDates(); // 100ms for 5400 entries + await loadDates(); _initialized = true; debugPrint('$runtimeType init done, elapsed=${stopwatch.elapsed}'); } @@ -49,15 +49,15 @@ class MediaStoreSource extends CollectionSource { stateNotifier.value = SourceState.loading; clearEntries(); - final oldEntries = await metadataDb.loadEntries(); // 400ms for 5500 entries + final oldEntries = await metadataDb.loadEntries(); final knownDateById = Map.fromEntries(oldEntries.map((entry) => MapEntry(entry.contentId!, entry.dateModifiedSecs!))); final obsoleteContentIds = (await mediaStoreService.checkObsoleteContentIds(knownDateById.keys.toList())).toSet(); oldEntries.removeWhere((entry) => obsoleteContentIds.contains(entry.contentId)); // show known entries addEntries(oldEntries); - await loadCatalogMetadata(); // 600ms for 5500 entries - await loadAddresses(); // 200ms for 3000 entries + await loadCatalogMetadata(); + await loadAddresses(); debugPrint('$runtimeType refresh loaded ${oldEntries.length} known entries, elapsed=${stopwatch.elapsed}'); // clean up obsolete entries @@ -94,7 +94,7 @@ class MediaStoreSource extends CollectionSource { addPendingEntries(); debugPrint('$runtimeType refresh loaded ${allNewEntries.length} new entries, elapsed=${stopwatch.elapsed}'); - await metadataDb.saveEntries(allNewEntries); // 700ms for 5500 entries + await metadataDb.saveEntries(allNewEntries); if (allNewEntries.isNotEmpty) { // new entries include existing entries with obsolete paths diff --git a/lib/model/source/tag.dart b/lib/model/source/tag.dart index bea5b66eb..505179730 100644 --- a/lib/model/source/tag.dart +++ b/lib/model/source/tag.dart @@ -15,10 +15,8 @@ mixin TagMixin on SourceBase { Future loadCatalogMetadata() async { final stopwatch = Stopwatch()..start(); final saved = await metadataDb.loadMetadataEntries(); - visibleEntries.forEach((entry) { - final contentId = entry.contentId; - entry.catalogMetadata = saved.firstWhereOrNull((metadata) => metadata.contentId == contentId); - }); + final idMap = entryById; + saved.forEach((metadata) => idMap[metadata.contentId]?.catalogMetadata = metadata); debugPrint('$runtimeType loadCatalogMetadata complete in ${stopwatch.elapsed.inMilliseconds}ms for ${saved.length} entries'); onCatalogMetadataChanged(); } diff --git a/lib/widgets/debug/app_debug_page.dart b/lib/widgets/debug/app_debug_page.dart index fc5c9480a..cf09dc132 100644 --- a/lib/widgets/debug/app_debug_page.dart +++ b/lib/widgets/debug/app_debug_page.dart @@ -95,6 +95,14 @@ class _AppDebugPageState extends State { }, title: const Text('Show tasks overlay'), ), + ElevatedButton( + onPressed: () async { + final source = context.read(); + await source.init(); + await source.refresh(); + }, + child: const Text('Source full refresh'), + ), const Divider(), Padding( padding: const EdgeInsets.only(left: 8, right: 8, bottom: 8), diff --git a/lib/widgets/debug/database.dart b/lib/widgets/debug/database.dart index 2b9eca4fe..a3f97fb7e 100644 --- a/lib/widgets/debug/database.dart +++ b/lib/widgets/debug/database.dart @@ -17,7 +17,7 @@ class DebugAppDatabaseSection extends StatefulWidget { class _DebugAppDatabaseSectionState extends State with AutomaticKeepAliveClientMixin { late Future _dbFileSizeLoader; late Future> _dbEntryLoader; - late Future> _dbDateLoader; + late Future> _dbDateLoader; late Future> _dbMetadataLoader; late Future> _dbAddressLoader; late Future> _dbFavouritesLoader; @@ -82,7 +82,7 @@ class _DebugAppDatabaseSectionState extends State with ); }, ), - FutureBuilder( + FutureBuilder>( future: _dbDateLoader, builder: (context, snapshot) { if (snapshot.hasError) return Text(snapshot.error.toString()); diff --git a/lib/widgets/viewer/debug/db.dart b/lib/widgets/viewer/debug/db.dart index a1a9161b8..af5f8bc3b 100644 --- a/lib/widgets/viewer/debug/db.dart +++ b/lib/widgets/viewer/debug/db.dart @@ -18,7 +18,7 @@ class DbTab extends StatefulWidget { } class _DbTabState extends State { - late Future _dbDateLoader; + late Future _dbDateLoader; late Future _dbEntryLoader; late Future _dbMetadataLoader; late Future _dbAddressLoader; @@ -33,7 +33,7 @@ class _DbTabState extends State { void _loadDatabase() { final contentId = entry.contentId; - _dbDateLoader = metadataDb.loadDates().then((values) => values.firstWhereOrNull((row) => row.contentId == contentId)); + _dbDateLoader = metadataDb.loadDates().then((values) => values[contentId]); _dbEntryLoader = metadataDb.loadEntries().then((values) => values.firstWhereOrNull((row) => row.contentId == contentId)); _dbMetadataLoader = metadataDb.loadMetadataEntries().then((values) => values.firstWhereOrNull((row) => row.contentId == contentId)); _dbAddressLoader = metadataDb.loadAddresses().then((values) => values.firstWhereOrNull((row) => row.contentId == contentId)); @@ -45,7 +45,7 @@ class _DbTabState extends State { return ListView( padding: const EdgeInsets.all(16), children: [ - FutureBuilder( + FutureBuilder( future: _dbDateLoader, builder: (context, snapshot) { if (snapshot.hasError) return Text(snapshot.error.toString()); @@ -58,7 +58,7 @@ class _DbTabState extends State { if (data != null) InfoRowGroup( info: { - 'dateMillis': '${data.dateMillis}', + 'dateMillis': '$data', }, ), ], diff --git a/test/fake/metadata_db.dart b/test/fake/metadata_db.dart index e5a17d99c..3584897d5 100644 --- a/test/fake/metadata_db.dart +++ b/test/fake/metadata_db.dart @@ -24,7 +24,7 @@ class FakeMetadataDb extends Fake implements MetadataDb { Future updateEntryId(int oldId, AvesEntry entry) => SynchronousFuture(null); @override - Future> loadDates() => SynchronousFuture([]); + Future> loadDates() => SynchronousFuture({}); @override Future> loadMetadataEntries() => SynchronousFuture([]);