Fabio/aves
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

#45 collection: find entries with obsolete paths

Browse source
This commit is contained in:
Thibault Deckers 2021-02-10 11:32:14 +09:00
parent b5d800edc2
commit 55acafc1ab
7 changed files with 175 additions and 87 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
android/app/src/main/kotlin/deckers/thibault/aves/MainActivity.kt
View file

@ -33,6 +33,7 @@ class MainActivity : FlutterActivity() {
MethodChannel(messenger, AppAdapterHandler.CHANNEL).setMethodCallHandler(AppAdapterHandler(this))
MethodChannel(messenger, AppShortcutHandler.CHANNEL).setMethodCallHandler(AppShortcutHandler(this))
MethodChannel(messenger, ImageFileHandler.CHANNEL).setMethodCallHandler(ImageFileHandler(this))
MethodChannel(messenger, MediaStoreHandler.CHANNEL).setMethodCallHandler(MediaStoreHandler(this))
MethodChannel(messenger, MetadataHandler.CHANNEL).setMethodCallHandler(MetadataHandler(this))
MethodChannel(messenger, StorageHandler.CHANNEL).setMethodCallHandler(StorageHandler(this))
MethodChannel(messenger, DebugHandler.CHANNEL).setMethodCallHandler(DebugHandler(this))

55
android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/ImageFileHandler.kt
View file

@ -14,7 +14,6 @@ import deckers.thibault.aves.model.ExifOrientationOp
import deckers.thibault.aves.model.FieldMap
import deckers.thibault.aves.model.provider.ImageProvider.ImageOpCallback
import deckers.thibault.aves.model.provider.ImageProviderFactory.getProvider
import deckers.thibault.aves.model.provider.MediaStoreImageProvider
import deckers.thibault.aves.utils.MimeTypes
import io.flutter.plugin.common.MethodCall
import io.flutter.plugin.common.MethodChannel
@ -31,25 +30,35 @@ class ImageFileHandler(private val activity: Activity) : MethodCallHandler {
override fun onMethodCall(call: MethodCall, result: MethodChannel.Result) {
when (call.method) {
"getObsoleteEntries" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::getObsoleteEntries) }
"getEntry" -> GlobalScope.launch(Dispatchers.IO) { safesus(call, result, ::getEntry) }
"getThumbnail" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::getThumbnail) }
"getRegion" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::getRegion) }
"clearSizedThumbnailDiskCache" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::clearSizedThumbnailDiskCache) }
"rename" -> GlobalScope.launch(Dispatchers.IO) { safesus(call, result, ::rename) }
"rotate" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::rotate) }
"flip" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::flip) }
"clearSizedThumbnailDiskCache" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::clearSizedThumbnailDiskCache) }
else -> result.notImplemented()
}
}
private fun getObsoleteEntries(call: MethodCall, result: MethodChannel.Result) {
val known = call.argument<List<Int>>("knownContentIds")
if (known == null) {
result.error("getObsoleteEntries-args", "failed because of missing arguments", null)
private suspend fun getEntry(call: MethodCall, result: MethodChannel.Result) {
val mimeType = call.argument<String>("mimeType") // MIME type is optional
val uri = call.argument<String>("uri")?.let { Uri.parse(it) }
if (uri == null) {
result.error("getEntry-args", "failed because of missing arguments", null)
return
}
result.success(MediaStoreImageProvider().getObsoleteContentIds(activity, known))
val provider = getProvider(uri)
if (provider == null) {
result.error("getEntry-provider", "failed to find provider for uri=$uri", null)
return
}
provider.fetchSingle(activity, uri, mimeType, object : ImageOpCallback {
override fun onSuccess(fields: FieldMap) = result.success(fields)
override fun onFailure(throwable: Throwable) = result.error("getEntry-failure", "failed to get entry for uri=$uri", throwable.message)
})
}
private fun getThumbnail(call: MethodCall, result: MethodChannel.Result) {
@ -122,31 +131,6 @@ class ImageFileHandler(private val activity: Activity) : MethodCallHandler {
}
}
private suspend fun getEntry(call: MethodCall, result: MethodChannel.Result) {
val mimeType = call.argument<String>("mimeType") // MIME type is optional
val uri = call.argument<String>("uri")?.let { Uri.parse(it) }
if (uri == null) {
result.error("getEntry-args", "failed because of missing arguments", null)
return
}
val provider = getProvider(uri)
if (provider == null) {
result.error("getEntry-provider", "failed to find provider for uri=$uri", null)
return
}
provider.fetchSingle(activity, uri, mimeType, object : ImageOpCallback {
override fun onSuccess(fields: FieldMap) = result.success(fields)
override fun onFailure(throwable: Throwable) = result.error("getEntry-failure", "failed to get entry for uri=$uri", throwable.message)
})
}
private fun clearSizedThumbnailDiskCache(@Suppress("UNUSED_PARAMETER") call: MethodCall, result: MethodChannel.Result) {
Glide.get(activity).clearDiskCache()
result.success(null)
}
private suspend fun rename(call: MethodCall, result: MethodChannel.Result) {
val entryMap = call.argument<FieldMap>("entry")
val newName = call.argument<String>("newName")
@ -217,6 +201,11 @@ class ImageFileHandler(private val activity: Activity) : MethodCallHandler {
})
}
private fun clearSizedThumbnailDiskCache(@Suppress("UNUSED_PARAMETER") call: MethodCall, result: MethodChannel.Result) {
Glide.get(activity).clearDiskCache()
result.success(null)
}
companion object {
const val CHANNEL = "deckers.thibault/aves/image"
}

43
android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MediaStoreHandler.kt Normal file
View file

@ -0,0 +1,43 @@
package deckers.thibault.aves.channel.calls
import android.app.Activity
import deckers.thibault.aves.channel.calls.Coresult.Companion.safe
import deckers.thibault.aves.model.provider.MediaStoreImageProvider
import io.flutter.plugin.common.MethodCall
import io.flutter.plugin.common.MethodChannel
import io.flutter.plugin.common.MethodChannel.MethodCallHandler
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.GlobalScope
import kotlinx.coroutines.launch
class MediaStoreHandler(private val activity: Activity) : MethodCallHandler {
override fun onMethodCall(call: MethodCall, result: MethodChannel.Result) {
when (call.method) {
"checkObsoleteContentIds" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::checkObsoleteContentIds) }
"checkObsoletePaths" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::checkObsoletePaths) }
else -> result.notImplemented()
}
}
private fun checkObsoleteContentIds(call: MethodCall, result: MethodChannel.Result) {
val knownContentIds = call.argument<List<Int>>("knownContentIds")
if (knownContentIds == null) {
result.error("checkObsoleteContentIds-args", "failed because of missing arguments", null)
return
}
result.success(MediaStoreImageProvider().checkObsoleteContentIds(activity, knownContentIds))
}
private fun checkObsoletePaths(call: MethodCall, result: MethodChannel.Result) {
val knownPathById = call.argument<Map<Int, String>>("knownPathById")
if (knownPathById == null) {
result.error("checkObsoletePaths-args", "failed because of missing arguments", null)
return
}
result.success(MediaStoreImageProvider().checkObsoletePaths(activity, knownPathById))
}
companion object {
const val CHANNEL = "deckers.thibault/aves/mediastore"
}
}

60
android/app/src/main/kotlin/deckers/thibault/aves/model/provider/MediaStoreImageProvider.kt
View file

@ -23,6 +23,7 @@ import deckers.thibault.aves.utils.UriUtils.tryParseId
import kotlinx.coroutines.delay
import java.io.File
import java.util.*
import kotlin.collections.ArrayList
class MediaStoreImageProvider : ImageProvider() {
suspend fun fetchAll(context: Context, knownEntries: Map<Int, Int?>, handleNewEntry: NewEntryHandler) {
@ -59,30 +60,53 @@ class MediaStoreImageProvider : ImageProvider() {
callback.onFailure(Exception("failed to fetch entry at uri=$uri"))
}
fun getObsoleteContentIds(context: Context, knownContentIds: List<Int>): List<Int> {
val current = arrayListOf<Int>().apply {
addAll(getContentIdList(context, IMAGE_CONTENT_URI))
addAll(getContentIdList(context, VIDEO_CONTENT_URI))
fun checkObsoleteContentIds(context: Context, knownContentIds: List<Int>): List<Int> {
val foundContentIds = ArrayList<Int>()
fun check(context: Context, contentUri: Uri) {
val projection = arrayOf(MediaStore.MediaColumns._ID)
try {
val cursor = context.contentResolver.query(contentUri, projection, null, null, null)
if (cursor != null) {
val idColumn = cursor.getColumnIndexOrThrow(MediaStore.MediaColumns._ID)
while (cursor.moveToNext()) {
foundContentIds.add(cursor.getInt(idColumn))
}
cursor.close()
}
} catch (e: Exception) {
Log.e(LOG_TAG, "failed to get content IDs for contentUri=$contentUri", e)
}
}
return knownContentIds.filter { id: Int -> !current.contains(id) }.toList()
check(context, IMAGE_CONTENT_URI)
check(context, VIDEO_CONTENT_URI)
return knownContentIds.filter { id: Int -> !foundContentIds.contains(id) }.toList()
}
private fun getContentIdList(context: Context, contentUri: Uri): List<Int> {
val foundContentIds = ArrayList<Int>()
val projection = arrayOf(MediaStore.MediaColumns._ID)
try {
val cursor = context.contentResolver.query(contentUri, projection, null, null, null)
if (cursor != null) {
val idColumn = cursor.getColumnIndexOrThrow(MediaStore.MediaColumns._ID)
while (cursor.moveToNext()) {
foundContentIds.add(cursor.getInt(idColumn))
fun checkObsoletePaths(context: Context, knownPathById: Map<Int, String>): List<Int> {
val obsoleteIds = ArrayList<Int>()
fun check(context: Context, contentUri: Uri) {
val projection = arrayOf(MediaStore.MediaColumns._ID, MediaColumns.PATH)
try {
val cursor = context.contentResolver.query(contentUri, projection, null, null, null)
if (cursor != null) {
val idColumn = cursor.getColumnIndexOrThrow(MediaStore.MediaColumns._ID)
val pathColumn = cursor.getColumnIndexOrThrow(MediaColumns.PATH)
while (cursor.moveToNext()) {
val id = cursor.getInt(idColumn)
val path = cursor.getString(pathColumn)
if (knownPathById.containsKey(id) && knownPathById[id] != path) {
obsoleteIds.add(id)
}
}
cursor.close()
}
cursor.close()
} catch (e: Exception) {
Log.e(LOG_TAG, "failed to get content IDs for contentUri=$contentUri", e)
}
} catch (e: Exception) {
Log.e(LOG_TAG, "failed to get content IDs for contentUri=$contentUri", e)
}
return foundContentIds
check(context, IMAGE_CONTENT_URI)
check(context, VIDEO_CONTENT_URI)
return obsoleteIds
}
private suspend fun fetchFrom(

19
lib/model/source/media_store_source.dart
View file

@ -7,6 +7,7 @@ import 'package:aves/model/metadata_db.dart';
import 'package:aves/model/settings/settings.dart';
import 'package:aves/model/source/collection_source.dart';
import 'package:aves/services/image_file_service.dart';
import 'package:aves/services/media_store_service.dart';
import 'package:aves/utils/android_file_utils.dart';
import 'package:aves/utils/math_utils.dart';
import 'package:firebase_analytics/firebase_analytics.dart';
@ -49,8 +50,8 @@ class MediaStoreSource extends CollectionSource {
clearEntries();
final oldEntries = await metadataDb.loadEntries(); // 400ms for 5500 entries
final knownEntryMap = Map.fromEntries(oldEntries.map((entry) => MapEntry(entry.contentId, entry.dateModifiedSecs)));
final obsoleteContentIds = (await ImageFileService.getObsoleteEntries(knownEntryMap.keys.toList())).toSet();
final knownDateById = Map.fromEntries(oldEntries.map((entry) => MapEntry(entry.contentId, entry.dateModifiedSecs)));
final obsoleteContentIds = (await MediaStoreService.checkObsoleteContentIds(knownDateById.keys.toList())).toSet();
oldEntries.removeWhere((entry) => obsoleteContentIds.contains(entry.contentId));
// show known entries
@ -62,6 +63,13 @@ class MediaStoreSource extends CollectionSource {
// clean up obsolete entries
metadataDb.removeIds(obsoleteContentIds, updateFavourites: true);
// verify paths because some apps move files without updating their `last modified date`
final knownPathById = Map.fromEntries(allEntries.map((entry) => MapEntry(entry.contentId, entry.path)));
final movedContentIds = (await MediaStoreService.checkObsoletePaths(knownPathById)).toSet();
movedContentIds.forEach((contentId) {
knownDateById[contentId] = 0;
});
// fetch new entries
// refresh after the first 10 entries, then after 100 more, then every 1000 entries
var refreshCount = 10;
@ -73,7 +81,7 @@ class MediaStoreSource extends CollectionSource {
pendingNewEntries.clear();
}
ImageFileService.getEntries(knownEntryMap).listen(
MediaStoreService.getEntries(knownDateById).listen(
(entry) {
pendingNewEntries.add(entry);
if (pendingNewEntries.length >= refreshCount) {
@ -124,7 +132,7 @@ class MediaStoreSource extends CollectionSource {
}).where((kv) => kv != null));
// clean up obsolete entries
final obsoleteContentIds = (await ImageFileService.getObsoleteEntries(uriByContentId.keys.toList())).toSet();
final obsoleteContentIds = (await MediaStoreService.checkObsoleteContentIds(uriByContentId.keys.toList())).toSet();
final obsoleteUris = obsoleteContentIds.map((contentId) => uriByContentId[contentId]).toSet();
removeEntries(obsoleteUris);
obsoleteContentIds.forEach(uriByContentId.remove);
@ -138,7 +146,8 @@ class MediaStoreSource extends CollectionSource {
final sourceEntry = await ImageFileService.getEntry(uri, null);
if (sourceEntry != null) {
final existingEntry = allEntries.firstWhere((entry) => entry.contentId == contentId, orElse: () => null);
if (existingEntry == null || sourceEntry.dateModifiedSecs > existingEntry.dateModifiedSecs) {
// compare paths because some apps move files without updating their `last modified date`
if (existingEntry == null || sourceEntry.dateModifiedSecs > existingEntry.dateModifiedSecs || sourceEntry.path != existingEntry.path) {
final volume = androidFileUtils.getStorageVolume(sourceEntry.path);
if (volume != null) {
newEntries.add(sourceEntry);

37
lib/services/image_file_service.dart
View file

@ -13,9 +13,8 @@ import 'package:streams_channel/streams_channel.dart';
class ImageFileService {
static const platform = MethodChannel('deckers.thibault/aves/image');
static final StreamsChannel mediaStoreChannel = StreamsChannel('deckers.thibault/aves/mediastorestream');
static final StreamsChannel byteChannel = StreamsChannel('deckers.thibault/aves/imagebytestream');
static final StreamsChannel opChannel = StreamsChannel('deckers.thibault/aves/imageopstream');
static final StreamsChannel _byteStreamChannel = StreamsChannel('deckers.thibault/aves/imagebytestream');
static final StreamsChannel _opStreamChannel = StreamsChannel('deckers.thibault/aves/imageopstream');
static const double thumbnailDefaultSize = 64.0;
static Map<String, dynamic> _toPlatformEntryMap(AvesEntry entry) {
@ -32,30 +31,6 @@ class ImageFileService {
};
}
// knownEntries: map of contentId -> dateModifiedSecs
static Stream<AvesEntry> getEntries(Map<int, int> knownEntries) {
try {
return mediaStoreChannel.receiveBroadcastStream(<String, dynamic>{
'knownEntries': knownEntries,
}).map((event) => AvesEntry.fromMap(event));
} on PlatformException catch (e) {
debugPrint('getEntries failed with code=${e.code}, exception=${e.message}, details=${e.details}');
return Stream.error(e);
}
}
static Future<List<int>> getObsoleteEntries(List<int> knownContentIds) async {
try {
final result = await platform.invokeMethod('getObsoleteEntries', <String, dynamic>{
'knownContentIds': knownContentIds,
});
return (result as List).cast<int>();
} on PlatformException catch (e) {
debugPrint('getObsoleteEntries failed with code=${e.code}, exception=${e.message}, details=${e.details}');
}
return [];
}
static Future<AvesEntry> getEntry(String uri, String mimeType) async {
try {
final result = await platform.invokeMethod('getEntry', <String, dynamic>{
@ -97,7 +72,7 @@ class ImageFileService {
final completer = Completer<Uint8List>.sync();
final sink = _OutputBuffer();
var bytesReceived = 0;
byteChannel.receiveBroadcastStream(<String, dynamic>{
_byteStreamChannel.receiveBroadcastStream(<String, dynamic>{
'uri': uri,
'mimeType': mimeType,
'rotationDegrees': rotationDegrees ?? 0,
@ -225,7 +200,7 @@ class ImageFileService {
static Stream<ImageOpEvent> delete(Iterable<AvesEntry> entries) {
try {
return opChannel.receiveBroadcastStream(<String, dynamic>{
return _opStreamChannel.receiveBroadcastStream(<String, dynamic>{
'op': 'delete',
'entries': entries.map(_toPlatformEntryMap).toList(),
}).map((event) => ImageOpEvent.fromMap(event));
@ -241,7 +216,7 @@ class ImageFileService {
@required String destinationAlbum,
}) {
try {
return opChannel.receiveBroadcastStream(<String, dynamic>{
return _opStreamChannel.receiveBroadcastStream(<String, dynamic>{
'op': 'move',
'entries': entries.map(_toPlatformEntryMap).toList(),
'copy': copy,
@ -259,7 +234,7 @@ class ImageFileService {
@required String destinationAlbum,
}) {
try {
return opChannel.receiveBroadcastStream(<String, dynamic>{
return _opStreamChannel.receiveBroadcastStream(<String, dynamic>{
'op': 'export',
'entries': entries.map(_toPlatformEntryMap).toList(),
'mimeType': mimeType,

47
lib/services/media_store_service.dart Normal file
View file

@ -0,0 +1,47 @@
import 'dart:async';
import 'package:aves/model/entry.dart';
import 'package:flutter/services.dart';
import 'package:flutter/widgets.dart';
import 'package:streams_channel/streams_channel.dart';
class MediaStoreService {
static const platform = MethodChannel('deckers.thibault/aves/mediastore');
static final StreamsChannel _streamChannel = StreamsChannel('deckers.thibault/aves/mediastorestream');
static Future<List<int>> checkObsoleteContentIds(List<int> knownContentIds) async {
try {
final result = await platform.invokeMethod('checkObsoleteContentIds', <String, dynamic>{
'knownContentIds': knownContentIds,
});
return (result as List).cast<int>();
} on PlatformException catch (e) {
debugPrint('checkObsoleteContentIds failed with code=${e.code}, exception=${e.message}, details=${e.details}');
}
return [];
}
static Future<List<int>> checkObsoletePaths(Map<int, String> knownPathById) async {
try {
final result = await platform.invokeMethod('checkObsoletePaths', <String, dynamic>{
'knownPathById': knownPathById,
});
return (result as List).cast<int>();
} on PlatformException catch (e) {
debugPrint('checkObsoletePaths failed with code=${e.code}, exception=${e.message}, details=${e.details}');
}
return [];
}
// knownEntries: map of contentId -> dateModifiedSecs
static Stream<AvesEntry> getEntries(Map<int, int> knownEntries) {
try {
return _streamChannel.receiveBroadcastStream(<String, dynamic>{
'knownEntries': knownEntries,
}).map((event) => AvesEntry.fromMap(event));
} on PlatformException catch (e) {
debugPrint('getEntries failed with code=${e.code}, exception=${e.message}, details=${e.details}');
return Stream.error(e);
}
}
}