From e1362fc40c183c2da733f17de38d40d4af8bc16b Mon Sep 17 00:00:00 2001 From: Thibault Deckers Date: Mon, 19 Apr 2021 16:10:34 +0900 Subject: [PATCH] perf: fewer allocations when decoding images --- .../aves/channel/calls/AppAdapterHandler.kt | 5 +- .../aves/channel/calls/ImageFileHandler.kt | 8 +-- .../aves/channel/calls/MetadataHandler.kt | 5 +- .../channel/calls/fetchers/RegionFetcher.kt | 2 +- .../calls/fetchers/ThumbnailFetcher.kt | 2 +- .../calls/fetchers/TiffRegionFetcher.kt | 2 +- .../channel/streams/ImageByteStreamHandler.kt | 6 +- .../aves/decoder/VideoThumbnailGlideModule.kt | 65 ++++++++++--------- .../thibault/aves/utils/BitmapUtils.kt | 28 ++++++-- 9 files changed, 75 insertions(+), 48 deletions(-) diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/AppAdapterHandler.kt b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/AppAdapterHandler.kt index f93f043e7..6c1956ef4 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/AppAdapterHandler.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/AppAdapterHandler.kt @@ -12,6 +12,7 @@ import com.bumptech.glide.Glide import com.bumptech.glide.load.DecodeFormat import com.bumptech.glide.request.RequestOptions import deckers.thibault.aves.channel.calls.Coresult.Companion.safe +import deckers.thibault.aves.channel.calls.Coresult.Companion.safesus import deckers.thibault.aves.model.FieldMap import deckers.thibault.aves.utils.BitmapUtils.getBytes import deckers.thibault.aves.utils.LogUtils @@ -30,7 +31,7 @@ class AppAdapterHandler(private val context: Context) : MethodCallHandler { override fun onMethodCall(call: MethodCall, result: MethodChannel.Result) { when (call.method) { "getPackages" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::getPackages) } - "getAppIcon" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::getAppIcon) } + "getAppIcon" -> GlobalScope.launch(Dispatchers.IO) { safesus(call, result, ::getAppIcon) } "edit" -> { val title = call.argument("title") val uri = call.argument("uri")?.let { Uri.parse(it) } @@ -109,7 +110,7 @@ class AppAdapterHandler(private val context: Context) : MethodCallHandler { result.success(ArrayList(packages.values)) } - private fun getAppIcon(call: MethodCall, result: MethodChannel.Result) { + private suspend fun getAppIcon(call: MethodCall, result: MethodChannel.Result) { val packageName = call.argument("packageName") val sizeDip = call.argument("sizeDip") if (packageName == null || sizeDip == null) { diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/ImageFileHandler.kt b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/ImageFileHandler.kt index 67f6afec8..70118a26e 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/ImageFileHandler.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/ImageFileHandler.kt @@ -31,8 +31,8 @@ class ImageFileHandler(private val activity: Activity) : MethodCallHandler { override fun onMethodCall(call: MethodCall, result: MethodChannel.Result) { when (call.method) { "getEntry" -> GlobalScope.launch(Dispatchers.IO) { safesus(call, result, ::getEntry) } - "getThumbnail" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::getThumbnail) } - "getRegion" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::getRegion) } + "getThumbnail" -> GlobalScope.launch(Dispatchers.IO) { safesus(call, result, ::getThumbnail) } + "getRegion" -> GlobalScope.launch(Dispatchers.IO) { safesus(call, result, ::getRegion) } "rename" -> GlobalScope.launch(Dispatchers.IO) { safesus(call, result, ::rename) } "rotate" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::rotate) } "flip" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::flip) } @@ -61,7 +61,7 @@ class ImageFileHandler(private val activity: Activity) : MethodCallHandler { }) } - private fun getThumbnail(call: MethodCall, result: MethodChannel.Result) { + private suspend fun getThumbnail(call: MethodCall, result: MethodChannel.Result) { val uri = call.argument("uri") val mimeType = call.argument("mimeType") val dateModifiedSecs = call.argument("dateModifiedSecs")?.toLong() @@ -93,7 +93,7 @@ class ImageFileHandler(private val activity: Activity) : MethodCallHandler { ).fetch() } - private fun getRegion(call: MethodCall, result: MethodChannel.Result) { + private suspend fun getRegion(call: MethodCall, result: MethodChannel.Result) { val uri = call.argument("uri")?.let { Uri.parse(it) } val mimeType = call.argument("mimeType") val pageId = call.argument("pageId") diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MetadataHandler.kt b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MetadataHandler.kt index 9f1ad02fe..466d23d84 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MetadataHandler.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MetadataHandler.kt @@ -28,6 +28,7 @@ import com.drew.metadata.png.PngDirectory import com.drew.metadata.webp.WebpDirectory import com.drew.metadata.xmp.XmpDirectory import deckers.thibault.aves.channel.calls.Coresult.Companion.safe +import deckers.thibault.aves.channel.calls.Coresult.Companion.safesus import deckers.thibault.aves.metadata.* import deckers.thibault.aves.metadata.ExifInterfaceHelper.describeAll import deckers.thibault.aves.metadata.ExifInterfaceHelper.getSafeDateMillis @@ -88,7 +89,7 @@ class MetadataHandler(private val context: Context) : MethodCallHandler { "getMultiPageInfo" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::getMultiPageInfo) } "getPanoramaInfo" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::getPanoramaInfo) } "getContentResolverProp" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::getContentResolverProp) } - "getExifThumbnails" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::getExifThumbnails) } + "getExifThumbnails" -> GlobalScope.launch(Dispatchers.IO) { safesus(call, result, ::getExifThumbnails) } "extractVideoEmbeddedPicture" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::extractVideoEmbeddedPicture) } "extractXmpDataProp" -> GlobalScope.launch(Dispatchers.IO) { safe(call, result, ::extractXmpDataProp) } else -> result.notImplemented() @@ -745,7 +746,7 @@ class MetadataHandler(private val context: Context) : MethodCallHandler { result.success(value?.toString()) } - private fun getExifThumbnails(call: MethodCall, result: MethodChannel.Result) { + private suspend fun getExifThumbnails(call: MethodCall, result: MethodChannel.Result) { val mimeType = call.argument("mimeType") val uri = call.argument("uri")?.let { Uri.parse(it) } val sizeBytes = call.argument("sizeBytes")?.toLong() diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/RegionFetcher.kt b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/RegionFetcher.kt index 47d7a60f7..d977c1316 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/RegionFetcher.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/RegionFetcher.kt @@ -30,7 +30,7 @@ class RegionFetcher internal constructor( .diskCacheStrategy(DiskCacheStrategy.NONE) .skipMemoryCache(true) - fun fetch( + suspend fun fetch( uri: Uri, mimeType: String, pageId: Int?, diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/ThumbnailFetcher.kt b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/ThumbnailFetcher.kt index 5b0b1f8c7..2696f59b7 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/ThumbnailFetcher.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/ThumbnailFetcher.kt @@ -45,7 +45,7 @@ class ThumbnailFetcher internal constructor( private val multiTrackFetch = isHeic(mimeType) && pageId != null private val customFetch = tiffFetch || multiTrackFetch - fun fetch() { + suspend fun fetch() { var bitmap: Bitmap? = null var exception: Exception? = null diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/TiffRegionFetcher.kt b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/TiffRegionFetcher.kt index 502553422..12666d371 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/TiffRegionFetcher.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/fetchers/TiffRegionFetcher.kt @@ -11,7 +11,7 @@ import org.beyka.tiffbitmapfactory.TiffBitmapFactory class TiffRegionFetcher internal constructor( private val context: Context, ) { - fun fetch( + suspend fun fetch( uri: Uri, page: Int, sampleSize: Int, diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/channel/streams/ImageByteStreamHandler.kt b/android/app/src/main/kotlin/deckers/thibault/aves/channel/streams/ImageByteStreamHandler.kt index c7c4355e5..9d4e73edc 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/channel/streams/ImageByteStreamHandler.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/channel/streams/ImageByteStreamHandler.kt @@ -76,7 +76,7 @@ class ImageByteStreamHandler(private val activity: Activity, private val argumen // - Flutter (as of v1.20): JPEG, PNG, GIF, Animated GIF, WebP, Animated WebP, BMP, and WBMP // - Android: https://developer.android.com/guide/topics/media/media-formats#image-formats // - Glide: https://github.com/bumptech/glide/blob/master/library/src/main/java/com/bumptech/glide/load/ImageHeaderParser.java - private fun streamImage() { + private suspend fun streamImage() { if (arguments !is Map<*, *>) { endOfStream() return @@ -114,7 +114,7 @@ class ImageByteStreamHandler(private val activity: Activity, private val argumen } } - private fun streamImageByGlide(uri: Uri, pageId: Int?, mimeType: String, rotationDegrees: Int, isFlipped: Boolean) { + private suspend fun streamImageByGlide(uri: Uri, pageId: Int?, mimeType: String, rotationDegrees: Int, isFlipped: Boolean) { val model: Any = if (isHeic(mimeType) && pageId != null) { MultiTrackImage(activity, uri, pageId) } else if (mimeType == MimeTypes.TIFF) { @@ -145,7 +145,7 @@ class ImageByteStreamHandler(private val activity: Activity, private val argumen } } - private fun streamVideoByGlide(uri: Uri) { + private suspend fun streamVideoByGlide(uri: Uri) { val target = Glide.with(activity) .asBitmap() .apply(glideOptions) diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/decoder/VideoThumbnailGlideModule.kt b/android/app/src/main/kotlin/deckers/thibault/aves/decoder/VideoThumbnailGlideModule.kt index 278e55cc6..37bc13071 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/decoder/VideoThumbnailGlideModule.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/decoder/VideoThumbnailGlideModule.kt @@ -19,6 +19,9 @@ import com.bumptech.glide.module.LibraryGlideModule import com.bumptech.glide.signature.ObjectKey import deckers.thibault.aves.utils.BitmapUtils.getBytes import deckers.thibault.aves.utils.StorageUtils.openMetadataRetriever +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.GlobalScope +import kotlinx.coroutines.launch import java.io.ByteArrayInputStream import java.io.InputStream @@ -47,40 +50,42 @@ internal class VideoThumbnailLoader : ModelLoader { internal class VideoThumbnailFetcher(private val model: VideoThumbnail) : DataFetcher { override fun loadData(priority: Priority, callback: DataCallback) { - val retriever = openMetadataRetriever(model.context, model.uri) - if (retriever != null) { - try { - var bytes = retriever.embeddedPicture - if (bytes == null) { - // try to match the thumbnails returned by the content resolver / Media Store - // the following strategies are from empirical evidence from a few test devices: - // - API 29: sync frame closest to the middle - // - API 26/27: default representative frame at any time position - var timeMillis: Long? = null - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { - val durationMillis = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION)?.toLongOrNull() - if (durationMillis != null) { - timeMillis = durationMillis / 2 + GlobalScope.launch(Dispatchers.IO) { + val retriever = openMetadataRetriever(model.context, model.uri) + if (retriever != null) { + try { + var bytes = retriever.embeddedPicture + if (bytes == null) { + // try to match the thumbnails returned by the content resolver / Media Store + // the following strategies are from empirical evidence from a few test devices: + // - API 29: sync frame closest to the middle + // - API 26/27: default representative frame at any time position + var timeMillis: Long? = null + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + val durationMillis = retriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION)?.toLongOrNull() + if (durationMillis != null) { + timeMillis = durationMillis / 2 + } } + val frame = if (timeMillis != null) { + retriever.getFrameAtTime(timeMillis * 1000) + } else { + retriever.frameAtTime + } + bytes = frame?.getBytes(canHaveAlpha = false, recycle = false) } - val frame = if (timeMillis != null) { - retriever.getFrameAtTime(timeMillis * 1000) - } else { - retriever.frameAtTime - } - bytes = frame?.getBytes(canHaveAlpha = false, recycle = false) - } - if (bytes != null) { - callback.onDataReady(ByteArrayInputStream(bytes)) - } else { - callback.onLoadFailed(Exception("failed to get embedded picture or any frame")) + if (bytes != null) { + callback.onDataReady(ByteArrayInputStream(bytes)) + } else { + callback.onLoadFailed(Exception("failed to get embedded picture or any frame")) + } + } catch (e: Exception) { + callback.onLoadFailed(e) + } finally { + // cannot rely on `MediaMetadataRetriever` being `AutoCloseable` on older APIs + retriever.release() } - } catch (e: Exception) { - callback.onLoadFailed(e) - } finally { - // cannot rely on `MediaMetadataRetriever` being `AutoCloseable` on older APIs - retriever.release() } } } diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/utils/BitmapUtils.kt b/android/app/src/main/kotlin/deckers/thibault/aves/utils/BitmapUtils.kt index 0e90819c0..a84cce1ef 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/utils/BitmapUtils.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/utils/BitmapUtils.kt @@ -6,14 +6,29 @@ import android.util.Log import com.bumptech.glide.Glide import com.bumptech.glide.load.resource.bitmap.TransformationUtils import deckers.thibault.aves.metadata.Metadata.getExifCode +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock import java.io.ByteArrayOutputStream object BitmapUtils { private val LOG_TAG = LogUtils.createTag() + private const val INITIAL_BUFFER_SIZE = 2 shl 17 // 256kB - fun Bitmap.getBytes(canHaveAlpha: Boolean = false, quality: Int = 100, recycle: Boolean = true): ByteArray? { + private val freeBaos = ArrayList() + private val mutex = Mutex() + + suspend fun Bitmap.getBytes(canHaveAlpha: Boolean = false, quality: Int = 100, recycle: Boolean): ByteArray? { + val stream: ByteArrayOutputStream + mutex.withLock { + // this method is called a lot, so we try and reuse output streams + // to reduce inner array allocations, and make the GC run less frequently + stream = if (freeBaos.isNotEmpty()) { + freeBaos.removeAt(0) + } else { + ByteArrayOutputStream(INITIAL_BUFFER_SIZE) + } + } try { - val stream = ByteArrayOutputStream() // we compress the bitmap because Flutter cannot decode the raw bytes // `Bitmap.CompressFormat.PNG` is slower than `JPEG`, but it allows transparency if (canHaveAlpha) { @@ -22,7 +37,12 @@ object BitmapUtils { this.compress(Bitmap.CompressFormat.JPEG, quality, stream) } if (recycle) this.recycle() - return stream.toByteArray() + val byteArray = stream.toByteArray() + stream.reset() + mutex.withLock { + freeBaos.add(stream) + } + return byteArray } catch (e: IllegalStateException) { Log.e(LOG_TAG, "failed to get bytes from bitmap", e) } @@ -42,4 +62,4 @@ object BitmapUtils { } fun getBitmapPool(context: Context) = Glide.get(context).bitmapPool -} \ No newline at end of file +}