From b228fcf55de60bfc19dfa312959435ad81487853 Mon Sep 17 00:00:00 2001 From: Thibault Deckers Date: Tue, 21 Apr 2020 17:19:52 +0900 Subject: [PATCH] fullscreen: stream image in chunks instead of reading and sending all at once --- android/app/build.gradle | 2 + .../deckers/thibault/aves/MainActivity.java | 5 + .../channelhandlers/ImageFileHandler.java | 98 ------------- .../channelhandlers/ImageStreamHandler.java | 135 ++++++++++++++++++ lib/services/image_file_service.dart | 22 ++- lib/widgets/album/empty.dart | 4 +- lib/widgets/fullscreen/image_view.dart | 9 +- pubspec.lock | 7 + pubspec.yaml | 1 + 9 files changed, 177 insertions(+), 106 deletions(-) create mode 100644 android/app/src/main/java/deckers/thibault/aves/channelhandlers/ImageStreamHandler.java diff --git a/android/app/build.gradle b/android/app/build.gradle index 5a8f1d831..1b74c5e92 100644 --- a/android/app/build.gradle +++ b/android/app/build.gradle @@ -62,4 +62,6 @@ dependencies { testImplementation 'junit:junit:4.13' androidTestImplementation 'androidx.test:runner:1.2.0' androidTestImplementation 'androidx.test.espresso:espresso-core:3.2.0' + + compileOnly rootProject.findProject(':streams_channel') } diff --git a/android/app/src/main/java/deckers/thibault/aves/MainActivity.java b/android/app/src/main/java/deckers/thibault/aves/MainActivity.java index 346dfa84e..b93b763bb 100644 --- a/android/app/src/main/java/deckers/thibault/aves/MainActivity.java +++ b/android/app/src/main/java/deckers/thibault/aves/MainActivity.java @@ -8,9 +8,11 @@ import android.util.Log; import java.util.HashMap; import java.util.Map; +import app.loup.streams_channel.StreamsChannel; import deckers.thibault.aves.channelhandlers.AppAdapterHandler; import deckers.thibault.aves.channelhandlers.FileAdapterHandler; import deckers.thibault.aves.channelhandlers.ImageFileHandler; +import deckers.thibault.aves.channelhandlers.ImageStreamHandler; import deckers.thibault.aves.channelhandlers.MediaStoreStreamHandler; import deckers.thibault.aves.channelhandlers.MetadataHandler; import deckers.thibault.aves.utils.Constants; @@ -46,6 +48,9 @@ public class MainActivity extends FlutterActivity { new MethodChannel(messenger, MetadataHandler.CHANNEL).setMethodCallHandler(new MetadataHandler(this)); new EventChannel(messenger, MediaStoreStreamHandler.CHANNEL).setStreamHandler(mediaStoreStreamHandler); + final StreamsChannel imageStreamChannel = new StreamsChannel(messenger, ImageStreamHandler.CHANNEL); + imageStreamChannel.setStreamHandlerFactory(arguments -> new ImageStreamHandler(this, arguments)); + new MethodChannel(messenger, VIEWER_CHANNEL).setMethodCallHandler( (call, result) -> { if (call.method.contentEquals("getSharedEntry")) { diff --git a/android/app/src/main/java/deckers/thibault/aves/channelhandlers/ImageFileHandler.java b/android/app/src/main/java/deckers/thibault/aves/channelhandlers/ImageFileHandler.java index 1376d00ec..e8d0c74eb 100644 --- a/android/app/src/main/java/deckers/thibault/aves/channelhandlers/ImageFileHandler.java +++ b/android/app/src/main/java/deckers/thibault/aves/channelhandlers/ImageFileHandler.java @@ -1,32 +1,17 @@ package deckers.thibault.aves.channelhandlers; import android.app.Activity; -import android.content.ContentResolver; -import android.graphics.Bitmap; -import android.graphics.ImageDecoder; import android.net.Uri; -import android.os.Build; import android.os.Handler; import android.os.Looper; import androidx.annotation.NonNull; -import com.bumptech.glide.Glide; -import com.bumptech.glide.load.engine.DiskCacheStrategy; -import com.bumptech.glide.request.FutureTarget; -import com.bumptech.glide.request.RequestOptions; -import com.bumptech.glide.request.target.Target; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; import java.util.Map; -import deckers.thibault.aves.decoder.VideoThumbnail; import deckers.thibault.aves.model.ImageEntry; import deckers.thibault.aves.model.provider.ImageProvider; import deckers.thibault.aves.model.provider.ImageProviderFactory; -import deckers.thibault.aves.utils.MimeTypes; import io.flutter.plugin.common.MethodCall; import io.flutter.plugin.common.MethodChannel; @@ -51,9 +36,6 @@ public class ImageFileHandler implements MethodChannel.MethodCallHandler { case "getImageEntry": new Thread(() -> getImageEntry(call, new MethodResultWrapper(result))).start(); break; - case "getImage": - new Thread(() -> getImage(call, new MethodResultWrapper(result))).start(); - break; case "getThumbnail": new Thread(() -> getThumbnail(call, new MethodResultWrapper(result))).start(); break; @@ -72,71 +54,6 @@ public class ImageFileHandler implements MethodChannel.MethodCallHandler { } } - private void getImage(@NonNull MethodCall call, @NonNull MethodChannel.Result result) { - String mimeType = call.argument("mimeType"); - String uriString = call.argument("uri"); - - Uri uri = Uri.parse(uriString); - - byte[] data = null; - if (mimeType != null && mimeType.startsWith(MimeTypes.VIDEO)) { - RequestOptions options = new RequestOptions() - .diskCacheStrategy(DiskCacheStrategy.RESOURCE); - FutureTarget target = Glide.with(activity) - .asBitmap() - .apply(options) - .load(new VideoThumbnail(activity, uri)) - .submit(Target.SIZE_ORIGINAL, Target.SIZE_ORIGINAL); - try { - Bitmap bitmap = target.get(); - if (bitmap != null) { - ByteArrayOutputStream stream = new ByteArrayOutputStream(); - // we compress the bitmap because Dart Image.memory cannot decode the raw bytes - // Bitmap.CompressFormat.PNG is slower than JPEG - bitmap.compress(Bitmap.CompressFormat.JPEG, 90, stream); - data = stream.toByteArray(); - } - } catch (Exception e) { - result.error("getImage-video-exception", "failed to get image from uri=" + uri, e.getMessage()); - return; - } - Glide.with(activity).clear(target); - } else { - ContentResolver cr = activity.getContentResolver(); - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P && (MimeTypes.HEIC.equals(mimeType) || MimeTypes.HEIF.equals(mimeType))) { - // as of Flutter v1.15.17, Dart Image.memory cannot decode HEIF/HEIC images - // so we convert the image using Android native decoder - try { - ImageDecoder.Source source = ImageDecoder.createSource(cr, uri); - Bitmap bitmap = ImageDecoder.decodeBitmap(source); - ByteArrayOutputStream stream = new ByteArrayOutputStream(); - // we compress the bitmap because Dart Image.memory cannot decode the raw bytes - // Bitmap.CompressFormat.PNG is slower than JPEG - bitmap.compress(Bitmap.CompressFormat.JPEG, 90, stream); - data = stream.toByteArray(); - } catch (IOException e) { - result.error("getImage-decode-exception", "failed to decode image from uri=" + uri, e.getMessage()); - return; - } - } else { - try (InputStream is = cr.openInputStream(uri)) { - if (is != null) { - data = getBytes(is); - } - } catch (IOException e) { - result.error("getImage-read-exception", "failed to get image from uri=" + uri, e.getMessage()); - return; - } - } - } - - if (data != null) { - result.success(data); - } else { - result.error("getImage-null", "failed to get image from uri=" + uri, null); - } - } - private void getThumbnail(@NonNull MethodCall call, @NonNull MethodChannel.Result result) { Map entryMap = call.argument("entry"); Integer width = call.argument("width"); @@ -261,19 +178,4 @@ public class ImageFileHandler implements MethodChannel.MethodCallHandler { } }); } - - // convenience methods - - // InputStream.readAllBytes is only available from Java 9+ - private byte[] getBytes(InputStream inputStream) throws IOException { - ByteArrayOutputStream byteBuffer = new ByteArrayOutputStream(); - int bufferSize = 1024; - byte[] buffer = new byte[bufferSize]; - - int len; - while ((len = inputStream.read(buffer)) != -1) { - byteBuffer.write(buffer, 0, len); - } - return byteBuffer.toByteArray(); - } } \ No newline at end of file diff --git a/android/app/src/main/java/deckers/thibault/aves/channelhandlers/ImageStreamHandler.java b/android/app/src/main/java/deckers/thibault/aves/channelhandlers/ImageStreamHandler.java new file mode 100644 index 000000000..aae7de800 --- /dev/null +++ b/android/app/src/main/java/deckers/thibault/aves/channelhandlers/ImageStreamHandler.java @@ -0,0 +1,135 @@ +package deckers.thibault.aves.channelhandlers; + +import android.app.Activity; +import android.content.ContentResolver; +import android.graphics.Bitmap; +import android.graphics.ImageDecoder; +import android.net.Uri; +import android.os.Build; +import android.os.Handler; +import android.os.Looper; + +import com.bumptech.glide.Glide; +import com.bumptech.glide.load.engine.DiskCacheStrategy; +import com.bumptech.glide.request.FutureTarget; +import com.bumptech.glide.request.RequestOptions; +import com.bumptech.glide.request.target.Target; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Map; + +import deckers.thibault.aves.decoder.VideoThumbnail; +import deckers.thibault.aves.utils.MimeTypes; +import io.flutter.plugin.common.EventChannel; + +public class ImageStreamHandler implements EventChannel.StreamHandler { + public static final String CHANNEL = "deckers.thibault/aves/imagestream"; + + private Activity activity; + private Uri uri; + private String mimeType; + private EventChannel.EventSink eventSink; + private Handler handler; + + public ImageStreamHandler(Activity activity, Object arguments) { + this.activity = activity; + if (arguments instanceof Map) { + Map argMap = (Map) arguments; + this.mimeType = (String) argMap.get("mimeType"); + this.uri = Uri.parse((String) argMap.get("uri")); + } + } + + @Override + public void onListen(Object o, final EventChannel.EventSink eventSink) { + this.eventSink = eventSink; + this.handler = new Handler(Looper.getMainLooper()); + new Thread(this::getImage).start(); + } + + @Override + public void onCancel(Object o) { + } + + private void success(final byte[] bytes) { + handler.post(() -> eventSink.success(bytes)); + } + + private void error(final String errorCode, final String errorMessage, final Object errorDetails) { + handler.post(() -> eventSink.error(errorCode, errorMessage, errorDetails)); + } + + private void endOfStream() { + handler.post(() -> eventSink.endOfStream()); + } + + private void getImage() { + if (mimeType != null && mimeType.startsWith(MimeTypes.VIDEO)) { + RequestOptions options = new RequestOptions() + .diskCacheStrategy(DiskCacheStrategy.RESOURCE); + FutureTarget target = Glide.with(activity) + .asBitmap() + .apply(options) + .load(new VideoThumbnail(activity, uri)) + .submit(Target.SIZE_ORIGINAL, Target.SIZE_ORIGINAL); + try { + Bitmap bitmap = target.get(); + if (bitmap != null) { + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + // we compress the bitmap because Dart Image.memory cannot decode the raw bytes + // Bitmap.CompressFormat.PNG is slower than JPEG + bitmap.compress(Bitmap.CompressFormat.JPEG, 90, stream); + success(stream.toByteArray()); + } else { + error("getImage-video-null", "failed to get image from uri=" + uri, null); + } + } catch (Exception e) { + error("getImage-video-exception", "failed to get image from uri=" + uri, e.getMessage()); + return; + } + Glide.with(activity).clear(target); + } else { + ContentResolver cr = activity.getContentResolver(); + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P && (MimeTypes.HEIC.equals(mimeType) || MimeTypes.HEIF.equals(mimeType))) { + // as of Flutter v1.15.17, Dart Image.memory cannot decode HEIF/HEIC images + // so we convert the image using Android native decoder + try { + ImageDecoder.Source source = ImageDecoder.createSource(cr, uri); + Bitmap bitmap = ImageDecoder.decodeBitmap(source); + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + // we compress the bitmap because Dart Image.memory cannot decode the raw bytes + // Bitmap.CompressFormat.PNG is slower than JPEG + bitmap.compress(Bitmap.CompressFormat.JPEG, 90, stream); + success(stream.toByteArray()); + } catch (IOException e) { + error("getImage-image-decode-exception", "failed to decode image from uri=" + uri, e.getMessage()); + } + } else { + try (InputStream is = cr.openInputStream(uri)) { + if (is != null) { + streamBytes(is); + } else { + error("getImage-image-read-null", "failed to get image from uri=" + uri, null); + } + } catch (IOException e) { + error("getImage-image-read-exception", "failed to get image from uri=" + uri, e.getMessage()); + } + } + } + endOfStream(); + } + + private void streamBytes(InputStream inputStream) throws IOException { + int bufferSize = 2 << 17; // ~250k + byte[] buffer = new byte[bufferSize]; + int len; + while ((len = inputStream.read(buffer)) != -1) { + // cannot decode image on Flutter side when using `buffer` directly... + byte[] sub = new byte[len]; + System.arraycopy(buffer, 0, sub, 0, len); + success(sub); + } + } +} diff --git a/lib/services/image_file_service.dart b/lib/services/image_file_service.dart index d26610f5d..39184536b 100644 --- a/lib/services/image_file_service.dart +++ b/lib/services/image_file_service.dart @@ -1,12 +1,16 @@ +import 'dart:async'; +import 'dart:io'; import 'dart:typed_data'; import 'package:aves/model/image_entry.dart'; import 'package:aves/services/service_policy.dart'; import 'package:flutter/foundation.dart'; import 'package:flutter/services.dart'; +import 'package:streams_channel/streams_channel.dart'; class ImageFileService { static const platform = MethodChannel('deckers.thibault/aves/image'); + static final StreamsChannel streamsChannel = StreamsChannel('deckers.thibault/aves/imagestream'); static Future getImageEntries() async { try { @@ -30,24 +34,30 @@ class ImageFileService { return null; } - static Future getImage(String uri, String mimeType) async { + static Future getImage(String uri, String mimeType) { try { - final result = await platform.invokeMethod('getImage', { + final completer = Completer(); + final bytesBuilder = BytesBuilder(copy: false); + streamsChannel.receiveBroadcastStream({ 'uri': uri, 'mimeType': mimeType, - }); - return result as Uint8List; + }).listen( + (data) => bytesBuilder.add(data as Uint8List), + onError: completer.completeError, + onDone: () => completer.complete(bytesBuilder.takeBytes()), + cancelOnError: true, + ); + return completer.future; } on PlatformException catch (e) { debugPrint('getImage failed with code=${e.code}, exception=${e.message}, details=${e.details}'); } - return Uint8List(0); + return Future.sync(() => Uint8List(0)); } static Future getThumbnail(ImageEntry entry, int width, int height, {Object cancellationKey}) { return servicePolicy.call( () async { if (width > 0 && height > 0) { -// debugPrint('getThumbnail width=$width path=${entry.path}'); try { final result = await platform.invokeMethod('getThumbnail', { 'entry': entry.toMap(), diff --git a/lib/widgets/album/empty.dart b/lib/widgets/album/empty.dart index c5bf58b3c..58568a2b2 100644 --- a/lib/widgets/album/empty.dart +++ b/lib/widgets/album/empty.dart @@ -4,17 +4,19 @@ import 'package:outline_material_icons/outline_material_icons.dart'; class EmptyContent extends StatelessWidget { final IconData icon; final String text; + final AlignmentGeometry alignment; const EmptyContent({ this.icon = OMIcons.photo, this.text = 'Nothing!', + this.alignment = const FractionalOffset(.5, .35), }); @override Widget build(BuildContext context) { const color = Color(0xFF607D8B); return Align( - alignment: const FractionalOffset(.5, .35), + alignment: alignment, child: Column( mainAxisSize: MainAxisSize.min, children: [ diff --git a/lib/widgets/fullscreen/image_view.dart b/lib/widgets/fullscreen/image_view.dart index 3ab3d3ed6..98aaee2d5 100644 --- a/lib/widgets/fullscreen/image_view.dart +++ b/lib/widgets/fullscreen/image_view.dart @@ -1,12 +1,14 @@ import 'package:aves/model/image_entry.dart'; import 'package:aves/utils/constants.dart'; +import 'package:aves/widgets/album/empty.dart'; import 'package:aves/widgets/common/image_providers/thumbnail_provider.dart'; import 'package:aves/widgets/common/image_providers/uri_image_provider.dart'; import 'package:aves/widgets/common/image_providers/uri_picture_provider.dart'; import 'package:aves/widgets/fullscreen/video_view.dart'; -import 'package:flutter_ijkplayer/flutter_ijkplayer.dart'; import 'package:flutter/material.dart'; +import 'package:flutter_ijkplayer/flutter_ijkplayer.dart'; import 'package:flutter_svg/flutter_svg.dart'; +import 'package:outline_material_icons/outline_material_icons.dart'; import 'package:photo_view/photo_view.dart'; import 'package:tuple/tuple.dart'; @@ -101,6 +103,11 @@ class ImageView extends StatelessWidget { image: uriImage, ) : thumbnailLoadingBuilder(context), + loadFailedChild: EmptyContent( + icon: OMIcons.errorOutline, + text: 'Oops!', + alignment: Alignment.center, + ), backgroundDecoration: backgroundDecoration, scaleStateChangedCallback: onScaleChanged, minScale: PhotoViewComputedScale.contained, diff --git a/pubspec.lock b/pubspec.lock index aab9e4719..3c4458c6a 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -399,6 +399,13 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "2.0.0" + streams_channel: + dependency: "direct main" + description: + name: streams_channel + url: "https://pub.dartlang.org" + source: hosted + version: "0.3.0" string_scanner: dependency: transitive description: diff --git a/pubspec.yaml b/pubspec.yaml index a97bb3759..37bfab0b4 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -70,6 +70,7 @@ dependencies: screen: shared_preferences: sqflite: + streams_channel: tuple: uuid: