tiling (WIP)

android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/ImageFileHandler.kt

@@ -1,6 +1,7 @@
 package deckers.thibault.aves.channel.calls
 
 import android.app.Activity
+import android.graphics.Rect
 import android.net.Uri
 import com.bumptech.glide.Glide
 import deckers.thibault.aves.model.ExifOrientationOp
@@ -24,6 +25,7 @@ class ImageFileHandler(private val activity: Activity) : MethodCallHandler {
             "getObsoleteEntries" -> GlobalScope.launch { getObsoleteEntries(call, Coresult(result)) }
             "getImageEntry" -> GlobalScope.launch { getImageEntry(call, Coresult(result)) }
             "getThumbnail" -> GlobalScope.launch { getThumbnail(call, Coresult(result)) }
+            "getRegion" -> GlobalScope.launch { getRegion(call, Coresult(result)) }
             "clearSizedThumbnailDiskCache" -> {
                 GlobalScope.launch { Glide.get(activity).clearDiskCache() }
                 result.success(null)
@@ -53,13 +55,13 @@ class ImageFileHandler(private val activity: Activity) : MethodCallHandler {
         val widthDip = call.argument<Double>("widthDip")
         val heightDip = call.argument<Double>("heightDip")
         val defaultSizeDip = call.argument<Double>("defaultSizeDip")
+
         if (uri == null || mimeType == null || dateModifiedSecs == null || rotationDegrees == null || isFlipped == null || widthDip == null || heightDip == null || defaultSizeDip == null) {
             result.error("getThumbnail-args", "failed because of missing arguments", null)
             return
         }
 
         // convert DIP to physical pixels here, instead of using `devicePixelRatio` in Flutter
-        GlobalScope.launch {
         ThumbnailFetcher(
             activity,
             uri,
@@ -70,9 +72,32 @@ class ImageFileHandler(private val activity: Activity) : MethodCallHandler {
             width = (widthDip * density).roundToInt(),
             height = (heightDip * density).roundToInt(),
             defaultSize = (defaultSizeDip * density).roundToInt(),
-                Coresult(result),
+            result,
         ).fetch()
     }
+
+    private fun getRegion(call: MethodCall, result: MethodChannel.Result) {
+        val uri = call.argument<String>("uri")?.let { Uri.parse(it) }
+        val mimeType = call.argument<String>("mimeType")
+        val sampleSize = call.argument<Int>("sampleSize")
+        val x = call.argument<Int>("x")
+        val y = call.argument<Int>("y")
+        val width = call.argument<Int>("width")
+        val height = call.argument<Int>("height")
+
+        if (uri == null || mimeType == null || sampleSize == null || x == null || y == null || width == null || height == null) {
+            result.error("getRegion-args", "failed because of missing arguments", null)
+            return
+        }
+
+        RegionFetcher(
+            activity,
+            uri,
+            mimeType,
+            sampleSize,
+            Rect(x, y, x + width, y + height),
+            result,
+        ).fetch()
     }
 
     private suspend fun getImageEntry(call: MethodCall, result: MethodChannel.Result) {

android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/RegionFetcher.kt

@@ -0,0 +1,50 @@
+package deckers.thibault.aves.channel.calls
+
+import android.app.Activity
+import android.graphics.Bitmap
+import android.graphics.BitmapFactory
+import android.graphics.BitmapRegionDecoder
+import android.graphics.Rect
+import android.net.Uri
+import deckers.thibault.aves.utils.MimeTypes
+import deckers.thibault.aves.utils.StorageUtils
+import io.flutter.plugin.common.MethodChannel
+import java.io.ByteArrayOutputStream
+
+class RegionFetcher internal constructor(
+    private val activity: Activity,
+    private val uri: Uri,
+    private val mimeType: String,
+    private val sampleSize: Int,
+    private val rect: Rect,
+    private val result: MethodChannel.Result,
+) {
+
+    fun fetch() {
+        val options = BitmapFactory.Options().apply { inSampleSize = sampleSize }
+
+        try {
+            StorageUtils.openInputStream(activity, uri).use { input ->
+                val decoder = BitmapRegionDecoder.newInstance(input, false)
+                val data = decoder.decodeRegion(rect, options)?.let {
+                    val stream = ByteArrayOutputStream()
+                    // we compress the bitmap because Dart Image.memory cannot decode the raw bytes
+                    // Bitmap.CompressFormat.PNG is slower than JPEG, but it allows transparency
+                    if (MimeTypes.canHaveAlpha(mimeType)) {
+                        it.compress(Bitmap.CompressFormat.PNG, 0, stream)
+                    } else {
+                        it.compress(Bitmap.CompressFormat.JPEG, 100, stream)
+                    }
+                    stream.toByteArray()
+                }
+                if (data != null) {
+                    result.success(data)
+                } else {
+                    result.error("getRegion-null", "failed to decode region for uri=$uri rect=$rect", null)
+                }
+            }
+        } catch (e: Exception) {
+            result.error("getRegion-read-exception", "failed to get image from uri=$uri", e.message)
+        }
+    }
+}

lib/model/image_entry.dart

@@ -169,6 +169,8 @@ class ImageEntry {
   // guess whether this is a photo, according to file type (used as a hint to e.g. display megapixels)
   bool get isPhoto => [MimeTypes.heic, MimeTypes.heif, MimeTypes.jpeg].contains(mimeType) || isRaw;
 
+  bool get canTile => !isVideo && !isAnimated && ![MimeTypes.gif].contains(mimeType);
+
   bool get isRaw => MimeTypes.rawImages.contains(mimeType);
 
   bool get isVideo => mimeType.startsWith('video');

lib/services/image_file_service.dart

@@ -113,6 +113,39 @@ class ImageFileService {
     return Future.sync(() => null);
   }
 
+  static Future<Uint8List> getRegion(
+    String uri,
+    String mimeType,
+    int rotationDegrees,
+    bool isFlipped,
+    int sampleSize,
+    Rect rect, {
+    Object taskKey,
+    int priority,
+  }) {
+    return servicePolicy.call(
+      () async {
+        try {
+          final result = await platform.invokeMethod('getRegion', <String, dynamic>{
+            'uri': uri,
+            'mimeType': mimeType,
+            'sampleSize': sampleSize,
+            'x': rect.left.toInt(),
+            'y': rect.top.toInt(),
+            'width': rect.width.toInt(),
+            'height': rect.height.toInt(),
+          });
+          return result as Uint8List;
+        } on PlatformException catch (e) {
+          debugPrint('getRegion failed with code=${e.code}, exception=${e.message}, details=${e.details}');
+        }
+        return null;
+      },
+      priority: priority ?? ServiceCallPriority.getRegion,
+      key: taskKey,
+    );
+  }
+
   static Future<Uint8List> getThumbnail(
     String uri,
     String mimeType,

lib/services/service_policy.dart

@@ -104,6 +104,7 @@ class CancelledException {}
 
 class ServiceCallPriority {
   static const int getFastThumbnail = 100;
+  static const int getRegion = 150;
   static const int getSizedThumbnail = 200;
   static const int normal = 500;
   static const int getMetadata = 1000;

lib/utils/geo_utils.dart

@@ -1,10 +1,10 @@
-import 'dart:math' as math;
+import 'dart:math';
 
 import 'package:intl/intl.dart';
 import 'package:tuple/tuple.dart';
 
 String _decimal2sexagesimal(final double degDecimal) {
-  double _round(final double value, {final int decimals = 6}) => (value * math.pow(10, decimals)).round() / math.pow(10, decimals);
+  double _round(final double value, {final int decimals = 6}) => (value * pow(10, decimals)).round() / pow(10, decimals);
 
   List<int> _split(final double value) {
     // NumberFormat is necessary to create digit after comma if the value

lib/utils/math_utils.dart

@@ -0,0 +1,7 @@
+import 'dart:math';
+
+const double _piOver180 = pi / 180.0;
+
+double toDegrees(double radians) => radians / _piOver180;
+
+double toRadians(double degrees) => degrees * _piOver180;

lib/widgets/common/image_providers/uri_image_provider.dart

@@ -73,7 +73,7 @@ class UriImage extends ImageProvider<UriImage> {
   @override
   bool operator ==(Object other) {
     if (other.runtimeType != runtimeType) return false;
-    return other is UriImage && other.uri == uri && other.mimeType == mimeType && other.scale == scale;
+    return other is UriImage && other.uri == uri && other.scale == scale;
   }
 
   @override

lib/widgets/common/image_providers/uri_region_provider.dart

@@ -0,0 +1,81 @@
+import 'dart:async';
+import 'dart:ui' as ui show Codec;
+
+import 'package:aves/services/image_file_service.dart';
+import 'package:flutter/foundation.dart';
+import 'package:flutter/material.dart';
+import 'package:pedantic/pedantic.dart';
+
+class UriRegion extends ImageProvider<UriRegion> {
+  const UriRegion({
+    @required this.uri,
+    @required this.mimeType,
+    @required this.rotationDegrees,
+    @required this.isFlipped,
+    @required this.sampleSize,
+    @required this.rect,
+    this.scale = 1.0,
+  })  : assert(uri != null),
+        assert(scale != null);
+
+  final String uri, mimeType;
+  final int rotationDegrees, sampleSize;
+  final bool isFlipped;
+  final Rect rect;
+  final double scale;
+
+  @override
+  Future<UriRegion> obtainKey(ImageConfiguration configuration) {
+    return SynchronousFuture<UriRegion>(this);
+  }
+
+  @override
+  ImageStreamCompleter load(UriRegion key, DecoderCallback decode) {
+    final chunkEvents = StreamController<ImageChunkEvent>();
+
+    return MultiFrameImageStreamCompleter(
+      codec: _loadAsync(key, decode, chunkEvents),
+      scale: key.scale,
+      chunkEvents: chunkEvents.stream,
+      informationCollector: () sync* {
+        yield ErrorDescription('uri=$uri, mimeType=$mimeType');
+      },
+    );
+  }
+
+  Future<ui.Codec> _loadAsync(UriRegion key, DecoderCallback decode, StreamController<ImageChunkEvent> chunkEvents) async {
+    assert(key == this);
+
+    try {
+      final bytes = await ImageFileService.getRegion(
+        uri,
+        mimeType,
+        rotationDegrees,
+        isFlipped,
+        sampleSize,
+        rect,
+      );
+      if (bytes == null) {
+        throw StateError('$uri ($mimeType) loading failed');
+      }
+      return await decode(bytes);
+    } catch (error) {
+      debugPrint('$runtimeType _loadAsync failed with mimeType=$mimeType, uri=$uri, error=$error');
+      throw StateError('$mimeType decoding failed');
+    } finally {
+      unawaited(chunkEvents.close());
+    }
+  }
+
+  @override
+  bool operator ==(Object other) {
+    if (other.runtimeType != runtimeType) return false;
+    return other is UriRegion && other.uri == uri && other.sampleSize == sampleSize && other.rect == rect && other.scale == scale;
+  }
+
+  @override
+  int get hashCode => hashValues(uri, sampleSize, rect, scale);
+
+  @override
+  String toString() => '${objectRuntimeType(this, 'UriRegion')}(uri=$uri, mimeType=$mimeType, scale=$scale)';
+}

lib/widgets/fullscreen/image_view.dart

@@ -70,7 +70,7 @@ class _ImageViewState extends State<ImageView> {
     } else if (entry.isSvg) {
       child = _buildSvgView();
     } else if (entry.canDecode) {
-      if (isLargeImage) {
+      if (useTile) {
         child = _buildTiledImageView();
       } else {
         child = _buildImageView();
@@ -98,7 +98,7 @@ class _ImageViewState extends State<ImageView> {
   // the images loaded by `PhotoView` cannot have a width or height larger than 8192
   // so the reported offset and scale does not match expected values derived from the original dimensions
   // besides, large images should be tiled to be memory-friendly
-  bool get isLargeImage => entry.width > 4096 || entry.height > 4096;
+  bool get useTile => entry.canTile && (entry.width > 4096 || entry.height > 4096);
 
   ImageProvider get fastThumbnailProvider => ThumbnailProvider(ThumbnailProviderKey.fromEntry(entry));
 

lib/widgets/fullscreen/info/maps/scalebar_utils.dart

@@ -1,17 +1,8 @@
 import 'dart:math';
 
+import 'package:aves/utils/math_utils.dart';
 import 'package:latlong/latlong.dart';
 
-const double piOver180 = PI / 180.0;
-
-double toDegrees(double radians) {
-  return radians / piOver180;
-}
-
-double toRadians(double degrees) {
-  return degrees * piOver180;
-}
-
 LatLng calculateEndingGlobalCoordinates(LatLng start, double startBearing, double distance) {
   var mSemiMajorAxis = 6378137.0; //WGS84 major axis
   var mSemiMinorAxis = (1.0 - 1.0 / 298.257223563) * 6378137.0;

lib/widgets/fullscreen/tiled_view.dart

@@ -1,7 +1,8 @@
 import 'dart:math';
 
 import 'package:aves/model/image_entry.dart';
-import 'package:aves/widgets/common/image_providers/uri_image_provider.dart';
+import 'package:aves/utils/math_utils.dart';
+import 'package:aves/widgets/common/image_providers/uri_region_provider.dart';
 import 'package:aves/widgets/fullscreen/image_view.dart';
 import 'package:flutter/foundation.dart';
 import 'package:flutter/material.dart';
@@ -26,34 +27,111 @@ class TiledImageView extends StatefulWidget {
 }
 
 class _TiledImageViewState extends State<TiledImageView> {
+  double _initialScale;
+  int _maxSampleSize;
+
   ImageEntry get entry => widget.entry;
 
   Size get viewportSize => widget.viewportSize;
 
   ValueNotifier<ViewState> get viewStateNotifier => widget.viewStateNotifier;
 
+  static const tileSide = 200.0;
+
+  // margin around visible area to fetch surrounding tiles in advance
+  static const preFetchMargin = 50.0;
+
+  @override
+  void initState() {
+    super.initState();
+    _init();
+  }
+
+  @override
+  void didUpdateWidget(TiledImageView oldWidget) {
+    super.didUpdateWidget(oldWidget);
+
+    if (oldWidget.viewportSize != widget.viewportSize || oldWidget.entry.displaySize != widget.entry.displaySize) {
+      _init();
+    }
+  }
+
+  void _init() {
+    _initialScale = min(viewportSize.width / entry.displaySize.width, viewportSize.height / entry.displaySize.height);
+    _maxSampleSize = _sampleSizeForScale(_initialScale);
+  }
+
   @override
   Widget build(BuildContext context) {
     if (viewStateNotifier == null) return SizedBox.shrink();
 
-    final uriImage = UriImage(
+    final displayWidth = entry.displaySize.width;
-      uri: entry.uri,
+    final displayHeight = entry.displaySize.height;
-      mimeType: entry.mimeType,
+    final rotationDegrees = entry.rotationDegrees;
-      rotationDegrees: entry.rotationDegrees,
+    final isFlipped = entry.isFlipped;
-      isFlipped: entry.isFlipped,
+    Matrix4 transform;
-      expectedContentLength: entry.sizeBytes,
+    if (rotationDegrees != 0 || isFlipped) {
-    );
+      transform = Matrix4.identity()
+        ..translate(entry.width / 2.0, entry.height / 2.0)
+        ..scale(isFlipped ? -1.0 : 1.0, 1.0, 1.0)
+        ..rotateZ(-toRadians(rotationDegrees.toDouble()))
+        ..translate(-displayWidth / 2.0, -displayHeight / 2.0);
+    }
 
     return AnimatedBuilder(
         animation: viewStateNotifier,
         builder: (context, child) {
-          final displayWidth = entry.displaySize.width;
+          final viewState = viewStateNotifier.value;
-          final displayHeight = entry.displaySize.height;
+          var scale = viewState.scale;
-          var scale = viewStateNotifier.value.scale;
           if (scale == 0.0) {
             // for initial scale as `PhotoViewComputedScale.contained`
-            scale = min(viewportSize.width / displayWidth, viewportSize.height / displayHeight);
+            scale = _initialScale;
           }
+
+          final centerOffset = viewState.position;
+          final viewOrigin = Offset(
+            ((displayWidth * scale - viewportSize.width) / 2 - centerOffset.dx),
+            ((displayHeight * scale - viewportSize.height) / 2 - centerOffset.dy),
+          );
+          final viewRect = (viewOrigin & viewportSize).inflate(preFetchMargin);
+
+          final tiles = <Widget>[];
+          var minSampleSize = min(_sampleSizeForScale(scale), _maxSampleSize);
+          for (var sampleSize = _maxSampleSize; sampleSize >= minSampleSize; sampleSize = (sampleSize / 2).floor()) {
+            final layerRegionSize = Size.square(tileSide * sampleSize);
+            for (var x = 0.0; x < displayWidth; x += layerRegionSize.width) {
+              for (var y = 0.0; y < displayHeight; y += layerRegionSize.height) {
+                final regionOrigin = Offset(x, y);
+                final nextOrigin = regionOrigin.translate(layerRegionSize.width, layerRegionSize.height);
+                final thisRegionSize = Size(
+                  layerRegionSize.width - (nextOrigin.dx >= displayWidth ? nextOrigin.dx - displayWidth : 0),
+                  layerRegionSize.height - (nextOrigin.dy >= displayHeight ? nextOrigin.dy - displayHeight : 0),
+                );
+                final tileRect = regionOrigin * scale & thisRegionSize * scale;
+
+                // only build visible tiles
+                if (viewRect.overlaps(tileRect)) {
+                  var regionRect = regionOrigin & thisRegionSize;
+
+                  // apply EXIF orientation
+                  if (transform != null) {
+                    regionRect = Rect.fromPoints(
+                      MatrixUtils.transformPoint(transform, regionRect.topLeft),
+                      MatrixUtils.transformPoint(transform, regionRect.bottomRight),
+                    );
+                  }
+
+                  tiles.add(RegionTile(
+                    entry: entry,
+                    tileRect: tileRect,
+                    regionRect: regionRect,
+                    sampleSize: sampleSize,
+                  ));
+                }
+              }
+            }
+          }
+
           return Stack(
             alignment: Alignment.center,
             children: [
@@ -62,16 +140,89 @@ class _TiledImageViewState extends State<TiledImageView> {
                 height: displayHeight * scale,
                 child: widget.baseChild,
               ),
-              Image(
+              ...tiles,
-                image: uriImage,
-                width: displayWidth * scale,
-                height: displayHeight * scale,
-                errorBuilder: widget.errorBuilder,
-                fit: BoxFit.contain,
-              ),
-              // TODO TLAD positioned tiles according to scale/sampleSize
             ],
           );
         });
   }
+
+  int _sampleSizeForScale(double scale) {
+    var sample = 0;
+    if (0 < scale && scale < 1) {
+      sample = pow(2, (log(1 / scale) / log(2)).floor());
+    }
+    return max<int>(1, sample);
+  }
+}
+
+class RegionTile extends StatelessWidget {
+  final ImageEntry entry;
+  // `tileRect` uses Flutter view coordinates
+  // `regionRect` uses the raw image pixel coordinates
+  final Rect tileRect, regionRect;
+  final int sampleSize;
+
+  const RegionTile({
+    @required this.entry,
+    @required this.tileRect,
+    @required this.regionRect,
+    @required this.sampleSize,
+  });
+
+  @override
+  Widget build(BuildContext context) {
+    Widget child = Image(
+      image: UriRegion(
+        uri: entry.uri,
+        mimeType: entry.mimeType,
+        rotationDegrees: entry.rotationDegrees,
+        isFlipped: entry.isFlipped,
+        sampleSize: sampleSize,
+        rect: regionRect,
+      ),
+      width: tileRect.width,
+      height: tileRect.height,
+      fit: BoxFit.fill,
+      // TODO TLAD remove when done with tiling
+      // color: Color.fromARGB((0xff / sampleSize).floor(), 0, 0, 0xff),
+      // colorBlendMode: BlendMode.color,
+    );
+
+    // child = Container(
+    //   foregroundDecoration: BoxDecoration(
+    //     border: Border.all(
+    //       color: Colors.cyan,
+    //     ),
+    //   ),
+    //   // child: Text('$sampleSize'),
+    //   child: child,
+    // );
+
+    // apply EXIF orientation
+    final quarterTurns = entry.rotationDegrees ~/ 90;
+    if (entry.isFlipped) {
+      final rotated = quarterTurns % 2 != 0;
+      final w = (rotated ? tileRect.height : tileRect.width) / 2.0;
+      final h = (rotated ? tileRect.width : tileRect.height) / 2.0;
+      final flipper = Matrix4.identity()
+        ..translate(w, h)
+        ..scale(-1.0, 1.0, 1.0)
+        ..translate(-w, -h);
+      child = Transform(
+        transform: flipper,
+        child: child,
+      );
+    }
+    if (quarterTurns != 0) {
+      child = RotatedBox(
+        quarterTurns: quarterTurns,
+        child: child,
+      );
+    }
+
+    return Positioned.fromRect(
+      rect: tileRect,
+      child: child,
+    );
+  }
 }