tiling: handle raw images with different decoding size

fixed fetching dimensions of raw images
This commit is contained in:
Thibault Deckers 2020-11-10 17:25:21 +09:00
parent 318fa0577a
commit b42201dec0
12 changed files with 197 additions and 64 deletions

View file

@ -3,6 +3,7 @@ package deckers.thibault.aves.channel.calls
import android.app.Activity
import android.graphics.Rect
import android.net.Uri
import android.util.Size
import com.bumptech.glide.Glide
import deckers.thibault.aves.model.ExifOrientationOp
import deckers.thibault.aves.model.provider.FieldMap
@ -82,12 +83,14 @@ class ImageFileHandler(private val activity: Activity) : MethodCallHandler {
val uri = call.argument<String>("uri")?.let { Uri.parse(it) }
val mimeType = call.argument<String>("mimeType")
val sampleSize = call.argument<Int>("sampleSize")
val x = call.argument<Int>("x")
val y = call.argument<Int>("y")
val width = call.argument<Int>("width")
val height = call.argument<Int>("height")
val x = call.argument<Int>("regionX")
val y = call.argument<Int>("regionY")
val width = call.argument<Int>("regionWidth")
val height = call.argument<Int>("regionHeight")
val imageWidth = call.argument<Int>("imageWidth")
val imageHeight = call.argument<Int>("imageHeight")
if (uri == null || mimeType == null || sampleSize == null || x == null || y == null || width == null || height == null) {
if (uri == null || mimeType == null || sampleSize == null || x == null || y == null || width == null || height == null || imageWidth == null || imageHeight == null) {
result.error("getRegion-args", "failed because of missing arguments", null)
return
}
@ -97,6 +100,7 @@ class ImageFileHandler(private val activity: Activity) : MethodCallHandler {
mimeType,
sampleSize,
Rect(x, y, x + width, y + height),
Size(imageWidth, imageHeight),
result,
)
}

View file

@ -5,6 +5,7 @@ import android.content.ContentUris
import android.content.Context
import android.database.Cursor
import android.graphics.Bitmap
import android.graphics.BitmapFactory
import android.media.MediaMetadataRetriever
import android.net.Uri
import android.os.Build
@ -58,6 +59,7 @@ import io.flutter.plugin.common.MethodChannel.MethodCallHandler
import kotlinx.coroutines.GlobalScope
import kotlinx.coroutines.launch
import java.io.ByteArrayOutputStream
import java.io.IOException
import java.util.*
import kotlin.math.roundToLong
@ -70,6 +72,7 @@ class MetadataHandler(private val context: Context) : MethodCallHandler {
"getContentResolverMetadata" -> GlobalScope.launch { getContentResolverMetadata(call, Coresult(result)) }
"getExifInterfaceMetadata" -> GlobalScope.launch { getExifInterfaceMetadata(call, Coresult(result)) }
"getMediaMetadataRetrieverMetadata" -> GlobalScope.launch { getMediaMetadataRetrieverMetadata(call, Coresult(result)) }
"getBitmapFactoryInfo" -> GlobalScope.launch { getBitmapFactoryInfo(call, Coresult(result)) }
"getEmbeddedPictures" -> GlobalScope.launch { getEmbeddedPictures(call, Coresult(result)) }
"getExifThumbnails" -> GlobalScope.launch { getExifThumbnails(call, Coresult(result)) }
"getXmpThumbnails" -> GlobalScope.launch { getXmpThumbnails(call, Coresult(result)) }
@ -483,6 +486,34 @@ class MetadataHandler(private val context: Context) : MethodCallHandler {
result.success(metadataMap)
}
private fun getBitmapFactoryInfo(call: MethodCall, result: MethodChannel.Result) {
val uri = call.argument<String>("uri")?.let { Uri.parse(it) }
if (uri == null) {
result.error("getBitmapDecoderInfo-args", "failed because of missing arguments", null)
return
}
val metadataMap = HashMap<String, String>()
try {
StorageUtils.openInputStream(context, uri)?.use { input ->
val options = BitmapFactory.Options().apply {
inJustDecodeBounds = true
}
BitmapFactory.decodeStream(input, null, options)
options.outMimeType?.let { metadataMap["MimeType"] = it }
options.outWidth.let { metadataMap["Width"] = it.toString() }
options.outHeight.let { metadataMap["Height"] = it.toString() }
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
options.outColorSpace?.let { metadataMap["ColorSpace"] = it.toString() }
options.outConfig?.let { metadataMap["Config"] = it.toString() }
}
}
} catch (e: IOException) {
// ignore
}
result.success(metadataMap)
}
private fun getEmbeddedPictures(call: MethodCall, result: MethodChannel.Result) {
val uri = call.argument<String>("uri")?.let { Uri.parse(it) }
if (uri == null) {

View file

@ -6,22 +6,24 @@ import android.graphics.BitmapFactory
import android.graphics.BitmapRegionDecoder
import android.graphics.Rect
import android.net.Uri
import android.util.Log
import android.util.Size
import deckers.thibault.aves.utils.MimeTypes
import deckers.thibault.aves.utils.StorageUtils
import io.flutter.plugin.common.MethodChannel
import java.io.ByteArrayOutputStream
import kotlin.math.roundToInt
class RegionFetcher internal constructor(
private val context: Context,
) {
private var lastDecoderRef: Pair<Uri, BitmapRegionDecoder>? = null
private var lastDecoderRef: LastDecoderRef? = null
fun fetch(
uri: Uri,
mimeType: String,
sampleSize: Int,
rect: Rect,
regionRect: Rect,
imageSize: Size,
result: MethodChannel.Result,
) {
val options = BitmapFactory.Options().apply {
@ -29,8 +31,8 @@ class RegionFetcher internal constructor(
}
var currentDecoderRef = lastDecoderRef
if (currentDecoderRef != null && currentDecoderRef.first != uri) {
currentDecoderRef.second.recycle()
if (currentDecoderRef != null && currentDecoderRef.uri != uri) {
currentDecoderRef.decoder.recycle()
currentDecoderRef = null
}
@ -39,12 +41,27 @@ class RegionFetcher internal constructor(
val newDecoder = StorageUtils.openInputStream(context, uri).use { input ->
BitmapRegionDecoder.newInstance(input, false)
}
currentDecoderRef = Pair(uri, newDecoder)
currentDecoderRef = LastDecoderRef(uri, newDecoder)
}
val decoder = currentDecoderRef.second
val decoder = currentDecoderRef.decoder
lastDecoderRef = currentDecoderRef
val data = decoder.decodeRegion(rect, options)?.let {
// with raw images, the known image size may not match the decoded image size
// so we scale the requested region accordingly
val effectiveRect = if (imageSize.width != decoder.width || imageSize.height != decoder.height) {
val xf = decoder.width.toDouble() / imageSize.width
val yf = decoder.height.toDouble() / imageSize.height
Rect(
(regionRect.left * xf).roundToInt(),
(regionRect.top * yf).roundToInt(),
(regionRect.right * xf).roundToInt(),
(regionRect.bottom * yf).roundToInt(),
)
} else {
regionRect
}
val data = decoder.decodeRegion(effectiveRect, options)?.let {
val stream = ByteArrayOutputStream()
// we compress the bitmap because Dart Image.memory cannot decode the raw bytes
// Bitmap.CompressFormat.PNG is slower than JPEG, but it allows transparency
@ -58,10 +75,15 @@ class RegionFetcher internal constructor(
if (data != null) {
result.success(data)
} else {
result.error("getRegion-null", "failed to decode region for uri=$uri rect=$rect", null)
result.error("getRegion-null", "failed to decode region for uri=$uri regionRect=$regionRect", null)
}
} catch (e: Exception) {
result.error("getRegion-read-exception", "failed to initialize region decoder for uri=$uri", e.message)
result.error("getRegion-read-exception", "failed to initialize region decoder for uri=$uri regionRect=$regionRect", e.message)
}
}
}
}
private data class LastDecoderRef(
val uri: Uri,
val decoder: BitmapRegionDecoder,
)

View file

@ -123,7 +123,8 @@ class SourceImageEntry {
fillVideoByMediaMetadataRetriever(context)
if (isSized && hasDuration) return this
}
if (MimeTypes.isSupportedByMetadataExtractor(sourceMimeType)) {
// skip metadata-extractor for raw images because it reports the decoded dimensions instead of the raw dimensions
if (!MimeTypes.isRaw(sourceMimeType) && MimeTypes.isSupportedByMetadataExtractor(sourceMimeType)) {
fillByMetadataExtractor(context)
if (isSized && foundExif) return this
}
@ -224,8 +225,9 @@ class SourceImageEntry {
private fun fillByBitmapDecode(context: Context) {
try {
StorageUtils.openInputStream(context, uri)?.use { input ->
val options = BitmapFactory.Options()
options.inJustDecodeBounds = true
val options = BitmapFactory.Options().apply {
inJustDecodeBounds = true
}
BitmapFactory.decodeStream(input, null, options)
width = options.outWidth
height = options.outHeight

View file

@ -144,11 +144,14 @@ class MediaStoreImageProvider : ImageProvider() {
"contentId" to contentId,
)
if ((width <= 0 || height <= 0) && needSize(mimeType)
if (MimeTypes.isRaw(mimeType)
|| (width <= 0 || height <= 0) && needSize(mimeType)
|| durationMillis == 0L && needDuration
) {
// some images are incorrectly registered in the Media Store,
// they are valid but miss some attributes, such as width, height, orientation
// Some images are incorrectly registered in the Media Store,
// missing some attributes such as width, height, orientation.
// Also, the reported size of raw images is inconsistent across devices
// and Android versions (sometimes the raw size, sometimes the decoded size).
val entry = SourceImageEntry(entryMap).fillPreCatalogMetadata(context)
entryMap = entry.toMap()
}

View file

@ -16,7 +16,16 @@ object MimeTypes {
const val WEBP = "image/webp"
// raw raster
private const val ARW = "image/x-sony-arw"
private const val CR2 = "image/x-canon-cr2"
private const val DNG = "image/x-adobe-dng"
private const val NEF = "image/x-nikon-nef"
private const val NRW = "image/x-nikon-nrw"
private const val ORF = "image/x-olympus-orf"
private const val PEF = "image/x-pentax-pef"
private const val RAF = "image/x-fuji-raf"
private const val RW2 = "image/x-panasonic-rw2"
private const val SRW = "image/x-samsung-srw"
// vector
const val SVG = "image/svg+xml"
@ -35,6 +44,13 @@ object MimeTypes {
else -> isVideo(mimeType)
}
fun isRaw(mimeType: String?): Boolean {
return when (mimeType) {
ARW, CR2, DNG, NEF, NRW, ORF, PEF, RAF, RW2, SRW -> true
else -> false
}
}
// returns whether the specified MIME type represents
// a raster image format that allows an alpha channel
fun canHaveAlpha(mimeType: String?) = when (mimeType) {

View file

@ -169,7 +169,26 @@ class ImageEntry {
// guess whether this is a photo, according to file type (used as a hint to e.g. display megapixels)
bool get isPhoto => [MimeTypes.heic, MimeTypes.heif, MimeTypes.jpeg].contains(mimeType) || isRaw;
bool get canTile => !isVideo && !isAnimated && ![MimeTypes.gif].contains(mimeType);
// Android's `BitmapRegionDecoder` documentation states that "only the JPEG and PNG formats are supported"
// but in practice (tested on API 25, 27, 29), it successfully decodes the formats listed below,
// and it actually fails to decode GIF, DNG and animated WEBP. Other formats were not tested.
bool get canTile =>
[
MimeTypes.heic,
MimeTypes.heif,
MimeTypes.jpeg,
MimeTypes.webp,
MimeTypes.arw,
MimeTypes.cr2,
MimeTypes.nef,
MimeTypes.nrw,
MimeTypes.orf,
MimeTypes.pef,
MimeTypes.raf,
MimeTypes.rw2,
MimeTypes.srw,
].contains(mimeType) &&
!isAnimated;
bool get isRaw => MimeTypes.rawImages.contains(mimeType);

View file

@ -1,5 +1,6 @@
import 'dart:async';
import 'dart:convert';
import 'dart:math';
import 'dart:typed_data';
import 'package:aves/model/image_entry.dart';
@ -113,13 +114,15 @@ class ImageFileService {
return Future.sync(() => null);
}
// `rect`: region to decode, with coordinates in reference to `imageSize`
static Future<Uint8List> getRegion(
String uri,
String mimeType,
int rotationDegrees,
bool isFlipped,
int sampleSize,
Rect rect, {
Rectangle<int> regionRect,
Size imageSize, {
Object taskKey,
int priority,
}) {
@ -130,10 +133,12 @@ class ImageFileService {
'uri': uri,
'mimeType': mimeType,
'sampleSize': sampleSize,
'x': rect.left.toInt(),
'y': rect.top.toInt(),
'width': rect.width.toInt(),
'height': rect.height.toInt(),
'regionX': regionRect.left,
'regionY': regionRect.top,
'regionWidth': regionRect.width,
'regionHeight': regionRect.height,
'imageWidth': imageSize.width.toInt(),
'imageHeight': imageSize.height.toInt(),
});
return result as Uint8List;
} on PlatformException catch (e) {

View file

@ -76,6 +76,19 @@ class MetadataService {
return null;
}
static Future<Map> getBitmapFactoryInfo(ImageEntry entry) async {
try {
// return map with all data available when decoding image bounds with `BitmapFactory`
final result = await platform.invokeMethod('getBitmapFactoryInfo', <String, dynamic>{
'uri': entry.uri,
}) as Map;
return result;
} on PlatformException catch (e) {
debugPrint('getBitmapFactoryInfo failed with code=${e.code}, exception=${e.message}, details=${e.details}');
}
return {};
}
static Future<Map> getContentResolverMetadata(ImageEntry entry) async {
try {
// return map with all data available from the content resolver
@ -92,7 +105,7 @@ class MetadataService {
static Future<Map> getExifInterfaceMetadata(ImageEntry entry) async {
try {
// return map with all data available from the ExifInterface library
// return map with all data available from the `ExifInterface` library
final result = await platform.invokeMethod('getExifInterfaceMetadata', <String, dynamic>{
'uri': entry.uri,
}) as Map;
@ -105,7 +118,7 @@ class MetadataService {
static Future<Map> getMediaMetadataRetrieverMetadata(ImageEntry entry) async {
try {
// return map with all data available from the MediaMetadataRetriever
// return map with all data available from `MediaMetadataRetriever`
final result = await platform.invokeMethod('getMediaMetadataRetrieverMetadata', <String, dynamic>{
'uri': entry.uri,
}) as Map;

View file

@ -1,4 +1,5 @@
import 'dart:async';
import 'dart:math';
import 'dart:ui' as ui show Codec;
import 'package:aves/model/image_entry.dart';
@ -22,7 +23,7 @@ class RegionProvider extends ImageProvider<RegionProviderKey> {
codec: _loadAsync(key, decode),
scale: key.scale,
informationCollector: () sync* {
yield ErrorDescription('uri=${key.uri}, rect=${key.rect}');
yield ErrorDescription('uri=${key.uri}, regionRect=${key.regionRect}');
},
);
}
@ -37,7 +38,8 @@ class RegionProvider extends ImageProvider<RegionProviderKey> {
key.rotationDegrees,
key.isFlipped,
key.sampleSize,
key.rect,
key.regionRect,
key.imageSize,
taskKey: key,
);
if (bytes == null) {
@ -63,7 +65,8 @@ class RegionProviderKey {
final String uri, mimeType;
final int rotationDegrees, sampleSize;
final bool isFlipped;
final Rect rect;
final Rectangle<int> regionRect;
final Size imageSize;
final double scale;
const RegionProviderKey({
@ -72,14 +75,16 @@ class RegionProviderKey {
@required this.rotationDegrees,
@required this.isFlipped,
@required this.sampleSize,
@required this.rect,
@required this.regionRect,
@required this.imageSize,
this.scale = 1.0,
}) : assert(uri != null),
assert(mimeType != null),
assert(rotationDegrees != null),
assert(isFlipped != null),
assert(sampleSize != null),
assert(rect != null),
assert(regionRect != null),
assert(imageSize != null),
assert(scale != null);
// do not store the entry as it is, because the key should be constant
@ -87,7 +92,7 @@ class RegionProviderKey {
factory RegionProviderKey.fromEntry(
ImageEntry entry, {
@required int sampleSize,
@required Rect rect,
@required Rectangle<int> rect,
}) {
return RegionProviderKey(
uri: entry.uri,
@ -95,14 +100,15 @@ class RegionProviderKey {
rotationDegrees: entry.rotationDegrees,
isFlipped: entry.isFlipped,
sampleSize: sampleSize,
rect: rect,
regionRect: rect,
imageSize: Size(entry.width.toDouble(), entry.height.toDouble()),
);
}
@override
bool operator ==(Object other) {
if (other.runtimeType != runtimeType) return false;
return other is RegionProviderKey && other.uri == uri && other.mimeType == mimeType && other.rotationDegrees == rotationDegrees && other.isFlipped == isFlipped && other.sampleSize == sampleSize && other.rect == rect && other.scale == scale;
return other is RegionProviderKey && other.uri == uri && other.mimeType == mimeType && other.rotationDegrees == rotationDegrees && other.isFlipped == isFlipped && other.sampleSize == sampleSize && other.regionRect == regionRect && other.imageSize == imageSize && other.scale == scale;
}
@override
@ -113,12 +119,13 @@ class RegionProviderKey {
isFlipped,
mimeType,
sampleSize,
rect,
regionRect,
imageSize,
scale,
);
@override
String toString() {
return 'RegionProviderKey(uri=$uri, mimeType=$mimeType, rotationDegrees=$rotationDegrees, isFlipped=$isFlipped, sampleSize=$sampleSize, rect=$rect, scale=$scale)';
return 'RegionProviderKey(uri=$uri, mimeType=$mimeType, rotationDegrees=$rotationDegrees, isFlipped=$isFlipped, sampleSize=$sampleSize, regionRect=$regionRect, imageSize=$imageSize, scale=$scale)';
}
}

View file

@ -18,7 +18,7 @@ class MetadataTab extends StatefulWidget {
}
class _MetadataTabState extends State<MetadataTab> {
Future<Map> _contentResolverMetadataLoader, _exifInterfaceMetadataLoader, _mediaMetadataLoader;
Future<Map> _bitmapFactoryLoader, _contentResolverMetadataLoader, _exifInterfaceMetadataLoader, _mediaMetadataLoader;
// MediaStore timestamp keys
static const secondTimestampKeys = ['date_added', 'date_modified', 'date_expires', 'isPlayed'];
@ -33,6 +33,7 @@ class _MetadataTabState extends State<MetadataTab> {
}
void _loadMetadata() {
_bitmapFactoryLoader = MetadataService.getBitmapFactoryInfo(entry);
_contentResolverMetadataLoader = MetadataService.getContentResolverMetadata(entry);
_exifInterfaceMetadataLoader = MetadataService.getExifInterfaceMetadata(entry);
_mediaMetadataLoader = MetadataService.getMediaMetadataRetrieverMetadata(entry);
@ -77,6 +78,10 @@ class _MetadataTabState extends State<MetadataTab> {
return ListView(
padding: EdgeInsets.all(8),
children: [
FutureBuilder<Map>(
future: _bitmapFactoryLoader,
builder: (context, snapshot) => builder(context, snapshot, 'Bitmap Factory'),
),
FutureBuilder<Map>(
future: _contentResolverMetadataLoader,
builder: (context, snapshot) => builder(context, snapshot, 'Content Resolver'),

View file

@ -37,9 +37,6 @@ class _TiledImageViewState extends State<TiledImageView> {
ValueNotifier<ViewState> get viewStateNotifier => widget.viewStateNotifier;
// margin around visible area to fetch surrounding tiles in advance
static const preFetchMargin = 0.0;
// magic number used to derive sample size from scale
static const scaleFactor = 2.0;
@ -79,8 +76,8 @@ class _TiledImageViewState extends State<TiledImageView> {
Widget build(BuildContext context) {
if (viewStateNotifier == null) return SizedBox.shrink();
final displayWidth = entry.displaySize.width;
final displayHeight = entry.displaySize.height;
final displayWidth = entry.displaySize.width.round();
final displayHeight = entry.displaySize.height.round();
return AnimatedBuilder(
animation: viewStateNotifier,
@ -97,32 +94,40 @@ class _TiledImageViewState extends State<TiledImageView> {
((displayWidth * scale - viewportSize.width) / 2 - centerOffset.dx),
((displayHeight * scale - viewportSize.height) / 2 - centerOffset.dy),
);
final viewRect = (viewOrigin & viewportSize).inflate(preFetchMargin);
final viewRect = viewOrigin & viewportSize;
final tiles = <RegionTile>[];
var minSampleSize = min(_sampleSizeForScale(scale), _maxSampleSize);
for (var sampleSize = _maxSampleSize; sampleSize >= minSampleSize; sampleSize = (sampleSize / 2).floor()) {
final layerRegionSize = Size.square(_tileSide * sampleSize);
for (var x = 0.0; x < displayWidth; x += layerRegionSize.width) {
for (var y = 0.0; y < displayHeight; y += layerRegionSize.height) {
final regionOrigin = Offset(x, y);
final nextOrigin = regionOrigin.translate(layerRegionSize.width, layerRegionSize.height);
final thisRegionSize = Size(
layerRegionSize.width - (nextOrigin.dx >= displayWidth ? nextOrigin.dx - displayWidth : 0),
layerRegionSize.height - (nextOrigin.dy >= displayHeight ? nextOrigin.dy - displayHeight : 0),
);
final tileRect = regionOrigin * scale & thisRegionSize * scale;
// for the largest sample size (matching the initial scale), the whole image is in view
// so we subsample the whole image instead of splitting it in tiles
final useTiles = sampleSize != _maxSampleSize;
final regionSide = (_tileSide * sampleSize).round();
final layerRegionWidth = useTiles ? regionSide : displayWidth;
final layerRegionHeight = useTiles ? regionSide : displayHeight;
for (var x = 0; x < displayWidth; x += layerRegionWidth) {
for (var y = 0; y < displayHeight; y += layerRegionHeight) {
final nextX = x + layerRegionWidth;
final nextY = y + layerRegionHeight;
final thisRegionWidth = layerRegionWidth - (nextX >= displayWidth ? nextX - displayWidth : 0);
final thisRegionHeight = layerRegionHeight - (nextY >= displayHeight ? nextY - displayHeight : 0);
final tileRect = Rect.fromLTWH(x * scale, y * scale, thisRegionWidth * scale, thisRegionHeight * scale);
// only build visible tiles
if (viewRect.overlaps(tileRect)) {
var regionRect = regionOrigin & thisRegionSize;
Rectangle<int> regionRect;
// apply EXIF orientation
if (_transform != null) {
regionRect = Rect.fromPoints(
MatrixUtils.transformPoint(_transform, regionRect.topLeft),
MatrixUtils.transformPoint(_transform, regionRect.bottomRight),
// apply EXIF orientation
final regionRectDouble = Rect.fromLTWH(x.toDouble(), y.toDouble(), thisRegionWidth.toDouble(), thisRegionHeight.toDouble());
final tl = MatrixUtils.transformPoint(_transform, regionRectDouble.topLeft);
final br = MatrixUtils.transformPoint(_transform, regionRectDouble.bottomRight);
regionRect = Rectangle<int>.fromPoints(
Point<int>(tl.dx.round(), tl.dy.round()),
Point<int>(br.dx.round(), br.dy.round()),
);
} else {
regionRect = Rectangle<int>(x, y, thisRegionWidth, thisRegionHeight);
}
tiles.add(RegionTile(
@ -164,7 +169,8 @@ class RegionTile extends StatefulWidget {
// `tileRect` uses Flutter view coordinates
// `regionRect` uses the raw image pixel coordinates
final Rect tileRect, regionRect;
final Rect tileRect;
final Rectangle<int> regionRect;
final int sampleSize;
const RegionTile({
@ -268,6 +274,6 @@ class _RegionTileState extends State<RegionTile> {
super.debugFillProperties(properties);
properties.add(IntProperty('contentId', widget.entry.contentId));
properties.add(IntProperty('sampleSize', widget.sampleSize));
properties.add(DiagnosticsProperty<Rect>('regionRect', widget.regionRect));
properties.add(DiagnosticsProperty<Rectangle<int>>('regionRect', widget.regionRect));
}
}