#838 info: open MPF embedded images

CHANGELOG.md

@@ -6,6 +6,7 @@ All notable changes to this project will be documented in this file.
 
 - Cataloguing: detect/filter `Ultra HDR`
 - Info: show metadata from JPEG MPF
+- Info: open images embedded via JPEG MPF
 
 ### Changed
 

android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/EmbeddedDataHandler.kt

@@ -11,21 +11,30 @@ import com.bumptech.glide.load.resource.bitmap.TransformationUtils
 import com.drew.metadata.xmp.XmpDirectory
 import deckers.thibault.aves.channel.calls.Coresult.Companion.safe
 import deckers.thibault.aves.channel.calls.Coresult.Companion.safeSuspend
-import deckers.thibault.aves.metadata.*
+import deckers.thibault.aves.metadata.GoogleDeviceContainer
+import deckers.thibault.aves.metadata.Metadata
+import deckers.thibault.aves.metadata.MultiPage
+import deckers.thibault.aves.metadata.XMP
 import deckers.thibault.aves.metadata.XMP.doesPropPathExist
 import deckers.thibault.aves.metadata.XMP.getSafeStructField
+import deckers.thibault.aves.metadata.XMPPropName
 import deckers.thibault.aves.metadata.metadataextractor.Helper
+import deckers.thibault.aves.metadata.metadataextractor.mpf.MpEntry
+import deckers.thibault.aves.metadata.metadataextractor.mpf.MpEntryDirectory
 import deckers.thibault.aves.model.FieldMap
 import deckers.thibault.aves.model.provider.ContentImageProvider
 import deckers.thibault.aves.model.provider.ImageProvider
-import deckers.thibault.aves.utils.*
+import deckers.thibault.aves.utils.BitmapUtils
 import deckers.thibault.aves.utils.BitmapUtils.getBytes
 import deckers.thibault.aves.utils.FileUtils.transferFrom
+import deckers.thibault.aves.utils.LogUtils
+import deckers.thibault.aves.utils.MimeTypes
 import deckers.thibault.aves.utils.MimeTypes.canReadWithExifInterface
 import deckers.thibault.aves.utils.MimeTypes.canReadWithMetadataExtractor
 import deckers.thibault.aves.utils.MimeTypes.extensionFor
 import deckers.thibault.aves.utils.MimeTypes.isImage
 import deckers.thibault.aves.utils.MimeTypes.isVideo
+import deckers.thibault.aves.utils.StorageUtils
 import io.flutter.plugin.common.MethodCall
 import io.flutter.plugin.common.MethodChannel
 import io.flutter.plugin.common.MethodChannel.MethodCallHandler
@@ -42,6 +51,7 @@ class EmbeddedDataHandler(private val context: Context) : MethodCallHandler {
         when (call.method) {
             "getExifThumbnails" -> ioScope.launch { safeSuspend(call, result, ::getExifThumbnails) }
             "extractGoogleDeviceItem" -> ioScope.launch { safe(call, result, ::extractGoogleDeviceItem) }
+            "extractJpegMultiPictureFormat" -> ioScope.launch { safe(call, result, ::extractJpegMultiPictureFormat) }
             "extractMotionPhotoImage" -> ioScope.launch { safe(call, result, ::extractMotionPhotoImage) }
             "extractMotionPhotoVideo" -> ioScope.launch { safe(call, result, ::extractMotionPhotoVideo) }
             "extractVideoEmbeddedPicture" -> ioScope.launch { safe(call, result, ::extractVideoEmbeddedPicture) }
@@ -141,6 +151,50 @@ class EmbeddedDataHandler(private val context: Context) : MethodCallHandler {
         result.error("extractGoogleDeviceItem-empty", "failed to extract item from Google Device XMP at uri=$uri dataUri=$dataUri", null)
     }
 
+    private fun extractJpegMultiPictureFormat(call: MethodCall, result: MethodChannel.Result) {
+        val mimeType = call.argument<String>("mimeType")
+        val uri = call.argument<String>("uri")?.let { Uri.parse(it) }
+        val sizeBytes = call.argument<Number>("sizeBytes")?.toLong()
+        val displayName = call.argument<String>("displayName")
+        val id = call.argument<Int>("id")
+        if (mimeType == null || uri == null || sizeBytes == null || id == null) {
+            result.error("extractJpegMultiPictureFormat-args", "missing arguments", null)
+            return
+        }
+
+        if (canReadWithMetadataExtractor(mimeType)) {
+            try {
+                Metadata.openSafeInputStream(context, uri, mimeType, sizeBytes)?.use { input ->
+                    val metadata = Helper.safeRead(input)
+                    metadata.getDirectoriesOfType(MpEntryDirectory::class.java).first { it.id == id }?.let { dir ->
+                        val mpEntry = dir.entry
+                        MpEntry.getMimeType(dir.entry.format)?.let { embedMimeType ->
+                            var dataOffset = mpEntry.dataOffset
+                            if (dataOffset > 0) {
+                                val baseOffset = MultiPage.getJpegMultiPictureFormatBaseOffset(context, uri, sizeBytes)
+                                if (baseOffset != null) {
+                                    dataOffset += baseOffset
+                                }
+                            }
+                            StorageUtils.openInputStream(context, uri)?.let { input ->
+                                input.skip(dataOffset)
+                                copyEmbeddedBytes(result, embedMimeType, displayName, input, mpEntry.size)
+                            }
+                            return
+                        }
+                    }
+                }
+            } catch (e: Exception) {
+                Log.w(LOG_TAG, "failed to extract file from MPF", e)
+            } catch (e: NoClassDefFoundError) {
+                Log.w(LOG_TAG, "failed to extract file from MPF", e)
+            } catch (e: AssertionError) {
+                Log.w(LOG_TAG, "failed to extract file from MPF", e)
+            }
+        }
+        result.error("extractJpegMultiPictureFormat-empty", "failed to extract file index=$id from MPF at uri=$uri", null)
+    }
+
     private fun extractMotionPhotoImage(call: MethodCall, result: MethodChannel.Result) {
         val mimeType = call.argument<String>("mimeType")
         val uri = call.argument<String>("uri")?.let { Uri.parse(it) }

android/app/src/main/kotlin/deckers/thibault/aves/metadata/MultiPage.kt

@@ -8,6 +8,7 @@ import android.os.Build
 import android.os.ParcelFileDescriptor
 import android.util.Log
 import com.adobe.internal.xmp.XMPMeta
+import com.drew.imaging.jpeg.JpegSegmentType
 import com.drew.metadata.xmp.XmpDirectory
 import deckers.thibault.aves.metadata.XMP.countPropArrayItems
 import deckers.thibault.aves.metadata.XMP.doesPropExist
@@ -203,6 +204,36 @@ object MultiPage {
         return offsetFromEnd
     }
 
+    // starts after `[APP2 marker (1 byte)] [segment size (2 bytes)] [MPF marker (4 bytes)]`
+    fun getJpegMultiPictureFormatBaseOffset(context: Context, uri: Uri, sizeBytes: Long): Int? {
+        val app2Marker = JpegSegmentType.APP2.byteValue
+        val mpfMarker = "MPF".toByteArray() + 0x00
+
+        try {
+            Metadata.openSafeInputStream(context, uri, MimeTypes.JPEG, sizeBytes)?.use { input ->
+                var offset = 0
+                while (true) {
+                    do {
+                        val b = input.read().toByte()
+                        offset++
+                    } while (b != app2Marker)
+                    // skip 2 bytes for segment size
+                    input.skip(2)
+                    offset += 2
+                    val marker = ByteArray(4)
+                    input.read(marker, 0, marker.size)
+                    offset += 4
+                    if (marker.contentEquals(mpfMarker)) {
+                        return offset
+                    }
+                }
+            }
+        } catch (e: Exception) {
+            Log.w(LOG_TAG, "failed to get MPF base offset from uri=$uri", e)
+        }
+        return null
+    }
+
     fun getTiffPages(context: Context, uri: Uri): ArrayList<FieldMap> {
         fun toMap(page: Int, options: TiffBitmapFactory.Options): FieldMap {
             return hashMapOf(

android/app/src/main/kotlin/deckers/thibault/aves/metadata/metadataextractor/mpf/MpEntryDirectory.kt

@@ -47,10 +47,7 @@ class MpEntryDescriptor(directory: MpEntryDirectory?) : TagDescriptor<MpEntryDir
     }
 
     fun getFormatDescription(format: Int): String {
-        return when (format) {
+        return MpEntry.getMimeType(format) ?: "Unknown ($format)"
-            0 -> MimeTypes.JPEG
-            else -> "Unknown ($format)"
-        }
     }
 
     fun getTypeDescription(type: Int): String {
@@ -73,4 +70,13 @@ class MpEntryDescriptor(directory: MpEntryDirectory?) : TagDescriptor<MpEntryDir
     }
 }
 
-class MpEntry(val flags: Int, val format: Int, val type: Int, val size: Long, val dataOffset: Long, val dep1: Short, val dep2: Short)
+class MpEntry(val flags: Int, val format: Int, val type: Int, val size: Long, val dataOffset: Long, val dep1: Short, val dep2: Short) {
+    companion object {
+        fun getMimeType(format: Int): String? {
+            return when (format) {
+                0 -> MimeTypes.JPEG
+                else -> null
+            }
+        }
+    }
+}

lib/services/media/embedded_data_service.dart

@@ -12,6 +12,8 @@ abstract class EmbeddedDataService {
 
   Future<Map> extractMotionPhotoVideo(AvesEntry entry);
 
+  Future<Map> extractJpegMultiPictureFormat(AvesEntry entry, int index);
+
   Future<Map> extractVideoEmbeddedPicture(AvesEntry entry);
 
   Future<Map> extractXmpDataProp(AvesEntry entry, List<dynamic>? props, String? propMimeType);
@@ -84,6 +86,23 @@ class PlatformEmbeddedDataService implements EmbeddedDataService {
     return {};
   }
 
+  @override
+  Future<Map> extractJpegMultiPictureFormat(AvesEntry entry, int id) async {
+    try {
+      final result = await _platform.invokeMethod('extractJpegMultiPictureFormat', <String, dynamic>{
+        'mimeType': entry.mimeType,
+        'uri': entry.uri,
+        'sizeBytes': entry.sizeBytes,
+        'displayName': ['${entry.bestTitle}', 'MPF #$id'].join(AText.separator),
+        'id': id,
+      });
+      if (result != null) return result as Map;
+    } on PlatformException catch (e, stack) {
+      await reportService.recordError(e, stack);
+    }
+    return {};
+  }
+
   @override
   Future<Map> extractVideoEmbeddedPicture(AvesEntry entry) async {
     try {

lib/widgets/common/action_mixins/feedback.dart

@@ -340,6 +340,7 @@ class _FeedbackMessageState extends State<_FeedbackMessage> with SingleTickerPro
         theme.textTheme.bodyMedium!.copyWith(
           color: colorScheme.onInverseSurface,
         );
+    final contentTextFontSize = contentTextStyle.fontSize ?? theme.textTheme.bodyMedium!.fontSize!;
     final timerChangeShadowColor = colorScheme.primary;
 
     return Row(
@@ -347,7 +348,7 @@ class _FeedbackMessageState extends State<_FeedbackMessage> with SingleTickerPro
         if (widget.type == FeedbackType.warn) ...[
           CustomPaint(
             painter: const _WarnIndicator(AColors.warning),
-            size: Size(4, textScaler.scale(contentTextStyle.fontSize!)),
+            size: Size(4, textScaler.scale(contentTextFontSize)),
           ),
           const SizedBox(width: 8),
         ],

lib/widgets/viewer/info/embedded/embedded_data_opener.dart

@@ -44,6 +44,8 @@ class EmbeddedDataOpener extends StatelessWidget with FeedbackMixin {
         fields = await embeddedDataService.extractGoogleDeviceItem(entry, notification.dataUri!);
       case EmbeddedDataSource.motionPhotoVideo:
         fields = await embeddedDataService.extractMotionPhotoVideo(entry);
+      case EmbeddedDataSource.mpf:
+        fields = await embeddedDataService.extractJpegMultiPictureFormat(entry, notification.mpfId!);
       case EmbeddedDataSource.videoCover:
         fields = await embeddedDataService.extractVideoEmbeddedPicture(entry);
       case EmbeddedDataSource.xmp:

lib/widgets/viewer/info/embedded/notifications.dart

@@ -1,19 +1,21 @@
 import 'package:flutter/foundation.dart';
 import 'package:flutter/material.dart';
 
-enum EmbeddedDataSource { googleDevice, motionPhotoVideo, videoCover, xmp }
+enum EmbeddedDataSource { googleDevice, motionPhotoVideo, mpf, videoCover, xmp }
 
 @immutable
 class OpenEmbeddedDataNotification extends Notification {
   final EmbeddedDataSource source;
   final List<dynamic>? props;
   final String? mimeType, dataUri;
+  final int? mpfId;
 
   const OpenEmbeddedDataNotification._private({
     required this.source,
     this.props,
     this.mimeType,
     this.dataUri,
+    this.mpfId,
   });
 
   factory OpenEmbeddedDataNotification.googleDevice({
@@ -28,6 +30,11 @@ class OpenEmbeddedDataNotification extends Notification {
         source: EmbeddedDataSource.motionPhotoVideo,
       );
 
+  factory OpenEmbeddedDataNotification.mpf(int id) => OpenEmbeddedDataNotification._private(
+        source: EmbeddedDataSource.mpf,
+        mpfId: id,
+      );
+
   factory OpenEmbeddedDataNotification.videoCover() => const OpenEmbeddedDataNotification._private(
         source: EmbeddedDataSource.videoCover,
       );
@@ -43,5 +50,5 @@ class OpenEmbeddedDataNotification extends Notification {
       );
 
   @override
-  String toString() => '$runtimeType#${shortHash(this)}{source=$source, props=$props, mimeType=$mimeType, dataUri=$dataUri}';
+  String toString() => '$runtimeType#${shortHash(this)}{source=$source, props=$props, mimeType=$mimeType, dataUri=$dataUri, index=$mpfId}';
 }

lib/widgets/viewer/info/metadata/metadata_dir.dart

@@ -13,9 +13,10 @@ class MetadataDirectory {
   // special directory names
   static const exifThumbnailDirectory = 'Exif Thumbnail'; // from metadata-extractor
   static const xmpDirectory = 'XMP'; // from metadata-extractor
-  static const mediaDirectory = 'Media'; // custom
   static const coverDirectory = 'Cover'; // custom
   static const geoTiffDirectory = 'GeoTIFF'; // custom
+  static const mediaDirectory = 'Media'; // custom
+  static const mpfImageDirectoryPrefix = 'MPF Image #'; // custom
 
   const MetadataDirectory(
     this.name,

lib/widgets/viewer/info/metadata/metadata_dir_tile.dart

@@ -6,6 +6,7 @@ import 'package:aves/services/metadata/svg_metadata_service.dart';
 import 'package:aves/theme/colors.dart';
 import 'package:aves/widgets/common/extensions/build_context.dart';
 import 'package:aves/widgets/common/identity/aves_expansion_tile.dart';
+import 'package:aves/widgets/common/identity/buttons/outlined_button.dart';
 import 'package:aves/widgets/viewer/info/common.dart';
 import 'package:aves/widgets/viewer/info/embedded/notifications.dart';
 import 'package:aves/widgets/viewer/info/metadata/geotiff.dart';
@@ -109,6 +110,19 @@ class MetadataDirTileBody extends StatelessWidget {
 
       children = [
         if (showThumbnails && dirName == MetadataDirectory.exifThumbnailDirectory) MetadataThumbnails(entry: entry),
+        if (dirName.startsWith(MetadataDirectory.mpfImageDirectoryPrefix))
+          Padding(
+            padding: const EdgeInsets.symmetric(horizontal: 8),
+            child: AvesOutlinedButton(
+              label: context.l10n.viewerInfoOpenLinkText,
+              onPressed: () {
+                final id = int.tryParse(dirName.substring(MetadataDirectory.mpfImageDirectoryPrefix.length));
+                if (id != null) {
+                  OpenEmbeddedDataNotification.mpf(id).dispatch(context);
+                }
+              },
+            ),
+          ),
         Padding(
           padding: const EdgeInsets.only(left: 8, right: 8, bottom: 8),
           child: InfoRowGroup(