info: show metadata from MP4 user data box

CHANGELOG.md

@@ -10,6 +10,7 @@ All notable changes to this project will be documented in this file.
 - Video: action to lock viewer
 - Info: improved state/place display (requires rescan, limited to AU/GB/IN/US)
 - Info: edit tags with state placeholder
+- Info: show metadata from MP4 user data box
 - Countries: show states for selected countries
 - Tags: delete selected tags from all media in collection
 - improved support for system font scale

android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/DebugHandler.kt

@@ -38,10 +38,6 @@ import kotlinx.coroutines.SupervisorJob
 import kotlinx.coroutines.launch
 import org.beyka.tiffbitmapfactory.TiffBitmapFactory
 import org.mp4parser.IsoFile
-import org.mp4parser.PropertyBoxParserImpl
-import org.mp4parser.boxes.iso14496.part12.FreeBox
-import org.mp4parser.boxes.iso14496.part12.MediaDataBox
-import org.mp4parser.boxes.iso14496.part12.SampleTableBox
 import java.io.FileInputStream
 import java.io.IOException
 
@@ -341,23 +337,7 @@ class DebugHandler(private val context: Context) : MethodCallHandler {
                 pfd.use {
                     FileInputStream(it.fileDescriptor).use { stream ->
                         stream.channel.use { channel ->
-                            val boxParser = PropertyBoxParserImpl().apply {
+                            IsoFile(channel, Mp4ParserHelper.metadataBoxParser()).use { isoFile ->
-                                val skippedTypes = listOf(
-                                    // parsing `MediaDataBox` can take a long time
-                                    MediaDataBox.TYPE,
-                                    // parsing `SampleTableBox` or `FreeBox` may yield OOM
-                                    SampleTableBox.TYPE, FreeBox.TYPE,
-                                    // some files are padded with `0` but the parser does not stop, reads type "0000",
-                                    // then a large size from following "0000", which may yield OOM
-                                    "0000",
-                                )
-                                setBoxSkipper { type, size ->
-                                    if (skippedTypes.contains(type)) return@setBoxSkipper true
-                                    if (size > Mp4ParserHelper.BOX_SIZE_DANGER_THRESHOLD) throw Exception("box (type=$type size=$size) is too large")
-                                    false
-                                }
-                            }
-                            IsoFile(channel, boxParser).use { isoFile ->
                                 isoFile.dumpBoxes(sb)
                             }
                         }

android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MetadataFetchHandler.kt

@@ -160,9 +160,11 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler {
                     thisDirName = "Spherical Video"
                     metadataMap[thisDirName] = HashMap(GSpherical(bytes).describe())
                 }
+
                 QuickTimeMetadata.PROF_UUID -> {
                     // redundant with info derived on the Dart side
                 }
+
                 QuickTimeMetadata.USMT_UUID -> {
                     val bytes = dir.getByteArray(Mp4UuidBoxDirectory.TAG_USER_DATA)
                     val blocks = QuickTimeMetadata.parseUuidUsmt(bytes)
@@ -187,6 +189,7 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler {
                         }
                     }
                 }
+
                 else -> {
                     val uuidPart = uuid.substringBefore('-')
                     thisDirName = "${dir.name} $uuidPart"
@@ -268,11 +271,13 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler {
                                                         // skip `Geo double/ascii params`, as their content is split and presented through various GeoTIFF keys
                                                         ExifGeoTiffTags.TAG_GEO_DOUBLE_PARAMS,
                                                         ExifGeoTiffTags.TAG_GEO_ASCII_PARAMS -> ArrayList()
+
                                                         else -> listOf(exifTagMapper(tag))
                                                     }
                                                 }?.let { geoTiffDirMap.putAll(it) }
                                                 byGeoTiff[false]?.map { exifTagMapper(it) }?.let { dirMap.putAll(it) }
                                             }
+
                                             mimeType == MimeTypes.DNG -> {
                                                 // split DNG tags in their own directory
                                                 val dngDirMap = metadataMap[DIR_DNG] ?: HashMap()
@@ -281,9 +286,11 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler {
                                                 byDng[true]?.map { exifTagMapper(it) }?.let { dngDirMap.putAll(it) }
                                                 byDng[false]?.map { exifTagMapper(it) }?.let { dirMap.putAll(it) }
                                             }
+
                                             else -> dirMap.putAll(tags.map { exifTagMapper(it) })
                                         }
                                     }
+
                                     dir.isPngTextDir() -> {
                                         metadataMap.remove(thisDirName)
                                         dirMap = metadataMap[DIR_PNG_TEXTUAL_DATA] ?: HashMap()
@@ -332,6 +339,7 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler {
                                             }
                                         }
                                     }
+
                                     else -> dirMap.putAll(tags.map { Pair(it.tagName, it.description) })
                                 }
                             }
@@ -406,6 +414,12 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler {
         }
 
         if (isVideo(mimeType)) {
+            // `metadata-extractor` do not extract custom tags in user data box
+            val userDataDir = Mp4ParserHelper.getUserData(context, mimeType, uri)
+            if (userDataDir.isNotEmpty()) {
+                metadataMap[Metadata.DIR_MP4_USER_DATA] = userDataDir
+            }
+
             // this is used as fallback when the video metadata cannot be found on the Dart side
             // and to identify whether there is an accessible cover image
             // do not include HEIC here
@@ -641,12 +655,14 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler {
                                 }
                             }
                         }
+
                         MimeTypes.GIF -> {
                             // identification of animated GIF
                             if (metadata.containsDirectoryOfType(GifAnimationDirectory::class.java)) {
                                 flags = flags or MASK_IS_ANIMATED
                             }
                         }
+
                         MimeTypes.WEBP -> {
                             // identification of animated WEBP
                             for (dir in metadata.getDirectoriesOfType(WebpDirectory::class.java)) {
@@ -655,6 +671,7 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler {
                                 }
                             }
                         }
+
                         MimeTypes.TIFF -> {
                             // identification of GeoTIFF
                             for (dir in metadata.getDirectoriesOfType(ExifIFD0Directory::class.java)) {
@@ -1119,16 +1136,19 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler {
                                 }
                             }
                         }
+
                         ExifSubIFDDirectory.TAG_DATETIME_DIGITIZED -> {
                             for (dir in metadata.getDirectoriesOfType(ExifSubIFDDirectory::class.java)) {
                                 dir.getDateDigitizedMillis { dateMillis = it }
                             }
                         }
+
                         ExifSubIFDDirectory.TAG_DATETIME_ORIGINAL -> {
                             for (dir in metadata.getDirectoriesOfType(ExifSubIFDDirectory::class.java)) {
                                 dir.getDateOriginalMillis { dateMillis = it }
                             }
                         }
+
                         GpsDirectory.TAG_DATE_STAMP -> {
                             for (dir in metadata.getDirectoriesOfType(GpsDirectory::class.java)) {
                                 dir.gpsDate?.let { dateMillis = it.time }

android/app/src/main/kotlin/deckers/thibault/aves/metadata/Metadata.kt

@@ -33,6 +33,7 @@ object Metadata {
     const val DIR_DNG = "DNG" // custom
     const val DIR_EXIF_GEOTIFF = "GeoTIFF" // custom
     const val DIR_PNG_TEXTUAL_DATA = "PNG Textual Data" // custom
+    const val DIR_MP4_USER_DATA = "User Data" // custom
 
     // types of metadata
     const val TYPE_COMMENT = "comment"

android/app/src/main/kotlin/deckers/thibault/aves/metadata/Mp4ParserHelper.kt

@@ -2,11 +2,22 @@ package deckers.thibault.aves.metadata
 
 import android.content.Context
 import android.net.Uri
+import android.util.Log
+import deckers.thibault.aves.utils.LogUtils
+import deckers.thibault.aves.utils.MimeTypes
 import deckers.thibault.aves.utils.StorageUtils
+import deckers.thibault.aves.utils.toByteArray
+import deckers.thibault.aves.utils.toHex
 import org.mp4parser.*
+import org.mp4parser.boxes.UnknownBox
 import org.mp4parser.boxes.UserBox
+import org.mp4parser.boxes.apple.AppleCoverBox
 import org.mp4parser.boxes.apple.AppleGPSCoordinatesBox
+import org.mp4parser.boxes.apple.AppleItemListBox
+import org.mp4parser.boxes.apple.AppleVariableSignedIntegerBox
+import org.mp4parser.boxes.apple.Utf8AppleDataBox
 import org.mp4parser.boxes.iso14496.part12.*
+import org.mp4parser.boxes.threegpp.ts26244.AuthorBox
 import org.mp4parser.support.AbstractBox
 import org.mp4parser.support.Matrix
 import org.mp4parser.tools.Path
@@ -15,8 +26,10 @@ import java.io.FileInputStream
 import java.nio.channels.Channels
 
 object Mp4ParserHelper {
+    private val LOG_TAG = LogUtils.createTag<Mp4ParserHelper>()
+
     // arbitrary size to detect boxes that may yield an OOM
-    const val BOX_SIZE_DANGER_THRESHOLD = 3 * (1 shl 20) // MB
+    private const val BOX_SIZE_DANGER_THRESHOLD = 3 * (1 shl 20) // MB
 
     fun computeEdits(context: Context, uri: Uri, modifier: (isoFile: IsoFile) -> Unit): List<Pair<Long, ByteArray>> {
         // we can skip uninteresting boxes with a seekable data source
@@ -214,10 +227,8 @@ object Mp4ParserHelper {
                         sb.appendLine("${"\t".repeat(indent)}[$boxType] ${box.javaClass.simpleName}")
                         box.dumpBoxes(sb, indent + 1)
                     }
-                    is UserBox -> {
+
-                        val userTypeHex = box.userType.joinToString("") { "%02x".format(it) }
+                    is UserBox -> sb.appendLine("${"\t".repeat(indent)}[$boxType] userType=${box.userType.toHex()} $box")
-                        sb.appendLine("${"\t".repeat(indent)}[$boxType] userType=$userTypeHex $box")
-                    }
                     else -> sb.appendLine("${"\t".repeat(indent)}[$boxType] $box")
                 }
             } catch (e: Exception) {
@@ -231,6 +242,120 @@ object Mp4ParserHelper {
         Channels.newChannel(stream).use { getBox(it) }
         return stream.toByteArray()
     }
+
+    fun metadataBoxParser() = PropertyBoxParserImpl().apply {
+        val skippedTypes = listOf(
+            // parsing `MediaDataBox` can take a long time
+            MediaDataBox.TYPE,
+            // parsing `SampleTableBox` or `FreeBox` may yield OOM
+            SampleTableBox.TYPE, FreeBox.TYPE,
+            // some files are padded with `0` but the parser does not stop, reads type "0000",
+            // then a large size from following "0000", which may yield OOM
+            "0000",
+        )
+        setBoxSkipper { type, size ->
+            if (skippedTypes.contains(type)) return@setBoxSkipper true
+            if (size > BOX_SIZE_DANGER_THRESHOLD) throw Exception("box (type=$type size=$size) is too large")
+            false
+        }
+    }
+
+    fun getUserData(
+        context: Context,
+        mimeType: String,
+        uri: Uri,
+    ): MutableMap<String, String> {
+        val fields = HashMap<String, String>()
+        if (mimeType != MimeTypes.MP4) return fields
+        try {
+            // we can skip uninteresting boxes with a seekable data source
+            val pfd = StorageUtils.openInputFileDescriptor(context, uri) ?: throw Exception("failed to open file descriptor for uri=$uri")
+            pfd.use {
+                FileInputStream(it.fileDescriptor).use { stream ->
+                    stream.channel.use { channel ->
+                        // creating `IsoFile` with a `File` or a `File.inputStream()` yields `No such device`
+                        IsoFile(channel, metadataBoxParser()).use { isoFile ->
+                            val userDataBox = Path.getPath<UserDataBox>(isoFile.movieBox, UserDataBox.TYPE)
+                            fields.putAll(extractBoxFields(userDataBox))
+                        }
+                    }
+                }
+            }
+        } catch (e: Exception) {
+            Log.w(LOG_TAG, "failed to get User Data box by MP4 parser for mimeType=$mimeType uri=$uri", e)
+        }
+        return fields
+    }
+
+    private fun extractBoxFields(container: Container): HashMap<String, String> {
+        val fields = HashMap<String, String>()
+        for (box in container.boxes) {
+            if (box is AbstractBox && !box.isParsed) {
+                box.parseDetails()
+            }
+            val type = box.type
+            val key = boxTypeMetadataKey(type)
+            when (box) {
+                is AuthorBox -> fields[key] = box.author
+                is AppleCoverBox -> fields[key] = "[${box.coverData.size} bytes]"
+                is AppleGPSCoordinatesBox -> fields[key] = box.value
+                is AppleItemListBox -> fields.putAll(extractBoxFields(box))
+                is AppleVariableSignedIntegerBox -> fields[key] = box.value.toString()
+                is Utf8AppleDataBox -> fields[key] = box.value
+
+                is HandlerBox -> {}
+                is MetaBox -> {
+                    val handlerBox = Path.getPath<HandlerBox>(box, HandlerBox.TYPE).apply { parseDetails() }
+                    when (val handlerType = handlerBox?.handlerType ?: MetaBox.TYPE) {
+                        "mdir" -> fields.putAll(extractBoxFields(box))
+                        else -> fields.putAll(extractBoxFields(box).map { Pair("$handlerType/${it.key}", it.value) }.toMap())
+                    }
+                }
+
+                is UnknownBox -> {
+                    val byteBuffer = box.data
+                    val remaining = byteBuffer.remaining()
+                    if (remaining > 512) {
+                        fields[key] = "[$remaining bytes]"
+                    } else {
+                        val bytes = byteBuffer.toByteArray()
+                        when (type) {
+                            "SDLN",
+                            "smrd" -> fields[key] = String(bytes)
+
+                            else -> fields[key] = "0x${bytes.toHex()}"
+                        }
+                    }
+                }
+
+                else -> fields[key] = box.toString()
+            }
+        }
+        return fields
+    }
+
+    // cf https://exiftool.org/TagNames/QuickTime.html
+    private fun boxTypeMetadataKey(type: String) = when (type) {
+        "auth" -> "Author"
+        "catg" -> "Category"
+        "covr" -> "Cover Art"
+        "keyw" -> "Keyword"
+        "mcvr" -> "Preview Image"
+        "pcst" -> "Podcast"
+        "SDLN" -> "Play Mode"
+        "stik" -> "Media Type"
+        "©alb" -> "Album"
+        "©ART" -> "Artist"
+        "©aut" -> "Author"
+        "©cmt" -> "Comment"
+        "©day" -> "Year"
+        "©des" -> "Description"
+        "©gen" -> "Genre"
+        "©nam" -> "Title"
+        "©too" -> "Encoder"
+        "©xyz" -> "GPS Coordinates"
+        else -> type
+    }
 }
 
 class Mp4TooLargeException(val type: String, message: String) : RuntimeException(message)

android/app/src/main/kotlin/deckers/thibault/aves/metadata/QuickTimeMetadata.kt

@@ -1,5 +1,6 @@
 package deckers.thibault.aves.metadata
 
+import deckers.thibault.aves.utils.toHex
 import java.math.BigInteger
 import java.nio.charset.Charset
 import java.util.*
@@ -51,7 +52,7 @@ object QuickTimeMetadata {
                 // 0x01: string
                 0x01 -> String(payload, Charset.forName("UTF-16BE")).trim()
                 // 0x101: artwork/icon
-                else -> "0x${payload.joinToString("") { "%02x".format(it) }}"
+                else -> "0x${payload.toHex()}"
             }
 
             val blockTypeString = when (blockType) {
@@ -61,7 +62,7 @@ object QuickTimeMetadata {
                 0x0A -> "Track property"
                 0x0B -> "Time zone"
                 0x0C -> "Modification Time"
-                else -> "0x${"%02x".format(blockType)}"
+                else -> "0x${blockType.toByte().toHex()}"
             }
 
             blocks.add(

android/app/src/main/kotlin/deckers/thibault/aves/metadata/XMP.kt

@@ -21,13 +21,9 @@ import deckers.thibault.aves.utils.MemoryUtils
 import deckers.thibault.aves.utils.MimeTypes
 import deckers.thibault.aves.utils.StorageUtils
 import org.mp4parser.IsoFile
-import org.mp4parser.PropertyBoxParserImpl
 import org.mp4parser.boxes.UserBox
-import org.mp4parser.boxes.iso14496.part12.FreeBox
-import org.mp4parser.boxes.iso14496.part12.MediaDataBox
-import org.mp4parser.boxes.iso14496.part12.SampleTableBox
 import java.io.FileInputStream
-import java.util.*
+import java.util.TimeZone
 
 object XMP {
     private val LOG_TAG = LogUtils.createTag<XMP>()
@@ -156,26 +152,12 @@ object XMP {
             pfd.use {
                 FileInputStream(it.fileDescriptor).use { stream ->
                     stream.channel.use { channel ->
-                        val boxParser = PropertyBoxParserImpl().apply {
-                            val skippedTypes = listOf(
-                                // parsing `MediaDataBox` can take a long time
-                                MediaDataBox.TYPE,
-                                // parsing `SampleTableBox` or `FreeBox` may yield OOM
-                                SampleTableBox.TYPE, FreeBox.TYPE,
-                            )
-                            setBoxSkipper { type, size ->
-                                if (skippedTypes.contains(type)) return@setBoxSkipper true
-                                if (size > Mp4ParserHelper.BOX_SIZE_DANGER_THRESHOLD) throw Exception("box (type=$type size=$size) is too large")
-                                false
-                            }
-                        }
-                        // creating `IsoFile` with a `File` or a `File.inputStream()` yields `No such device`
-
                         // TODO TLAD [mp4] `IsoFile` init may fail if a skipped box has a `org.mp4parser.boxes.iso14496.part12.MetaBox` as parent,
                         // because `MetaBox.parse()` changes the argument `dataSource` to a `RewindableReadableByteChannel`,
                         // so it is no longer a seekable `FileChannel`, which is a requirement to skip boxes.
 
-                        IsoFile(channel, boxParser).use { isoFile ->
+                        // creating `IsoFile` with a `File` or a `File.inputStream()` yields `No such device`
+                        IsoFile(channel, Mp4ParserHelper.metadataBoxParser()).use { isoFile ->
                             isoFile.processBoxes(UserBox::class.java, true) { box, _ ->
                                 val boxSize = box.size
                                 if (MemoryUtils.canAllocate(boxSize)) {

android/app/src/main/kotlin/deckers/thibault/aves/utils/ByteUtils.kt

@@ -0,0 +1,13 @@
+package deckers.thibault.aves.utils
+
+import java.nio.ByteBuffer
+
+fun ByteBuffer.toByteArray(): ByteArray {
+    val bytes = ByteArray(remaining())
+    get(bytes, 0, bytes.size)
+    return bytes
+}
+
+fun ByteArray.toHex(): String = joinToString(separator = "") { it.toHex() }
+
+fun Byte.toHex(): String = "%02x".format(this)

lib/model/video/metadata.dart

@@ -211,6 +211,12 @@ class VideoMetadataFormatter {
               final captureFps = double.parse(value);
               save('Capture Frame Rate', '${roundToPrecision(captureFps, decimals: 3).toString()} FPS');
               break;
+            case Keys.androidManufacturer:
+              save('Android Manufacturer', value);
+              break;
+            case Keys.androidModel:
+              save('Android Model', value);
+              break;
             case Keys.androidVersion:
               save('Android Version', value);
               break;
@@ -316,6 +322,16 @@ class VideoMetadataFormatter {
             case Keys.minorVersion:
               if (value != '0') save('Minor Version', value);
               break;
+            case Keys.quicktimeLocationAccuracyHorizontal:
+              save('QuickTime Location Horizontal Accuracy', value);
+              break;
+            case Keys.quicktimeCreationDate:
+            case Keys.quicktimeLocationIso6709:
+            case Keys.quicktimeMake:
+            case Keys.quicktimeModel:
+            case Keys.quicktimeSoftware:
+              // redundant with `QuickTime Metadata` directory
+              break;
             case Keys.rotate:
               save('Rotation', '$value°');
               break;
@@ -346,6 +362,9 @@ class VideoMetadataFormatter {
             case Keys.width:
               save('Width', '$value pixels');
               break;
+            case Keys.xiaomiSlowMoment:
+              save('Xiaomi Slow Moment', value);
+              break;
             default:
               save(key.toSentenceCase(), value.toString());
           }

plugins/aves_model/lib/src/video/keys.dart

@@ -3,6 +3,8 @@
 // that write additional metadata to media files
 class Keys {
   static const androidCaptureFramerate = 'com.android.capture.fps';
+  static const androidManufacturer = 'com.android.manufacturer';
+  static const androidModel = 'com.android.model';
   static const androidVersion = 'com.android.version';
   static const bps = 'bps';
   static const bitrate = 'bitrate';
@@ -31,6 +33,12 @@ class Keys {
   static const mediaFormat = 'format';
   static const mediaType = 'media_type';
   static const minorVersion = 'minor_version';
+  static const quicktimeCreationDate = 'com.apple.quicktime.creationdate';
+  static const quicktimeLocationAccuracyHorizontal = 'com.apple.quicktime.location.accuracy.horizontal';
+  static const quicktimeLocationIso6709 = 'com.apple.quicktime.location.iso6709';
+  static const quicktimeMake = 'com.apple.quicktime.make';
+  static const quicktimeModel = 'com.apple.quicktime.model';
+  static const quicktimeSoftware = 'com.apple.quicktime.software';
   static const rotate = 'rotate';
   static const sampleRate = 'sample_rate';
   static const sarDen = 'sar_den';
@@ -50,4 +58,5 @@ class Keys {
   static const title = 'title';
   static const track = 'track';
   static const width = 'width';
+  static const xiaomiSlowMoment = 'com.xiaomi.slow_moment';
 }