From 4f67f55c1a7e0991025628cccdcf888462f6c719 Mon Sep 17 00:00:00 2001 From: Thibault Deckers Date: Tue, 18 Apr 2023 16:23:14 +0200 Subject: [PATCH] info: show metadata from MP4 user data box --- CHANGELOG.md | 1 + .../aves/channel/calls/DebugHandler.kt | 22 +-- .../channel/calls/MetadataFetchHandler.kt | 20 +++ .../thibault/aves/metadata/Metadata.kt | 1 + .../thibault/aves/metadata/Mp4ParserHelper.kt | 135 +++++++++++++++++- .../aves/metadata/QuickTimeMetadata.kt | 5 +- .../deckers/thibault/aves/metadata/XMP.kt | 24 +--- .../deckers/thibault/aves/utils/ByteUtils.kt | 13 ++ lib/model/video/metadata.dart | 19 +++ plugins/aves_model/lib/src/video/keys.dart | 9 ++ 10 files changed, 200 insertions(+), 49 deletions(-) create mode 100644 android/app/src/main/kotlin/deckers/thibault/aves/utils/ByteUtils.kt diff --git a/CHANGELOG.md b/CHANGELOG.md index 089ef8e22..c7ca48588 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ All notable changes to this project will be documented in this file. - Video: action to lock viewer - Info: improved state/place display (requires rescan, limited to AU/GB/IN/US) - Info: edit tags with state placeholder +- Info: show metadata from MP4 user data box - Countries: show states for selected countries - Tags: delete selected tags from all media in collection - improved support for system font scale diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/DebugHandler.kt b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/DebugHandler.kt index 0ec5ace5c..812b86e91 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/DebugHandler.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/DebugHandler.kt @@ -38,10 +38,6 @@ import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.launch import org.beyka.tiffbitmapfactory.TiffBitmapFactory import org.mp4parser.IsoFile -import org.mp4parser.PropertyBoxParserImpl -import org.mp4parser.boxes.iso14496.part12.FreeBox -import org.mp4parser.boxes.iso14496.part12.MediaDataBox -import org.mp4parser.boxes.iso14496.part12.SampleTableBox import java.io.FileInputStream import java.io.IOException @@ -341,23 +337,7 @@ class DebugHandler(private val context: Context) : MethodCallHandler { pfd.use { FileInputStream(it.fileDescriptor).use { stream -> stream.channel.use { channel -> - val boxParser = PropertyBoxParserImpl().apply { - val skippedTypes = listOf( - // parsing `MediaDataBox` can take a long time - MediaDataBox.TYPE, - // parsing `SampleTableBox` or `FreeBox` may yield OOM - SampleTableBox.TYPE, FreeBox.TYPE, - // some files are padded with `0` but the parser does not stop, reads type "0000", - // then a large size from following "0000", which may yield OOM - "0000", - ) - setBoxSkipper { type, size -> - if (skippedTypes.contains(type)) return@setBoxSkipper true - if (size > Mp4ParserHelper.BOX_SIZE_DANGER_THRESHOLD) throw Exception("box (type=$type size=$size) is too large") - false - } - } - IsoFile(channel, boxParser).use { isoFile -> + IsoFile(channel, Mp4ParserHelper.metadataBoxParser()).use { isoFile -> isoFile.dumpBoxes(sb) } } diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MetadataFetchHandler.kt b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MetadataFetchHandler.kt index 1358fdb69..feda79f93 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MetadataFetchHandler.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MetadataFetchHandler.kt @@ -160,9 +160,11 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler { thisDirName = "Spherical Video" metadataMap[thisDirName] = HashMap(GSpherical(bytes).describe()) } + QuickTimeMetadata.PROF_UUID -> { // redundant with info derived on the Dart side } + QuickTimeMetadata.USMT_UUID -> { val bytes = dir.getByteArray(Mp4UuidBoxDirectory.TAG_USER_DATA) val blocks = QuickTimeMetadata.parseUuidUsmt(bytes) @@ -187,6 +189,7 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler { } } } + else -> { val uuidPart = uuid.substringBefore('-') thisDirName = "${dir.name} $uuidPart" @@ -268,11 +271,13 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler { // skip `Geo double/ascii params`, as their content is split and presented through various GeoTIFF keys ExifGeoTiffTags.TAG_GEO_DOUBLE_PARAMS, ExifGeoTiffTags.TAG_GEO_ASCII_PARAMS -> ArrayList() + else -> listOf(exifTagMapper(tag)) } }?.let { geoTiffDirMap.putAll(it) } byGeoTiff[false]?.map { exifTagMapper(it) }?.let { dirMap.putAll(it) } } + mimeType == MimeTypes.DNG -> { // split DNG tags in their own directory val dngDirMap = metadataMap[DIR_DNG] ?: HashMap() @@ -281,9 +286,11 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler { byDng[true]?.map { exifTagMapper(it) }?.let { dngDirMap.putAll(it) } byDng[false]?.map { exifTagMapper(it) }?.let { dirMap.putAll(it) } } + else -> dirMap.putAll(tags.map { exifTagMapper(it) }) } } + dir.isPngTextDir() -> { metadataMap.remove(thisDirName) dirMap = metadataMap[DIR_PNG_TEXTUAL_DATA] ?: HashMap() @@ -332,6 +339,7 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler { } } } + else -> dirMap.putAll(tags.map { Pair(it.tagName, it.description) }) } } @@ -406,6 +414,12 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler { } if (isVideo(mimeType)) { + // `metadata-extractor` do not extract custom tags in user data box + val userDataDir = Mp4ParserHelper.getUserData(context, mimeType, uri) + if (userDataDir.isNotEmpty()) { + metadataMap[Metadata.DIR_MP4_USER_DATA] = userDataDir + } + // this is used as fallback when the video metadata cannot be found on the Dart side // and to identify whether there is an accessible cover image // do not include HEIC here @@ -641,12 +655,14 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler { } } } + MimeTypes.GIF -> { // identification of animated GIF if (metadata.containsDirectoryOfType(GifAnimationDirectory::class.java)) { flags = flags or MASK_IS_ANIMATED } } + MimeTypes.WEBP -> { // identification of animated WEBP for (dir in metadata.getDirectoriesOfType(WebpDirectory::class.java)) { @@ -655,6 +671,7 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler { } } } + MimeTypes.TIFF -> { // identification of GeoTIFF for (dir in metadata.getDirectoriesOfType(ExifIFD0Directory::class.java)) { @@ -1119,16 +1136,19 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler { } } } + ExifSubIFDDirectory.TAG_DATETIME_DIGITIZED -> { for (dir in metadata.getDirectoriesOfType(ExifSubIFDDirectory::class.java)) { dir.getDateDigitizedMillis { dateMillis = it } } } + ExifSubIFDDirectory.TAG_DATETIME_ORIGINAL -> { for (dir in metadata.getDirectoriesOfType(ExifSubIFDDirectory::class.java)) { dir.getDateOriginalMillis { dateMillis = it } } } + GpsDirectory.TAG_DATE_STAMP -> { for (dir in metadata.getDirectoriesOfType(GpsDirectory::class.java)) { dir.gpsDate?.let { dateMillis = it.time } diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/metadata/Metadata.kt b/android/app/src/main/kotlin/deckers/thibault/aves/metadata/Metadata.kt index 54ac83280..699dc4009 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/metadata/Metadata.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/metadata/Metadata.kt @@ -33,6 +33,7 @@ object Metadata { const val DIR_DNG = "DNG" // custom const val DIR_EXIF_GEOTIFF = "GeoTIFF" // custom const val DIR_PNG_TEXTUAL_DATA = "PNG Textual Data" // custom + const val DIR_MP4_USER_DATA = "User Data" // custom // types of metadata const val TYPE_COMMENT = "comment" diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/metadata/Mp4ParserHelper.kt b/android/app/src/main/kotlin/deckers/thibault/aves/metadata/Mp4ParserHelper.kt index 8edb548df..1c89c00a1 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/metadata/Mp4ParserHelper.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/metadata/Mp4ParserHelper.kt @@ -2,11 +2,22 @@ package deckers.thibault.aves.metadata import android.content.Context import android.net.Uri +import android.util.Log +import deckers.thibault.aves.utils.LogUtils +import deckers.thibault.aves.utils.MimeTypes import deckers.thibault.aves.utils.StorageUtils +import deckers.thibault.aves.utils.toByteArray +import deckers.thibault.aves.utils.toHex import org.mp4parser.* +import org.mp4parser.boxes.UnknownBox import org.mp4parser.boxes.UserBox +import org.mp4parser.boxes.apple.AppleCoverBox import org.mp4parser.boxes.apple.AppleGPSCoordinatesBox +import org.mp4parser.boxes.apple.AppleItemListBox +import org.mp4parser.boxes.apple.AppleVariableSignedIntegerBox +import org.mp4parser.boxes.apple.Utf8AppleDataBox import org.mp4parser.boxes.iso14496.part12.* +import org.mp4parser.boxes.threegpp.ts26244.AuthorBox import org.mp4parser.support.AbstractBox import org.mp4parser.support.Matrix import org.mp4parser.tools.Path @@ -15,8 +26,10 @@ import java.io.FileInputStream import java.nio.channels.Channels object Mp4ParserHelper { + private val LOG_TAG = LogUtils.createTag() + // arbitrary size to detect boxes that may yield an OOM - const val BOX_SIZE_DANGER_THRESHOLD = 3 * (1 shl 20) // MB + private const val BOX_SIZE_DANGER_THRESHOLD = 3 * (1 shl 20) // MB fun computeEdits(context: Context, uri: Uri, modifier: (isoFile: IsoFile) -> Unit): List> { // we can skip uninteresting boxes with a seekable data source @@ -214,10 +227,8 @@ object Mp4ParserHelper { sb.appendLine("${"\t".repeat(indent)}[$boxType] ${box.javaClass.simpleName}") box.dumpBoxes(sb, indent + 1) } - is UserBox -> { - val userTypeHex = box.userType.joinToString("") { "%02x".format(it) } - sb.appendLine("${"\t".repeat(indent)}[$boxType] userType=$userTypeHex $box") - } + + is UserBox -> sb.appendLine("${"\t".repeat(indent)}[$boxType] userType=${box.userType.toHex()} $box") else -> sb.appendLine("${"\t".repeat(indent)}[$boxType] $box") } } catch (e: Exception) { @@ -231,6 +242,120 @@ object Mp4ParserHelper { Channels.newChannel(stream).use { getBox(it) } return stream.toByteArray() } + + fun metadataBoxParser() = PropertyBoxParserImpl().apply { + val skippedTypes = listOf( + // parsing `MediaDataBox` can take a long time + MediaDataBox.TYPE, + // parsing `SampleTableBox` or `FreeBox` may yield OOM + SampleTableBox.TYPE, FreeBox.TYPE, + // some files are padded with `0` but the parser does not stop, reads type "0000", + // then a large size from following "0000", which may yield OOM + "0000", + ) + setBoxSkipper { type, size -> + if (skippedTypes.contains(type)) return@setBoxSkipper true + if (size > BOX_SIZE_DANGER_THRESHOLD) throw Exception("box (type=$type size=$size) is too large") + false + } + } + + fun getUserData( + context: Context, + mimeType: String, + uri: Uri, + ): MutableMap { + val fields = HashMap() + if (mimeType != MimeTypes.MP4) return fields + try { + // we can skip uninteresting boxes with a seekable data source + val pfd = StorageUtils.openInputFileDescriptor(context, uri) ?: throw Exception("failed to open file descriptor for uri=$uri") + pfd.use { + FileInputStream(it.fileDescriptor).use { stream -> + stream.channel.use { channel -> + // creating `IsoFile` with a `File` or a `File.inputStream()` yields `No such device` + IsoFile(channel, metadataBoxParser()).use { isoFile -> + val userDataBox = Path.getPath(isoFile.movieBox, UserDataBox.TYPE) + fields.putAll(extractBoxFields(userDataBox)) + } + } + } + } + } catch (e: Exception) { + Log.w(LOG_TAG, "failed to get User Data box by MP4 parser for mimeType=$mimeType uri=$uri", e) + } + return fields + } + + private fun extractBoxFields(container: Container): HashMap { + val fields = HashMap() + for (box in container.boxes) { + if (box is AbstractBox && !box.isParsed) { + box.parseDetails() + } + val type = box.type + val key = boxTypeMetadataKey(type) + when (box) { + is AuthorBox -> fields[key] = box.author + is AppleCoverBox -> fields[key] = "[${box.coverData.size} bytes]" + is AppleGPSCoordinatesBox -> fields[key] = box.value + is AppleItemListBox -> fields.putAll(extractBoxFields(box)) + is AppleVariableSignedIntegerBox -> fields[key] = box.value.toString() + is Utf8AppleDataBox -> fields[key] = box.value + + is HandlerBox -> {} + is MetaBox -> { + val handlerBox = Path.getPath(box, HandlerBox.TYPE).apply { parseDetails() } + when (val handlerType = handlerBox?.handlerType ?: MetaBox.TYPE) { + "mdir" -> fields.putAll(extractBoxFields(box)) + else -> fields.putAll(extractBoxFields(box).map { Pair("$handlerType/${it.key}", it.value) }.toMap()) + } + } + + is UnknownBox -> { + val byteBuffer = box.data + val remaining = byteBuffer.remaining() + if (remaining > 512) { + fields[key] = "[$remaining bytes]" + } else { + val bytes = byteBuffer.toByteArray() + when (type) { + "SDLN", + "smrd" -> fields[key] = String(bytes) + + else -> fields[key] = "0x${bytes.toHex()}" + } + } + } + + else -> fields[key] = box.toString() + } + } + return fields + } + + // cf https://exiftool.org/TagNames/QuickTime.html + private fun boxTypeMetadataKey(type: String) = when (type) { + "auth" -> "Author" + "catg" -> "Category" + "covr" -> "Cover Art" + "keyw" -> "Keyword" + "mcvr" -> "Preview Image" + "pcst" -> "Podcast" + "SDLN" -> "Play Mode" + "stik" -> "Media Type" + "©alb" -> "Album" + "©ART" -> "Artist" + "©aut" -> "Author" + "©cmt" -> "Comment" + "©day" -> "Year" + "©des" -> "Description" + "©gen" -> "Genre" + "©nam" -> "Title" + "©too" -> "Encoder" + "©xyz" -> "GPS Coordinates" + else -> type + } } class Mp4TooLargeException(val type: String, message: String) : RuntimeException(message) diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/metadata/QuickTimeMetadata.kt b/android/app/src/main/kotlin/deckers/thibault/aves/metadata/QuickTimeMetadata.kt index 6341ffe53..0c198ad29 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/metadata/QuickTimeMetadata.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/metadata/QuickTimeMetadata.kt @@ -1,5 +1,6 @@ package deckers.thibault.aves.metadata +import deckers.thibault.aves.utils.toHex import java.math.BigInteger import java.nio.charset.Charset import java.util.* @@ -51,7 +52,7 @@ object QuickTimeMetadata { // 0x01: string 0x01 -> String(payload, Charset.forName("UTF-16BE")).trim() // 0x101: artwork/icon - else -> "0x${payload.joinToString("") { "%02x".format(it) }}" + else -> "0x${payload.toHex()}" } val blockTypeString = when (blockType) { @@ -61,7 +62,7 @@ object QuickTimeMetadata { 0x0A -> "Track property" 0x0B -> "Time zone" 0x0C -> "Modification Time" - else -> "0x${"%02x".format(blockType)}" + else -> "0x${blockType.toByte().toHex()}" } blocks.add( diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/metadata/XMP.kt b/android/app/src/main/kotlin/deckers/thibault/aves/metadata/XMP.kt index f618c0df1..939bd7076 100644 --- a/android/app/src/main/kotlin/deckers/thibault/aves/metadata/XMP.kt +++ b/android/app/src/main/kotlin/deckers/thibault/aves/metadata/XMP.kt @@ -21,13 +21,9 @@ import deckers.thibault.aves.utils.MemoryUtils import deckers.thibault.aves.utils.MimeTypes import deckers.thibault.aves.utils.StorageUtils import org.mp4parser.IsoFile -import org.mp4parser.PropertyBoxParserImpl import org.mp4parser.boxes.UserBox -import org.mp4parser.boxes.iso14496.part12.FreeBox -import org.mp4parser.boxes.iso14496.part12.MediaDataBox -import org.mp4parser.boxes.iso14496.part12.SampleTableBox import java.io.FileInputStream -import java.util.* +import java.util.TimeZone object XMP { private val LOG_TAG = LogUtils.createTag() @@ -156,26 +152,12 @@ object XMP { pfd.use { FileInputStream(it.fileDescriptor).use { stream -> stream.channel.use { channel -> - val boxParser = PropertyBoxParserImpl().apply { - val skippedTypes = listOf( - // parsing `MediaDataBox` can take a long time - MediaDataBox.TYPE, - // parsing `SampleTableBox` or `FreeBox` may yield OOM - SampleTableBox.TYPE, FreeBox.TYPE, - ) - setBoxSkipper { type, size -> - if (skippedTypes.contains(type)) return@setBoxSkipper true - if (size > Mp4ParserHelper.BOX_SIZE_DANGER_THRESHOLD) throw Exception("box (type=$type size=$size) is too large") - false - } - } - // creating `IsoFile` with a `File` or a `File.inputStream()` yields `No such device` - // TODO TLAD [mp4] `IsoFile` init may fail if a skipped box has a `org.mp4parser.boxes.iso14496.part12.MetaBox` as parent, // because `MetaBox.parse()` changes the argument `dataSource` to a `RewindableReadableByteChannel`, // so it is no longer a seekable `FileChannel`, which is a requirement to skip boxes. - IsoFile(channel, boxParser).use { isoFile -> + // creating `IsoFile` with a `File` or a `File.inputStream()` yields `No such device` + IsoFile(channel, Mp4ParserHelper.metadataBoxParser()).use { isoFile -> isoFile.processBoxes(UserBox::class.java, true) { box, _ -> val boxSize = box.size if (MemoryUtils.canAllocate(boxSize)) { diff --git a/android/app/src/main/kotlin/deckers/thibault/aves/utils/ByteUtils.kt b/android/app/src/main/kotlin/deckers/thibault/aves/utils/ByteUtils.kt new file mode 100644 index 000000000..f45236ba4 --- /dev/null +++ b/android/app/src/main/kotlin/deckers/thibault/aves/utils/ByteUtils.kt @@ -0,0 +1,13 @@ +package deckers.thibault.aves.utils + +import java.nio.ByteBuffer + +fun ByteBuffer.toByteArray(): ByteArray { + val bytes = ByteArray(remaining()) + get(bytes, 0, bytes.size) + return bytes +} + +fun ByteArray.toHex(): String = joinToString(separator = "") { it.toHex() } + +fun Byte.toHex(): String = "%02x".format(this) \ No newline at end of file diff --git a/lib/model/video/metadata.dart b/lib/model/video/metadata.dart index 2691bb6fe..eab4c5031 100644 --- a/lib/model/video/metadata.dart +++ b/lib/model/video/metadata.dart @@ -211,6 +211,12 @@ class VideoMetadataFormatter { final captureFps = double.parse(value); save('Capture Frame Rate', '${roundToPrecision(captureFps, decimals: 3).toString()} FPS'); break; + case Keys.androidManufacturer: + save('Android Manufacturer', value); + break; + case Keys.androidModel: + save('Android Model', value); + break; case Keys.androidVersion: save('Android Version', value); break; @@ -316,6 +322,16 @@ class VideoMetadataFormatter { case Keys.minorVersion: if (value != '0') save('Minor Version', value); break; + case Keys.quicktimeLocationAccuracyHorizontal: + save('QuickTime Location Horizontal Accuracy', value); + break; + case Keys.quicktimeCreationDate: + case Keys.quicktimeLocationIso6709: + case Keys.quicktimeMake: + case Keys.quicktimeModel: + case Keys.quicktimeSoftware: + // redundant with `QuickTime Metadata` directory + break; case Keys.rotate: save('Rotation', '$value°'); break; @@ -346,6 +362,9 @@ class VideoMetadataFormatter { case Keys.width: save('Width', '$value pixels'); break; + case Keys.xiaomiSlowMoment: + save('Xiaomi Slow Moment', value); + break; default: save(key.toSentenceCase(), value.toString()); } diff --git a/plugins/aves_model/lib/src/video/keys.dart b/plugins/aves_model/lib/src/video/keys.dart index 47a84bc84..9cb9e3fbd 100644 --- a/plugins/aves_model/lib/src/video/keys.dart +++ b/plugins/aves_model/lib/src/video/keys.dart @@ -3,6 +3,8 @@ // that write additional metadata to media files class Keys { static const androidCaptureFramerate = 'com.android.capture.fps'; + static const androidManufacturer = 'com.android.manufacturer'; + static const androidModel = 'com.android.model'; static const androidVersion = 'com.android.version'; static const bps = 'bps'; static const bitrate = 'bitrate'; @@ -31,6 +33,12 @@ class Keys { static const mediaFormat = 'format'; static const mediaType = 'media_type'; static const minorVersion = 'minor_version'; + static const quicktimeCreationDate = 'com.apple.quicktime.creationdate'; + static const quicktimeLocationAccuracyHorizontal = 'com.apple.quicktime.location.accuracy.horizontal'; + static const quicktimeLocationIso6709 = 'com.apple.quicktime.location.iso6709'; + static const quicktimeMake = 'com.apple.quicktime.make'; + static const quicktimeModel = 'com.apple.quicktime.model'; + static const quicktimeSoftware = 'com.apple.quicktime.software'; static const rotate = 'rotate'; static const sampleRate = 'sample_rate'; static const sarDen = 'sar_den'; @@ -50,4 +58,5 @@ class Keys { static const title = 'title'; static const track = 'track'; static const width = 'width'; + static const xiaomiSlowMoment = 'com.xiaomi.slow_moment'; }