Fabio/aves
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

info: show metadata from MP4 user data box

Browse source
This commit is contained in:
Thibault Deckers 2023-04-18 16:23:14 +02:00
parent 10756f5156
commit 4f67f55c1a
10 changed files with 200 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
CHANGELOG.md
View file

@ -10,6 +10,7 @@ All notable changes to this project will be documented in this file.
- Video: action to lock viewer
- Info: improved state/place display (requires rescan, limited to AU/GB/IN/US)
- Info: edit tags with state placeholder
- Info: show metadata from MP4 user data box
- Countries: show states for selected countries
- Tags: delete selected tags from all media in collection
- improved support for system font scale

22
android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/DebugHandler.kt
View file

@ -38,10 +38,6 @@ import kotlinx.coroutines.SupervisorJob
import kotlinx.coroutines.launch
import org.beyka.tiffbitmapfactory.TiffBitmapFactory
import org.mp4parser.IsoFile
import org.mp4parser.PropertyBoxParserImpl
import org.mp4parser.boxes.iso14496.part12.FreeBox
import org.mp4parser.boxes.iso14496.part12.MediaDataBox
import org.mp4parser.boxes.iso14496.part12.SampleTableBox
import java.io.FileInputStream
import java.io.IOException
@ -341,23 +337,7 @@ class DebugHandler(private val context: Context) : MethodCallHandler {
pfd.use {
FileInputStream(it.fileDescriptor).use { stream ->
stream.channel.use { channel ->
val boxParser = PropertyBoxParserImpl().apply {
val skippedTypes = listOf(
// parsing `MediaDataBox` can take a long time
MediaDataBox.TYPE,
// parsing `SampleTableBox` or `FreeBox` may yield OOM
SampleTableBox.TYPE, FreeBox.TYPE,
// some files are padded with `0` but the parser does not stop, reads type "0000",
// then a large size from following "0000", which may yield OOM
"0000",
)
setBoxSkipper { type, size ->
if (skippedTypes.contains(type)) return@setBoxSkipper true
if (size > Mp4ParserHelper.BOX_SIZE_DANGER_THRESHOLD) throw Exception("box (type=$type size=$size) is too large")
false
}
}
IsoFile(channel, boxParser).use { isoFile ->
IsoFile(channel, Mp4ParserHelper.metadataBoxParser()).use { isoFile ->
isoFile.dumpBoxes(sb)
}
}

20
android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MetadataFetchHandler.kt
View file

@ -160,9 +160,11 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler {
thisDirName = "Spherical Video"
metadataMap[thisDirName] = HashMap(GSpherical(bytes).describe())
}
QuickTimeMetadata.PROF_UUID -> {
// redundant with info derived on the Dart side
}
QuickTimeMetadata.USMT_UUID -> {
val bytes = dir.getByteArray(Mp4UuidBoxDirectory.TAG_USER_DATA)
val blocks = QuickTimeMetadata.parseUuidUsmt(bytes)
@ -187,6 +189,7 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler {
}
}
}
else -> {
val uuidPart = uuid.substringBefore('-')
thisDirName = "${dir.name} $uuidPart"
@ -268,11 +271,13 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler {
// skip `Geo double/ascii params`, as their content is split and presented through various GeoTIFF keys
ExifGeoTiffTags.TAG_GEO_DOUBLE_PARAMS,
ExifGeoTiffTags.TAG_GEO_ASCII_PARAMS -> ArrayList()
else -> listOf(exifTagMapper(tag))
}
}?.let { geoTiffDirMap.putAll(it) }
byGeoTiff[false]?.map { exifTagMapper(it) }?.let { dirMap.putAll(it) }
}
mimeType == MimeTypes.DNG -> {
// split DNG tags in their own directory
val dngDirMap = metadataMap[DIR_DNG] ?: HashMap()
@ -281,9 +286,11 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler {
byDng[true]?.map { exifTagMapper(it) }?.let { dngDirMap.putAll(it) }
byDng[false]?.map { exifTagMapper(it) }?.let { dirMap.putAll(it) }
}
else -> dirMap.putAll(tags.map { exifTagMapper(it) })
}
}
dir.isPngTextDir() -> {
metadataMap.remove(thisDirName)
dirMap = metadataMap[DIR_PNG_TEXTUAL_DATA] ?: HashMap()
@ -332,6 +339,7 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler {
}
}
}
else -> dirMap.putAll(tags.map { Pair(it.tagName, it.description) })
}
}
@ -406,6 +414,12 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler {
}
if (isVideo(mimeType)) {
// `metadata-extractor` do not extract custom tags in user data box
val userDataDir = Mp4ParserHelper.getUserData(context, mimeType, uri)
if (userDataDir.isNotEmpty()) {
metadataMap[Metadata.DIR_MP4_USER_DATA] = userDataDir
}
// this is used as fallback when the video metadata cannot be found on the Dart side
// and to identify whether there is an accessible cover image
// do not include HEIC here
@ -641,12 +655,14 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler {
}
}
}
MimeTypes.GIF -> {
// identification of animated GIF
if (metadata.containsDirectoryOfType(GifAnimationDirectory::class.java)) {
flags = flags or MASK_IS_ANIMATED
}
}
MimeTypes.WEBP -> {
// identification of animated WEBP
for (dir in metadata.getDirectoriesOfType(WebpDirectory::class.java)) {
@ -655,6 +671,7 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler {
}
}
}
MimeTypes.TIFF -> {
// identification of GeoTIFF
for (dir in metadata.getDirectoriesOfType(ExifIFD0Directory::class.java)) {
@ -1119,16 +1136,19 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler {
}
}
}
ExifSubIFDDirectory.TAG_DATETIME_DIGITIZED -> {
for (dir in metadata.getDirectoriesOfType(ExifSubIFDDirectory::class.java)) {
dir.getDateDigitizedMillis { dateMillis = it }
}
}
ExifSubIFDDirectory.TAG_DATETIME_ORIGINAL -> {
for (dir in metadata.getDirectoriesOfType(ExifSubIFDDirectory::class.java)) {
dir.getDateOriginalMillis { dateMillis = it }
}
}
GpsDirectory.TAG_DATE_STAMP -> {
for (dir in metadata.getDirectoriesOfType(GpsDirectory::class.java)) {
dir.gpsDate?.let { dateMillis = it.time }

1
android/app/src/main/kotlin/deckers/thibault/aves/metadata/Metadata.kt
View file

@ -33,6 +33,7 @@ object Metadata {
const val DIR_DNG = "DNG" // custom
const val DIR_EXIF_GEOTIFF = "GeoTIFF" // custom
const val DIR_PNG_TEXTUAL_DATA = "PNG Textual Data" // custom
const val DIR_MP4_USER_DATA = "User Data" // custom
// types of metadata
const val TYPE_COMMENT = "comment"

135
android/app/src/main/kotlin/deckers/thibault/aves/metadata/Mp4ParserHelper.kt
View file

@ -2,11 +2,22 @@ package deckers.thibault.aves.metadata
import android.content.Context
import android.net.Uri
import android.util.Log
import deckers.thibault.aves.utils.LogUtils
import deckers.thibault.aves.utils.MimeTypes
import deckers.thibault.aves.utils.StorageUtils
import deckers.thibault.aves.utils.toByteArray
import deckers.thibault.aves.utils.toHex
import org.mp4parser.*
import org.mp4parser.boxes.UnknownBox
import org.mp4parser.boxes.UserBox
import org.mp4parser.boxes.apple.AppleCoverBox
import org.mp4parser.boxes.apple.AppleGPSCoordinatesBox
import org.mp4parser.boxes.apple.AppleItemListBox
import org.mp4parser.boxes.apple.AppleVariableSignedIntegerBox
import org.mp4parser.boxes.apple.Utf8AppleDataBox
import org.mp4parser.boxes.iso14496.part12.*
import org.mp4parser.boxes.threegpp.ts26244.AuthorBox
import org.mp4parser.support.AbstractBox
import org.mp4parser.support.Matrix
import org.mp4parser.tools.Path
@ -15,8 +26,10 @@ import java.io.FileInputStream
import java.nio.channels.Channels
object Mp4ParserHelper {
private val LOG_TAG = LogUtils.createTag<Mp4ParserHelper>()
// arbitrary size to detect boxes that may yield an OOM
const val BOX_SIZE_DANGER_THRESHOLD = 3 * (1 shl 20) // MB
private const val BOX_SIZE_DANGER_THRESHOLD = 3 * (1 shl 20) // MB
fun computeEdits(context: Context, uri: Uri, modifier: (isoFile: IsoFile) -> Unit): List<Pair<Long, ByteArray>> {
// we can skip uninteresting boxes with a seekable data source
@ -214,10 +227,8 @@ object Mp4ParserHelper {
sb.appendLine("${"\t".repeat(indent)}[$boxType] ${box.javaClass.simpleName}")
box.dumpBoxes(sb, indent + 1)
}
is UserBox -> {
val userTypeHex = box.userType.joinToString("") { "%02x".format(it) }
sb.appendLine("${"\t".repeat(indent)}[$boxType] userType=$userTypeHex $box")
}
is UserBox -> sb.appendLine("${"\t".repeat(indent)}[$boxType] userType=${box.userType.toHex()} $box")
else -> sb.appendLine("${"\t".repeat(indent)}[$boxType] $box")
}
} catch (e: Exception) {
@ -231,6 +242,120 @@ object Mp4ParserHelper {
Channels.newChannel(stream).use { getBox(it) }
return stream.toByteArray()
}
fun metadataBoxParser() = PropertyBoxParserImpl().apply {
val skippedTypes = listOf(
// parsing `MediaDataBox` can take a long time
MediaDataBox.TYPE,
// parsing `SampleTableBox` or `FreeBox` may yield OOM
SampleTableBox.TYPE, FreeBox.TYPE,
// some files are padded with `0` but the parser does not stop, reads type "0000",
// then a large size from following "0000", which may yield OOM
"0000",
)
setBoxSkipper { type, size ->
if (skippedTypes.contains(type)) return@setBoxSkipper true
if (size > BOX_SIZE_DANGER_THRESHOLD) throw Exception("box (type=$type size=$size) is too large")
false
}
}
fun getUserData(
context: Context,
mimeType: String,
uri: Uri,
): MutableMap<String, String> {
val fields = HashMap<String, String>()
if (mimeType != MimeTypes.MP4) return fields
try {
// we can skip uninteresting boxes with a seekable data source
val pfd = StorageUtils.openInputFileDescriptor(context, uri) ?: throw Exception("failed to open file descriptor for uri=$uri")
pfd.use {
FileInputStream(it.fileDescriptor).use { stream ->
stream.channel.use { channel ->
// creating `IsoFile` with a `File` or a `File.inputStream()` yields `No such device`
IsoFile(channel, metadataBoxParser()).use { isoFile ->
val userDataBox = Path.getPath<UserDataBox>(isoFile.movieBox, UserDataBox.TYPE)
fields.putAll(extractBoxFields(userDataBox))
}
}
}
}
} catch (e: Exception) {
Log.w(LOG_TAG, "failed to get User Data box by MP4 parser for mimeType=$mimeType uri=$uri", e)
}
return fields
}
private fun extractBoxFields(container: Container): HashMap<String, String> {
val fields = HashMap<String, String>()
for (box in container.boxes) {
if (box is AbstractBox && !box.isParsed) {
box.parseDetails()
}
val type = box.type
val key = boxTypeMetadataKey(type)
when (box) {
is AuthorBox -> fields[key] = box.author
is AppleCoverBox -> fields[key] = "[${box.coverData.size} bytes]"
is AppleGPSCoordinatesBox -> fields[key] = box.value
is AppleItemListBox -> fields.putAll(extractBoxFields(box))
is AppleVariableSignedIntegerBox -> fields[key] = box.value.toString()
is Utf8AppleDataBox -> fields[key] = box.value
is HandlerBox -> {}
is MetaBox -> {
val handlerBox = Path.getPath<HandlerBox>(box, HandlerBox.TYPE).apply { parseDetails() }
when (val handlerType = handlerBox?.handlerType ?: MetaBox.TYPE) {
"mdir" -> fields.putAll(extractBoxFields(box))
else -> fields.putAll(extractBoxFields(box).map { Pair("$handlerType/${it.key}", it.value) }.toMap())
}
}
is UnknownBox -> {
val byteBuffer = box.data
val remaining = byteBuffer.remaining()
if (remaining > 512) {
fields[key] = "[$remaining bytes]"
} else {
val bytes = byteBuffer.toByteArray()
when (type) {
"SDLN",
"smrd" -> fields[key] = String(bytes)
else -> fields[key] = "0x${bytes.toHex()}"
}
}
}
else -> fields[key] = box.toString()
}
}
return fields
}
// cf https://exiftool.org/TagNames/QuickTime.html
private fun boxTypeMetadataKey(type: String) = when (type) {
"auth" -> "Author"
"catg" -> "Category"
"covr" -> "Cover Art"
"keyw" -> "Keyword"
"mcvr" -> "Preview Image"
"pcst" -> "Podcast"
"SDLN" -> "Play Mode"
"stik" -> "Media Type"
"©alb" -> "Album"
"©ART" -> "Artist"
"©aut" -> "Author"
"©cmt" -> "Comment"
"©day" -> "Year"
"©des" -> "Description"
"©gen" -> "Genre"
"©nam" -> "Title"
"©too" -> "Encoder"
"©xyz" -> "GPS Coordinates"
else -> type
}
}
class Mp4TooLargeException(val type: String, message: String) : RuntimeException(message)

5
android/app/src/main/kotlin/deckers/thibault/aves/metadata/QuickTimeMetadata.kt
View file

@ -1,5 +1,6 @@
package deckers.thibault.aves.metadata
import deckers.thibault.aves.utils.toHex
import java.math.BigInteger
import java.nio.charset.Charset
import java.util.*
@ -51,7 +52,7 @@ object QuickTimeMetadata {
// 0x01: string
0x01 -> String(payload, Charset.forName("UTF-16BE")).trim()
// 0x101: artwork/icon
else -> "0x${payload.joinToString("") { "%02x".format(it) }}"
else -> "0x${payload.toHex()}"
}
val blockTypeString = when (blockType) {
@ -61,7 +62,7 @@ object QuickTimeMetadata {
0x0A -> "Track property"
0x0B -> "Time zone"
0x0C -> "Modification Time"
else -> "0x${"%02x".format(blockType)}"
else -> "0x${blockType.toByte().toHex()}"
}
blocks.add(

24
android/app/src/main/kotlin/deckers/thibault/aves/metadata/XMP.kt
View file

@ -21,13 +21,9 @@ import deckers.thibault.aves.utils.MemoryUtils
import deckers.thibault.aves.utils.MimeTypes
import deckers.thibault.aves.utils.StorageUtils
import org.mp4parser.IsoFile
import org.mp4parser.PropertyBoxParserImpl
import org.mp4parser.boxes.UserBox
import org.mp4parser.boxes.iso14496.part12.FreeBox
import org.mp4parser.boxes.iso14496.part12.MediaDataBox
import org.mp4parser.boxes.iso14496.part12.SampleTableBox
import java.io.FileInputStream
import java.util.*
import java.util.TimeZone
object XMP {
private val LOG_TAG = LogUtils.createTag<XMP>()
@ -156,26 +152,12 @@ object XMP {
pfd.use {
FileInputStream(it.fileDescriptor).use { stream ->
stream.channel.use { channel ->
val boxParser = PropertyBoxParserImpl().apply {
val skippedTypes = listOf(
// parsing `MediaDataBox` can take a long time
MediaDataBox.TYPE,
// parsing `SampleTableBox` or `FreeBox` may yield OOM
SampleTableBox.TYPE, FreeBox.TYPE,
)
setBoxSkipper { type, size ->
if (skippedTypes.contains(type)) return@setBoxSkipper true
if (size > Mp4ParserHelper.BOX_SIZE_DANGER_THRESHOLD) throw Exception("box (type=$type size=$size) is too large")
false
}
}
// creating `IsoFile` with a `File` or a `File.inputStream()` yields `No such device`
// TODO TLAD [mp4] `IsoFile` init may fail if a skipped box has a `org.mp4parser.boxes.iso14496.part12.MetaBox` as parent,
// because `MetaBox.parse()` changes the argument `dataSource` to a `RewindableReadableByteChannel`,
// so it is no longer a seekable `FileChannel`, which is a requirement to skip boxes.
IsoFile(channel, boxParser).use { isoFile ->
// creating `IsoFile` with a `File` or a `File.inputStream()` yields `No such device`
IsoFile(channel, Mp4ParserHelper.metadataBoxParser()).use { isoFile ->
isoFile.processBoxes(UserBox::class.java, true) { box, _ ->
val boxSize = box.size
if (MemoryUtils.canAllocate(boxSize)) {

13
android/app/src/main/kotlin/deckers/thibault/aves/utils/ByteUtils.kt Normal file
View file

@ -0,0 +1,13 @@
package deckers.thibault.aves.utils
import java.nio.ByteBuffer
fun ByteBuffer.toByteArray(): ByteArray {
val bytes = ByteArray(remaining())
get(bytes, 0, bytes.size)
return bytes
}
fun ByteArray.toHex(): String = joinToString(separator = "") { it.toHex() }
fun Byte.toHex(): String = "%02x".format(this)

19
lib/model/video/metadata.dart
View file

@ -211,6 +211,12 @@ class VideoMetadataFormatter {
final captureFps = double.parse(value);
save('Capture Frame Rate', '${roundToPrecision(captureFps, decimals: 3).toString()} FPS');
break;
case Keys.androidManufacturer:
save('Android Manufacturer', value);
break;
case Keys.androidModel:
save('Android Model', value);
break;
case Keys.androidVersion:
save('Android Version', value);
break;
@ -316,6 +322,16 @@ class VideoMetadataFormatter {
case Keys.minorVersion:
if (value != '0') save('Minor Version', value);
break;
case Keys.quicktimeLocationAccuracyHorizontal:
save('QuickTime Location Horizontal Accuracy', value);
break;
case Keys.quicktimeCreationDate:
case Keys.quicktimeLocationIso6709:
case Keys.quicktimeMake:
case Keys.quicktimeModel:
case Keys.quicktimeSoftware:
// redundant with `QuickTime Metadata` directory
break;
case Keys.rotate:
save('Rotation', '$value°');
break;
@ -346,6 +362,9 @@ class VideoMetadataFormatter {
case Keys.width:
save('Width', '$value pixels');
break;
case Keys.xiaomiSlowMoment:
save('Xiaomi Slow Moment', value);
break;
default:
save(key.toSentenceCase(), value.toString());
}

9
plugins/aves_model/lib/src/video/keys.dart
View file

@ -3,6 +3,8 @@
// that write additional metadata to media files
class Keys {
static const androidCaptureFramerate = 'com.android.capture.fps';
static const androidManufacturer = 'com.android.manufacturer';
static const androidModel = 'com.android.model';
static const androidVersion = 'com.android.version';
static const bps = 'bps';
static const bitrate = 'bitrate';
@ -31,6 +33,12 @@ class Keys {
static const mediaFormat = 'format';
static const mediaType = 'media_type';
static const minorVersion = 'minor_version';
static const quicktimeCreationDate = 'com.apple.quicktime.creationdate';
static const quicktimeLocationAccuracyHorizontal = 'com.apple.quicktime.location.accuracy.horizontal';
static const quicktimeLocationIso6709 = 'com.apple.quicktime.location.iso6709';
static const quicktimeMake = 'com.apple.quicktime.make';
static const quicktimeModel = 'com.apple.quicktime.model';
static const quicktimeSoftware = 'com.apple.quicktime.software';
static const rotate = 'rotate';
static const sampleRate = 'sample_rate';
static const sarDen = 'sar_den';
@ -50,4 +58,5 @@ class Keys {
static const title = 'title';
static const track = 'track';
static const width = 'width';
static const xiaomiSlowMoment = 'com.xiaomi.slow_moment';
}