Fabio/aves
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

#1423 support for Samsung HEIC motion photos embedding video in sefd box

Browse source
This commit is contained in:
Thibault Deckers 2025-02-09 11:32:27 +01:00
parent e2e0ee706f
commit 026cfebd49
7 changed files with 179 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
CHANGELOG.md
View file

@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
## <a id="unreleased"></a>[Unreleased]
### Added
- support for Samsung HEIC motion photos embedding video in sefd box
## <a id="v1.12.3"></a>[v1.12.3] - 2025-02-06
### Added

2
android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/DebugHandler.kt
View file

@ -316,7 +316,7 @@ class DebugHandler(private val context: Context) : MethodCallHandler {
}
val sb = StringBuilder()
if (mimeType == MimeTypes.MP4) {
if (mimeType == MimeTypes.MP4 || MimeTypes.isHeic(mimeType)) {
try {
// we can skip uninteresting boxes with a seekable data source
val pfd = StorageUtils.openInputFileDescriptor(context, uri) ?: throw Exception("failed to open file descriptor for uri=$uri")

9
android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/EmbeddedDataHandler.kt
View file

@ -186,7 +186,7 @@ class EmbeddedDataHandler(private val context: Context) : MethodCallHandler {
return
}
MultiPage.getMotionPhotoVideoSize(context, uri, mimeType, sizeBytes)?.let { videoSizeBytes ->
MultiPage.getTrailerVideoSize(context, uri, mimeType, sizeBytes)?.let { videoSizeBytes ->
val imageSizeBytes = sizeBytes - videoSizeBytes
StorageUtils.openInputStream(context, uri)?.let { input ->
copyEmbeddedBytes(result, mimeType, displayName, input, imageSizeBytes)
@ -207,11 +207,10 @@ class EmbeddedDataHandler(private val context: Context) : MethodCallHandler {
return
}
MultiPage.getMotionPhotoVideoSize(context, uri, mimeType, sizeBytes)?.let { videoSizeBytes ->
val videoStartOffset = sizeBytes - videoSizeBytes
MultiPage.getMotionPhotoVideoSizing(context, uri, mimeType, sizeBytes)?.let { (videoOffset, videoSize) ->
StorageUtils.openInputStream(context, uri)?.let { input ->
input.skip(videoStartOffset)
copyEmbeddedBytes(result, MimeTypes.MP4, displayName, input, videoSizeBytes)
input.skip(videoOffset)
copyEmbeddedBytes(result, MimeTypes.MP4, displayName, input, videoSize)
}
return
}

26
android/app/src/main/kotlin/deckers/thibault/aves/channel/calls/MetadataFetchHandler.kt
View file

@ -6,6 +6,7 @@ import android.media.MediaMetadataRetriever
import android.net.Uri
import android.os.Build
import android.util.Log
import androidx.core.net.toUri
import com.adobe.internal.xmp.XMPException
import com.adobe.internal.xmp.XMPMeta
import com.adobe.internal.xmp.XMPMetaFactory
@ -13,6 +14,7 @@ import com.adobe.internal.xmp.options.SerializeOptions
import com.adobe.internal.xmp.properties.XMPPropertyInfo
import com.drew.lang.KeyValuePair
import com.drew.lang.Rational
import com.drew.lang.SequentialByteArrayReader
import com.drew.metadata.Tag
import com.drew.metadata.avi.AviDirectory
import com.drew.metadata.exif.ExifDirectoryBase
@ -107,7 +109,6 @@ import java.util.Locale
import kotlin.math.roundToInt
import kotlin.math.roundToLong
import androidx.exifinterface.media.ExifInterfaceFork as ExifInterface
import androidx.core.net.toUri
class MetadataFetchHandler(private val context: Context) : MethodCallHandler {
private val ioScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
@ -470,6 +471,22 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler {
// Android's `MediaExtractor` and `MediaPlayer` cannot be used for details
// about embedded images as they do not list them as separate tracks
// and only identify at most one
} else if (isHeic(mimeType)) {
Mp4ParserHelper.getSamsungSefd(context, uri)?.let { (_, bytes) ->
val dir = hashMapOf(
"Size" to bytes.size.toString(),
)
val reader = SequentialByteArrayReader(bytes).apply {
isMotorolaByteOrder = false
}
val start = reader.uInt16
val tag = reader.uInt16
if (start == 0 && tag == Mp4ParserHelper.SEFD_EMBEDDED_VIDEO_TAG) {
val nameSize = reader.uInt32
dir["Embedded Video Type"] = reader.getString(nameSize.toInt())
}
metadataMap[Mp4ParserHelper.SAMSUNG_MAKERNOTE_BOX_TYPE] = dir
}
}
if (metadataMap.isNotEmpty()) {
@ -531,6 +548,13 @@ class MetadataFetchHandler(private val context: Context) : MethodCallHandler {
getMultimediaCatalogMetadataByMediaMetadataRetriever(mimeType, uri, metadataMap)
}
if (isHeic(mimeType)) {
val flags = (metadataMap[KEY_FLAGS] ?: 0) as Int
if ((flags and MASK_IS_MOTION_PHOTO == 0) && MultiPage.isHeicSefdMotionPhoto(context, uri)) {
metadataMap[KEY_FLAGS] = flags or MASK_IS_MULTIPAGE or MASK_IS_MOTION_PHOTO
}
}
// report success even when empty
result.success(metadataMap)
}

40
android/app/src/main/kotlin/deckers/thibault/aves/metadata/Mp4ParserHelper.kt
View file

@ -45,6 +45,16 @@ object Mp4ParserHelper {
// arbitrary size to detect boxes that may yield an OOM
private const val BOX_SIZE_DANGER_THRESHOLD = 3 * (1 shl 20) // MB
const val SAMSUNG_MAKERNOTE_BOX_TYPE = "sefd"
const val SEFD_EMBEDDED_VIDEO_TAG = 0x0a30
const val SEFD_MOTION_PHOTO_NAME = "MotionPhoto_Data"
private val largerTypeWhitelist = listOf(
// HEIC motion photo may contain Samsung maker notes in `sefd` box,
// including a video larger than the danger threshold
SAMSUNG_MAKERNOTE_BOX_TYPE,
)
fun computeEdits(context: Context, uri: Uri, modifier: (isoFile: IsoFile) -> Unit): List<Pair<Long, ByteArray>> {
// we can skip uninteresting boxes with a seekable data source
val pfd = StorageUtils.openInputFileDescriptor(context, uri) ?: throw Exception("failed to open file descriptor for uri=$uri")
@ -133,6 +143,34 @@ object Mp4ParserHelper {
return false
}
fun getSamsungSefd(context: Context, uri: Uri): Pair<Long, ByteArray>? {
try {
// we can skip uninteresting boxes with a seekable data source
val pfd = StorageUtils.openInputFileDescriptor(context, uri) ?: throw Exception("failed to open file descriptor for uri=$uri")
pfd.use {
FileInputStream(it.fileDescriptor).use { stream ->
stream.channel.use { channel ->
IsoFile(channel, metadataBoxParser()).use { isoFile ->
var offset = 0L
for (box in isoFile.boxes) {
if (box is UnknownBox && box.type == SAMSUNG_MAKERNOTE_BOX_TYPE) {
if (!box.isParsed) {
box.parseDetails()
}
return Pair(offset + 8, box.data.toByteArray()) // skip 8 bytes for box header
}
offset += box.size
}
}
}
}
}
} catch (e: Exception) {
Log.w(LOG_TAG, "failed to read sefd box", e)
}
return null
}
// extensions
fun IsoFile.updateLocation(locationIso6709: String?) {
@ -272,7 +310,7 @@ object Mp4ParserHelper {
)
setBoxSkipper { type, size ->
if (skippedTypes.contains(type)) return@setBoxSkipper true
if (size > BOX_SIZE_DANGER_THRESHOLD) throw Exception("box (type=$type size=$size) is too large")
if (size > BOX_SIZE_DANGER_THRESHOLD && !largerTypeWhitelist.contains(type)) throw Exception("box (type=$type size=$size) is too large")
false
}
}

71
android/app/src/main/kotlin/deckers/thibault/aves/metadata/MultiPage.kt
View file

@ -11,6 +11,7 @@ import android.os.ParcelFileDescriptor
import android.util.Log
import com.adobe.internal.xmp.XMPMeta
import com.drew.imaging.jpeg.JpegSegmentType
import com.drew.lang.SequentialByteArrayReader
import com.drew.metadata.exif.ExifDirectoryBase
import com.drew.metadata.exif.ExifIFD0Directory
import com.drew.metadata.xmp.XmpDirectory
@ -37,6 +38,8 @@ import androidx.exifinterface.media.ExifInterfaceFork as ExifInterface
object MultiPage {
private val LOG_TAG = LogUtils.createTag<MultiPage>()
// TODO TLAD more generic support, (e.g. 0x00000014 + `ftyp` + `qt `)
// atom length (variable, e.g. `0x00000018`) + atom type (`ftyp`) + type (variable, e.g. `mp42`, `qt`)
private val heicMotionPhotoVideoStartIndicator = byteArrayOf(0x00, 0x00, 0x00, 0x18) + "ftypmp42".toByteArray()
// page info
@ -84,6 +87,22 @@ object MultiPage {
return tracks
}
fun isHeicSefdMotionPhoto(context: Context, uri: Uri): Boolean {
Mp4ParserHelper.getSamsungSefd(context, uri)?.let { (_, bytes) ->
val reader = SequentialByteArrayReader(bytes).apply {
isMotorolaByteOrder = false
}
val start = reader.uInt16
val tag = reader.uInt16
if (start == 0 && tag == Mp4ParserHelper.SEFD_EMBEDDED_VIDEO_TAG) {
val nameSize = reader.uInt32
val name = reader.getString(nameSize.toInt())
return name == Mp4ParserHelper.SEFD_MOTION_PHOTO_NAME
}
}
return false
}
private fun getJpegMpfPrimaryRotation(context: Context, uri: Uri, sizeBytes: Long): Int {
val mimeType = MimeTypes.JPEG
var rotationDegrees = 0
@ -245,8 +264,7 @@ object MultiPage {
fun getMotionPhotoPages(context: Context, uri: Uri, mimeType: String, sizeBytes: Long): ArrayList<FieldMap> {
val pages = ArrayList<FieldMap>()
getMotionPhotoVideoSize(context, uri, mimeType, sizeBytes)?.let { videoSizeBytes ->
getTrailerVideoInfo(context, uri, fileSizeBytes = sizeBytes, videoSizeBytes = videoSizeBytes)?.let { videoInfo ->
getMotionPhotoVideoInfo(context, uri, mimeType, sizeBytes)?.let { videoInfo ->
// set the original image as the first and default track
var pageIndex = 0
pages.add(
@ -274,11 +292,10 @@ object MultiPage {
}
}
}
}
return pages
}
fun getMotionPhotoVideoSize(context: Context, uri: Uri, mimeType: String, sizeBytes: Long): Long? {
fun getTrailerVideoSize(context: Context, uri: Uri, mimeType: String, sizeBytes: Long): Long? {
if (MimeTypes.isHeic(mimeType)) {
// XMP in HEIC motion photos (as taken with a Samsung Camera v12.0.01.50) indicates an `Item:Length` of 68 bytes for the video.
// This item does not contain the video itself, but only some kind of metadata (no doc, no spec),
@ -325,15 +342,25 @@ object MultiPage {
return offsetFromEnd
}
fun getTrailerVideoInfo(context: Context, uri: Uri, fileSizeBytes: Long, videoSizeBytes: Long): MediaFormat? {
private fun getMotionPhotoVideoInfo(context: Context, uri: Uri, mimeType: String, sizeBytes: Long): MediaFormat? {
getMotionPhotoVideoSizing(context, uri, mimeType, sizeBytes)?.let { (videoOffset, videoSize) ->
return getEmbedVideoInfo(context, uri, videoOffset, videoSize)
}
return null
}
fun getTrailerVideoInfo(context: Context, uri: Uri, fileSize: Long, videoSize: Long): MediaFormat? {
return getEmbedVideoInfo(context, uri, videoOffset = fileSize - videoSize, videoSize = videoSize)
}
private fun getEmbedVideoInfo(context: Context, uri: Uri, videoOffset: Long, videoSize: Long): MediaFormat? {
var format: MediaFormat? = null
val extractor = MediaExtractor()
var pfd: ParcelFileDescriptor? = null
try {
val videoStartOffset = fileSizeBytes - videoSizeBytes
pfd = context.contentResolver.openFileDescriptor(uri, "r")
pfd?.fileDescriptor?.let { fd ->
extractor.setDataSource(fd, videoStartOffset, videoSizeBytes)
extractor.setDataSource(fd, videoOffset, videoSize)
if (extractor.trackCount > 0) {
// only consider the first track to represent the appended video
val trackIndex = 0
@ -353,6 +380,36 @@ object MultiPage {
return format
}
fun getMotionPhotoVideoSizing(context: Context, uri: Uri, mimeType: String, sizeBytes: Long): Pair<Long, Long>? {
// default to trailer videos
getTrailerVideoSize(context, uri, mimeType, sizeBytes)?.let { videoSize ->
val videoOffset = sizeBytes - videoSize
return Pair(videoOffset, videoSize)
}
if (MimeTypes.isHeic(mimeType)) {
// fallback to video within Samsung SEFD box
Mp4ParserHelper.getSamsungSefd(context, uri)?.let { (sefdOffset, bytes) ->
val reader = SequentialByteArrayReader(bytes).apply {
isMotorolaByteOrder = false
}
val start = reader.uInt16
val tag = reader.uInt16
if (start == 0 && tag == Mp4ParserHelper.SEFD_EMBEDDED_VIDEO_TAG) {
val nameSize = reader.uInt32
val name = reader.getString(nameSize.toInt())
if (name == Mp4ParserHelper.SEFD_MOTION_PHOTO_NAME) {
val videoOffset = sefdOffset + reader.position
val videoSize = reader.available().toLong()
return Pair(videoOffset, videoSize)
}
}
}
}
return null
}
fun getTiffPages(context: Context, uri: Uri): ArrayList<FieldMap> {
fun toMap(pageIndex: Int, options: TiffBitmapFactory.Options): FieldMap {
return hashMapOf(

40
android/app/src/main/kotlin/deckers/thibault/aves/model/provider/ImageProvider.kt
View file

@ -648,13 +648,13 @@ abstract class ImageProvider {
val originalFileSize = File(path).length()
var trailerVideoBytes: ByteArray? = null
val editableFile = StorageUtils.createTempFile(context).apply {
val videoSize = MultiPage.getMotionPhotoVideoSize(context, uri, mimeType, originalFileSize)?.let { it + trailerDiff }
val isTrailerVideoValid = videoSize != null && MultiPage.getTrailerVideoInfo(context, uri, originalFileSize, videoSize) != null
val trailerVideoSize = MultiPage.getTrailerVideoSize(context, uri, mimeType, originalFileSize)?.let { it + trailerDiff }
val isTrailerVideoValid = trailerVideoSize != null && MultiPage.getTrailerVideoInfo(context, uri, originalFileSize, trailerVideoSize) != null
try {
if (videoSize != null && isTrailerVideoValid) {
if (trailerVideoSize != null && isTrailerVideoValid) {
// handle motion photo and embedded video separately
val imageSize = (originalFileSize - videoSize).toInt()
val videoByteSize = videoSize.toInt()
val imageSize = (originalFileSize - trailerVideoSize).toInt()
val videoByteSize = trailerVideoSize.toInt()
trailerVideoBytes = ByteArray(videoByteSize)
StorageUtils.openInputStream(context, uri)?.let { input ->
@ -733,13 +733,13 @@ abstract class ImageProvider {
val originalFileSize = File(path).length()
var trailerVideoBytes: ByteArray? = null
val editableFile = StorageUtils.createTempFile(context).apply {
val videoSize = MultiPage.getMotionPhotoVideoSize(context, uri, mimeType, originalFileSize)?.let { it + trailerDiff }
val isTrailerVideoValid = videoSize != null && MultiPage.getTrailerVideoInfo(context, uri, originalFileSize, videoSize) != null
val trailerVideoSize = MultiPage.getTrailerVideoSize(context, uri, mimeType, originalFileSize)?.let { it + trailerDiff }
val isTrailerVideoValid = trailerVideoSize != null && MultiPage.getTrailerVideoInfo(context, uri, originalFileSize, trailerVideoSize) != null
try {
if (videoSize != null && isTrailerVideoValid) {
if (trailerVideoSize != null && isTrailerVideoValid) {
// handle motion photo and embedded video separately
val imageSize = (originalFileSize - videoSize).toInt()
val videoByteSize = videoSize.toInt()
val imageSize = (originalFileSize - trailerVideoSize).toInt()
val videoByteSize = trailerVideoSize.toInt()
trailerVideoBytes = ByteArray(videoByteSize)
StorageUtils.openInputStream(context, uri)?.let { input ->
@ -899,7 +899,7 @@ abstract class ImageProvider {
}
val originalFileSize = File(path).length()
val videoSize = MultiPage.getMotionPhotoVideoSize(context, uri, mimeType, originalFileSize)?.let { it.toInt() + trailerDiff }
val trailerVideoSize = MultiPage.getTrailerVideoSize(context, uri, mimeType, originalFileSize)?.let { it.toInt() + trailerDiff }
val editableFile = StorageUtils.createTempFile(context).apply {
try {
editXmpWithPixy(
@ -921,7 +921,7 @@ abstract class ImageProvider {
// copy the edited temporary file back to the original
editableFile.transferTo(outputStream(context, mimeType, uri, path))
if (autoCorrectTrailerOffset && !checkTrailerOffset(context, path, uri, mimeType, videoSize, editableFile, callback)) {
if (autoCorrectTrailerOffset && !checkTrailerOffset(context, path, uri, mimeType, trailerVideoSize, editableFile, callback)) {
return false
}
editableFile.delete()
@ -1262,15 +1262,15 @@ abstract class ImageProvider {
callback: ImageOpCallback,
) {
val originalFileSize = File(path).length()
val videoSize = MultiPage.getMotionPhotoVideoSize(context, uri, mimeType, originalFileSize)
if (videoSize == null) {
val trailerVideoSize = MultiPage.getTrailerVideoSize(context, uri, mimeType, originalFileSize)
if (trailerVideoSize == null) {
callback.onFailure(Exception("failed to get trailer video size"))
return
}
val isTrailerVideoValid = MultiPage.getTrailerVideoInfo(context, uri, fileSizeBytes = originalFileSize, videoSizeBytes = videoSize) != null
val isTrailerVideoValid = MultiPage.getTrailerVideoInfo(context, uri, fileSize = originalFileSize, videoSize = trailerVideoSize) != null
if (!isTrailerVideoValid) {
callback.onFailure(Exception("failed to open trailer video with size=$videoSize"))
callback.onFailure(Exception("failed to open trailer video with size=$trailerVideoSize"))
return
}
@ -1278,7 +1278,7 @@ abstract class ImageProvider {
try {
val inputStream = StorageUtils.openInputStream(context, uri)
// partial copy
transferFrom(inputStream, originalFileSize - videoSize)
transferFrom(inputStream, originalFileSize - trailerVideoSize)
} catch (e: Exception) {
Log.d(LOG_TAG, "failed to remove trailer video", e)
callback.onFailure(e)
@ -1313,8 +1313,8 @@ abstract class ImageProvider {
}
val originalFileSize = File(path).length()
val videoSize = MultiPage.getMotionPhotoVideoSize(context, uri, mimeType, originalFileSize)
val isTrailerVideoValid = videoSize != null && MultiPage.getTrailerVideoInfo(context, uri, originalFileSize, videoSize) != null
val trailerVideoSize = MultiPage.getTrailerVideoSize(context, uri, mimeType, originalFileSize)
val isTrailerVideoValid = trailerVideoSize != null && MultiPage.getTrailerVideoInfo(context, uri, originalFileSize, trailerVideoSize) != null
val editableFile = StorageUtils.createTempFile(context).apply {
try {
outputStream().use { output ->
@ -1334,7 +1334,7 @@ abstract class ImageProvider {
// copy the edited temporary file back to the original
editableFile.transferTo(outputStream(context, mimeType, uri, path))
if (!types.contains(TYPE_XMP) && isTrailerVideoValid && !checkTrailerOffset(context, path, uri, mimeType, videoSize, editableFile, callback)) {
if (!types.contains(TYPE_XMP) && isTrailerVideoValid && !checkTrailerOffset(context, path, uri, mimeType, trailerVideoSize, editableFile, callback)) {
return
}
editableFile.delete()