music: add exoplayer backend [#128]

Add an ExoPlayer-based metadata backend.

This backend finally allows Auxio to parse metadata internally, which
has been a highly requested feature given how many unfixable MediaStore
issues it would fix.

However, this is not fully ready for production yet. The loading
process becomes much larger with manual parsing, the current
implementation still allocates picture metadata (terrible for
efficiency), and there is no good indicator in the app to keep
track of music loading.

As a result, this backend is disabled until it can be fully integrated
into Auxio's architecture.
This commit is contained in:
OxygenCobalt 2022-05-28 19:48:24 -06:00
parent 87bdf50d39
commit 47726c3e02
No known key found for this signature in database
GPG key ID: 37DBE3621FE9AD47
7 changed files with 293 additions and 56 deletions

View file

@ -27,10 +27,10 @@ import java.lang.Exception
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import org.oxycblt.auxio.music.indexer.Indexer
import org.oxycblt.auxio.music.indexer.useQuery
import org.oxycblt.auxio.util.contentResolverSafe
import org.oxycblt.auxio.util.logD
import org.oxycblt.auxio.util.logE
import org.oxycblt.auxio.util.useQuery
/**
* The main storage for music items. Getting an instance of this object is more complicated as it

View file

@ -25,9 +25,7 @@ import androidx.lifecycle.viewModelScope
import kotlinx.coroutines.launch
import org.oxycblt.auxio.util.logD
/**
* A [ViewModel] that represents the current music indexing state.
*/
/** A [ViewModel] that represents the current music indexing state. */
class MusicViewModel : ViewModel(), MusicStore.Callback {
private val musicStore = MusicStore.getInstance()
@ -45,7 +43,7 @@ class MusicViewModel : ViewModel(), MusicStore.Callback {
* navigated to and because SnackBars will have the best UX here.
*/
fun loadMusic(context: Context) {
if (_loaderResponse.value != null || isBusy) {
if (musicStore.library != null || isBusy) {
logD("Loader is busy/already completed, not reloading")
return
}

View file

@ -0,0 +1,183 @@
/*
* Copyright (c) 2022 Auxio Project
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.oxycblt.auxio.music.indexer
import android.content.ContentUris
import android.content.Context
import android.database.Cursor
import android.provider.MediaStore
import com.google.android.exoplayer2.MediaItem
import com.google.android.exoplayer2.MetadataRetriever
import com.google.android.exoplayer2.metadata.Metadata
import com.google.android.exoplayer2.metadata.id3.TextInformationFrame
import com.google.android.exoplayer2.metadata.vorbis.VorbisComment
import com.google.android.exoplayer2.source.TrackGroupArray
import com.google.common.util.concurrent.FutureCallback
import com.google.common.util.concurrent.Futures
import java.nio.charset.StandardCharsets
import java.util.concurrent.ConcurrentLinkedQueue
import java.util.concurrent.Executors
import java.util.concurrent.Future
import org.oxycblt.auxio.music.Song
import org.oxycblt.auxio.util.logW
/**
* A [Indexer.Backend] that leverages ExoPlayer's metadata retrieval system to index metadata.
*
* TODO: This class is currently not used, as there are a number of technical improvements that must
* be made first before it is practical.
*
* @author OxygenCobalt
*/
class ExoPlayerBackend(private val inner: MediaStoreBackend) : Indexer.Backend {
private val runningTasks: Array<Future<TrackGroupArray>?> = arrayOfNulls(TASK_CAPACITY)
override fun query(context: Context) = inner.query(context)
override fun loadSongs(context: Context, cursor: Cursor): Collection<Song> {
// Metadata retrieval with ExoPlayer is asynchronous, so a callback may at any point
// add a completed song to the list. To prevent a crash in that case, we use the
// concurrent counterpart to a typical mutable list.
val songs = ConcurrentLinkedQueue<Song>()
while (cursor.moveToNext()) {
val audio = inner.buildAudio(context, cursor)
val audioUri =
ContentUris.withAppendedId(
MediaStore.Audio.Media.EXTERNAL_CONTENT_URI, requireNotNull(audio.id))
while (true) {
// Spin until a task slot opens up, then start trying to parse metadata.
val idx = runningTasks.indexOfFirst { it == null }
if (idx != -1) {
val task =
MetadataRetriever.retrieveMetadata(context, MediaItem.fromUri(audioUri))
Futures.addCallback(
task, AudioCallback(audio, songs, idx), Executors.newSingleThreadExecutor())
runningTasks[idx] = task
break
}
}
}
while (runningTasks.any { it != null }) {
// Spin until all tasks are complete
}
return songs
}
private inner class AudioCallback(
private val audio: MediaStoreBackend.Audio,
private val dest: ConcurrentLinkedQueue<Song>,
private val taskIdx: Int
) : FutureCallback<TrackGroupArray> {
override fun onSuccess(result: TrackGroupArray) {
val metadata = result[0].getFormat(0).metadata
if (metadata != null) {
completeAudio(audio, metadata)
} else {
logW("No metadata was found for ${audio.title}")
}
dest.add(audio.toSong())
runningTasks[taskIdx] = null
}
override fun onFailure(t: Throwable) {
logW("Unable to extract metadata for ${audio.title}")
logW(t.stackTraceToString())
dest.add(audio.toSong())
runningTasks[taskIdx] = null
}
}
private fun completeAudio(audio: MediaStoreBackend.Audio, metadata: Metadata) {
for (i in 0 until metadata.length()) {
when (val tag = metadata.get(i)) {
// ID3v2 text information frame.
is TextInformationFrame ->
if (tag.value.isNotEmpty()) {
handleId3TextFrame(tag.id, tag.value.sanitize(), audio)
}
// Vorbis comment. It is assumed that a Metadata can only have vorbis
// comments/pictures or ID3 frames, never both.
is VorbisComment ->
if (tag.value.isNotEmpty()) {
handleVorbisComment(tag.key, tag.value.sanitize(), audio)
}
}
}
}
/**
* Copies and sanitizes this string under the assumption that it is UTF-8.
*
* Sometimes ExoPlayer emits weird UTF-8. Worse still, sometimes it emits strings backed by data
* allocated by some native function. This could easily cause a terrible crash if you even look
* at the malformed string the wrong way.
*
* This function mitigates it by first encoding the string as UTF-8 bytes (replacing malformed
* characters with the replacement in the process), and then re-interpreting it as a new string,
* which hopefully fixes encoding sanity while also copying the string out of dodgy native
* memory.
*/
private fun String.sanitize() = String(encodeToByteArray(), StandardCharsets.UTF_8)
private fun handleId3TextFrame(id: String, value: String, audio: MediaStoreBackend.Audio) {
// It's assumed that duplicate frames are eliminated by ExoPlayer's metadata parser.
when (id) {
"TIT2" -> audio.title = value // Title
"TRCK" -> value.no?.let { audio.track = it } // Track, as NN/TT
"TPOS" -> value.no?.let { audio.disc = it } // Disc, as NN/TT
"TYER" -> value.toIntOrNull()?.let { audio.year = it } // ID3v2.3 year, should be digits
"TDRC" -> value.iso8601year?.let { audio.year = it } // ID3v2.4 date, parse year field
"TALB" -> audio.album = value // Album
"TPE1" -> audio.artist = value // Artist
"TPE2" -> audio.albumArtist = value // Album artist
"TCON" -> audio.genre = value // Genre, with the weird ID3v2 rules
}
}
private fun handleVorbisComment(key: String, value: String, audio: MediaStoreBackend.Audio) {
// It's assumed that duplicate tags are eliminated by ExoPlayer's metadata parser.
when (key) {
"TITLE" -> audio.title = value // Title, presumably as NN/TT
"TRACKNUMBER" -> value.no?.let { audio.track = it } // Track, presumably as NN/TT
"DISCNUMBER" -> value.no?.let { audio.disc = it } // Disc, presumably as NN/TT
"DATE" -> value.iso8601year?.let { audio.year = it } // Date, presumably as ISO-8601
"ALBUM" -> audio.album = value // Album
"ARTIST" -> audio.artist = value // Artist
"ALBUMARTIST" -> audio.albumArtist = value // Album artist
"GENRE" -> audio.genre = value // Genre, assumed that ID3v2 rules will apply here too.
}
}
companion object {
/**
* The amount of tasks this backend can run efficiently at one time. Eight was chosen here
* as higher values made little difference in speed, and lower values generally caused
* bottlenecks.
*/
private const val TASK_CAPACITY = 8
}
}

View file

@ -20,7 +20,6 @@ package org.oxycblt.auxio.music.indexer
import android.content.Context
import android.database.Cursor
import android.os.Build
import android.provider.MediaStore
import org.oxycblt.auxio.music.Album
import org.oxycblt.auxio.music.Artist
import org.oxycblt.auxio.music.Genre
@ -29,18 +28,44 @@ import org.oxycblt.auxio.music.Song
import org.oxycblt.auxio.ui.Sort
import org.oxycblt.auxio.util.logD
/**
* Auxio's media indexer.
*
* Auxio's media indexer is somewhat complicated, as it has grown to support a variety of use cases
* (and hacky garbage) in order to produce the best possible experience. It is split into three
* distinct steps:
*
* 1. Finding a [Backend] to use and then querying the media database with it.
* 2. Using the [Backend] and the media data to create songs
* 3. Using the songs to build the library, which primarily involves linking up all data objects
* with their corresponding parents/children.
*
* This class in particular handles 3 primarily. For the code that handles 1 and 2, see the other
* files in the module.
*
* @author OxygenCobalt
*/
object Indexer {
fun index(context: Context): MusicStore.Library? {
// Establish the backend to use when initially loading songs.
val backend =
val mediaStoreBackend =
when {
Build.VERSION.SDK_INT >= Build.VERSION_CODES.R -> Api30MediaStoreBackend()
else -> Api21MediaStoreBackend()
}
val songs = buildSongs(context, backend)
// Disabled until direct parsing is fully capable of integration into Auxio's
// architecture.
// val backend = if (settingsManager.useQualityMetadata) {
// ExoPlayerBackend(mediaStoreBackend)
// } else {
// mediaStoreBackend
// }
val songs = buildSongs(context, mediaStoreBackend)
if (songs.isEmpty()) return null
val buildStart = System.currentTimeMillis()
val albums = buildAlbums(songs)
val artists = buildArtists(albums)
val genres = buildGenres(songs)
@ -56,17 +81,27 @@ object Indexer {
}
}
logD("Successfully built library in ${System.currentTimeMillis() - buildStart}ms")
return MusicStore.Library(genres, artists, albums, songs)
}
/**
* Does the initial query over the song database using [backend]. The songs
* returned by this function are **not** well-formed. The companion [buildAlbums],
* [buildArtists], and [buildGenres] functions must be called with the returned list so that all
* songs are properly linked up.
* Does the initial query over the song database using [backend]. The songs returned by this
* function are **not** well-formed. The companion [buildAlbums], [buildArtists], and
* [buildGenres] functions must be called with the returned list so that all songs are properly
* linked up.
*/
private fun buildSongs(context: Context, backend: Backend): List<Song> {
var songs = backend.query(context).use { cursor -> backend.loadSongs(context, cursor) }
val queryStart = System.currentTimeMillis()
var songs =
backend.query(context).use { cursor ->
val loadStart = System.currentTimeMillis()
logD("Successfully queried media database in ${loadStart - queryStart}ms")
val songs = backend.loadSongs(context, cursor)
logD("Successfully loaded songs in ${System.currentTimeMillis() - loadStart}ms")
songs
}
// Deduplicate songs to prevent (most) deformed music clones
songs =
@ -125,8 +160,8 @@ object Indexer {
rawName = templateSong._albumName,
year = templateSong._year,
albumCoverUri = templateSong._albumCoverUri,
_artistGroupingName = templateSong._artistGroupingName,
songs = entry.value))
songs = entry.value,
_artistGroupingName = templateSong._artistGroupingName))
}
logD("Successfully built ${albums.size} albums")
@ -146,10 +181,7 @@ object Indexer {
// The first album will suffice for template metadata.
val templateAlbum = entry.value[0]
artists.add(Artist(
rawName = templateAlbum._artistGroupingName,
albums = entry.value
))
artists.add(Artist(rawName = templateAlbum._artistGroupingName, albums = entry.value))
}
logD("Successfully built ${artists.size} artists")
@ -157,9 +189,7 @@ object Indexer {
return artists
}
/**
* Build genres and link them to their particular songs.
*/
/** Build genres and link them to their particular songs. */
private fun buildGenres(songs: List<Song>): List<Genre> {
val genres = mutableListOf<Genre>()
val songsByGenre = songs.groupBy { it._genreName?.hashCode() }
@ -174,6 +204,7 @@ object Indexer {
return genres
}
/** Represents a backend that metadata can be extracted from. */
interface Backend {
/** Query the media database for an initial cursor. */
fun query(context: Context): Cursor

View file

@ -0,0 +1,50 @@
/*
* Copyright (c) 2022 Auxio Project
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.oxycblt.auxio.music.indexer
import android.content.ContentResolver
import android.database.Cursor
import android.net.Uri
/** Shortcut for making a [ContentResolver] query with less superfluous arguments. */
fun ContentResolver.queryCursor(
uri: Uri,
projection: Array<out String>,
selector: String? = null,
args: Array<String>? = null
) = query(uri, projection, selector, args, null)
/** Shortcut for making a [ContentResolver] query and using the particular cursor with [use]. */
fun <R> ContentResolver.useQuery(
uri: Uri,
projection: Array<out String>,
selector: String? = null,
args: Array<String>? = null,
block: (Cursor) -> R
): R? = queryCursor(uri, projection, selector, args)?.use(block)
/**
* Parse out the number field from an NN/TT string that is typically found in DISC_NUMBER and
* CD_TRACK_NUMBER.
*/
val String.no: Int?
get() = split('/', limit = 2).getOrNull(0)?.toIntOrNull()
/** Parse out the year field from a (presumably) ISO-8601-like date. */
val String.iso8601year: Int?
get() = split(":", limit = 2).getOrNull(0)?.toIntOrNull()

View file

@ -29,8 +29,6 @@ import androidx.core.database.getStringOrNull
import org.oxycblt.auxio.music.Song
import org.oxycblt.auxio.music.excluded.ExcludedDatabase
import org.oxycblt.auxio.util.contentResolverSafe
import org.oxycblt.auxio.util.queryCursor
import org.oxycblt.auxio.util.useQuery
/*
* This file acts as the base for most the black magic required to get a remotely sensible music
@ -271,17 +269,19 @@ abstract class MediaStoreBackend : Indexer.Backend {
Song(
rawName = requireNotNull(title) { "Malformed audio: No title" },
fileName = requireNotNull(displayName) { "Malformed audio: No file name" },
uri = ContentUris.withAppendedId(
MediaStore.Audio.Media.EXTERNAL_CONTENT_URI,
requireNotNull(id) { "Malformed audio: No song id" }),
uri =
ContentUris.withAppendedId(
MediaStore.Audio.Media.EXTERNAL_CONTENT_URI,
requireNotNull(id) { "Malformed audio: No song id" }),
durationMs = requireNotNull(duration) { "Malformed audio: No duration" },
track = track,
disc = disc,
_year = year,
_albumName = requireNotNull(album) { "Malformed song: No album name" },
_albumCoverUri = ContentUris.withAppendedId(
EXTERNAL_ALBUM_ART_URI,
requireNotNull(albumId) { "Malformed song: No album id" }),
_albumCoverUri =
ContentUris.withAppendedId(
EXTERNAL_ALBUM_ART_URI,
requireNotNull(albumId) { "Malformed song: No album id" }),
_artistName = artist,
_albumArtistName = albumArtist,
_genreName = genre)
@ -384,6 +384,7 @@ class Api30MediaStoreBackend : MediaStoreBackend() {
override fun buildAudio(context: Context, cursor: Cursor): Audio {
val audio = super.buildAudio(context, cursor)
// Populate our indices if we have not already.
if (trackIndex == -1) {
trackIndex = cursor.getColumnIndexOrThrow(MediaStore.Audio.AudioColumns.CD_TRACK_NUMBER)
discIndex = cursor.getColumnIndexOrThrow(MediaStore.Audio.AudioColumns.DISC_NUMBER)
@ -399,11 +400,4 @@ class Api30MediaStoreBackend : MediaStoreBackend() {
return audio
}
/**
* Parse out the number field from an NN/TT string that is typically found in DISC_NUMBER and
* CD_TRACK_NUMBER.
*/
private val String.no: Int?
get() = split('/', limit = 2).getOrNull(0)?.toIntOrNull()
}

View file

@ -17,7 +17,6 @@
package org.oxycblt.auxio.util
import android.content.ContentResolver
import android.content.Context
import android.content.res.ColorStateList
import android.database.Cursor
@ -25,7 +24,6 @@ import android.database.sqlite.SQLiteDatabase
import android.graphics.Insets
import android.graphics.Rect
import android.graphics.drawable.Drawable
import android.net.Uri
import android.os.Build
import android.view.View
import android.view.WindowInsets
@ -156,23 +154,6 @@ fun Fragment.requireAttached() = check(!isDetached) { "Fragment is detached from
fun <R> SQLiteDatabase.queryAll(tableName: String, block: (Cursor) -> R) =
query(tableName, null, null, null, null, null, null)?.use(block)
/** Shortcut for making a [ContentResolver] query with less superfluous arguments. */
fun ContentResolver.queryCursor(
uri: Uri,
projection: Array<out String>,
selector: String? = null,
args: Array<String>? = null
) = query(uri, projection, selector, args, null)
/** Shortcut for making a [ContentResolver] query and using the particular cursor with [use]. */
fun <R> ContentResolver.useQuery(
uri: Uri,
projection: Array<out String>,
selector: String? = null,
args: Array<String>? = null,
block: (Cursor) -> R
): R? = queryCursor(uri, projection, selector, args)?.use(block)
/**
* Resolve system bar insets in a version-aware manner. This can be used to apply padding to a view
* that properly follows all the frustrating changes that were made between Android 8-11.