musikr: standardize internal song data structure

This commit is contained in:
Alexander Capehart 2024-12-13 19:44:02 -07:00
parent 9ab4dc5595
commit c70c27a7b4
No known key found for this signature in database
GPG key ID: 37DBE3621FE9AD47
6 changed files with 151 additions and 164 deletions

View file

@ -18,15 +18,13 @@
package org.oxycblt.musikr.cache
import org.oxycblt.ktaglib.Properties
import org.oxycblt.musikr.cover.Cover
import org.oxycblt.musikr.fs.query.DeviceFile
import org.oxycblt.musikr.tag.parse.ParsedTags
import org.oxycblt.musikr.pipeline.RawSong
interface Cache {
suspend fun read(file: DeviceFile): CachedSong?
suspend fun read(file: DeviceFile): CacheResult
suspend fun write(file: DeviceFile, song: CachedSong)
suspend fun write(song: RawSong)
companion object {
fun full(db: CacheDatabase): Cache = FullCache(db.cachedSongsDao())
@ -35,23 +33,25 @@ interface Cache {
}
}
data class CachedSong(
val parsedTags: ParsedTags,
val cover: Cover.Single?,
val properties: Properties
)
sealed interface CacheResult {
data class Hit(val song: RawSong) : CacheResult
data class Miss(val file: DeviceFile) : CacheResult
}
private class FullCache(private val cacheInfoDao: CacheInfoDao) : Cache {
override suspend fun read(file: DeviceFile) =
cacheInfoDao.selectInfo(file.uri.toString(), file.lastModified)?.intoCachedSong()
cacheInfoDao.selectSong(file.uri.toString(), file.lastModified)?.let {
CacheResult.Hit(it.intoRawSong(file))
} ?: CacheResult.Miss(file)
override suspend fun write(file: DeviceFile, song: CachedSong) =
cacheInfoDao.updateInfo(CachedInfo.fromCachedSong(file, song))
override suspend fun write(song: RawSong) =
cacheInfoDao.updateSong(CachedSong.fromRawSong(song))
}
private class WriteOnlyCache(private val cacheInfoDao: CacheInfoDao) : Cache {
override suspend fun read(file: DeviceFile) = null
override suspend fun read(file: DeviceFile) = CacheResult.Miss(file)
override suspend fun write(file: DeviceFile, song: CachedSong) =
cacheInfoDao.updateInfo(CachedInfo.fromCachedSong(file, song))
override suspend fun write(song: RawSong) =
cacheInfoDao.updateSong(CachedSong.fromRawSong(song))
}

View file

@ -33,12 +33,13 @@ import androidx.room.TypeConverters
import org.oxycblt.ktaglib.Properties
import org.oxycblt.musikr.cover.Cover
import org.oxycblt.musikr.fs.query.DeviceFile
import org.oxycblt.musikr.pipeline.RawSong
import org.oxycblt.musikr.tag.Date
import org.oxycblt.musikr.tag.parse.ParsedTags
import org.oxycblt.musikr.tag.util.correctWhitespace
import org.oxycblt.musikr.tag.util.splitEscaped
@Database(entities = [CachedInfo::class], version = 50, exportSchema = false)
@Database(entities = [CachedSong::class], version = 50, exportSchema = false)
abstract class CacheDatabase : RoomDatabase() {
internal abstract fun cachedSongsDao(): CacheInfoDao
@ -53,23 +54,21 @@ abstract class CacheDatabase : RoomDatabase() {
@Dao
internal interface CacheInfoDao {
@Query("SELECT * FROM CachedInfo WHERE uri = :uri AND dateModified = :dateModified")
suspend fun selectInfo(uri: String, dateModified: Long): CachedInfo?
@Query("SELECT * FROM CachedSong WHERE uri = :uri AND dateModified = :dateModified")
suspend fun selectSong(uri: String, dateModified: Long): CachedSong?
@Insert(onConflict = OnConflictStrategy.REPLACE) suspend fun updateInfo(cachedInfo: CachedInfo)
@Insert(onConflict = OnConflictStrategy.REPLACE) suspend fun updateSong(cachedSong: CachedSong)
}
@Entity
@TypeConverters(CachedInfo.Converters::class)
internal data class CachedInfo(
/**
* The Uri of the [AudioFile]'s audio file, obtained from SAF. This should ideally be a black
* box only used for comparison.
*/
@TypeConverters(CachedSong.Converters::class)
internal data class CachedSong(
@PrimaryKey val uri: String,
val dateModified: Long,
val replayGainTrackAdjustment: Float?,
val replayGainAlbumAdjustment: Float?,
val mimeType: String,
val durationMs: Long,
val bitrate: Int,
val sampleRate: Int,
val musicBrainzId: String?,
val name: String,
val sortName: String?,
@ -88,21 +87,19 @@ internal data class CachedInfo(
val albumArtistNames: List<String>,
val albumArtistSortNames: List<String>,
val genreNames: List<String>,
val replayGainTrackAdjustment: Float?,
val replayGainAlbumAdjustment: Float?,
val cover: Cover.Single?,
val mimeType: String,
val durationMs: Long,
val bitrate: Int,
val sampleRate: Int,
) {
fun intoCachedSong() =
CachedSong(
fun intoRawSong(file: DeviceFile) =
RawSong(
file,
Properties(mimeType, durationMs, bitrate, sampleRate),
ParsedTags(
musicBrainzId = musicBrainzId,
name = name,
sortName = sortName,
durationMs = durationMs,
replayGainTrackAdjustment = replayGainTrackAdjustment,
replayGainAlbumAdjustment = replayGainAlbumAdjustment,
track = track,
disc = disc,
subtitle = subtitle,
@ -117,9 +114,10 @@ internal data class CachedInfo(
albumArtistMusicBrainzIds = albumArtistMusicBrainzIds,
albumArtistNames = albumArtistNames,
albumArtistSortNames = albumArtistSortNames,
genreNames = genreNames),
cover,
Properties(mimeType, durationMs, bitrate, sampleRate))
genreNames = genreNames,
replayGainTrackAdjustment = replayGainTrackAdjustment,
replayGainAlbumAdjustment = replayGainAlbumAdjustment),
cover)
object Converters {
@TypeConverter
@ -139,34 +137,34 @@ internal data class CachedInfo(
}
companion object {
fun fromCachedSong(deviceFile: DeviceFile, cachedSong: CachedSong) =
CachedInfo(
uri = deviceFile.uri.toString(),
dateModified = deviceFile.lastModified,
musicBrainzId = cachedSong.parsedTags.musicBrainzId,
name = cachedSong.parsedTags.name,
sortName = cachedSong.parsedTags.sortName,
durationMs = cachedSong.parsedTags.durationMs,
replayGainTrackAdjustment = cachedSong.parsedTags.replayGainTrackAdjustment,
replayGainAlbumAdjustment = cachedSong.parsedTags.replayGainAlbumAdjustment,
track = cachedSong.parsedTags.track,
disc = cachedSong.parsedTags.disc,
subtitle = cachedSong.parsedTags.subtitle,
date = cachedSong.parsedTags.date,
albumMusicBrainzId = cachedSong.parsedTags.albumMusicBrainzId,
albumName = cachedSong.parsedTags.albumName,
albumSortName = cachedSong.parsedTags.albumSortName,
releaseTypes = cachedSong.parsedTags.releaseTypes,
artistMusicBrainzIds = cachedSong.parsedTags.artistMusicBrainzIds,
artistNames = cachedSong.parsedTags.artistNames,
artistSortNames = cachedSong.parsedTags.artistSortNames,
albumArtistMusicBrainzIds = cachedSong.parsedTags.albumArtistMusicBrainzIds,
albumArtistNames = cachedSong.parsedTags.albumArtistNames,
albumArtistSortNames = cachedSong.parsedTags.albumArtistSortNames,
genreNames = cachedSong.parsedTags.genreNames,
cover = cachedSong.cover,
mimeType = cachedSong.properties.mimeType,
bitrate = cachedSong.properties.bitrate,
sampleRate = cachedSong.properties.sampleRate)
fun fromRawSong(rawSong: RawSong) =
CachedSong(
uri = rawSong.file.uri.toString(),
dateModified = rawSong.file.lastModified,
musicBrainzId = rawSong.tags.musicBrainzId,
name = rawSong.tags.name,
sortName = rawSong.tags.sortName,
durationMs = rawSong.tags.durationMs,
track = rawSong.tags.track,
disc = rawSong.tags.disc,
subtitle = rawSong.tags.subtitle,
date = rawSong.tags.date,
albumMusicBrainzId = rawSong.tags.albumMusicBrainzId,
albumName = rawSong.tags.albumName,
albumSortName = rawSong.tags.albumSortName,
releaseTypes = rawSong.tags.releaseTypes,
artistMusicBrainzIds = rawSong.tags.artistMusicBrainzIds,
artistNames = rawSong.tags.artistNames,
artistSortNames = rawSong.tags.artistSortNames,
albumArtistMusicBrainzIds = rawSong.tags.albumArtistMusicBrainzIds,
albumArtistNames = rawSong.tags.albumArtistNames,
albumArtistSortNames = rawSong.tags.albumArtistSortNames,
genreNames = rawSong.tags.genreNames,
replayGainTrackAdjustment = rawSong.tags.replayGainTrackAdjustment,
replayGainAlbumAdjustment = rawSong.tags.replayGainAlbumAdjustment,
cover = rawSong.cover,
mimeType = rawSong.properties.mimeType,
bitrate = rawSong.properties.bitrate,
sampleRate = rawSong.properties.sampleRate)
}
}

View file

@ -53,10 +53,7 @@ private class EvaluateStepImpl(
val preSongs =
extractedMusic
.filterIsInstance<ExtractedMusic.Song>()
.map {
tagInterpreter.interpret(
it.file, it.tags, it.cover, it.properties, interpretation)
}
.map { tagInterpreter.interpret(it.song, interpretation) }
.flowOn(Dispatchers.Main)
.buffer(Channel.UNLIMITED)
val graphBuilder = MusicGraph.builder()

View file

@ -30,13 +30,12 @@ import kotlinx.coroutines.flow.mapNotNull
import kotlinx.coroutines.flow.merge
import org.oxycblt.ktaglib.Properties
import org.oxycblt.musikr.Storage
import org.oxycblt.musikr.cache.CachedSong
import org.oxycblt.musikr.cache.CacheResult
import org.oxycblt.musikr.cover.Cover
import org.oxycblt.musikr.fs.query.DeviceFile
import org.oxycblt.musikr.metadata.MetadataExtractor
import org.oxycblt.musikr.tag.parse.ParsedTags
import org.oxycblt.musikr.tag.parse.TagParser
import timber.log.Timber as L
interface ExtractStep {
fun extract(storage: Storage, nodes: Flow<ExploreNode>): Flow<ExtractedMusic>
@ -55,28 +54,27 @@ private class ExtractStepImpl(
val cacheResults =
nodes
.filterIsInstance<ExploreNode.Audio>()
.map {
val tags = storage.cache.read(it.file)
MaybeCachedSong(it.file, tags)
}
.map { storage.cache.read(it.file) }
.flowOn(Dispatchers.IO)
.buffer(Channel.UNLIMITED)
val (cachedSongs, uncachedSongs) =
cacheResults.mapPartition {
it.cachedSong?.let { song ->
ExtractedMusic.Song(it.file, song.properties, song.parsedTags, song.cover)
val divertedFlow =
cacheResults.divert {
when (it) {
is CacheResult.Hit -> Divert.Left(it.song)
is CacheResult.Miss -> Divert.Right(it.file)
}
}
val split = uncachedSongs.distribute(16)
val cachedSongs = divertedFlow.left.map { ExtractedMusic.Song(it) }
val uncachedSongs = divertedFlow.right
val distributedFlow = uncachedSongs.distribute(16)
val extractedSongs =
Array(split.hot.size) { i ->
split.hot[i]
.mapNotNull { node ->
val metadata =
metadataExtractor.extract(node.file) ?: return@mapNotNull null
val tags = tagParser.parse(node.file, metadata)
Array(distributedFlow.flows.size) { i ->
distributedFlow.flows[i]
.mapNotNull { file ->
val metadata = metadataExtractor.extract(file) ?: return@mapNotNull null
val tags = tagParser.parse(file, metadata)
val cover = metadata.cover?.let { storage.storedCovers.write(it) }
ExtractedMusic.Song(node.file, metadata.properties, tags, cover)
RawSong(file, metadata.properties, tags, cover)
}
.flowOn(Dispatchers.IO)
.buffer(Channel.UNLIMITED)
@ -84,26 +82,23 @@ private class ExtractStepImpl(
val writtenSongs =
merge(*extractedSongs)
.map {
storage.cache.write(it.file, CachedSong(it.tags, it.cover, it.properties))
it
storage.cache.write(it)
ExtractedMusic.Song(it)
}
.flowOn(Dispatchers.IO)
.buffer(Channel.UNLIMITED)
return merge<ExtractedMusic>(
cachedSongs,
split.cold,
writtenSongs,
)
divertedFlow.manager, cachedSongs, distributedFlow.manager, writtenSongs)
}
}
data class MaybeCachedSong(val file: DeviceFile, val cachedSong: CachedSong?)
}
sealed interface ExtractedMusic {
data class Song(
data class RawSong(
val file: DeviceFile,
val properties: Properties,
val tags: ParsedTags,
val cover: Cover.Single?
) : ExtractedMusic
)
sealed interface ExtractedMusic {
data class Song(val song: RawSong) : ExtractedMusic
}

View file

@ -20,38 +20,47 @@ package org.oxycblt.musikr.pipeline
import kotlinx.coroutines.channels.Channel
import kotlinx.coroutines.flow.Flow
import kotlinx.coroutines.flow.consumeAsFlow
import kotlinx.coroutines.flow.flow
import kotlinx.coroutines.flow.map
import kotlinx.coroutines.flow.receiveAsFlow
import kotlinx.coroutines.flow.withIndex
data class HotCold<H, C>(val hot: H, val cold: Flow<C>)
sealed interface Divert<L, R> {
data class Left<L, R>(val value: L) : Divert<L, R>
inline fun <T, R> Flow<T>.mapPartition(crossinline predicate: (T) -> R?): HotCold<Flow<R>, T> {
val passChannel = Channel<R>(Channel.UNLIMITED)
val passFlow = passChannel.consumeAsFlow()
val failFlow = flow {
data class Right<L, R>(val value: R) : Divert<L, R>
}
class DivertedFlow<L, R>(val manager: Flow<Nothing>, val left: Flow<L>, val right: Flow<R>)
inline fun <T, L, R> Flow<T>.divert(
crossinline predicate: (T) -> Divert<L, R>
): DivertedFlow<L, R> {
val leftChannel = Channel<L>(Channel.UNLIMITED)
val rightChannel = Channel<R>(Channel.UNLIMITED)
val managedFlow =
flow<Nothing> {
collect {
val result = predicate(it)
if (result != null) {
passChannel.send(result)
} else {
emit(it)
when (val result = predicate(it)) {
is Divert.Left -> leftChannel.send(result.value)
is Divert.Right -> rightChannel.send(result.value)
}
}
passChannel.close()
leftChannel.close()
rightChannel.close()
}
return HotCold(passFlow, failFlow)
return DivertedFlow(managedFlow, leftChannel.receiveAsFlow(), rightChannel.receiveAsFlow())
}
class DistributedFlow<T>(val manager: Flow<Nothing>, val flows: Array<Flow<T>>)
/**
* Equally "distributes" the values of some flow across n new flows.
*
* Note that this function requires the "manager" flow to be consumed alongside the split flows in
* order to function. Without this, all of the newly split flows will simply block.
*/
fun <T> Flow<T>.distribute(n: Int): HotCold<Array<Flow<T>>, Nothing> {
fun <T> Flow<T>.distribute(n: Int): DistributedFlow<T> {
val posChannels = Array(n) { Channel<T>(Channel.UNLIMITED) }
val managerFlow =
flow<Nothing> {
@ -64,5 +73,5 @@ fun <T> Flow<T>.distribute(n: Int): HotCold<Array<Flow<T>>, Nothing> {
}
}
val hotFlows = posChannels.map { it.receiveAsFlow() }.toTypedArray()
return HotCold(hotFlows, managerFlow)
return DistributedFlow(managerFlow, hotFlows)
}

View file

@ -21,11 +21,10 @@ package org.oxycblt.musikr.tag.interpret
import org.oxycblt.auxio.R
import org.oxycblt.auxio.playback.replaygain.ReplayGainAdjustment
import org.oxycblt.auxio.util.toUuidOrNull
import org.oxycblt.ktaglib.Properties
import org.oxycblt.musikr.Interpretation
import org.oxycblt.musikr.cover.Cover
import org.oxycblt.musikr.fs.Format
import org.oxycblt.musikr.fs.query.DeviceFile
import org.oxycblt.musikr.pipeline.RawSong
import org.oxycblt.musikr.tag.Disc
import org.oxycblt.musikr.tag.Name
import org.oxycblt.musikr.tag.ReleaseType
@ -33,13 +32,7 @@ import org.oxycblt.musikr.tag.parse.ParsedTags
import org.oxycblt.musikr.tag.util.parseId3GenreNames
interface TagInterpreter {
fun interpret(
file: DeviceFile,
parsedTags: ParsedTags,
cover: Cover.Single?,
properties: Properties,
interpretation: Interpretation
): PreSong
fun interpret(song: RawSong, interpretation: Interpretation): PreSong
companion object {
fun new(): TagInterpreter = TagInterpreterImpl
@ -47,56 +40,51 @@ interface TagInterpreter {
}
private data object TagInterpreterImpl : TagInterpreter {
override fun interpret(
file: DeviceFile,
parsedTags: ParsedTags,
cover: Cover.Single?,
properties: Properties,
interpretation: Interpretation
): PreSong {
override fun interpret(song: RawSong, interpretation: Interpretation): PreSong {
val individualPreArtists =
makePreArtists(
parsedTags.artistMusicBrainzIds,
parsedTags.artistNames,
parsedTags.artistSortNames,
song.tags.artistMusicBrainzIds,
song.tags.artistNames,
song.tags.artistSortNames,
interpretation)
val albumPreArtists =
makePreArtists(
parsedTags.albumArtistMusicBrainzIds,
parsedTags.albumArtistNames,
parsedTags.albumArtistSortNames,
song.tags.albumArtistMusicBrainzIds,
song.tags.albumArtistNames,
song.tags.albumArtistSortNames,
interpretation)
val preAlbum =
makePreAlbum(file, parsedTags, individualPreArtists, albumPreArtists, interpretation)
makePreAlbum(
song.file, song.tags, individualPreArtists, albumPreArtists, interpretation)
val rawArtists =
individualPreArtists.ifEmpty { albumPreArtists }.ifEmpty { listOf(unknownPreArtist()) }
val rawGenres =
makePreGenres(parsedTags, interpretation).ifEmpty { listOf(unknownPreGenre()) }
val uri = file.uri
makePreGenres(song.tags, interpretation).ifEmpty { listOf(unknownPreGenre()) }
val uri = song.file.uri
return PreSong(
musicBrainzId = parsedTags.musicBrainzId?.toUuidOrNull(),
name = interpretation.nameFactory.parse(parsedTags.name, parsedTags.sortName),
rawName = parsedTags.name,
track = parsedTags.track,
disc = parsedTags.disc?.let { Disc(it, parsedTags.subtitle) },
date = parsedTags.date,
uri = uri,
path = file.path,
size = file.size,
durationMs = parsedTags.durationMs,
path = song.file.path,
size = song.file.size,
format = Format.infer(song.file.mimeType, song.properties.mimeType),
lastModified = song.file.lastModified,
// TODO: Figure out what to do with date added
dateAdded = song.file.lastModified,
musicBrainzId = song.tags.musicBrainzId?.toUuidOrNull(),
name = interpretation.nameFactory.parse(song.tags.name, song.tags.sortName),
rawName = song.tags.name,
track = song.tags.track,
disc = song.tags.disc?.let { Disc(it, song.tags.subtitle) },
date = song.tags.date,
durationMs = song.tags.durationMs,
replayGainAdjustment =
ReplayGainAdjustment(
parsedTags.replayGainTrackAdjustment,
parsedTags.replayGainAlbumAdjustment,
song.tags.replayGainTrackAdjustment,
song.tags.replayGainAlbumAdjustment,
),
format = Format.infer(file.mimeType, properties.mimeType),
lastModified = file.lastModified,
// TODO: Figure out what to do with date added
dateAdded = file.lastModified,
preAlbum = preAlbum,
preArtists = rawArtists,
preGenres = rawGenres,
cover = cover)
cover = song.cover)
}
private fun makePreAlbum(