music: parallelize loading

Parallelize music loading.

- Queries over Media DB and Cache are ran parallel
- MediaStoreExtractor can now continue extracting songs independent
of MetadataExtractor's task pool capacity
- Library and Cache are saved in parallel

Resolves #343.

This should result in some pretty signifigant performance gains
due to the operations now being ran in parallel instead of
sequentially.

app/src/main/java/org/oxycblt/auxio/detail/DetailViewModel.kt

@@ -17,9 +17,7 @@
  
 package org.oxycblt.auxio.detail
 
-import android.app.Application
 import androidx.annotation.StringRes
-import androidx.lifecycle.AndroidViewModel
 import androidx.lifecycle.ViewModel
 import androidx.lifecycle.viewModelScope
 import dagger.hilt.android.lifecycle.HiltViewModel
@@ -44,10 +42,8 @@ import org.oxycblt.auxio.playback.PlaybackSettings
 import org.oxycblt.auxio.util.*
 
 /**
- * [AndroidViewModel] that manages the Song, Album, Artist, and Genre detail views. Keeps track of
+ * [ViewModel] that manages the Song, Album, Artist, and Genre detail views. Keeps track of the
- * the current item they are showing, sub-data to display, and configuration. Since this ViewModel
+ * current item they are showing, sub-data to display, and configuration.
- * requires a context, it must be instantiated [AndroidViewModel]'s Factory.
- * @param application [Application] context required to initialize certain information.
  * @author Alexander Capehart (OxygenCobalt)
  */
 @HiltViewModel

app/src/main/java/org/oxycblt/auxio/music/extractor/ExtractionResult.kt

@@ -1,31 +0,0 @@
-/*
- * Copyright (c) 2022 Auxio Project
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
- 
-package org.oxycblt.auxio.music.extractor
-
-/**
- * Represents the result of an extraction operation.
- * @author Alexander Capehart (OxygenCobalt)
- */
-enum class ExtractionResult {
-    /** A raw song was successfully extracted from the cache. */
-    CACHED,
-    /** A raw song was successfully extracted from parsing it's file. */
-    PARSED,
-    /** A raw song could not be parsed. */
-    NONE
-}

app/src/main/java/org/oxycblt/auxio/music/extractor/MediaStoreExtractor.kt

@@ -27,6 +27,7 @@ import androidx.annotation.RequiresApi
 import androidx.core.database.getIntOrNull
 import androidx.core.database.getStringOrNull
 import java.io.File
+import kotlinx.coroutines.channels.Channel
 import org.oxycblt.auxio.music.MusicSettings
 import org.oxycblt.auxio.music.library.RealSong
 import org.oxycblt.auxio.music.metadata.Date
@@ -47,14 +48,45 @@ import org.oxycblt.auxio.util.logD
  * music extraction process and primarily intended for redundancy for files not natively supported
  * by [MetadataExtractor]. Solely relying on this is not recommended, as it often produces bad
  * metadata.
- * @param context [Context] required to query the media database.
- * @param cacheExtractor [CacheExtractor] implementation for cache optimizations.
  * @author Alexander Capehart (OxygenCobalt)
  */
-abstract class MediaStoreExtractor(
+interface MediaStoreExtractor {
-    private val context: Context,
+    /**
-    private val cacheExtractor: CacheExtractor
+     * Query the media database, initializing this instance in the process.
-) {
+     * @return The new [Cursor] returned by the media databases.
+     */
+    suspend fun query(): Cursor
+
+    /**
+     * Consume the [Cursor] loaded after [query].
+     * @param cache A [MetadataCache] used to avoid extracting metadata for cached songs, or null if
+     * no [MetadataCache] was available.
+     * @param incompleteSongs A channel where songs that could not be retrieved from the
+     * [MetadataCache] should be sent to.
+     * @param completeSongs A channel where completed songs should be sent to.
+     */
+    suspend fun consume(
+        cache: MetadataCache?,
+        incompleteSongs: Channel<RealSong.Raw>,
+        completeSongs: Channel<RealSong.Raw>
+    )
+
+    companion object {
+        /**
+         * Create a framework-backed instance.
+         * @param context [Context] required.
+         * @return A new [RealMediaStoreExtractor] that will work best on the device's API level.
+         */
+        fun from(context: Context): MediaStoreExtractor =
+            when {
+                Build.VERSION.SDK_INT >= Build.VERSION_CODES.R -> Api30MediaStoreExtractor(context)
+                Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q -> Api29MediaStoreExtractor(context)
+                else -> Api21MediaStoreExtractor(context)
+            }
+    }
+}
+
+private abstract class RealMediaStoreExtractor(private val context: Context) : MediaStoreExtractor {
     private var cursor: Cursor? = null
     private var idIndex = -1
     private var titleIndex = -1
@@ -78,14 +110,8 @@ abstract class MediaStoreExtractor(
     protected var volumes = listOf<StorageVolume>()
         private set
 
-    /**
+    override suspend fun query(): Cursor {
-     * Initialize this instance. This involves setting up the required sub-extractors and querying
-     * the media database for music files.
-     * @return A [Cursor] of the music data returned from the database.
-     */
-    open suspend fun init(): Cursor {
         val start = System.currentTimeMillis()
-        cacheExtractor.init()
         val musicSettings = MusicSettings.from(context)
         val storageManager = context.getSystemServiceCompat(StorageManager::class)
 
@@ -190,42 +216,27 @@ abstract class MediaStoreExtractor(
         return cursor
     }
 
-    /**
+    override suspend fun consume(
-     * Finalize the Extractor by writing the newly-loaded [RealSong.Raw]s back into the cache,
+        cache: MetadataCache?,
-     * alongside freeing up memory.
+        incompleteSongs: Channel<RealSong.Raw>,
-     * @param rawSongs The songs to write into the cache.
+        completeSongs: Channel<RealSong.Raw>
-     */
+    ) {
-    open suspend fun finalize(rawSongs: List<RealSong.Raw>) {
+        val cursor = requireNotNull(cursor) { "Must call query first before running consume" }
-        // Free the cursor (and it's resources)
+        while (cursor.moveToNext()) {
-        cursor?.close()
+            val rawSong = RealSong.Raw()
-        cursor = null
+            populateFileData(cursor, rawSong)
-        cacheExtractor.finalize(rawSongs)
+            if (cache?.populate(rawSong) == true) {
-    }
+                completeSongs.send(rawSong)
-
+            } else {
-    /**
+                populateMetadata(cursor, rawSong)
-     * Populate a [RealSong.Raw] with the next [Cursor] value provided by [MediaStore].
+                incompleteSongs.send(rawSong)
-     * @param raw The [RealSong.Raw] to populate.
+            }
-     * @return An [ExtractionResult] signifying the result of the operation. Will return
-     * [ExtractionResult.CACHED] if [CacheExtractor] returned it.
-     */
-    fun populate(raw: RealSong.Raw): ExtractionResult {
-        val cursor = requireNotNull(cursor) { "MediaStoreLayer is not properly initialized" }
-        // Move to the next cursor, stopping if we have exhausted it.
-        if (!cursor.moveToNext()) {
-            logD("Cursor is exhausted")
-            return ExtractionResult.NONE
         }
-
+        // Free the cursor and signal that no more incomplete songs will be produced by
-        // Populate the minimum required columns to maybe obtain a cache entry.
+        // this extractor.
-        populateFileData(cursor, raw)
+        cursor.close()
-        if (cacheExtractor.populate(raw) == ExtractionResult.CACHED) {
+        incompleteSongs.close()
-            // We found a valid cache entry, no need to fully read the entry.
+        this.cursor = null
-            return ExtractionResult.CACHED
-        }
-
-        // Could not load entry from cache, we have to read the rest of the metadata.
-        populateMetadata(cursor, raw)
-        return ExtractionResult.PARSED
     }
 
     /**
@@ -326,21 +337,6 @@ abstract class MediaStoreExtractor(
     }
 
     companion object {
-        /**
-         * Create a framework-backed instance.
-         * @param context [Context] required.
-         * @param cacheExtractor [CacheExtractor] to wrap.
-         * @return A new [MediaStoreExtractor] that will work best on the device's API level.
-         */
-        fun from(context: Context, cacheExtractor: CacheExtractor) =
-            when {
-                Build.VERSION.SDK_INT >= Build.VERSION_CODES.R ->
-                    Api30MediaStoreExtractor(context, cacheExtractor)
-                Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q ->
-                    Api29MediaStoreExtractor(context, cacheExtractor)
-                else -> Api21MediaStoreExtractor(context, cacheExtractor)
-            }
-
         /**
          * The base selector that works across all versions of android. Does not exclude
          * directories.
@@ -366,20 +362,12 @@ abstract class MediaStoreExtractor(
 // Note: The separation between version-specific backends may not be the cleanest. To preserve
 // speed, we only want to add redundancy on known issues, not with possible issues.
 
-/**
+private class Api21MediaStoreExtractor(context: Context) : RealMediaStoreExtractor(context) {
- * A [MediaStoreExtractor] that completes the music loading process in a way compatible from API 21
- * onwards to API 28.
- * @param context [Context] required to query the media database.
- * @param cacheExtractor [CacheExtractor] implementation for cache optimizations.
- * @author Alexander Capehart (OxygenCobalt)
- */
-private class Api21MediaStoreExtractor(context: Context, cacheExtractor: CacheExtractor) :
-    MediaStoreExtractor(context, cacheExtractor) {
     private var trackIndex = -1
     private var dataIndex = -1
 
-    override suspend fun init(): Cursor {
+    override suspend fun query(): Cursor {
-        val cursor = super.init()
+        val cursor = super.query()
         // Set up cursor indices for later use.
         trackIndex = cursor.getColumnIndexOrThrow(MediaStore.Audio.AudioColumns.TRACK)
         dataIndex = cursor.getColumnIndexOrThrow(MediaStore.Audio.AudioColumns.DATA)
@@ -447,19 +435,19 @@ private class Api21MediaStoreExtractor(context: Context, cacheExtractor: CacheEx
 }
 
 /**
- * A [MediaStoreExtractor] that implements common behavior supported from API 29 onwards.
+ * A [RealMediaStoreExtractor] that implements common behavior supported from API 29 onwards.
  * @param context [Context] required to query the media database.
- * @param cacheExtractor [CacheExtractor] implementation for cache optimizations.
+ * @param metadataCacheRepository [MetadataCacheRepository] implementation for cache optimizations.
  * @author Alexander Capehart (OxygenCobalt)
  */
 @RequiresApi(Build.VERSION_CODES.Q)
-private open class BaseApi29MediaStoreExtractor(context: Context, cacheExtractor: CacheExtractor) :
+private open class BaseApi29MediaStoreExtractor(context: Context) :
-    MediaStoreExtractor(context, cacheExtractor) {
+    RealMediaStoreExtractor(context) {
     private var volumeIndex = -1
     private var relativePathIndex = -1
 
-    override suspend fun init(): Cursor {
+    override suspend fun query(): Cursor {
-        val cursor = super.init()
+        val cursor = super.query()
         // Set up cursor indices for later use.
         volumeIndex = cursor.getColumnIndexOrThrow(MediaStore.Audio.AudioColumns.VOLUME_NAME)
         relativePathIndex =
@@ -509,19 +497,20 @@ private open class BaseApi29MediaStoreExtractor(context: Context, cacheExtractor
 }
 
 /**
- * A [MediaStoreExtractor] that completes the music loading process in a way compatible with at API
+ * A [RealMediaStoreExtractor] that completes the music loading process in a way compatible with at
+ * API
  * 29.
  * @param context [Context] required to query the media database.
- * @param cacheExtractor [CacheExtractor] implementation for cache functionality.
+ * @param metadataCacheRepository [MetadataCacheRepository] implementation for cache functionality.
  * @author Alexander Capehart (OxygenCobalt)
  */
 @RequiresApi(Build.VERSION_CODES.Q)
-private open class Api29MediaStoreExtractor(context: Context, cacheExtractor: CacheExtractor) :
+private open class Api29MediaStoreExtractor(context: Context) :
-    BaseApi29MediaStoreExtractor(context, cacheExtractor) {
+    BaseApi29MediaStoreExtractor(context) {
     private var trackIndex = -1
 
-    override suspend fun init(): Cursor {
+    override suspend fun query(): Cursor {
-        val cursor = super.init()
+        val cursor = super.query()
         // Set up cursor indices for later use.
         trackIndex = cursor.getColumnIndexOrThrow(MediaStore.Audio.AudioColumns.TRACK)
         return cursor
@@ -544,20 +533,19 @@ private open class Api29MediaStoreExtractor(context: Context, cacheExtractor: Ca
 }
 
 /**
- * A [MediaStoreExtractor] that completes the music loading process in a way compatible from API 30
+ * A [RealMediaStoreExtractor] that completes the music loading process in a way compatible from API
- * onwards.
+ * 30 onwards.
  * @param context [Context] required to query the media database.
- * @param cacheExtractor [CacheExtractor] implementation for cache optimizations.
+ * @param metadataCacheRepository [MetadataCacheRepository] implementation for cache optimizations.
  * @author Alexander Capehart (OxygenCobalt)
  */
 @RequiresApi(Build.VERSION_CODES.R)
-private class Api30MediaStoreExtractor(context: Context, cacheExtractor: CacheExtractor) :
+private class Api30MediaStoreExtractor(context: Context) : BaseApi29MediaStoreExtractor(context) {
-    BaseApi29MediaStoreExtractor(context, cacheExtractor) {
     private var trackIndex: Int = -1
     private var discIndex: Int = -1
 
-    override suspend fun init(): Cursor {
+    override suspend fun query(): Cursor {
-        val cursor = super.init()
+        val cursor = super.query()
         // Set up cursor indices for later use.
         trackIndex = cursor.getColumnIndexOrThrow(MediaStore.Audio.AudioColumns.CD_TRACK_NUMBER)
         discIndex = cursor.getColumnIndexOrThrow(MediaStore.Audio.AudioColumns.DISC_NUMBER)

app/src/main/java/org/oxycblt/auxio/music/extractor/MetadataCacheRepository.kt

@@ -36,149 +36,118 @@ import org.oxycblt.auxio.music.parsing.splitEscaped
 import org.oxycblt.auxio.util.*
 
 /**
- * Defines an Extractor that can load cached music. This is the first step in the music extraction
+ * A cache of music metadata obtained in prior music loading operations. Obtain an instance with
- * process and is an optimization to avoid the slow [MediaStoreExtractor] and [MetadataExtractor]
+ * [MetadataCacheRepository].
- * extraction process.
  * @author Alexander Capehart (OxygenCobalt)
  */
-interface CacheExtractor {
+interface MetadataCache {
-    /** Initialize the Extractor by reading the cache data into memory. */
+    /** Whether this cache has encountered a [RealSong.Raw] that did not have a cache entry. */
-    suspend fun init()
+    val invalidated: Boolean
 
     /**
-     * Finalize the Extractor by writing the newly-loaded [RealSong.Raw]s back into the cache,
+     * Populate a [RealSong.Raw] from a cache entry, if it exists.
-     * alongside freeing up memory.
+     * @param rawSong The [RealSong.Raw] to populate.
-     * @param rawSongs The songs to write into the cache.
+     * @return true if a cache entry could be applied to [rawSong], false otherwise.
      */
-    suspend fun finalize(rawSongs: List<RealSong.Raw>)
+    fun populate(rawSong: RealSong.Raw): Boolean
-
-    /**
-     * Use the cache to populate the given [RealSong.Raw].
-     * @param rawSong The [RealSong.Raw] to attempt to populate. Note that this [RealSong.Raw] will
-     * only contain the bare minimum information required to load a cache entry.
-     * @return An [ExtractionResult] representing the result of the operation.
-     * [ExtractionResult.PARSED] is not returned.
-     */
-    fun populate(rawSong: RealSong.Raw): ExtractionResult
-
-    companion object {
-        /**
-         * Create an instance with optional read-capacity.
-         * @param context [Context] required.
-         * @param readable Whether the new [CacheExtractor] should be able to read cached entries.
-         * @return A new [CacheExtractor] with the specified configuration.
-         */
-        fun from(context: Context, readable: Boolean): CacheExtractor =
-            if (readable) {
-                ReadWriteCacheExtractor(context)
-            } else {
-                WriteOnlyCacheExtractor(context)
-            }
-    }
 }
 
-/**
+private class RealMetadataCache(cachedSongs: List<CachedSong>) : MetadataCache {
- * A [CacheExtractor] only capable of writing to the cache. This can be used to load music with
+    private val cacheMap = buildMap {
- * without the cache if the user desires.
+        for (cachedSong in cachedSongs) {
- * @param context [Context] required to read the cache database.
+            put(cachedSong.mediaStoreId, cachedSong)
- * @see CacheExtractor
- * @author Alexander Capehart (OxygenCobalt)
- */
-private open class WriteOnlyCacheExtractor(private val context: Context) : CacheExtractor {
-    protected val cacheDao: CacheDao by lazy { CacheDatabase.getInstance(context).cacheDao() }
-
-    override suspend fun init() {
-        // Nothing to do.
-    }
-
-    override suspend fun finalize(rawSongs: List<RealSong.Raw>) {
-        try {
-            // Still write out whatever data was extracted.
-            cacheDao.nukeCache()
-            cacheDao.insertCache(rawSongs.map(CachedSong::fromRaw))
-        } catch (e: Exception) {
-            logE("Unable to save cache database.")
-            logE(e.stackTraceToString())
         }
     }
 
-    override fun populate(rawSong: RealSong.Raw) =
+    override var invalidated = false
-        // Nothing to do.
+    override fun populate(rawSong: RealSong.Raw): Boolean {
-        ExtractionResult.NONE
-}
-
-/**
- * A [CacheExtractor] that supports reading from and writing to the cache.
- * @param context [Context] required to load
- * @see CacheExtractor
- * @author Alexander Capehart
- */
-private class ReadWriteCacheExtractor(private val context: Context) :
-    WriteOnlyCacheExtractor(context) {
-    private var cacheMap: Map<Long, CachedSong>? = null
-    private var invalidate = false
-
-    override suspend fun init() {
-        try {
-            // Faster to load the whole database into memory than do a query on each
-            // populate call.
-            val cache = cacheDao.readCache()
-            cacheMap = buildMap {
-                for (cachedSong in cache) {
-                    put(cachedSong.mediaStoreId, cachedSong)
-                }
-            }
-        } catch (e: Exception) {
-            logE("Unable to load cache database.")
-            logE(e.stackTraceToString())
-        }
-    }
-
-    override suspend fun finalize(rawSongs: List<RealSong.Raw>) {
-        cacheMap = null
-        // Same some time by not re-writing the cache if we were able to create the entire
-        // library from it. If there is even just one song we could not populate from the
-        // cache, then we will re-write it.
-        if (invalidate) {
-            logD("Cache was invalidated during loading, rewriting")
-            super.finalize(rawSongs)
-        }
-    }
-
-    override fun populate(rawSong: RealSong.Raw): ExtractionResult {
-        val map = cacheMap ?: return ExtractionResult.NONE
 
         // For a cached raw song to be used, it must exist within the cache and have matching
         // addition and modification timestamps. Technically the addition timestamp doesn't
         // exist, but to safeguard against possible OEM-specific timestamp incoherence, we
         // check for it anyway.
-        val cachedSong = map[rawSong.mediaStoreId]
+        val cachedSong = cacheMap[rawSong.mediaStoreId]
         if (cachedSong != null &&
             cachedSong.dateAdded == rawSong.dateAdded &&
             cachedSong.dateModified == rawSong.dateModified) {
             cachedSong.copyToRaw(rawSong)
-            return ExtractionResult.CACHED
+            return true
         }
 
         // We could not populate this song. This means our cache is stale and should be
         // re-written with newly-loaded music.
-        invalidate = true
+        invalidated = true
-        return ExtractionResult.NONE
+        return false
+    }
+}
+
+/**
+ * A repository allowing access to cached metadata obtained in prior music loading operations.
+ * @author Alexander Capehart (OxygenCobalt)
+ */
+interface MetadataCacheRepository {
+    /**
+     * Read the current [MetadataCache], if it exists.
+     * @return The stored [MetadataCache], or null if it could not be obtained.
+     */
+    suspend fun readCache(): MetadataCache?
+
+    /**
+     * Write the list of newly-loaded [RealSong.Raw]s to the cache, replacing the prior data.
+     * @param rawSongs The [rawSongs] to write to the cache.
+     */
+    suspend fun writeCache(rawSongs: List<RealSong.Raw>)
+
+    companion object {
+        /**
+         * Create a framework-backed instance.
+         * @param context [Context] required.
+         * @return A new instance.
+         */
+        fun from(context: Context): MetadataCacheRepository = RealMetadataCacheRepository(context)
+    }
+}
+
+private class RealMetadataCacheRepository(private val context: Context) : MetadataCacheRepository {
+    private val cachedSongsDao: CachedSongsDao by lazy {
+        MetadataCacheDatabase.getInstance(context).cachedSongsDao()
+    }
+
+    override suspend fun readCache() =
+        try {
+            // Faster to load the whole database into memory than do a query on each
+            // populate call.
+            RealMetadataCache(cachedSongsDao.readSongs())
+        } catch (e: Exception) {
+            logE("Unable to load cache database.")
+            logE(e.stackTraceToString())
+            null
+        }
+
+    override suspend fun writeCache(rawSongs: List<RealSong.Raw>) {
+        try {
+            // Still write out whatever data was extracted.
+            cachedSongsDao.nukeSongs()
+            cachedSongsDao.insertSongs(rawSongs.map(CachedSong::fromRaw))
+        } catch (e: Exception) {
+            logE("Unable to save cache database.")
+            logE(e.stackTraceToString())
+        }
     }
 }
 
 @Database(entities = [CachedSong::class], version = 27, exportSchema = false)
-private abstract class CacheDatabase : RoomDatabase() {
+private abstract class MetadataCacheDatabase : RoomDatabase() {
-    abstract fun cacheDao(): CacheDao
+    abstract fun cachedSongsDao(): CachedSongsDao
 
     companion object {
-        @Volatile private var INSTANCE: CacheDatabase? = null
+        @Volatile private var INSTANCE: MetadataCacheDatabase? = null
 
         /**
          * Get/create the shared instance of this database.
          * @param context [Context] required.
          */
-        fun getInstance(context: Context): CacheDatabase {
+        fun getInstance(context: Context): MetadataCacheDatabase {
             val instance = INSTANCE
             if (instance != null) {
                 return instance
@@ -188,7 +157,7 @@ private abstract class CacheDatabase : RoomDatabase() {
                 val newInstance =
                     Room.databaseBuilder(
                             context.applicationContext,
-                            CacheDatabase::class.java,
+                            MetadataCacheDatabase::class.java,
                             "auxio_metadata_cache.db")
                         .fallbackToDestructiveMigration()
                         .fallbackToDestructiveMigrationFrom(0)
@@ -202,10 +171,10 @@ private abstract class CacheDatabase : RoomDatabase() {
 }
 
 @Dao
-private interface CacheDao {
+private interface CachedSongsDao {
-    @Query("SELECT * FROM ${CachedSong.TABLE_NAME}") suspend fun readCache(): List<CachedSong>
+    @Query("SELECT * FROM ${CachedSong.TABLE_NAME}") suspend fun readSongs(): List<CachedSong>
-    @Query("DELETE FROM ${CachedSong.TABLE_NAME}") suspend fun nukeCache()
+    @Query("DELETE FROM ${CachedSong.TABLE_NAME}") suspend fun nukeSongs()
-    @Insert suspend fun insertCache(songs: List<CachedSong>)
+    @Insert suspend fun insertSongs(songs: List<CachedSong>)
 }
 
 @Entity(tableName = CachedSong.TABLE_NAME)

app/src/main/java/org/oxycblt/auxio/music/extractor/MetadataExtractor.kt

@@ -21,7 +21,7 @@ import android.content.Context
 import androidx.core.text.isDigitsOnly
 import com.google.android.exoplayer2.MediaItem
 import com.google.android.exoplayer2.MetadataRetriever
-import kotlinx.coroutines.flow.flow
+import kotlinx.coroutines.channels.Channel
 import org.oxycblt.auxio.music.library.RealSong
 import org.oxycblt.auxio.music.metadata.Date
 import org.oxycblt.auxio.music.metadata.TextTags
@@ -34,86 +34,53 @@ import org.oxycblt.auxio.util.logW
 /**
  * The extractor that leverages ExoPlayer's [MetadataRetriever] API to parse metadata. This is the
  * last step in the music extraction process and is mostly responsible for papering over the bad
- * metadata that [MediaStoreExtractor] produces.
+ * metadata that [RealMediaStoreExtractor] produces.
  *
  * @param context [Context] required for reading audio files.
- * @param mediaStoreExtractor [MediaStoreExtractor] implementation for cache optimizations and
- * redundancy.
  * @author Alexander Capehart (OxygenCobalt)
  */
-class MetadataExtractor(
+class MetadataExtractor(private val context: Context) {
-    private val context: Context,
-    private val mediaStoreExtractor: MediaStoreExtractor
-) {
     // We can parallelize MetadataRetriever Futures to work around it's speed issues,
     // producing similar throughput's to other kinds of manual metadata extraction.
     private val taskPool: Array<Task?> = arrayOfNulls(TASK_CAPACITY)
 
-    /**
+    suspend fun consume(
-     * Initialize this extractor. This actually initializes the sub-extractors that this instance
+        incompleteSongs: Channel<RealSong.Raw>,
-     * relies on.
+        completeSongs: Channel<RealSong.Raw>
-     * @return The amount of music that is expected to be loaded.
+    ) {
-     */
-    suspend fun init() = mediaStoreExtractor.init().count
-
-    /**
-     * Finalize the Extractor by writing the newly-loaded [RealSong.Raw]s back into the cache,
-     * alongside freeing up memory.
-     * @param rawSongs The songs to write into the cache.
-     */
-    suspend fun finalize(rawSongs: List<RealSong.Raw>) = mediaStoreExtractor.finalize(rawSongs)
-
-    /**
-     * Returns a flow that parses all [RealSong.Raw] instances queued by the sub-extractors. This
-     * will first delegate to the sub-extractors before parsing the metadata itself.
-     * @return A flow of [RealSong.Raw] instances.
-     */
-    fun extract() = flow {
-        while (true) {
-            val raw = RealSong.Raw()
-            when (mediaStoreExtractor.populate(raw)) {
-                ExtractionResult.NONE -> break
-                ExtractionResult.PARSED -> {}
-                ExtractionResult.CACHED -> {
-                    // Avoid running the expensive parsing process on songs we can already
-                    // restore from the cache.
-                    emit(raw)
-                    continue
-                }
-            }
-
-            // Spin until there is an open slot we can insert a task in.
-            spin@ while (true) {
-                for (i in taskPool.indices) {
-                    val task = taskPool[i]
-                    if (task != null) {
-                        val finishedRaw = task.get()
-                        if (finishedRaw != null) {
-                            emit(finishedRaw)
-                            taskPool[i] = Task(context, raw)
-                            break@spin
-                        }
-                    } else {
-                        taskPool[i] = Task(context, raw)
-                        break@spin
-                    }
-                }
-            }
-        }
-
         spin@ while (true) {
-            // Spin until all of the remaining tasks are complete.
+            // Spin until there is an open slot we can insert a task in.
             for (i in taskPool.indices) {
                 val task = taskPool[i]
                 if (task != null) {
-                    val finishedRaw = task.get() ?: continue@spin
+                    completeSongs.send(task.get() ?: continue)
-                    emit(finishedRaw)
+                }
+                val result = incompleteSongs.tryReceive()
+                if (result.isClosed) {
                     taskPool[i] = null
+                    break@spin
+                }
+                taskPool[i] = result.getOrNull()?.let { Task(context, it) }
+            }
+        }
+
+        do {
+            var ongoingTasks = false
+            for (i in taskPool.indices) {
+                val task = taskPool[i]
+                if (task != null) {
+                    val finishedRawSong = task.get()
+                    if (finishedRawSong != null) {
+                        completeSongs.send(finishedRawSong)
+                        taskPool[i] = null
+                    } else {
+                        ongoingTasks = true
+                    }
                 }
             }
+        } while (ongoingTasks)
 
-            break
+        completeSongs.close()
-        }
     }
 
     private companion object {
@@ -127,7 +94,7 @@ class MetadataExtractor(
  * @param raw [RealSong.Raw] to process.
  * @author Alexander Capehart (OxygenCobalt)
  */
-class Task(context: Context, private val raw: RealSong.Raw) {
+private class Task(context: Context, private val raw: RealSong.Raw) {
     // Note that we do not leverage future callbacks. This is because errors in the
     // (highly fallible) extraction process will not bubble up to Indexer when a
     // listener is used, instead crashing the app entirely.
@@ -137,6 +104,8 @@ class Task(context: Context, private val raw: RealSong.Raw) {
             MediaItem.fromUri(
                 requireNotNull(raw.mediaStoreId) { "Invalid raw: No id" }.toAudioUri()))
 
+    init {}
+
     /**
      * Try to get a completed song from this [Task], if it has finished processing.
      * @return A [RealSong.Raw] instance if processing has completed, null otherwise.

app/src/main/java/org/oxycblt/auxio/music/library/RealMusic.kt

@@ -46,6 +46,8 @@ import org.oxycblt.auxio.util.nonZeroOrNull
 import org.oxycblt.auxio.util.toUuidOrNull
 import org.oxycblt.auxio.util.unlikelyToBeNull
 
+// TODO: Split off raw music and real music
+
 /**
  * Library-backed implementation of [RealSong].
  * @param raw The [Raw] to derive the member data from.

app/src/main/java/org/oxycblt/auxio/music/system/Indexer.kt

@@ -22,8 +22,13 @@ import android.content.Context
 import android.content.pm.PackageManager
 import android.os.Build
 import androidx.core.content.ContextCompat
+import java.util.LinkedList
 import kotlinx.coroutines.CancellationException
+import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.async
+import kotlinx.coroutines.channels.Channel
+import kotlinx.coroutines.launch
 import kotlinx.coroutines.withContext
 import kotlinx.coroutines.yield
 import org.oxycblt.auxio.BuildConfig
@@ -93,8 +98,9 @@ interface Indexer {
      * @param context [Context] required to load music.
      * @param withCache Whether to use the cache or not when loading. If false, the cache will still
      * be written, but no cache entries will be loaded into the new library.
+     * @param scope The [CoroutineScope] to run the indexing job in.
      */
-    suspend fun index(context: Context, withCache: Boolean)
+    fun index(context: Context, withCache: Boolean, scope: CoroutineScope)
 
     /**
      * Request that the music library should be reloaded. This should be used by components that do
@@ -287,26 +293,28 @@ private class RealIndexer : Indexer {
         this.listener = null
     }
 
-    override suspend fun index(context: Context, withCache: Boolean) {
+    override fun index(context: Context, withCache: Boolean, scope: CoroutineScope) {
-        val result =
+        scope.launch {
-            try {
+            val result =
-                val start = System.currentTimeMillis()
+                try {
-                val library = indexImpl(context, withCache)
+                    val start = System.currentTimeMillis()
-                logD(
+                    val library = indexImpl(context, withCache, this)
-                    "Music indexing completed successfully in " +
+                    logD(
-                        "${System.currentTimeMillis() - start}ms")
+                        "Music indexing completed successfully in " +
-                Result.success(library)
+                            "${System.currentTimeMillis() - start}ms")
-            } catch (e: CancellationException) {
+                    Result.success(library)
-                // Got cancelled, propagate upwards to top-level co-routine.
+                } catch (e: CancellationException) {
-                logD("Loading routine was cancelled")
+                    // Got cancelled, propagate upwards to top-level co-routine.
-                throw e
+                    logD("Loading routine was cancelled")
-            } catch (e: Exception) {
+                    throw e
-                // Music loading process failed due to something we have not handled.
+                } catch (e: Exception) {
-                logE("Music indexing failed")
+                    // Music loading process failed due to something we have not handled.
-                logE(e.stackTraceToString())
+                    logE("Music indexing failed")
-                Result.failure(e)
+                    logE(e.stackTraceToString())
-            }
+                    Result.failure(e)
-        emitCompletion(result)
+                }
+            emitCompletion(result)
+        }
     }
 
     @Synchronized
@@ -321,55 +329,62 @@ private class RealIndexer : Indexer {
         emitIndexing(null)
     }
 
-    private suspend fun indexImpl(context: Context, withCache: Boolean): Library {
+    private suspend fun indexImpl(
+        context: Context,
+        withCache: Boolean,
+        scope: CoroutineScope
+    ): Library {
         if (ContextCompat.checkSelfPermission(context, Indexer.PERMISSION_READ_AUDIO) ==
             PackageManager.PERMISSION_DENIED) {
             // No permissions, signal that we can't do anything.
             throw Indexer.NoPermissionException()
         }
 
-        // Create the chain of extractors. Each extractor builds on the previous and
-        // enables version-specific features in order to create the best possible music
-        // experience.
-        val cacheExtractor = CacheExtractor.from(context, withCache)
-        val mediaStoreExtractor = MediaStoreExtractor.from(context, cacheExtractor)
-        val metadataExtractor = MetadataExtractor(context, mediaStoreExtractor)
-        val rawSongs = loadRawSongs(metadataExtractor).ifEmpty { throw Indexer.NoMusicException() }
-        // Build the rest of the music library from the song list. This is much more powerful
-        // and reliable compared to using MediaStore to obtain grouping information.
-        val buildStart = System.currentTimeMillis()
-        val library = Library.from(rawSongs, MusicSettings.from(context))
-        logD("Successfully built library in ${System.currentTimeMillis() - buildStart}ms")
-        return library
-    }
-
-    private suspend fun loadRawSongs(metadataExtractor: MetadataExtractor): List<RealSong.Raw> {
-        logD("Starting indexing process")
-        val start = System.currentTimeMillis()
         // Start initializing the extractors. Use an indeterminate state, as there is no ETA on
         // how long a media database query will take.
         emitIndexing(Indexer.Indexing.Indeterminate)
-        val total = metadataExtractor.init()
-        yield()
 
-        // Note: We use a set here so we can eliminate song duplicates.
+        val metadataCacheRepository = MetadataCacheRepository.from(context)
-        val rawSongs = mutableListOf<RealSong.Raw>()
+        val mediaStoreExtractor = MediaStoreExtractor.from(context)
-        metadataExtractor.extract().collect { rawSong ->
+        val metadataExtractor = MetadataExtractor(context)
+
+        // Do the initial query of the cache and media databases in parallel.
+        val mediaStoreQueryJob = scope.async { mediaStoreExtractor.query() }
+        val cache =
+            if (withCache) {
+                metadataCacheRepository.readCache()
+            } else {
+                null
+            }
+        val total = mediaStoreQueryJob.await().count
+
+        // Now start processing the queried song information in parallel. Songs that can't be
+        // received from the cache are consisted incomplete and pushed to a separate channel
+        // that will eventually be processed into completed raw songs.
+        val completeSongs = Channel<RealSong.Raw>(Channel.UNLIMITED)
+        val incompleteSongs = Channel<RealSong.Raw>(Channel.UNLIMITED)
+        val mediaStoreJob =
+            scope.async { mediaStoreExtractor.consume(cache, incompleteSongs, completeSongs) }
+        val metadataJob = scope.async { metadataExtractor.consume(incompleteSongs, completeSongs) }
+
+        // Await completed raw songs as they are processed.
+        val rawSongs = LinkedList<RealSong.Raw>()
+        for (rawSong in completeSongs) {
             rawSongs.add(rawSong)
-            // Now we can signal a defined progress by showing how many songs we have
-            // loaded, and the projected amount of songs we found in the library
-            // (obtained by the extractors)
-            yield()
             emitIndexing(Indexer.Indexing.Songs(rawSongs.size, total))
         }
+        mediaStoreJob.await()
+        metadataJob.await()
 
-        // Finalize the extractors with the songs we have now loaded. There is no ETA
+        // Successfully loaded the library, now save the cache and create the library in
-        // on this process, so go back to an indeterminate state.
+        // parallel.
         emitIndexing(Indexer.Indexing.Indeterminate)
-        metadataExtractor.finalize(rawSongs)
+        val libraryJob =
-        logD(
+            scope.async(Dispatchers.Main) { Library.from(rawSongs, MusicSettings.from(context)) }
-            "Successfully loaded ${rawSongs.size} raw songs in ${System.currentTimeMillis() - start}ms")
+        if (cache == null || cache.invalidated) {
-        return rawSongs
+            metadataCacheRepository.writeCache(rawSongs)
+        }
+        return libraryJob.await()
     }
 
     /**

app/src/main/java/org/oxycblt/auxio/music/system/IndexerService.kt

@@ -30,7 +30,7 @@ import dagger.hilt.android.AndroidEntryPoint
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.Job
-import kotlinx.coroutines.launch
+import kotlinx.coroutines.cancel
 import org.oxycblt.auxio.BuildConfig
 import org.oxycblt.auxio.music.MusicRepository
 import org.oxycblt.auxio.music.MusicSettings
@@ -119,11 +119,11 @@ class IndexerService : Service(), Indexer.Controller, MusicSettings.Listener {
     override fun onStartIndexing(withCache: Boolean) {
         if (indexer.isIndexing) {
             // Cancel the previous music loading job.
-            currentIndexJob?.cancel()
+            indexScope.cancel()
             indexer.reset()
         }
         // Start a new music loading job on a co-routine.
-        currentIndexJob = indexScope.launch { indexer.index(this@IndexerService, withCache) }
+        indexer.index(this@IndexerService, withCache, indexScope)
     }
 
     override fun onIndexerStateChanged(state: Indexer.State?) {

app/src/main/java/org/oxycblt/auxio/playback/PlaybackViewModel.kt

@@ -17,7 +17,6 @@
  
 package org.oxycblt.auxio.playback
 
-import androidx.lifecycle.AndroidViewModel
 import androidx.lifecycle.ViewModel
 import androidx.lifecycle.viewModelScope
 import dagger.hilt.android.lifecycle.HiltViewModel
@@ -33,7 +32,7 @@ import org.oxycblt.auxio.playback.queue.Queue
 import org.oxycblt.auxio.playback.state.*
 
 /**
- * An [AndroidViewModel] that provides a safe UI frontend for the current playback state.
+ * An [ViewModel] that provides a safe UI frontend for the current playback state.
  * @author Alexander Capehart (OxygenCobalt)
  */
 @HiltViewModel

app/src/main/java/org/oxycblt/auxio/search/SearchViewModel.kt

@@ -18,7 +18,6 @@
 package org.oxycblt.auxio.search
 
 import androidx.annotation.IdRes
-import androidx.lifecycle.AndroidViewModel
 import androidx.lifecycle.ViewModel
 import androidx.lifecycle.viewModelScope
 import dagger.hilt.android.lifecycle.HiltViewModel
@@ -38,7 +37,7 @@ import org.oxycblt.auxio.playback.PlaybackSettings
 import org.oxycblt.auxio.util.logD
 
 /**
- * An [AndroidViewModel] that keeps performs search operations and tracks their results.
+ * An [ViewModel] that keeps performs search operations and tracks their results.
  * @author Alexander Capehart (OxygenCobalt)
  */
 @HiltViewModel

app/src/main/java/org/oxycblt/auxio/settings/categories/UIPreferenceFragment.kt

@@ -26,6 +26,10 @@ import org.oxycblt.auxio.settings.ui.WrappedDialogPreference
 import org.oxycblt.auxio.ui.UISettings
 import org.oxycblt.auxio.util.isNight
 
+/**
+ * Display preferences.
+ * @author Alexander Capehart (OxygenCobalt)
+ */
 class UIPreferenceFragment : BasePreferenceFragment(R.xml.preferences_ui) {
     override fun onOpenDialogPreference(preference: WrappedDialogPreference) {
         if (preference.key == getString(R.string.set_key_accent)) {

app/src/main/java/org/oxycblt/auxio/ui/UIModule.kt

@@ -1,31 +0,0 @@
-/*
- * Copyright (c) 2023 Auxio Project
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
- 
-package org.oxycblt.auxio.ui
-
-import android.content.Context
-import dagger.Module
-import dagger.hilt.InstallIn
-import dagger.hilt.android.qualifiers.ApplicationContext
-import dagger.hilt.components.SingletonComponent
-import org.oxycblt.auxio.image.ImageSettings
-
-@Module
-@InstallIn(SingletonComponent::class)
-class UIModule {
-    fun settings(@ApplicationContext context: Context) = ImageSettings.from(context)
-}