Add support for caching transcoded files

* Add Cli\CacheProcessCommand * Local catalogs only for now https://github.com/ampache/ampache/wiki/Transcode-Caching
ampache · Aug 4, 2021 · 01acc3c · 01acc3c
1 parent 8f7a805
commit 01acc3c
Show file tree

Hide file tree

Showing 15 changed files with 311 additions and 7 deletions.
diff --git a/bin/cli b/bin/cli
@@ -41,6 +41,7 @@ $app = new Ahc\Cli\Application(
 $app->add($dic->get(Cli\BroadcastCommand::class));
 $app->add($dic->get(Cli\ComputeCacheCommand::class));
 $app->add($dic->get(Cli\CronProcessCommand::class));
+$app->add($dic->get(Cli\CacheProcessCommand::class));
 $app->add($dic->get(Cli\ArtSizeCalculationCommand::class));
 $app->add($dic->get(Cli\RunChannelCommand::class));
 $app->add($dic->get(Cli\PrintTagsCommand::class));

diff --git a/config/ampache.cfg.php.dist b/config/ampache.cfg.php.dist
@@ -7,7 +7,7 @@
 ; if this config file is up to date
 ; this is compared against a value hard-coded
 ; into the init script
-config_version = 53
+config_version = 54
 
 ;#########################################################
 ; Auto Update                                            #
@@ -1058,7 +1058,9 @@ registration_mandatory_fields = "fullname"
 ; DEFAULT: 8
 ;min_bit_rate = 48
 
-;######################################################
+;#########################################################
+; Transcode Settings                                     #
+;#########################################################
 ; These are commands used to transcode non-streaming
 ; formats to the target file type for streaming.
 ; This can be useful in re-encoding file types that don't stream
@@ -1199,6 +1201,41 @@ encode_ss_duration = "-t %DURATION%"
 ; DEFAULT: "webplayer"
 send_full_stream = "webplayer"
 
+;#########################################################
+; Transcode Caching                                      #
+;#########################################################
+; These are commands used to pre-cache a file format for streaming.
+; This helps avoid waiting for transcodes to finish and makes
+; files immediately available to the client.
+
+; Path to your cache folder.
+; This is where the pre-transcoded files will be stored
+; DEFAULT: none
+;cache_path = "/tmp"
+
+; Default audio output format
+; DEFAULT: none
+;cache_target = "mp3"
+
+; Look in your catalogs for these file extensions and pre-cache to the
+; 'cache_path'. This could be helpful to reduce space needed on your
+; web server or to make sure that clients get files quickly.
+; Execute "bin/cli run:cacheProcess" to process these files.
+; DEFAULT: "false"
+;cache_m4a  = "true"
+;cache_flac = "true"
+;cache_mpc  = "true"
+;cache_ogg  = "true"
+;cache_oga  = "true"
+;cache_opus = "true"
+;cache_wav  = "true"
+;cache_wma  = "true"
+;cache_aif  = "true"
+;cache_aiff = "true"
+;cache_ape  = "true"
+;cache_shn  = "true"
+;cache_mp3  = "true"
+
 ;#########################################################
 ; Proxy Settings (optional)                              #
 ;#########################################################

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
@@ -20,6 +20,7 @@ This means Ampache now **requires** php-intl module/dll to be enabled.
 ### Added
 
 * Private catalogs! You can now set a public or per user catalog for your music folders
+* Cache transcoded files before a user requests them with [Transcode Caching](https://github.com/ampache/ampache/wiki/Transcode-Caching)
 * Added a CONTRIBUTING.md file
 * php-intl is now required for translation of date formats into your locale
 * Added %R (Release Status) to catalog pattern matching
@@ -54,13 +55,15 @@ This means Ampache now **requires** php-intl module/dll to be enabled.
   * use_original_year: Browse by Original Year for albums (falls back to Year)
   * hide_single_artist: Hide the Song Artist column for Albums with one Artist
   * show_license: Hiding the license column in song rows
-* Config version 53
+* Config version 54
 * NEW config options
   * composer_binary_path: Override the composer binary path to distinguish between multiple composer versions
   * write_tags: Write tag changes to file (including art if available)
   * art_zip_add: Include Album Art for zip downlaods
   * catalog_filter: Allow filtering catalogs to specific users
   * catalog_verify_by_time: Only verify the files that have been modified since the last verify
+  * cache_path: The folder where the pre-transcoded files will be stored
+  * cache_target: Target audio format for the cache
 
 ### Changed
 
@@ -79,6 +82,7 @@ This means Ampache now **requires** php-intl module/dll to be enabled.
 * Replace mt_rand with random_bytes for random token generation
 * Move user bandwidth calculations out of the user format function into the user_data table
 * All localplay links use the type (e.g. mpd/upnp) as the agent to fix muti-client access
+* updateCatalog now implies add when using -i / --import by itself
 * Subsonic
   * Wait a few seconds before allowing scrobbles to avoid collisions
   * Shift the last music play if gap is bigger than 5 repeated plays (over night, etc)
@@ -121,6 +125,8 @@ This means Ampache now **requires** php-intl module/dll to be enabled.
 * Some buttons and links in the light theme needed extra CSS
 * Updated the inotifywait.sh example to stop it trying to add the same file multiple times
 * Translations could break JS with apostrophes
+* Playlist imports with an empty web_path would never work
+* Playlist imports were importing nothing
 * Subsonic
   * Support a global user playqueue with getplayqueue, saveplayqueue
   * Incorrect header being set on art requests

diff --git a/src/Config/Init/InitializationHandlerConfig.php b/src/Config/Init/InitializationHandlerConfig.php
@@ -33,7 +33,7 @@
 final class InitializationHandlerConfig implements InitializationHandlerInterface
 {
     private const VERSION        = 'develop';
-    private const CONFIG_VERSION = '53';
+    private const CONFIG_VERSION = '54';
 
     public const CONFIG_FILE_PATH = __DIR__ . '/../../../config/ampache.cfg.php';
 

diff --git a/src/Module/Application/Playback/PlayAction.php b/src/Module/Application/Playback/PlayAction.php
@@ -598,9 +598,26 @@ public function run(ServerRequestInterface $request, GuiGatekeeperInterface $gat
         if ($transcode_to) {
             debug_event('play/index', 'Transcode to {' . (string) $transcode_to . '}', 5);
         }
+        // have you pre-cached?
+        $cache_file = false;
+        $cache_path = AmpConfig::get('cache_path', '');
+        if (is_dir($cache_path)) {
+            debug_event('play/index', 'Checking cache_path {' . $cache_path . '}', 5);
+            if (AmpConfig::get('cache_' . $media->type)) {
+                $cache_target = AmpConfig::get('cache_target', '');
+                $file_target  = rtrim(trim($cache_path), '/') . '/' . $media->id . '.' . $cache_target;
+                if (is_file($file_target)) {
+                    debug_event('play/index', 'Found pre-cached file {' . $file_target . '}', 5);
+                    $record_stats = false;
+                    $cache_file   = true;
+                    $media->file  = $file_target;
+                    $media->size  = Core::get_filesize($file_target);
+                }
+            }
+        }
 
-        // If custom play action, do not try to transcode
-        if (!$cpaction && !$original) {
+        // If custom play action or already cached, do not try to transcode
+        if (!$cpaction && !$original && !$cache_file) {
             $transcode_cfg = AmpConfig::get('transcode');
             $valid_types   = $media->get_stream_types($player);
             if (!is_array($valid_types)) {

diff --git a/src/Module/Beets/Catalog.php b/src/Module/Beets/Catalog.php
@@ -298,6 +298,14 @@ public function move_catalog_proc($new_path)
         return false;
     }
 
+    /**
+     * @return boolean
+     */
+    public function cache_catalog_proc()
+    {
+        return false;
+    }
+
     /**
      * Remove a song from the "to be deleted"-list if it was found.
      * @param array $song

diff --git a/src/Module/Catalog/Catalog_Seafile.php b/src/Module/Catalog/Catalog_Seafile.php
@@ -532,6 +532,14 @@ public function move_catalog_proc($new_path)
         return false;
     }
 
+    /**
+     * @return boolean
+     */
+    public function cache_catalog_proc()
+    {
+        return false;
+    }
+
     /**
      * check_remote_song
      *

diff --git a/src/Module/Catalog/Catalog_dropbox.php b/src/Module/Catalog/Catalog_dropbox.php
@@ -628,6 +628,14 @@ public function move_catalog_proc($new_path)
         return false;
     }
 
+    /**
+     * @return boolean
+     */
+    public function cache_catalog_proc()
+    {
+        return false;
+    }
+
     /**
      * check_remote_song
      *

diff --git a/src/Module/Catalog/Catalog_local.php b/src/Module/Catalog/Catalog_local.php
@@ -23,6 +23,7 @@
 namespace Ampache\Module\Catalog;
 
 use Ampache\Config\AmpConfig;
+use Ampache\Module\Playback\Stream;
 use Ampache\Module\Util\UtilityFactoryInterface;
 use Ampache\Repository\Model\Album;
 use Ampache\Repository\Model\Art;
@@ -1084,6 +1085,108 @@ public function move_catalog_proc($new_path)
         return true;
     } // move_catalog_proc
 
+    /**
+     * cache_catalog_proc
+     * @return boolean
+     */
+    public function cache_catalog_proc()
+    {
+        $m4a    = AmpConfig::get('cache_m4a');
+        $flac   = AmpConfig::get('cache_flac');
+        $mpc    = AmpConfig::get('cache_mpc');
+        $ogg    = AmpConfig::get('cache_ogg');
+        $oga    = AmpConfig::get('cache_oga');
+        $opus   = AmpConfig::get('cache_opus');
+        $wav    = AmpConfig::get('cache_wav');
+        $wma    = AmpConfig::get('cache_wma');
+        $aif    = AmpConfig::get('cache_aif');
+        $aiff   = AmpConfig::get('cache_aiff');
+        $ape    = AmpConfig::get('cache_ape');
+        $shn    = AmpConfig::get('cache_shn');
+        $mp3    = AmpConfig::get('cache_mp3');
+        $target = AmpConfig::get('cache_target');
+        $path   = AmpConfig::get('cache_path', '');
+        // need a destination and target filetype
+        if ((!is_dir($path) || !$target)) {
+            debug_event('local.catalog', 'Check your cache_path and cache_target settings', 5);
+
+            return false;
+        }
+        // need at least one type to transcode
+        if ($m4a && !$flac && !$mpc && !$ogg && !$oga && !$opus && !$wav && !$wma && !$aif && !$aiff && !$ape && !$shn && !$mp3) {
+            debug_event('local.catalog', 'You need to pick at least 1 file format to cache', 5);
+
+            return false;
+        }
+        $sql    = "SELECT `id` FROM `song` WHERE `catalog` = ? ";
+        $params = array($this->id);
+        $join   = 'AND (';
+        if ($m4a) {
+            $sql .= "$join `file` LIKE '%.m4a' ";
+            $join = 'OR';
+        }
+        if ($flac) {
+            $sql .= "$join `file` LIKE '%.flac' ";
+            $join = 'OR';
+        }
+        if ($mpc) {
+            $sql .= "$join `file` LIKE '%.mpc' ";
+            $join = 'OR';
+        }
+        if ($ogg) {
+            $sql .= "$join `file` LIKE '%.ogg' ";
+            $join = 'OR';
+        }
+        if ($oga) {
+            $sql .= "$join `file` LIKE '%.oga' ";
+            $join = 'OR';
+        }
+        if ($opus) {
+            $sql .= "$join `file` LIKE '%.opus' ";
+            $join = 'OR';
+        }
+        if ($wav) {
+            $sql .= "$join `file` LIKE '%.wav' ";
+            $join = 'OR';
+        }
+        if ($wma) {
+            $sql .= "$join `file` LIKE '%.wma' ";
+            $join = 'OR';
+        }
+        if ($aif) {
+            $sql .= "$join `file` LIKE '%.aif' ";
+            $join = 'OR';
+        }
+        if ($aiff) {
+            $sql .= "$join `file` LIKE '%.aiff' ";
+            $join = 'OR';
+        }
+        if ($ape) {
+            $sql .= "$join `file` LIKE '%.ape' ";
+            $join = 'OR';
+        }
+        if ($shn) {
+            $sql .= "$join `file` LIKE '%.shn' ";
+        }
+        $sql .= ');';
+        $results    = array();
+        $db_results = Dba::read($sql, $params);
+
+        while ($row = Dba::fetch_assoc($db_results)) {
+            $results[] = (int)$row['id'];
+        }
+        foreach ($results as $song_id) {
+            $target_file = rtrim(trim($path), '/') . '/' . $song_id . '.' . $target;
+            if (!is_file($target_file)) {
+                debug_event('local.catalog', 'Cache needed for: ' . $target_file, 5);
+                $song = new Song($song_id);
+                Stream::start_transcode($song, $target, 'cache_catalog_proc', array($target_file));
+            }
+        }
+
+        return true;
+    }
+
     /**
      * @deprecated Inject by constructor
      */

diff --git a/src/Module/Catalog/Catalog_remote.php b/src/Module/Catalog/Catalog_remote.php
@@ -371,6 +371,14 @@ public function move_catalog_proc($new_path)
         return false;
     }
 
+    /**
+     * @return boolean
+     */
+    public function cache_catalog_proc()
+    {
+        return false;
+    }
+
     /**
      * check_remote_song
      *

diff --git a/src/Module/Catalog/Catalog_soundcloud.php b/src/Module/Catalog/Catalog_soundcloud.php
@@ -428,6 +428,14 @@ public function move_catalog_proc($new_path)
         return false;
     }
 
+    /**
+     * @return boolean
+     */
+    public function cache_catalog_proc()
+    {
+        return false;
+    }
+
     /**
      * @param $url
      * @return integer

diff --git a/src/Module/Catalog/Catalog_subsonic.php b/src/Module/Catalog/Catalog_subsonic.php
@@ -386,6 +386,14 @@ public function move_catalog_proc($new_path)
         return false;
     }
 
+    /**
+     * @return boolean
+     */
+    public function cache_catalog_proc()
+    {
+        return false;
+    }
+
     /**
      * check_remote_song
      *