heimoshuiyu/gonic
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat(transcode): add cache pruning and config options

Browse Source 
* Added config option to set size of transcode cache and cadence to enforce that sizing via ejection.

* Added cache eject to contrib/config.

* Added error return for CacheEject(). Changed to use WalkDir() instead of Walk().

* Lint fix.

* Added universal lock for cache eject.

* Removed accidentally committed binary.
This commit is contained in:
brian-doherty
2024-09-15 09:04:28 -05:00
committed by GitHub
parent bfa0e130d4
commit bcb613c79c
5 changed files with 86 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@@ -71,6 +71,8 @@ password can then be changed from the web interface
| `GONIC_MULTI_VALUE_GENRE` | `-multi-value-genre` | **optional** setting for multi-valued genre tags when scanning ([see more](#multi-valued-tags-v016)) |
| `GONIC_MULTI_VALUE_ARTIST` | `-multi-value-artist` | **optional** setting for multi-valued artist tags when scanning ([see more](#multi-valued-tags-v016)) |
| `GONIC_MULTI_VALUE_ALBUM_ARTIST` | `-multi-value-album-artist` | **optional** setting for multi-valued album artist tags when scanning ([see more](#multi-valued-tags-v016)) |
| `GONIC_TRANSCODE_CACHE_SIZE` | `-transcode-cache-size` | **optional** size of the transcode cache in MB (0 = no limit) |
| `GONIC_TRANSCODE_EJECT_INTERVAL` | `-transcode-eject-interval` | **optional** interval (in minutes) to eject transcode cache (0 = never) |
| `GONIC_EXPVAR` | `-expvar` | **optional** enable the /debug/vars endpoint (exposes useful debugging attributes as well as database stats) |
## multi valued tags (v0.16+)

19
cmd/gonic/gonic.go
View File

@@ -93,6 +93,9 @@ func main() {
deprecatedConfGenreSplit := flag.String("genre-split", "", "(deprecated, see multi-value settings)")
confTranscodeCacheSize := flag.Int("transcode-cache-size", 0, "size of the transcode cache in MB (0 = no limit) (optional)")
confTranscodeEjectInterval := flag.Int("transcode-eject-interval", 0, "interval (in minutes) to eject transcode cache (0 = never) (optional)")
flag.Parse()
flagconf.ParseEnv()
flagconf.ParseConfig(*confConfigPath)
@@ -201,6 +204,7 @@ func main() {
transcoder := transcode.NewCachingTranscoder(
transcode.NewFFmpegTranscoder(),
cacheDirAudio,
*confTranscodeCacheSize,
)
lastfmClientKeySecretFunc := func() (string, string, error) {
@@ -404,6 +408,21 @@ func main() {
return nil
})
errgrp.Go(func() error {
if *confTranscodeEjectInterval == 0 || *confTranscodeCacheSize == 0 {
return nil
}
defer logJob("transcode cache eject")()
ctxTick(ctx, time.Duration(*confTranscodeEjectInterval)*time.Minute, func() {
if err := transcoder.CacheEject(); err != nil {
log.Printf("error ejecting transcode cache: %v", err)
}
})
return nil
})
errgrp.Go(func() error {
if *confScanIntervalMins == 0 {
return nil

4
contrib/config
View File

@@ -49,6 +49,10 @@ playlists-path <path to your m3u playlist dir>
# regenerated.
cache-path /var/cache/gonic
# Option to eject least recently used items from transcode cache.
#transcode-cache-size 5000 # in Mb (0 = no limit)
#transcode-eject-interval 1440 # in minutes (0 = never eject)
# Interval (in minutes) to check for new music. Default: don't scan
#scan-interval 0
#scan-at-start-enabled false

2
transcode/transcode_test.go
View File

@@ -113,7 +113,7 @@ func TestCachingParallelism(t *testing.T) {
callback: func() { realTranscodeCount.Add(1) },
}
cacheTranscoder := transcode.NewCachingTranscoder(transcoder, t.TempDir())
cacheTranscoder := transcode.NewCachingTranscoder(transcoder, t.TempDir(), 1024)
var wg sync.WaitGroup
for i := 0; i < 5; i++ {

62
transcode/transcoder_caching.go
View File

@@ -5,9 +5,12 @@ import (
"crypto/md5"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"sort"
"sync"
"time"
)
const perm = 0o644
@@ -15,16 +18,21 @@ const perm = 0o644
type CachingTranscoder struct {
cachePath string
transcoder Transcoder
limitMB int
locks keyedMutex
cleanLock sync.RWMutex
}
var _ Transcoder = (*CachingTranscoder)(nil)
func NewCachingTranscoder(t Transcoder, cachePath string) *CachingTranscoder {
return &CachingTranscoder{transcoder: t, cachePath: cachePath}
func NewCachingTranscoder(t Transcoder, cachePath string, limitMB int) *CachingTranscoder {
return &CachingTranscoder{transcoder: t, cachePath: cachePath, limitMB: limitMB}
}
func (t *CachingTranscoder) Transcode(ctx context.Context, profile Profile, in string, out io.Writer) error {
t.cleanLock.RLock()
defer t.cleanLock.RUnlock()
// don't try cache partial transcodes
if profile.Seek() > 0 {
return t.transcoder.Transcode(ctx, profile, in, out)
@@ -52,6 +60,7 @@ func (t *CachingTranscoder) Transcode(ctx context.Context, profile Profile, in s
if i, err := cf.Stat(); err == nil && i.Size() > 0 {
_, _ = io.Copy(out, cf)
_ = os.Chtimes(path, time.Now(), time.Now()) // Touch for LRU cache purposes
return nil
}
@@ -64,6 +73,55 @@ func (t *CachingTranscoder) Transcode(ctx context.Context, profile Profile, in s
return nil
}
func (t *CachingTranscoder) CacheEject() error {
t.cleanLock.Lock()
defer t.cleanLock.Unlock()
// Delete LRU cache files that exceed size limit. Use last modified time.
type file struct {
path string
info os.FileInfo
}
var files []file
var total int64 = 0
err := filepath.WalkDir(t.cachePath, func(path string, de fs.DirEntry, err error) error {
if err != nil {
return err
}
if !de.IsDir() {
info, err := de.Info()
if err != nil {
return fmt.Errorf("walk cache path for eject: %w", err)
}
files = append(files, file{path, info})
total += info.Size()
}
return nil
})
if err != nil {
return fmt.Errorf("walk cache path for eject: %w", err)
}
sort.Slice(files, func(i, j int) bool {
return files[i].info.ModTime().Before(files[j].info.ModTime())
})
for total > int64(t.limitMB)*1024*1024 {
curFile := files[0]
files = files[1:]
total -= curFile.info.Size()
err = os.Remove(curFile.path)
if err != nil {
return fmt.Errorf("remove cache file: %w", err)
}
}
return nil
}
func cacheKey(cmd string, args []string) string {
// the cache is invalid whenever transcode command (which includes the
// absolute filepath, bit rate args, replay gain args, etc.) changes