feat(subsonic): cache and use lastfm responses for covers, bios, top songs

artistinfocache/artistinfocache.go

@@ -0,0 +1,122 @@
+//nolint:revive
+package artistinfocache
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log"
+	"time"
+
+	"github.com/jinzhu/gorm"
+	"go.senan.xyz/gonic/db"
+	"go.senan.xyz/gonic/scrobble/lastfm"
+)
+
+const keepFor = 30 * time.Hour * 24
+
+type ArtistInfoCache struct {
+	db           *db.DB
+	lastfmClient *lastfm.Client
+}
+
+func New(db *db.DB, lastfmClient *lastfm.Client) *ArtistInfoCache {
+	return &ArtistInfoCache{db: db, lastfmClient: lastfmClient}
+}
+
+func (a *ArtistInfoCache) GetOrLookup(ctx context.Context, apiKey string, artistID int) (*db.ArtistInfo, error) {
+	var artist db.Artist
+	if err := a.db.Find(&artist, "id=?", artistID).Error; err != nil {
+		return nil, fmt.Errorf("find artist in db: %w", err)
+	}
+
+	var artistInfo db.ArtistInfo
+	if err := a.db.Find(&artistInfo, "id=?", artistID).Error; err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
+		return nil, fmt.Errorf("find artist info in db: %w", err)
+	}
+
+	if artistInfo.ID == 0 || artistInfo.Biography == "" /* prev not found maybe */ || time.Since(artistInfo.UpdatedAt) > keepFor {
+		return a.Lookup(ctx, apiKey, &artist)
+	}
+
+	return &artistInfo, nil
+}
+
+func (a *ArtistInfoCache) Get(ctx context.Context, artistID int) (*db.ArtistInfo, error) {
+	var artistInfo db.ArtistInfo
+	if err := a.db.Find(&artistInfo, "id=?", artistID).Error; err != nil {
+		return nil, fmt.Errorf("find artist info in db: %w", err)
+	}
+	return &artistInfo, nil
+}
+
+func (a *ArtistInfoCache) Lookup(ctx context.Context, apiKey string, artist *db.Artist) (*db.ArtistInfo, error) {
+	var artistInfo db.ArtistInfo
+	artistInfo.ID = artist.ID
+
+	if err := a.db.FirstOrCreate(&artistInfo, "id=?", artistInfo.ID).Error; err != nil {
+		return nil, fmt.Errorf("first or create artist info: %w", err)
+	}
+
+	info, err := a.lastfmClient.ArtistGetInfo(apiKey, artist.Name)
+	if err != nil {
+		return nil, fmt.Errorf("get upstream info: %w", err)
+	}
+
+	artistInfo.ID = artist.ID
+	artistInfo.Biography = info.Bio.Summary
+	artistInfo.MusicBrainzID = info.MBID
+	artistInfo.LastFMURL = info.URL
+
+	var similar []string
+	for _, sim := range info.Similar.Artists {
+		similar = append(similar, sim.Name)
+	}
+	artistInfo.SetSimilarArtists(similar)
+
+	url, _ := a.lastfmClient.StealArtistImage(info.URL)
+	artistInfo.ImageURL = url
+
+	topTracksResponse, err := a.lastfmClient.ArtistGetTopTracks(apiKey, artist.Name)
+	if err != nil {
+		return nil, fmt.Errorf("get top tracks: %w", err)
+	}
+	var topTracks []string
+	for _, tr := range topTracksResponse.Tracks {
+		topTracks = append(topTracks, tr.Name)
+	}
+	artistInfo.SetTopTracks(topTracks)
+
+	if err := a.db.Save(&artistInfo).Error; err != nil {
+		return nil, fmt.Errorf("save upstream info: %w", err)
+	}
+
+	return &artistInfo, nil
+}
+
+func (a *ArtistInfoCache) Refresh(apiKey string, interval time.Duration) error {
+	ticker := time.NewTicker(interval)
+	for range ticker.C {
+		q := a.db.
+			Where("artist_infos.id IS NULL OR artist_infos.updated_at<?", time.Now().Add(-keepFor)).
+			Joins("LEFT JOIN artist_infos ON artist_infos.id=artists.id")
+
+		var artist db.Artist
+		if err := q.Find(&artist).Error; err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
+			log.Printf("error finding non cached artist: %v", err)
+			continue
+		}
+		if artist.ID == 0 {
+			continue
+		}
+
+		if _, err := a.Lookup(context.Background(), apiKey, &artist); err != nil {
+			log.Printf("error looking up non cached artist %s: %v", artist.Name, err)
+			continue
+		}
+
+		log.Printf("cached artist info for %q", artist.Name)
+	}
+
+	return nil
+}

cmd/gonic/gonic.go

@@ -26,6 +26,7 @@ import (
 	"github.com/sentriz/gormstore"
 
 	"go.senan.xyz/gonic"
+	"go.senan.xyz/gonic/artistinfocache"
 	"go.senan.xyz/gonic/db"
 	"go.senan.xyz/gonic/jukebox"
 	"go.senan.xyz/gonic/playlist"
@@ -205,6 +206,8 @@ func main() {
 	sessDB.SessionOpts.HttpOnly = true
 	sessDB.SessionOpts.SameSite = http.SameSiteLaxMode
 
+	artistInfoCache := artistinfocache.New(dbc, lastfmClient)
+
 	ctrlBase := &ctrlbase.Controller{
 		DB:            dbc,
 		PlaylistStore: playlistStore,
@@ -222,6 +225,7 @@ func main() {
 		CacheAudioPath:  cacheDirAudio,
 		CacheCoverPath:  cacheDirCovers,
 		LastFMClient:    lastfmClient,
+		ArtistInfoCache: artistInfoCache,
 		Scrobblers: []scrobble.Scrobbler{
 			lastfm.NewScrobbler(dbc, lastfmClient),
 			listenbrainz.NewScrobbler(),
@@ -345,6 +349,14 @@ func main() {
 		})
 	}
 
+	lastfmAPIKey, _ := dbc.GetSetting("lastfm_api_key")
+	if lastfmAPIKey != "" {
+		g.Add(func() error {
+			log.Printf("starting job 'refresh artist info'\n")
+			return artistInfoCache.Refresh(lastfmAPIKey, 5*time.Second)
+		}, nil)
+	}
+
 	if *confScanAtStart {
 		if _, err := scannr.ScanAndClean(scanner.ScanOptions{}); err != nil {
 			log.Panicf("error scanning at start: %v\n", err)

db/migrations.go

@@ -58,6 +58,7 @@ func (db *DB) Migrate(ctx MigrationContext) error {
 		construct(ctx, "202305301718", migratePlayCountToLength),
 		construct(ctx, "202307281628", migrateAlbumArtistsMany2Many),
 		construct(ctx, "202309070009", migrateDeleteArtistCoverField),
+		construct(ctx, "202309131743", migrateArtistInfo),
 	}
 
 	return gormigrate.
@@ -605,3 +606,10 @@ func migrateDeleteArtistCoverField(tx *gorm.DB, _ MigrationContext) error {
 
 	return nil
 }
+
+func migrateArtistInfo(tx *gorm.DB, _ MigrationContext) error {
+	return tx.AutoMigrate(
+		ArtistInfo{},
+	).
+		Error
+}

db/model.go

@@ -7,6 +7,7 @@ package db
 // https://www.db-fiddle.com/f/wJ7z8L7mu6ZKaYmWk1xr1p/5
 
 import (
+	"fmt"
 	"path"
 	"path/filepath"
 	"sort"
@@ -32,7 +33,7 @@ func splitIDs(in, sep string) []specid.ID {
 	return ret
 }
 
-func joinIds(in []specid.ID, sep string) string {
+func join[T fmt.Stringer](in []T, sep string) string {
 	if in == nil {
 		return ""
 	}
@@ -270,7 +271,7 @@ func (p *PlayQueue) GetItems() []specid.ID {
 }
 
 func (p *PlayQueue) SetItems(items []specid.ID) {
-	p.Items = joinIds(items, ",")
+	p.Items = join(items, ",")
 }
 
 type TranscodePreference struct {
@@ -441,3 +442,21 @@ type InternetRadioStation struct {
 func (ir *InternetRadioStation) SID() *specid.ID {
 	return &specid.ID{Type: specid.InternetRadioStation, Value: ir.ID}
 }
+
+type ArtistInfo struct {
+	ID             int `gorm:"primary_key" sql:"type:int REFERENCES artists(id) ON DELETE CASCADE"`
+	CreatedAt      time.Time
+	UpdatedAt      time.Time `gorm:"index"`
+	Biography      string
+	MusicBrainzID  string
+	LastFMURL      string
+	ImageURL       string
+	SimilarArtists string
+	TopTracks      string
+}
+
+func (p *ArtistInfo) GetSimilarArtists() []string      { return strings.Split(p.SimilarArtists, ";") }
+func (p *ArtistInfo) SetSimilarArtists(items []string) { p.SimilarArtists = strings.Join(items, ";") }
+
+func (p *ArtistInfo) GetTopTracks() []string      { return strings.Split(p.TopTracks, ";") }
+func (p *ArtistInfo) SetTopTracks(items []string) { p.TopTracks = strings.Join(items, ";") }

server/ctrlsubsonic/ctrl.go

@@ -9,6 +9,7 @@ import (
 	"log"
 	"net/http"
 
+	"go.senan.xyz/gonic/artistinfocache"
 	"go.senan.xyz/gonic/jukebox"
 	"go.senan.xyz/gonic/podcasts"
 	"go.senan.xyz/gonic/scrobble"
@@ -50,6 +51,7 @@ type Controller struct {
 	Podcasts        *podcasts.Podcasts
 	Transcoder      transcode.Transcoder
 	LastFMClient    *lastfm.Client
+	ArtistInfoCache *artistinfocache.ArtistInfoCache
 }
 
 type metaResponse struct {

server/ctrlsubsonic/handlers_by_tags.go

@@ -322,41 +322,38 @@ func (c *Controller) ServeGetArtistInfoTwo(r *http.Request) *spec.Response {
 	if apiKey == "" {
 		return sub
 	}
-	info, err := c.LastFMClient.ArtistGetInfo(apiKey, artist.Name)
+
+	info, err := c.ArtistInfoCache.GetOrLookup(r.Context(), apiKey, artist.ID)
 	if err != nil {
 		return spec.NewError(0, "fetching artist info: %v", err)
 	}
 
-	sub.ArtistInfoTwo.Biography = info.Bio.Summary
+	sub.ArtistInfoTwo.Biography = info.Biography
-	sub.ArtistInfoTwo.MusicBrainzID = info.MBID
+	sub.ArtistInfoTwo.MusicBrainzID = info.MusicBrainzID
-	sub.ArtistInfoTwo.LastFMURL = info.URL
+	sub.ArtistInfoTwo.LastFMURL = info.LastFMURL
 
 	sub.ArtistInfoTwo.SmallImageURL = c.genArtistCoverURL(r, &artist, 64)
 	sub.ArtistInfoTwo.MediumImageURL = c.genArtistCoverURL(r, &artist, 126)
 	sub.ArtistInfoTwo.LargeImageURL = c.genArtistCoverURL(r, &artist, 256)
 
-	if url, _ := c.LastFMClient.StealArtistImage(info.URL); url != "" {
+	if info.ImageURL != "" {
-		sub.ArtistInfoTwo.SmallImageURL = url
+		sub.ArtistInfoTwo.SmallImageURL = info.ImageURL
-		sub.ArtistInfoTwo.MediumImageURL = url
+		sub.ArtistInfoTwo.MediumImageURL = info.ImageURL
-		sub.ArtistInfoTwo.LargeImageURL = url
+		sub.ArtistInfoTwo.LargeImageURL = info.ImageURL
-		sub.ArtistInfoTwo.ArtistImageURL = url
+		sub.ArtistInfoTwo.ArtistImageURL = info.ImageURL
 	}
 
 	count := params.GetOrInt("count", 20)
 	inclNotPresent := params.GetOrBool("includeNotPresent", false)
-	similarArtists, err := c.LastFMClient.ArtistGetSimilar(apiKey, artist.Name)
-	if err != nil {
-		return spec.NewError(0, "fetching artist similar: %v", err)
-	}
 
-	for i, similarInfo := range similarArtists.Artists {
+	for i, similarName := range info.GetSimilarArtists() {
 		if i == count {
 			break
 		}
 		var artist db.Artist
 		err = c.DB.
 			Select("artists.*, count(albums.id) album_count").
-			Where("name=?", similarInfo.Name).
+			Where("name=?", similarName).
 			Joins("LEFT JOIN album_artists ON album_artists.artist_id=artists.id").
 			Joins("LEFT JOIN albums ON albums.id=album_artists.album_id").
 			Group("artists.id").
@@ -372,7 +369,7 @@ func (c *Controller) ServeGetArtistInfoTwo(r *http.Request) *spec.Response {
 		}
 		sub.ArtistInfoTwo.SimilarArtist = append(sub.ArtistInfoTwo.SimilarArtist, &spec.SimilarArtist{
 			ID:         artistID,
-			Name:       similarInfo.Name,
+			Name:       similarName,
 			CoverArt:   artistID,
 			AlbumCount: artist.AlbumCount,
 		})
@@ -544,7 +541,7 @@ func (c *Controller) ServeGetTopSongs(r *http.Request) *spec.Response {
 	if apiKey == "" {
 		return spec.NewResponse()
 	}
-	topTracks, err := c.LastFMClient.ArtistGetTopTracks(apiKey, artist.Name)
+	info, err := c.ArtistInfoCache.GetOrLookup(r.Context(), apiKey, artist.ID)
 	if err != nil {
 		return spec.NewError(0, "fetching artist top tracks: %v", err)
 	}
@@ -554,15 +551,11 @@ func (c *Controller) ServeGetTopSongs(r *http.Request) *spec.Response {
 		Tracks: make([]*spec.TrackChild, 0),
 	}
 
-	if len(topTracks.Tracks) == 0 {
+	topTrackNames := info.GetTopTracks()
+	if len(topTrackNames) == 0 {
 		return sub
 	}
 
-	topTrackNames := make([]string, len(topTracks.Tracks))
-	for i, t := range topTracks.Tracks {
-		topTrackNames[i] = t.Name
-	}
-
 	var tracks []*db.Track
 	err = c.DB.
 		Preload("Album").

server/ctrlsubsonic/handlers_raw.go

@@ -2,8 +2,10 @@ package ctrlsubsonic
 
 import (
 	"bytes"
+	"context"
 	"errors"
 	"fmt"
+	"io"
 	"log"
 	"net/http"
 	"os"
@@ -13,6 +15,7 @@ import (
 	"github.com/disintegration/imaging"
 	"github.com/jinzhu/gorm"
 
+	"go.senan.xyz/gonic/artistinfocache"
 	"go.senan.xyz/gonic/db"
 	"go.senan.xyz/gonic/server/ctrlsubsonic/params"
 	"go.senan.xyz/gonic/server/ctrlsubsonic/spec"
@@ -107,115 +110,101 @@ const (
 
 var (
 	errCoverNotFound = errors.New("could not find a cover with that id")
-	errCoverEmpty    = errors.New("no cover found for that folder")
+	errCoverEmpty    = errors.New("no cover found")
 )
 
 // TODO: can we use specidpaths.Locate here?
-func coverGetPath(dbc *db.DB, podcastPath string, id specid.ID) (string, error) {
+func coverFor(dbc *db.DB, artistInfoCache *artistinfocache.ArtistInfoCache, podcastPath string, id specid.ID) (io.ReadCloser, error) {
 	switch id.Type {
 	case specid.Album:
-		return coverGetPathAlbum(dbc, id.Value)
+		return coverForAlbum(dbc, id.Value)
 	case specid.Artist:
-		return coverGetPathArtist(dbc, id.Value)
+		return coverForArtist(artistInfoCache, id.Value)
 	case specid.Podcast:
-		return coverGetPathPodcast(dbc, podcastPath, id.Value)
+		return coverForPodcast(dbc, podcastPath, id.Value)
 	case specid.PodcastEpisode:
 		return coverGetPathPodcastEpisode(dbc, podcastPath, id.Value)
 	default:
-		return "", errCoverNotFound
+		return nil, errCoverNotFound
 	}
 }
 
-func coverGetPathAlbum(dbc *db.DB, id int) (string, error) {
+func coverForAlbum(dbc *db.DB, id int) (*os.File, error) {
 	folder := &db.Album{}
 	err := dbc.DB.
 		Select("id, root_dir, left_path, right_path, cover").
 		First(folder, id).
 		Error
 	if err != nil {
-		return "", fmt.Errorf("select album: %w", err)
+		return nil, fmt.Errorf("select album: %w", err)
 	}
 	if folder.Cover == "" {
-		return "", errCoverEmpty
+		return nil, errCoverEmpty
 	}
-	return path.Join(
+	return os.Open(path.Join(folder.RootDir, folder.LeftPath, folder.RightPath, folder.Cover))
-		folder.RootDir,
-		folder.LeftPath,
-		folder.RightPath,
-		folder.Cover,
-	), nil
 }
 
-func coverGetPathArtist(dbc *db.DB, id int) (string, error) {
+func coverForArtist(artistInfoCache *artistinfocache.ArtistInfoCache, id int) (io.ReadCloser, error) {
-	folder := &db.Album{}
+	info, err := artistInfoCache.Get(context.Background(), id)
-	err := dbc.DB.
-		Select("albums.id, albums.root_dir, albums.left_path, albums.right_path, albums.cover").
-		Joins("JOIN album_artists ON album_artists.album_id=albums.id").
-		Where("album_artists.artist_id=?", id).
-		Group("albums.id").
-		Find(folder).
-		Error
 	if err != nil {
-		return "", fmt.Errorf("select guessed artist folder: %w", err)
+		return nil, fmt.Errorf("get artist info from cache: %w", err)
 	}
-	if folder.Cover == "" {
+	if info.ImageURL == "" {
-		return "", errCoverEmpty
+		return nil, fmt.Errorf("%w: cache miss", errCoverEmpty)
 	}
-	return path.Join(
+	resp, err := http.Get(info.ImageURL)
-		folder.RootDir,
+	if err != nil {
-		folder.LeftPath,
+		return nil, fmt.Errorf("req image from lastfm: %w", err)
-		folder.RightPath,
+	}
-		folder.Cover,
+	return resp.Body, nil
-	), nil
 }
 
-func coverGetPathPodcast(dbc *db.DB, podcastPath string, id int) (string, error) {
+func coverForPodcast(dbc *db.DB, podcastPath string, id int) (*os.File, error) {
 	podcast := &db.Podcast{}
 	err := dbc.
 		First(podcast, id).
 		Error
 	if err != nil {
-		return "", fmt.Errorf("select podcast: %w", err)
+		return nil, fmt.Errorf("select podcast: %w", err)
 	}
 	if podcast.ImagePath == "" {
-		return "", errCoverEmpty
+		return nil, errCoverEmpty
 	}
-	return path.Join(podcastPath, podcast.ImagePath), nil
+	return os.Open(path.Join(podcastPath, podcast.ImagePath))
 }
 
-func coverGetPathPodcastEpisode(dbc *db.DB, podcastPath string, id int) (string, error) {
+func coverGetPathPodcastEpisode(dbc *db.DB, podcastPath string, id int) (*os.File, error) {
 	episode := &db.PodcastEpisode{}
 	err := dbc.
 		First(episode, id).
 		Error
 	if err != nil {
-		return "", fmt.Errorf("select episode: %w", err)
+		return nil, fmt.Errorf("select episode: %w", err)
 	}
 	podcast := &db.Podcast{}
 	err = dbc.
 		First(podcast, episode.PodcastID).
 		Error
 	if err != nil {
-		return "", fmt.Errorf("select podcast: %w", err)
+		return nil, fmt.Errorf("select podcast: %w", err)
 	}
 	if podcast.ImagePath == "" {
-		return "", errCoverEmpty
+		return nil, errCoverEmpty
 	}
-	return path.Join(podcastPath, podcast.ImagePath), nil
+	return os.Open(path.Join(podcastPath, podcast.ImagePath))
 }
 
-func coverScaleAndSave(absPath, cachePath string, size int) error {
+func coverScaleAndSave(reader io.Reader, cachePath string, size int) error {
-	src, err := imaging.Open(absPath)
+	src, err := imaging.Decode(reader)
 	if err != nil {
-		return fmt.Errorf("resizing `%s`: %w", absPath, err)
+		return fmt.Errorf("resizing: %w", err)
 	}
 	width := size
 	if width > src.Bounds().Dx() {
 		// don't upscale images
 		width = src.Bounds().Dx()
 	}
-	err = imaging.Save(imaging.Resize(src, width, 0, imaging.Lanczos), cachePath)
+	if err = imaging.Save(imaging.Resize(src, width, 0, imaging.Lanczos), cachePath); err != nil {
-	if err != nil {
 		return fmt.Errorf("caching `%s`: %w", cachePath, err)
 	}
 	return nil
@@ -235,11 +224,13 @@ func (c *Controller) ServeGetCoverArt(w http.ResponseWriter, r *http.Request) *s
 	_, err = os.Stat(cachePath)
 	switch {
 	case os.IsNotExist(err):
-		coverPath, err := coverGetPath(c.DB, c.PodcastsPath, id)
+		reader, err := coverFor(c.DB, c.ArtistInfoCache, c.PodcastsPath, id)
 		if err != nil {
 			return spec.NewError(10, "couldn't find cover `%s`: %v", id, err)
 		}
-		if err := coverScaleAndSave(coverPath, cachePath, size); err != nil {
+		defer reader.Close()
+
+		if err := coverScaleAndSave(reader, cachePath, size); err != nil {
 			log.Printf("error scaling cover: %v", err)
 			return nil
 		}