refactor: move shared packages up a level

This commit is contained in:
sentriz
2022-04-13 00:09:10 +01:00
parent 165904c2bb
commit 8b803ecf20
53 changed files with 65 additions and 68 deletions

531
scanner/scanner.go Normal file
View File

@@ -0,0 +1,531 @@
package scanner
import (
"errors"
"fmt"
"io/fs"
"log"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"sync/atomic"
"syscall"
"time"
"github.com/jinzhu/gorm"
"github.com/rainycape/unidecode"
"go.senan.xyz/gonic/multierr"
"go.senan.xyz/gonic/db"
"go.senan.xyz/gonic/mime"
"go.senan.xyz/gonic/scanner/tags"
)
var (
ErrAlreadyScanning = errors.New("already scanning")
ErrReadingTags = errors.New("could not read tags")
)
type Scanner struct {
db *db.DB
musicDirs []string
genreSplit string
tagger tags.Reader
scanning *int32
}
func New(musicDirs []string, db *db.DB, genreSplit string, tagger tags.Reader) *Scanner {
return &Scanner{
db: db,
musicDirs: musicDirs,
genreSplit: genreSplit,
tagger: tagger,
scanning: new(int32),
}
}
func (s *Scanner) IsScanning() bool {
return atomic.LoadInt32(s.scanning) == 1
}
type ScanOptions struct {
IsFull bool
}
func (s *Scanner) ScanAndClean(opts ScanOptions) (*Context, error) {
if s.IsScanning() {
return nil, ErrAlreadyScanning
}
atomic.StoreInt32(s.scanning, 1)
defer atomic.StoreInt32(s.scanning, 0)
start := time.Now()
c := &Context{
errs: &multierr.Err{},
seenTracks: map[int]struct{}{},
seenAlbums: map[int]struct{}{},
isFull: opts.IsFull,
}
log.Println("starting scan")
defer func() {
log.Printf("finished scan in %s, +%d/%d tracks (%d err)\n",
durSince(start), c.SeenTracksNew(), c.SeenTracks(), c.errs.Len())
}()
for _, dir := range s.musicDirs {
err := filepath.WalkDir(dir, func(absPath string, d fs.DirEntry, err error) error {
return s.scanCallback(c, dir, absPath, d, err)
})
if err != nil {
return nil, fmt.Errorf("walk: %w", err)
}
}
if err := s.cleanTracks(c); err != nil {
return nil, fmt.Errorf("clean tracks: %w", err)
}
if err := s.cleanAlbums(c); err != nil {
return nil, fmt.Errorf("clean albums: %w", err)
}
if err := s.cleanArtists(c); err != nil {
return nil, fmt.Errorf("clean artists: %w", err)
}
if err := s.cleanGenres(c); err != nil {
return nil, fmt.Errorf("clean genres: %w", err)
}
if err := s.db.SetSetting("last_scan_time", strconv.FormatInt(time.Now().Unix(), 10)); err != nil {
return nil, fmt.Errorf("set scan time: %w", err)
}
if c.errs.Len() > 0 {
return c, c.errs
}
return c, nil
}
func (s *Scanner) scanCallback(c *Context, dir string, absPath string, d fs.DirEntry, err error) error {
if err != nil {
c.errs.Add(err)
return nil
}
if dir == absPath {
return nil
}
switch d.Type() {
case os.ModeDir:
case os.ModeSymlink:
eval, _ := filepath.EvalSymlinks(absPath)
return filepath.WalkDir(eval, func(subAbs string, d fs.DirEntry, err error) error {
subAbs = strings.Replace(subAbs, eval, absPath, 1)
return s.scanCallback(c, dir, subAbs, d, err)
})
default:
return nil
}
log.Printf("processing folder `%s`", absPath)
tx := s.db.Begin()
if err := s.scanDir(tx, c, dir, absPath); err != nil {
c.errs.Add(fmt.Errorf("%q: %w", absPath, err))
tx.Rollback()
return nil
}
if err := tx.Commit().Error; err != nil {
return fmt.Errorf("commit tx: %w", err)
}
return nil
}
func (s *Scanner) scanDir(tx *db.DB, c *Context, musicDir string, absPath string) error {
items, err := os.ReadDir(absPath)
if err != nil {
return err
}
var tracks []string
var cover string
for _, item := range items {
if isCover(item.Name()) {
cover = item.Name()
continue
}
if _, ok := mime.FromExtension(ext(item.Name())); ok {
tracks = append(tracks, item.Name())
continue
}
}
relPath, _ := filepath.Rel(musicDir, absPath)
pdir, pbasename := filepath.Split(filepath.Dir(relPath))
var parent db.Album
if err := tx.Where(db.Album{RootDir: musicDir, LeftPath: pdir, RightPath: pbasename}).FirstOrCreate(&parent).Error; err != nil {
return fmt.Errorf("first or create parent: %w", err)
}
c.seenAlbums[parent.ID] = struct{}{}
dir, basename := filepath.Split(relPath)
var album db.Album
if err := populateAlbumBasics(tx, musicDir, &parent, &album, dir, basename, cover); err != nil {
return fmt.Errorf("populate album basics: %w", err)
}
c.seenAlbums[album.ID] = struct{}{}
sort.Strings(tracks)
for i, basename := range tracks {
absPath := filepath.Join(musicDir, relPath, basename)
if err := s.populateTrackAndAlbumArtists(tx, c, i, &parent, &album, basename, absPath); err != nil {
return fmt.Errorf("populate track %q: %w", basename, err)
}
}
return nil
}
func (s *Scanner) populateTrackAndAlbumArtists(tx *db.DB, c *Context, i int, parent, album *db.Album, basename string, absPath string) error {
stat, err := os.Stat(absPath)
if err != nil {
return fmt.Errorf("stating %q: %w", basename, err)
}
track := &db.Track{AlbumID: album.ID, Filename: filepath.Base(basename)}
if err := tx.Where(track).First(track).Error; err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
return fmt.Errorf("query track: %w", err)
}
if !c.isFull && track.ID != 0 && stat.ModTime().Before(track.UpdatedAt) {
c.seenTracks[track.ID] = struct{}{}
return nil
}
trags, err := s.tagger.Read(absPath)
if err != nil {
return fmt.Errorf("%v: %w", err, ErrReadingTags)
}
genreNames := strings.Split(trags.SomeGenre(), s.genreSplit)
genreIDs, err := populateGenres(tx, track, genreNames)
if err != nil {
return fmt.Errorf("populate genres: %w", err)
}
// metadata for the album table comes only from the the first track's tags
if i == 0 || album.TagArtist == nil {
albumArtist, err := populateAlbumArtist(tx, album, parent, trags.SomeAlbumArtist())
if err != nil {
return fmt.Errorf("populate album artist: %w", err)
}
if err := populateAlbum(tx, album, albumArtist, trags, genreIDs, stat.ModTime(), statCreateTime(stat)); err != nil {
return fmt.Errorf("populate album: %w", err)
}
if err := populateAlbumGenres(tx, album, genreIDs); err != nil {
return fmt.Errorf("populate album genres: %w", err)
}
}
if err := populateTrack(tx, album, track, trags, basename, int(stat.Size())); err != nil {
return fmt.Errorf("process %q: %w", basename, err)
}
if err := populateTrackGenres(tx, track, genreIDs); err != nil {
return fmt.Errorf("populate track genres: %w", err)
}
c.seenTracks[track.ID] = struct{}{}
c.seenTracksNew++
return nil
}
func populateAlbum(tx *db.DB, album *db.Album, albumArtist *db.Artist, trags tags.Parser, genreIDs []int, modTime, createTime time.Time) error {
albumName := trags.SomeAlbum()
album.TagTitle = albumName
album.TagTitleUDec = decoded(albumName)
album.TagBrainzID = trags.AlbumBrainzID()
album.TagYear = trags.Year()
album.TagArtist = albumArtist
album.ModifiedAt = modTime
if !createTime.IsZero() {
album.CreatedAt = createTime
}
if err := tx.Save(&album).Error; err != nil {
return fmt.Errorf("saving album: %w", err)
}
return nil
}
func populateAlbumBasics(tx *db.DB, musicDir string, parent, album *db.Album, dir, basename string, cover string) error {
if err := tx.Where(db.Album{RootDir: musicDir, LeftPath: dir, RightPath: basename}).First(album).Error; err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
return fmt.Errorf("find album: %w", err)
}
// see if we can save ourselves from an extra write if it's found and nothing has changed
if album.ID != 0 && album.Cover == cover && album.ParentID == parent.ID {
return nil
}
album.RootDir = musicDir
album.LeftPath = dir
album.RightPath = basename
album.Cover = cover
album.RightPathUDec = decoded(basename)
album.ParentID = parent.ID
if err := tx.Save(&album).Error; err != nil {
return fmt.Errorf("saving album: %w", err)
}
return nil
}
func populateTrack(tx *db.DB, album *db.Album, track *db.Track, trags tags.Parser, absPath string, size int) error {
basename := filepath.Base(absPath)
track.Filename = basename
track.FilenameUDec = decoded(basename)
track.Size = size
track.AlbumID = album.ID
track.ArtistID = album.TagArtist.ID
track.TagTitle = trags.Title()
track.TagTitleUDec = decoded(trags.Title())
track.TagTrackArtist = trags.Artist()
track.TagTrackNumber = trags.TrackNumber()
track.TagDiscNumber = trags.DiscNumber()
track.TagBrainzID = trags.BrainzID()
track.Length = trags.Length() // these two should be calculated
track.Bitrate = trags.Bitrate() // ...from the file instead of tags
if err := tx.Save(&track).Error; err != nil {
return fmt.Errorf("saving track: %w", err)
}
return nil
}
func populateAlbumArtist(tx *db.DB, album, parent *db.Album, artistName string) (*db.Artist, error) {
var update db.Artist
update.Name = artistName
update.NameUDec = decoded(artistName)
if parent.Cover != "" {
update.Cover = parent.Cover
}
var artist db.Artist
if err := tx.Where("name=?", artistName).Assign(update).FirstOrCreate(&artist).Error; err != nil {
return nil, fmt.Errorf("find or create artist: %w", err)
}
return &artist, nil
}
func populateGenres(tx *db.DB, track *db.Track, names []string) ([]int, error) {
var filteredNames []string
for _, name := range names {
if clean := strings.TrimSpace(name); clean != "" {
filteredNames = append(filteredNames, clean)
}
}
if len(filteredNames) == 0 {
return []int{}, nil
}
var ids []int
for _, name := range filteredNames {
var genre db.Genre
if err := tx.FirstOrCreate(&genre, db.Genre{Name: name}).Error; err != nil {
return nil, fmt.Errorf("find or create genre: %w", err)
}
ids = append(ids, genre.ID)
}
return ids, nil
}
func populateTrackGenres(tx *db.DB, track *db.Track, genreIDs []int) error {
if err := tx.Where("track_id=?", track.ID).Delete(db.TrackGenre{}).Error; err != nil {
return fmt.Errorf("delete old track genre records: %w", err)
}
if err := tx.InsertBulkLeftMany("track_genres", []string{"track_id", "genre_id"}, track.ID, genreIDs); err != nil {
return fmt.Errorf("insert bulk track genres: %w", err)
}
return nil
}
func populateAlbumGenres(tx *db.DB, album *db.Album, genreIDs []int) error {
if err := tx.Where("album_id=?", album.ID).Delete(db.AlbumGenre{}).Error; err != nil {
return fmt.Errorf("delete old album genre records: %w", err)
}
if err := tx.InsertBulkLeftMany("album_genres", []string{"album_id", "genre_id"}, album.ID, genreIDs); err != nil {
return fmt.Errorf("insert bulk album genres: %w", err)
}
return nil
}
func (s *Scanner) cleanTracks(c *Context) error {
start := time.Now()
defer func() { log.Printf("finished clean tracks in %s, %d removed", durSince(start), c.TracksMissing()) }()
var all []int
err := s.db.
Model(&db.Track{}).
Pluck("id", &all).
Error
if err != nil {
return fmt.Errorf("plucking ids: %w", err)
}
for _, a := range all {
if _, ok := c.seenTracks[a]; !ok {
c.tracksMissing = append(c.tracksMissing, int64(a))
}
}
return s.db.TransactionChunked(c.tracksMissing, func(tx *gorm.DB, chunk []int64) error {
return tx.Where(chunk).Delete(&db.Track{}).Error
})
}
func (s *Scanner) cleanAlbums(c *Context) error {
start := time.Now()
defer func() { log.Printf("finished clean albums in %s, %d removed", durSince(start), c.AlbumsMissing()) }()
var all []int
err := s.db.
Model(&db.Album{}).
Pluck("id", &all).
Error
if err != nil {
return fmt.Errorf("plucking ids: %w", err)
}
for _, a := range all {
if _, ok := c.seenAlbums[a]; !ok {
c.albumsMissing = append(c.albumsMissing, int64(a))
}
}
return s.db.TransactionChunked(c.albumsMissing, func(tx *gorm.DB, chunk []int64) error {
return tx.Where(chunk).Delete(&db.Album{}).Error
})
}
func (s *Scanner) cleanArtists(c *Context) error {
start := time.Now()
defer func() { log.Printf("finished clean artists in %s, %d removed", durSince(start), c.ArtistsMissing()) }()
sub := s.db.
Select("artists.id").
Model(&db.Artist{}).
Joins("LEFT JOIN albums ON albums.tag_artist_id=artists.id").
Where("albums.id IS NULL").
SubQuery()
q := s.db.
Where("artists.id IN ?", sub).
Delete(&db.Artist{})
if err := q.Error; err != nil {
return err
}
c.artistsMissing = int(q.RowsAffected)
return nil
}
func (s *Scanner) cleanGenres(c *Context) error {
start := time.Now()
defer func() { log.Printf("finished clean genres in %s, %d removed", durSince(start), c.GenresMissing()) }()
subTrack := s.db.
Select("genres.id").
Model(&db.Genre{}).
Joins("LEFT JOIN track_genres ON track_genres.genre_id=genres.id").
Where("track_genres.genre_id IS NULL").
SubQuery()
subAlbum := s.db.
Select("genres.id").
Model(&db.Genre{}).
Joins("LEFT JOIN album_genres ON album_genres.genre_id=genres.id").
Where("album_genres.genre_id IS NULL").
SubQuery()
q := s.db.
Where("genres.id IN ? AND genres.id IN ?", subTrack, subAlbum).
Delete(&db.Genre{})
c.genresMissing = int(q.RowsAffected)
return nil
}
func ext(name string) string {
if ext := filepath.Ext(name); len(ext) > 0 {
return ext[1:]
}
return ""
}
func isCover(name string) bool {
switch path := strings.ToLower(name); path {
case
"cover.png", "cover.jpg", "cover.jpeg",
"folder.png", "folder.jpg", "folder.jpeg",
"album.png", "album.jpg", "album.jpeg",
"albumart.png", "albumart.jpg", "albumart.jpeg",
"front.png", "front.jpg", "front.jpeg",
"artist.png", "artist.jpg", "artist.jpeg":
return true
default:
return false
}
}
// decoded converts a string to it's latin equivalent.
// it will be used by the model's *UDec fields, and is only set if it
// differs from the original. the fields are used for searching.
func decoded(in string) string {
if u := unidecode.Unidecode(in); u != in {
return u
}
return ""
}
func durSince(t time.Time) time.Duration {
return time.Since(t).Truncate(10 * time.Microsecond)
}
type Context struct {
errs *multierr.Err
isFull bool
seenTracks map[int]struct{}
seenAlbums map[int]struct{}
seenTracksNew int
tracksMissing []int64
albumsMissing []int64
artistsMissing int
genresMissing int
}
func (c *Context) SeenTracks() int { return len(c.seenTracks) }
func (c *Context) SeenAlbums() int { return len(c.seenAlbums) }
func (c *Context) SeenTracksNew() int { return c.seenTracksNew }
func (c *Context) TracksMissing() int { return len(c.tracksMissing) }
func (c *Context) AlbumsMissing() int { return len(c.albumsMissing) }
func (c *Context) ArtistsMissing() int { return c.artistsMissing }
func (c *Context) GenresMissing() int { return c.genresMissing }
func statCreateTime(info fs.FileInfo) time.Time {
stat, ok := info.Sys().(*syscall.Stat_t)
if !ok {
return time.Time{}
}
if stat.Ctim.Sec == 0 {
return time.Time{}
}
//nolint:unconvert // Ctim.Sec/Nsec is int32 on arm/386, etc
return time.Unix(int64(stat.Ctim.Sec), int64(stat.Ctim.Nsec))
}

View File

@@ -0,0 +1,33 @@
package scanner_test
import (
"fmt"
"testing"
"go.senan.xyz/gonic/mockfs"
)
func BenchmarkScanIncremental(b *testing.B) {
m := mockfs.New(b)
for i := 0; i < 5; i++ {
m.AddItemsPrefix(fmt.Sprintf("t-%d", i))
}
m.ScanAndClean()
b.ResetTimer()
for i := 0; i < b.N; i++ {
m.ScanAndClean()
}
}
func BenchmarkScanFull(b *testing.B) {
for i := 0; i < b.N; i++ {
m := mockfs.New(b)
for i := 0; i < 5; i++ {
m.AddItemsPrefix(fmt.Sprintf("t-%d", i))
}
b.StartTimer()
m.ScanAndClean()
b.StopTimer()
}
}

View File

@@ -0,0 +1,83 @@
//go:build go1.18
// +build go1.18
package scanner_test
import (
"fmt"
"math/rand"
"reflect"
"testing"
_ "github.com/jinzhu/gorm/dialects/sqlite"
"github.com/matryer/is"
"go.senan.xyz/gonic/mockfs"
)
func FuzzScanner(f *testing.F) {
checkDelta := func(is *is.I, m *mockfs.MockFS, expSeen, expNew int) {
is.Helper()
ctx := m.ScanAndClean()
is.Equal(ctx.SeenTracks(), expSeen)
is.Equal(ctx.SeenTracksNew(), expNew)
is.Equal(ctx.TracksMissing(), 0)
is.Equal(ctx.AlbumsMissing(), 0)
is.Equal(ctx.ArtistsMissing(), 0)
is.Equal(ctx.GenresMissing(), 0)
}
f.Fuzz(func(t *testing.T, data []byte, seed int64) {
is := is.NewRelaxed(t)
m := mockfs.New(t)
const toAdd = 1000
for i := 0; i < toAdd; i++ {
path := fmt.Sprintf("artist-%d/album-%d/track-%d.flac", i/6, i/3, i)
m.AddTrack(path)
m.SetTags(path, func(tags *mockfs.Tags) error {
fuzzStruct(i, data, seed, tags)
return nil
})
}
checkDelta(is, m, toAdd, toAdd) // we added all tracks, 0 delta
checkDelta(is, m, toAdd, 0) // we added 0 tracks, 0 delta
})
}
func fuzzStruct(taken int, data []byte, seed int64, dest interface{}) {
if len(data) == 0 {
return
}
r := rand.New(rand.NewSource(seed))
v := reflect.ValueOf(dest)
for i := 0; i < v.Elem().NumField(); i++ {
if r.Float64() < 0.1 {
continue
}
take := int(r.Float64() * 12)
b := make([]byte, take)
for i := range b {
b[i] = data[(i+taken)%len(data)]
}
taken += take
switch f := v.Elem().Field(i); f.Kind() {
case reflect.Bool:
f.SetBool(b[0] < 128)
case reflect.String:
f.SetString(string(b))
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
f.SetInt(int64(b[0]))
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
f.SetUint(uint64(b[0]))
case reflect.Float32, reflect.Float64:
f.SetFloat(float64(b[0]))
case reflect.Struct:
fuzzStruct(taken, data, seed, f.Addr().Interface())
}
}
}

557
scanner/scanner_test.go Normal file
View File

@@ -0,0 +1,557 @@
package scanner_test
import (
"errors"
"fmt"
"io"
"log"
"os"
"path/filepath"
"testing"
"github.com/jinzhu/gorm"
_ "github.com/jinzhu/gorm/dialects/sqlite"
"github.com/matryer/is"
"go.senan.xyz/gonic/multierr"
"go.senan.xyz/gonic/db"
"go.senan.xyz/gonic/mockfs"
"go.senan.xyz/gonic/scanner"
)
func TestMain(m *testing.M) {
log.SetOutput(io.Discard)
os.Exit(m.Run())
}
func TestTableCounts(t *testing.T) {
t.Parallel()
is := is.NewRelaxed(t)
m := mockfs.New(t)
m.AddItems()
m.ScanAndClean()
var tracks int
is.NoErr(m.DB().Model(&db.Track{}).Count(&tracks).Error) // not all tracks
is.Equal(tracks, m.NumTracks())
var albums int
is.NoErr(m.DB().Model(&db.Album{}).Count(&albums).Error) // not all albums
is.Equal(albums, 13) // not all albums
var artists int
is.NoErr(m.DB().Model(&db.Artist{}).Count(&artists).Error) // not all artists
is.Equal(artists, 3) // not all artists
}
func TestParentID(t *testing.T) {
t.Parallel()
is := is.New(t)
m := mockfs.New(t)
m.AddItems()
m.ScanAndClean()
var nullParentAlbums []*db.Album
is.NoErr(m.DB().Where("parent_id IS NULL").Find(&nullParentAlbums).Error) // one parent_id=NULL which is root folder
is.Equal(len(nullParentAlbums), 1) // one parent_id=NULL which is root folder
is.Equal(nullParentAlbums[0].LeftPath, "")
is.Equal(nullParentAlbums[0].RightPath, ".")
is.Equal(m.DB().Where("id=parent_id").Find(&db.Album{}).Error, gorm.ErrRecordNotFound) // no self-referencing albums
var album db.Album
var parent db.Album
is.NoErr(m.DB().Find(&album, "left_path=? AND right_path=?", "artist-0/", "album-0").Error) // album has parent ID
is.NoErr(m.DB().Find(&parent, "right_path=?", "artist-0").Error) // album has parent ID
is.Equal(album.ParentID, parent.ID) // album has parent ID
}
func TestUpdatedCover(t *testing.T) {
t.Parallel()
is := is.NewRelaxed(t)
m := mockfs.New(t)
m.AddItems()
m.ScanAndClean()
m.AddCover("artist-0/album-0/cover.jpg")
m.ScanAndClean()
var album db.Album
is.NoErr(m.DB().Where("left_path=? AND right_path=?", "artist-0/", "album-0").Find(&album).Error) // album has cover
is.Equal(album.Cover, "cover.jpg") // album has cover
}
func TestCoverBeforeTracks(t *testing.T) {
t.Parallel()
is := is.New(t)
m := mockfs.New(t)
m.AddCover("artist-2/album-2/cover.jpg")
m.ScanAndClean()
m.AddItems()
m.ScanAndClean()
var album db.Album
is.NoErr(m.DB().Preload("TagArtist").Where("left_path=? AND right_path=?", "artist-2/", "album-2").Find(&album).Error) // album has cover
is.Equal(album.Cover, "cover.jpg") // album has cover
is.Equal(album.TagArtist.Name, "artist-2") // album artist
var tracks []*db.Track
is.NoErr(m.DB().Where("album_id=?", album.ID).Find(&tracks).Error) // album has tracks
is.Equal(len(tracks), 3) // album has tracks
}
func TestUpdatedTags(t *testing.T) {
t.Parallel()
is := is.New(t)
m := mockfs.New(t)
m.AddTrack("artist-10/album-10/track-10.flac")
m.SetTags("artist-10/album-10/track-10.flac", func(tags *mockfs.Tags) error {
tags.RawArtist = "artist"
tags.RawAlbumArtist = "album-artist"
tags.RawAlbum = "album"
tags.RawTitle = "title"
return nil
})
m.ScanAndClean()
var track db.Track
is.NoErr(m.DB().Preload("Album").Preload("Artist").Where("filename=?", "track-10.flac").Find(&track).Error) // track has tags
is.Equal(track.TagTrackArtist, "artist") // track has tags
is.Equal(track.Artist.Name, "album-artist") // track has tags
is.Equal(track.Album.TagTitle, "album") // track has tags
is.Equal(track.TagTitle, "title") // track has tags
m.SetTags("artist-10/album-10/track-10.flac", func(tags *mockfs.Tags) error {
tags.RawArtist = "artist-upd"
tags.RawAlbumArtist = "album-artist-upd"
tags.RawAlbum = "album-upd"
tags.RawTitle = "title-upd"
return nil
})
m.ScanAndClean()
var updated db.Track
is.NoErr(m.DB().Preload("Album").Preload("Artist").Where("filename=?", "track-10.flac").Find(&updated).Error) // updated has tags
is.Equal(updated.ID, track.ID) // updated has tags
is.Equal(updated.TagTrackArtist, "artist-upd") // updated has tags
is.Equal(updated.Artist.Name, "album-artist-upd") // updated has tags
is.Equal(updated.Album.TagTitle, "album-upd") // updated has tags
is.Equal(updated.TagTitle, "title-upd") // updated has tags
}
func TestDeleteAlbum(t *testing.T) {
t.Parallel()
is := is.NewRelaxed(t)
m := mockfs.New(t)
m.AddItems()
m.ScanAndClean()
is.NoErr(m.DB().Where("left_path=? AND right_path=?", "artist-2/", "album-2").Find(&db.Album{}).Error) // album exists
m.RemoveAll("artist-2/album-2")
m.ScanAndClean()
is.Equal(m.DB().Where("left_path=? AND right_path=?", "artist-2/", "album-2").Find(&db.Album{}).Error, gorm.ErrRecordNotFound) // album doesn't exist
}
func TestDeleteArtist(t *testing.T) {
t.Parallel()
is := is.NewRelaxed(t)
m := mockfs.New(t)
m.AddItems()
m.ScanAndClean()
is.NoErr(m.DB().Where("left_path=? AND right_path=?", "artist-2/", "album-2").Find(&db.Album{}).Error) // album exists
m.RemoveAll("artist-2")
m.ScanAndClean()
is.Equal(m.DB().Where("left_path=? AND right_path=?", "artist-2/", "album-2").Find(&db.Album{}).Error, gorm.ErrRecordNotFound) // album doesn't exist
is.Equal(m.DB().Where("name=?", "artist-2").Find(&db.Artist{}).Error, gorm.ErrRecordNotFound) // artist doesn't exist
}
func TestGenres(t *testing.T) {
t.Parallel()
is := is.New(t)
m := mockfs.New(t)
albumGenre := func(artist, album, genre string) error {
return m.DB().
Where("albums.left_path=? AND albums.right_path=? AND genres.name=?", artist, album, genre).
Joins("JOIN albums ON albums.id=album_genres.album_id").
Joins("JOIN genres ON genres.id=album_genres.genre_id").
Find(&db.AlbumGenre{}).
Error
}
isAlbumGenre := func(artist, album, genreName string) {
is.Helper()
is.NoErr(albumGenre(artist, album, genreName))
}
isAlbumGenreMissing := func(artist, album, genreName string) {
is.Helper()
is.Equal(albumGenre(artist, album, genreName), gorm.ErrRecordNotFound)
}
trackGenre := func(artist, album, filename, genreName string) error {
return m.DB().
Where("albums.left_path=? AND albums.right_path=? AND tracks.filename=? AND genres.name=?", artist, album, filename, genreName).
Joins("JOIN tracks ON tracks.id=track_genres.track_id").
Joins("JOIN genres ON genres.id=track_genres.genre_id").
Joins("JOIN albums ON albums.id=tracks.album_id").
Find(&db.TrackGenre{}).
Error
}
isTrackGenre := func(artist, album, filename, genreName string) {
is.Helper()
is.NoErr(trackGenre(artist, album, filename, genreName))
}
isTrackGenreMissing := func(artist, album, filename, genreName string) {
is.Helper()
is.Equal(trackGenre(artist, album, filename, genreName), gorm.ErrRecordNotFound)
}
genre := func(genre string) error {
return m.DB().Where("name=?", genre).Find(&db.Genre{}).Error
}
isGenre := func(genreName string) {
is.Helper()
is.NoErr(genre(genreName))
}
isGenreMissing := func(genreName string) {
is.Helper()
is.Equal(genre(genreName), gorm.ErrRecordNotFound)
}
m.AddItems()
m.SetTags("artist-0/album-0/track-0.flac", func(tags *mockfs.Tags) error { tags.RawGenre = "genre-a;genre-b"; return nil })
m.SetTags("artist-0/album-0/track-1.flac", func(tags *mockfs.Tags) error { tags.RawGenre = "genre-c;genre-d"; return nil })
m.SetTags("artist-1/album-2/track-0.flac", func(tags *mockfs.Tags) error { tags.RawGenre = "genre-e;genre-f"; return nil })
m.SetTags("artist-1/album-2/track-1.flac", func(tags *mockfs.Tags) error { tags.RawGenre = "genre-g;genre-h"; return nil })
m.ScanAndClean()
isGenre("genre-a") // genre exists
isGenre("genre-b") // genre exists
isGenre("genre-c") // genre exists
isGenre("genre-d") // genre exists
isTrackGenre("artist-0/", "album-0", "track-0.flac", "genre-a") // track genre exists
isTrackGenre("artist-0/", "album-0", "track-0.flac", "genre-b") // track genre exists
isTrackGenre("artist-0/", "album-0", "track-1.flac", "genre-c") // track genre exists
isTrackGenre("artist-0/", "album-0", "track-1.flac", "genre-d") // track genre exists
isTrackGenre("artist-1/", "album-2", "track-0.flac", "genre-e") // track genre exists
isTrackGenre("artist-1/", "album-2", "track-0.flac", "genre-f") // track genre exists
isTrackGenre("artist-1/", "album-2", "track-1.flac", "genre-g") // track genre exists
isTrackGenre("artist-1/", "album-2", "track-1.flac", "genre-h") // track genre exists
isAlbumGenre("artist-0/", "album-0", "genre-a") // album genre exists
isAlbumGenre("artist-0/", "album-0", "genre-b") // album genre exists
m.SetTags("artist-0/album-0/track-0.flac", func(tags *mockfs.Tags) error { tags.RawGenre = "genre-aa;genre-bb"; return nil })
m.ScanAndClean()
isTrackGenre("artist-0/", "album-0", "track-0.flac", "genre-aa") // updated track genre exists
isTrackGenre("artist-0/", "album-0", "track-0.flac", "genre-bb") // updated track genre exists
isTrackGenreMissing("artist-0/", "album-0", "track-0.flac", "genre-a") // old track genre missing
isTrackGenreMissing("artist-0/", "album-0", "track-0.flac", "genre-b") // old track genre missing
isAlbumGenreMissing("artist-0/", "album-0", "genre-a") // old album genre missing
isAlbumGenreMissing("artist-0/", "album-0", "genre-b") // old album genre missing
isGenreMissing("genre-a") // old genre missing
isGenreMissing("genre-b") // old genre missing
}
func TestMultiFolders(t *testing.T) {
t.Parallel()
is := is.New(t)
m := mockfs.NewWithDirs(t, []string{"m-1", "m-2", "m-3"})
m.AddItemsPrefix("m-1")
m.AddItemsPrefix("m-2")
m.AddItemsPrefix("m-3")
m.ScanAndClean()
var rootDirs []*db.Album
is.NoErr(m.DB().Where("parent_id IS NULL").Find(&rootDirs).Error)
is.Equal(len(rootDirs), 3)
for i, r := range rootDirs {
is.Equal(r.RootDir, filepath.Join(m.TmpDir(), fmt.Sprintf("m-%d", i+1)))
is.Equal(r.ParentID, 0)
is.Equal(r.LeftPath, "")
is.Equal(r.RightPath, ".")
}
m.AddCover("m-3/artist-0/album-0/cover.jpg")
m.ScanAndClean()
m.LogItems()
checkCover := func(root string, q string) {
is.Helper()
is.NoErr(m.DB().Where(q, filepath.Join(m.TmpDir(), root)).Find(&db.Album{}).Error)
}
checkCover("m-1", "root_dir=? AND cover IS NULL") // mf 1 no cover
checkCover("m-2", "root_dir=? AND cover IS NULL") // mf 2 no cover
checkCover("m-3", "root_dir=? AND cover='cover.jpg'") // mf 3 has cover
}
func TestNewAlbumForExistingArtist(t *testing.T) {
t.Parallel()
is := is.New(t)
m := mockfs.New(t)
m.AddItems()
m.ScanAndClean()
m.LogAlbums()
m.LogArtists()
var artist db.Artist
is.NoErr(m.DB().Where("name=?", "artist-2").Find(&artist).Error) // find orig artist
is.True(artist.ID > 0)
for tr := 0; tr < 3; tr++ {
m.AddTrack(fmt.Sprintf("artist-2/new-album/track-%d.mp3", tr))
m.SetTags(fmt.Sprintf("artist-2/new-album/track-%d.mp3", tr), func(tags *mockfs.Tags) error {
tags.RawArtist = "artist-2"
tags.RawAlbumArtist = "artist-2"
tags.RawAlbum = "new-album"
tags.RawTitle = fmt.Sprintf("title-%d", tr)
return nil
})
}
var updated db.Artist
is.NoErr(m.DB().Where("name=?", "artist-2").Find(&updated).Error) // find updated artist
is.Equal(artist.ID, updated.ID) // find updated artist
var all []*db.Artist
is.NoErr(m.DB().Find(&all).Error) // still only 3?
is.Equal(len(all), 3) // still only 3?
}
func TestMultiFolderWithSharedArtist(t *testing.T) {
t.Parallel()
is := is.New(t)
m := mockfs.NewWithDirs(t, []string{"m-0", "m-1"})
const artistName = "artist-a"
m.AddTrack(fmt.Sprintf("m-0/%s/album-a/track-1.flac", artistName))
m.SetTags(fmt.Sprintf("m-0/%s/album-a/track-1.flac", artistName), func(tags *mockfs.Tags) error {
tags.RawArtist = artistName
tags.RawAlbumArtist = artistName
tags.RawAlbum = "album-a"
tags.RawTitle = "track-1"
return nil
})
m.ScanAndClean()
m.AddTrack(fmt.Sprintf("m-1/%s/album-a/track-1.flac", artistName))
m.SetTags(fmt.Sprintf("m-1/%s/album-a/track-1.flac", artistName), func(tags *mockfs.Tags) error {
tags.RawArtist = artistName
tags.RawAlbumArtist = artistName
tags.RawAlbum = "album-a"
tags.RawTitle = "track-1"
return nil
})
m.ScanAndClean()
sq := func(db *gorm.DB) *gorm.DB {
return db.
Select("*, count(sub.id) child_count, sum(sub.length) duration").
Joins("LEFT JOIN tracks sub ON albums.id=sub.album_id").
Group("albums.id")
}
var artist db.Artist
is.NoErr(m.DB().Where("name=?", artistName).Preload("Albums", sq).First(&artist).Error)
is.Equal(artist.Name, artistName)
is.Equal(len(artist.Albums), 2)
for _, album := range artist.Albums {
is.True(album.TagYear > 0)
is.Equal(album.TagArtistID, artist.ID)
is.True(album.ChildCount > 0)
is.True(album.Duration > 0)
}
}
func TestSymlinkedAlbum(t *testing.T) {
t.Parallel()
is := is.New(t)
m := mockfs.NewWithDirs(t, []string{"scan"})
m.AddItemsPrefixWithCovers("temp")
tempAlbum0 := filepath.Join(m.TmpDir(), "temp", "artist-0", "album-0")
scanAlbum0 := filepath.Join(m.TmpDir(), "scan", "artist-sym", "album-0")
m.Symlink(tempAlbum0, scanAlbum0)
m.ScanAndClean()
m.LogTracks()
m.LogAlbums()
var track db.Track
is.NoErr(m.DB().Preload("Album.Parent").Find(&track).Error) // track exists
is.True(track.Album != nil) // track has album
is.True(track.Album.Cover != "") // album has cover
is.Equal(track.Album.Parent.RightPath, "artist-sym") // artist is sym
info, err := os.Stat(track.AbsPath())
is.NoErr(err) // track resolves
is.True(!info.IsDir()) // track resolves
is.True(!info.ModTime().IsZero()) // track resolves
}
func TestSymlinkedSubdiscs(t *testing.T) {
t.Parallel()
is := is.New(t)
m := mockfs.NewWithDirs(t, []string{"scan"})
addItem := func(prefix, artist, album, disc, track string) {
p := fmt.Sprintf("%s/%s/%s/%s/%s", prefix, artist, album, disc, track)
m.AddTrack(p)
m.SetTags(p, func(tags *mockfs.Tags) error {
tags.RawArtist = artist
tags.RawAlbumArtist = artist
tags.RawAlbum = album
tags.RawTitle = track
return nil
})
}
addItem("temp", "artist-a", "album-a", "disc-1", "track-1.flac")
addItem("temp", "artist-a", "album-a", "disc-1", "track-2.flac")
addItem("temp", "artist-a", "album-a", "disc-1", "track-3.flac")
addItem("temp", "artist-a", "album-a", "disc-2", "track-1.flac")
addItem("temp", "artist-a", "album-a", "disc-2", "track-2.flac")
addItem("temp", "artist-a", "album-a", "disc-2", "track-3.flac")
tempAlbum0 := filepath.Join(m.TmpDir(), "temp", "artist-a", "album-a")
scanAlbum0 := filepath.Join(m.TmpDir(), "scan", "artist-a", "album-sym")
m.Symlink(tempAlbum0, scanAlbum0)
m.ScanAndClean()
m.LogTracks()
m.LogAlbums()
var track db.Track
is.NoErr(m.DB().Preload("Album.Parent").Find(&track).Error) // track exists
is.True(track.Album != nil) // track has album
is.Equal(track.Album.Parent.RightPath, "album-sym") // artist is sym
info, err := os.Stat(track.AbsPath())
is.NoErr(err) // track resolves
is.True(!info.IsDir()) // track resolves
is.True(!info.ModTime().IsZero()) // track resolves
}
func TestArtistHasCover(t *testing.T) {
t.Parallel()
is := is.New(t)
m := mockfs.New(t)
m.AddItemsWithCovers()
m.AddCover("artist-2/artist.png")
m.ScanAndClean()
var artistWith db.Artist
is.NoErr(m.DB().Where("name=?", "artist-2").First(&artistWith).Error)
is.Equal(artistWith.Cover, "artist.png")
var artistWithout db.Artist
is.NoErr(m.DB().Where("name=?", "artist-0").First(&artistWithout).Error)
is.Equal(artistWithout.Cover, "")
}
func TestTagErrors(t *testing.T) {
t.Parallel()
is := is.New(t)
m := mockfs.New(t)
m.AddItemsWithCovers()
m.SetTags("artist-1/album-0/track-0.flac", func(tags *mockfs.Tags) error {
return scanner.ErrReadingTags
})
m.SetTags("artist-1/album-1/track-0.flac", func(tags *mockfs.Tags) error {
return scanner.ErrReadingTags
})
var errs *multierr.Err
ctx, err := m.ScanAndCleanErr()
is.True(errors.As(err, &errs))
is.Equal(errs.Len(), 2) // we have 2 dir errors
is.Equal(ctx.SeenTracks(), m.NumTracks()-(3*2)) // we saw all tracks bar 2 album contents
is.Equal(ctx.SeenTracksNew(), m.NumTracks()-(3*2)) // we have all tracks bar 2 album contents
ctx, err = m.ScanAndCleanErr()
is.True(errors.As(err, &errs))
is.Equal(errs.Len(), 2) // we have 2 dir errors
is.Equal(ctx.SeenTracks(), m.NumTracks()-(3*2)) // we saw all tracks bar 2 album contents
is.Equal(ctx.SeenTracksNew(), 0) // we have no new tracks
}
// https://github.com/sentriz/gonic/issues/185#issuecomment-1050092128
func TestCompilationAlbumWithoutAlbumArtist(t *testing.T) {
t.Parallel()
is := is.New(t)
m := mockfs.New(t)
const pathArtist = "various-artists"
const pathAlbum = "my-compilation"
const toAdd = 5
// add tracks to one folder with random artists and no album artist tag
for i := 0; i < toAdd; i++ {
p := fmt.Sprintf("%s/%s/track-%d.flac", pathArtist, pathAlbum, i)
m.AddTrack(p)
m.SetTags(p, func(tags *mockfs.Tags) error {
// don't set an album artist
tags.RawTitle = fmt.Sprintf("track %d", i)
tags.RawArtist = fmt.Sprintf("artist %d", i)
tags.RawAlbum = pathArtist
return nil
})
}
m.ScanAndClean()
var trackCount int
is.NoErr(m.DB().Model(&db.Track{}).Count(&trackCount).Error)
is.Equal(trackCount, 5)
var artists []*db.Artist
is.NoErr(m.DB().Preload("Albums").Find(&artists).Error)
is.Equal(len(artists), 1) // we only have one album artist
is.Equal(artists[0].Name, "artist 0") // it came from the first track's fallback to artist tag
is.Equal(len(artists[0].Albums), 1) // the artist has one album
is.Equal(artists[0].Albums[0].RightPath, pathAlbum)
is.Equal(artists[0].Albums[0].LeftPath, pathArtist+"/")
}
func TestIncrementalScanNoChangeNoUpdatedAt(t *testing.T) {
t.Parallel()
is := is.New(t)
m := mockfs.New(t)
m.AddItems()
m.ScanAndClean()
var albumA db.Album
is.NoErr(m.DB().Where("tag_artist_id NOT NULL").Order("updated_at DESC").Find(&albumA).Error)
m.ScanAndClean()
var albumB db.Album
is.NoErr(m.DB().Where("tag_artist_id NOT NULL").Order("updated_at DESC").Find(&albumB).Error)
is.Equal(albumA.UpdatedAt, albumB.UpdatedAt)
}

94
scanner/tags/tags.go Normal file
View File

@@ -0,0 +1,94 @@
package tags
import (
"strconv"
"strings"
"github.com/nicksellen/audiotags"
)
type TagReader struct{}
func (*TagReader) Read(abspath string) (Parser, error) {
raw, props, err := audiotags.Read(abspath)
return &Tagger{raw, props}, err
}
type Tagger struct {
raw map[string]string
props *audiotags.AudioProperties
}
func (t *Tagger) first(keys ...string) string {
for _, key := range keys {
if val, ok := t.raw[key]; ok {
return val
}
}
return ""
}
func (t *Tagger) Title() string { return t.first("title") }
func (t *Tagger) BrainzID() string { return t.first("musicbrainz_trackid") }
func (t *Tagger) Artist() string { return t.first("artist") }
func (t *Tagger) Album() string { return t.first("album") }
func (t *Tagger) AlbumArtist() string { return t.first("albumartist", "album artist") }
func (t *Tagger) AlbumBrainzID() string { return t.first("musicbrainz_albumid") }
func (t *Tagger) Genre() string { return t.first("genre") }
func (t *Tagger) TrackNumber() int { return intSep(t.first("tracknumber"), "/") } // eg. 5/12
func (t *Tagger) DiscNumber() int { return intSep(t.first("discnumber"), "/") } // eg. 1/2
func (t *Tagger) Length() int { return t.props.Length }
func (t *Tagger) Bitrate() int { return t.props.Bitrate }
func (t *Tagger) Year() int { return intSep(t.first("originaldate", "date", "year"), "-") }
func (t *Tagger) SomeAlbum() string { return first("Unknown Album", t.Album()) }
func (t *Tagger) SomeArtist() string { return first("Unknown Artist", t.Artist()) }
func (t *Tagger) SomeAlbumArtist() string {
return first("Unknown Artist", t.AlbumArtist(), t.Artist())
}
func (t *Tagger) SomeGenre() string { return first("Unknown Genre", t.Genre()) }
func first(or string, strs ...string) string {
for _, str := range strs {
if str != "" {
return str
}
}
return or
}
func intSep(in, sep string) int {
if in == "" {
return 0
}
start := strings.SplitN(in, sep, 2)[0]
out, err := strconv.Atoi(start)
if err != nil {
return 0
}
return out
}
type Reader interface {
Read(abspath string) (Parser, error)
}
type Parser interface {
Title() string
BrainzID() string
Artist() string
Album() string
AlbumArtist() string
AlbumBrainzID() string
Genre() string
TrackNumber() int
DiscNumber() int
Length() int
Bitrate() int
Year() int
SomeAlbum() string
SomeArtist() string
SomeAlbumArtist() string
SomeGenre() string
}

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,3 @@
go test fuzz v1
[]byte("001")
int64(3472329395739373616)

View File

@@ -0,0 +1,3 @@
go test fuzz v1
[]byte("001")
int64(3472329395739373616)

View File

@@ -0,0 +1,3 @@
go test fuzz v1
[]byte("")
int64(0)