refactor: update scanner, scanner tests, mockfs

closes #165
closes #163
This commit is contained in:
sentriz
2021-11-03 23:05:08 +00:00
parent b07b9a8be6
commit fa587fc7de
64 changed files with 3469 additions and 2373 deletions

View File

@@ -5,8 +5,8 @@ import (
"fmt"
"log"
"os"
"path"
"path/filepath"
"sort"
"strconv"
"strings"
"sync/atomic"
@@ -16,9 +16,9 @@ import (
"github.com/karrick/godirwalk"
"github.com/rainycape/unidecode"
"go.senan.xyz/gonic/multierr"
"go.senan.xyz/gonic/server/db"
"go.senan.xyz/gonic/server/mime"
"go.senan.xyz/gonic/server/scanner/stack"
"go.senan.xyz/gonic/server/scanner/tags"
)
@@ -28,70 +28,347 @@ var (
ErrReadingTags = errors.New("could not read tags")
)
func durSince(t time.Time) time.Duration {
return time.Since(t).Truncate(10 * time.Microsecond)
}
// decoded converts a string to it's latin equivalent.
// it will be used by the model's *UDec fields, and is only set if it
// differs from the original. the fields are used for searching.
func decoded(in string) string {
if u := unidecode.Unidecode(in); u != in {
return u
}
return ""
}
// isScanning acts as an atomic boolean semaphore. we don't
// want to have more than one scan going on at a time
var isScanning int32 //nolint:gochecknoglobals
func IsScanning() bool {
return atomic.LoadInt32(&isScanning) == 1
}
func SetScanning() func() {
atomic.StoreInt32(&isScanning, 1)
return func() {
atomic.StoreInt32(&isScanning, 0)
}
}
type Scanner struct {
db *db.DB
musicPath string
isFull bool
musicPaths []string
sorted bool
genreSplit string
// these two are for the transaction we do for every album.
// the boolean is there so we dont begin or commit multiple
// times in the handle album or post children callback
trTx *db.DB
trTxOpen bool
// these two are for keeping state between noted in the tree.
// eg. keep track of a parents album or the path to a cover
// we just saw that we need to commit in the post children
// callback
curAlbums *stack.Stack
curCover string
// then the rest are for stats and cleanup at the very end
seenTracks map[int]struct{} // set of p keys
seenAlbums map[int]struct{} // set of p keys
seenTracksNew int // n tracks not seen before
tagger tags.Reader
scanning *int32
}
func New(musicPath string, db *db.DB, genreSplit string) *Scanner {
func New(musicPaths []string, sorted bool, db *db.DB, genreSplit string, tagger tags.Reader) *Scanner {
return &Scanner{
db: db,
musicPath: musicPath,
musicPaths: musicPaths,
sorted: sorted,
genreSplit: genreSplit,
tagger: tagger,
scanning: new(int32),
}
}
// ## begin clean funcs
// ## begin clean funcs
// ## begin clean funcs
type ScanOptions struct {
IsFull bool
// TODO https://github.com/sentriz/gonic/issues/64
Path string
}
func (s *Scanner) cleanTracks() error {
func (s *Scanner) IsScanning() bool {
return atomic.LoadInt32(s.scanning) == 1
}
type ScanOptions struct {
IsFull bool
}
func (s *Scanner) ScanAndClean(opts ScanOptions) error {
c := &collected{
seenTracks: map[int]struct{}{},
seenAlbums: map[int]struct{}{},
}
if err := s.scan(c, opts.IsFull); err != nil {
return err
}
if err := s.clean(c); err != nil {
return err
}
return nil
}
func (s *Scanner) scan(c *collected, isFull bool) error {
if s.IsScanning() {
return ErrAlreadyScanning
}
atomic.StoreInt32(s.scanning, 1)
defer atomic.StoreInt32(s.scanning, 0)
start := time.Now()
itemErrs := multierr.Err{}
log.Println("starting scan")
defer func() {
log.Printf("finished scan in %s, +%d/%d tracks (%d err)\n",
durSince(start), c.seenTracksNew, len(c.seenTracks), itemErrs.Len())
}()
for _, musicPath := range s.musicPaths {
err := godirwalk.Walk(musicPath, &godirwalk.Options{
Callback: func(_ string, _ *godirwalk.Dirent) error {
return nil
},
PostChildrenCallback: func(itemPath string, _ *godirwalk.Dirent) error {
return s.callback(c, isFull, musicPath, itemPath)
},
Unsorted: !s.sorted,
FollowSymbolicLinks: true,
ErrorCallback: func(path string, err error) godirwalk.ErrorAction {
itemErrs.Add(fmt.Errorf("%q: %w", path, err))
return godirwalk.SkipNode
},
})
if err != nil {
return fmt.Errorf("walking filesystem: %w", err)
}
}
if err := s.db.SetSetting("last_scan_time", strconv.FormatInt(time.Now().Unix(), 10)); err != nil {
return fmt.Errorf("set scan time: %w", err)
}
if itemErrs.Len() > 0 {
return itemErrs
}
return nil
}
func (s *Scanner) clean(c *collected) error {
if err := s.cleanTracks(c.seenTracks); err != nil {
return fmt.Errorf("clean tracks: %w", err)
}
if err := s.cleanAlbums(c.seenAlbums); err != nil {
return fmt.Errorf("clean albums: %w", err)
}
if err := s.cleanArtists(); err != nil {
return fmt.Errorf("clean artists: %w", err)
}
if err := s.cleanGenres(); err != nil {
return fmt.Errorf("clean genres: %w", err)
}
return nil
}
func (s *Scanner) callback(c *collected, isFull bool, rootAbsPath string, itemAbsPath string) error {
if rootAbsPath == itemAbsPath {
return nil
}
relpath, _ := filepath.Rel(rootAbsPath, itemAbsPath)
gs, err := godirwalk.NewScanner(itemAbsPath)
if err != nil {
return err
}
var tracks []string
var cover string
for gs.Scan() {
if isCover(gs.Name()) {
cover = gs.Name()
continue
}
if _, ok := mime.FromExtension(ext(gs.Name())); ok {
tracks = append(tracks, gs.Name())
continue
}
}
tx := s.db.Begin()
defer tx.Commit()
pdir, pbasename := filepath.Split(filepath.Dir(relpath))
parent := &db.Album{}
if err := tx.Where(db.Album{RootDir: rootAbsPath, LeftPath: pdir, RightPath: pbasename}).FirstOrCreate(parent).Error; err != nil {
return fmt.Errorf("first or create parent: %w", err)
}
c.seenAlbums[parent.ID] = struct{}{}
dir, basename := filepath.Split(relpath)
album := &db.Album{}
if err := tx.Where(db.Album{RootDir: rootAbsPath, LeftPath: dir, RightPath: basename}).First(album).Error; err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
return fmt.Errorf("find album: %w", err)
}
if err := populateAlbumBasics(tx, rootAbsPath, parent, album, dir, basename, cover); err != nil {
return fmt.Errorf("populate album basics: %w", err)
}
c.seenAlbums[album.ID] = struct{}{}
sort.Strings(tracks)
for i, basename := range tracks {
abspath := filepath.Join(itemAbsPath, basename)
if err := s.populateTrackAndAlbumArtists(tx, c, i, album, basename, abspath, isFull); err != nil {
return fmt.Errorf("process %q: %w", "", err)
}
}
return nil
}
func (s *Scanner) populateTrackAndAlbumArtists(tx *db.DB, c *collected, i int, album *db.Album, basename string, abspath string, isFull bool) error {
track := &db.Track{AlbumID: album.ID, Filename: filepath.Base(basename)}
if err := tx.Where(track).First(track).Error; err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
return fmt.Errorf("query track: %w", err)
}
c.seenTracks[track.ID] = struct{}{}
stat, err := os.Stat(abspath)
if err != nil {
return fmt.Errorf("stating %q: %w", basename, err)
}
if !isFull && stat.ModTime().Before(track.UpdatedAt) {
return nil
}
trags, err := s.tagger.Read(abspath)
if err != nil {
return fmt.Errorf("%v: %w", err, ErrReadingTags)
}
artistName := trags.SomeAlbumArtist()
albumArtist, err := s.populateAlbumArtist(tx, artistName)
if err != nil {
return fmt.Errorf("populate artist: %w", err)
}
if err := populateTrack(tx, album, albumArtist, track, trags, basename, int(stat.Size())); err != nil {
return fmt.Errorf("process %q: %w", basename, err)
}
c.seenTracks[track.ID] = struct{}{}
c.seenTracksNew++
genreNames := strings.Split(trags.SomeGenre(), s.genreSplit)
genreIDs, err := s.populateGenres(tx, track, genreNames)
if err != nil {
return fmt.Errorf("populate genres: %w", err)
}
if err := s.populateTrackGenres(tx, track, genreIDs); err != nil {
return fmt.Errorf("propulate track genres: %w", err)
}
// metadata for the album table comes only from the the first track's tags
if i > 0 {
return nil
}
if err := populateAlbum(tx, album, albumArtist, trags, stat.ModTime()); err != nil {
return fmt.Errorf("propulate album: %w", err)
}
if err := populateAlbumGenres(tx, album, genreIDs); err != nil {
return fmt.Errorf("populate album genres: %w", err)
}
return nil
}
func populateAlbum(tx *db.DB, album *db.Album, albumArtist *db.Artist, trags tags.Parser, modTime time.Time) error {
albumName := trags.SomeAlbum()
album.TagTitle = albumName
album.TagTitleUDec = decoded(albumName)
album.TagBrainzID = trags.AlbumBrainzID()
album.TagYear = trags.Year()
album.TagArtistID = albumArtist.ID
album.ModifiedAt = modTime
if err := tx.Save(&album).Error; err != nil {
return fmt.Errorf("saving album: %w", err)
}
return nil
}
func populateAlbumBasics(tx *db.DB, rootAbsPath string, parent, album *db.Album, dir, basename string, cover string) error {
album.RootDir = rootAbsPath
album.LeftPath = dir
album.RightPath = basename
album.Cover = cover
album.RightPathUDec = decoded(basename)
album.ParentID = parent.ID
if err := tx.Save(&album).Error; err != nil {
return fmt.Errorf("saving album: %w", err)
}
return nil
}
func populateTrack(tx *db.DB, album *db.Album, albumArtist *db.Artist, track *db.Track, trags tags.Parser, abspath string, size int) error {
basename := filepath.Base(abspath)
track.Filename = basename
track.FilenameUDec = decoded(basename)
track.Size = size
track.AlbumID = album.ID
track.ArtistID = albumArtist.ID
track.TagTitle = trags.Title()
track.TagTitleUDec = decoded(trags.Title())
track.TagTrackArtist = trags.Artist()
track.TagTrackNumber = trags.TrackNumber()
track.TagDiscNumber = trags.DiscNumber()
track.TagBrainzID = trags.BrainzID()
track.Length = trags.Length() // these two should be calculated
track.Bitrate = trags.Bitrate() // ...from the file instead of tags
if err := tx.Save(&track).Error; err != nil {
return fmt.Errorf("saving track: %w", err)
}
return nil
}
func (s *Scanner) populateAlbumArtist(tx *db.DB, artistName string) (*db.Artist, error) {
var artist db.Artist
update := db.Artist{
Name: artistName,
NameUDec: decoded(artistName),
}
if err := tx.Where("name=?", artistName).Assign(update).FirstOrCreate(&artist).Error; err != nil {
return nil, fmt.Errorf("find or create artist: %w", err)
}
return &artist, nil
}
func (s *Scanner) populateGenres(tx *db.DB, track *db.Track, names []string) ([]int, error) {
var filteredNames []string
for _, name := range names {
if clean := strings.TrimSpace(name); clean != "" {
filteredNames = append(filteredNames, clean)
}
}
if len(filteredNames) == 0 {
return []int{}, nil
}
var ids []int
for _, name := range filteredNames {
var genre db.Genre
if err := tx.FirstOrCreate(&genre, db.Genre{Name: name}).Error; err != nil {
return nil, fmt.Errorf("find or create genre: %w", err)
}
ids = append(ids, genre.ID)
}
return ids, nil
}
func (s *Scanner) populateTrackGenres(tx *db.DB, track *db.Track, genreIDs []int) error {
if err := tx.Where("track_id=?", track.ID).Delete(db.TrackGenre{}).Error; err != nil {
return fmt.Errorf("delete old track genre records: %w", err)
}
if err := tx.InsertBulkLeftMany("track_genres", []string{"track_id", "genre_id"}, track.ID, genreIDs); err != nil {
return fmt.Errorf("insert bulk track genres: %w", err)
}
return nil
}
func populateAlbumGenres(tx *db.DB, album *db.Album, genreIDs []int) error {
if err := tx.Where("album_id=?", album.ID).Delete(db.AlbumGenre{}).Error; err != nil {
return fmt.Errorf("delete old album genre records: %w", err)
}
if err := tx.InsertBulkLeftMany("album_genres", []string{"album_id", "genre_id"}, album.ID, genreIDs); err != nil {
return fmt.Errorf("insert bulk album genres: %w", err)
}
return nil
}
func (s *Scanner) cleanTracks(seenTracks map[int]struct{}) error {
start := time.Now()
var previous []int
var missing []int64
@@ -103,7 +380,7 @@ func (s *Scanner) cleanTracks() error {
return fmt.Errorf("plucking ids: %w", err)
}
for _, prev := range previous {
if _, ok := s.seenTracks[prev]; !ok {
if _, ok := seenTracks[prev]; !ok {
missing = append(missing, int64(prev))
}
}
@@ -117,7 +394,7 @@ func (s *Scanner) cleanTracks() error {
return nil
}
func (s *Scanner) cleanAlbums() error {
func (s *Scanner) cleanAlbums(seenAlbums map[int]struct{}) error {
start := time.Now()
var previous []int
var missing []int64
@@ -129,7 +406,7 @@ func (s *Scanner) cleanAlbums() error {
return fmt.Errorf("plucking ids: %w", err)
}
for _, prev := range previous {
if _, ok := s.seenAlbums[prev]; !ok {
if _, ok := seenAlbums[prev]; !ok {
missing = append(missing, int64(prev))
}
}
@@ -176,383 +453,50 @@ func (s *Scanner) cleanGenres() error {
Where("album_genres.genre_id IS NULL").
SubQuery()
q := s.db.
Where("genres.id IN ?", subTrack).
Or("genres.id IN ?", subAlbum).
Where("genres.id IN ? AND genres.id IN ?", subTrack, subAlbum).
Delete(&db.Genre{})
log.Printf("finished clean genres in %s, %d removed", durSince(start), q.RowsAffected)
return nil
}
// ## begin entries
// ## begin entries
// ## begin entries
type ScanOptions struct {
IsFull bool
// TODO https://github.com/sentriz/gonic/issues/64
Path string
func ext(name string) string {
ext := filepath.Ext(name)
if len(ext) == 0 {
return ""
}
return ext[1:]
}
func (s *Scanner) Start(opts ScanOptions) error {
if IsScanning() {
return ErrAlreadyScanning
}
unSet := SetScanning()
defer unSet()
// reset state vars for the new scan
s.isFull = opts.IsFull
s.seenTracks = map[int]struct{}{}
s.seenAlbums = map[int]struct{}{}
s.curAlbums = &stack.Stack{}
s.seenTracksNew = 0
// begin walking
log.Println("starting scan")
var errCount int
start := time.Now()
err := godirwalk.Walk(s.musicPath, &godirwalk.Options{
Callback: s.callbackItem,
PostChildrenCallback: s.callbackPost,
Unsorted: true,
FollowSymbolicLinks: true,
ErrorCallback: func(path string, err error) godirwalk.ErrorAction {
log.Printf("error processing `%s`: %v", path, err)
errCount++
return godirwalk.SkipNode
},
})
if err != nil {
return fmt.Errorf("walking filesystem: %w", err)
}
log.Printf("finished scan in %s, +%d/%d tracks (%d err)\n",
durSince(start),
s.seenTracksNew,
len(s.seenTracks),
errCount,
)
if err := s.cleanTracks(); err != nil {
return fmt.Errorf("clean tracks: %w", err)
}
if err := s.cleanAlbums(); err != nil {
return fmt.Errorf("clean albums: %w", err)
}
if err := s.cleanArtists(); err != nil {
return fmt.Errorf("clean artists: %w", err)
}
if err := s.cleanGenres(); err != nil {
return fmt.Errorf("clean genres: %w", err)
}
// finish up
strNow := strconv.FormatInt(time.Now().Unix(), 10)
s.db.SetSetting("last_scan_time", strNow)
return nil
}
// items are passed to the handle*() functions
type item struct {
fullPath string
relPath string
directory string
filename string
stat os.FileInfo
}
func isCover(filename string) bool {
filename = strings.ToLower(filename)
known := map[string]struct{}{
"cover.png": {},
"cover.jpg": {},
"cover.jpeg": {},
"folder.png": {},
"folder.jpg": {},
"folder.jpeg": {},
"album.png": {},
"album.jpg": {},
"album.jpeg": {},
"albumart.png": {},
"albumart.jpg": {},
"albumart.jpeg": {},
"front.png": {},
"front.jpg": {},
"front.jpeg": {},
}
_, ok := known[filename]
return ok
}
// ## begin callbacks
// ## begin callbacks
// ## begin callbacks
func (s *Scanner) callbackItem(fullPath string, info *godirwalk.Dirent) error {
stat, err := os.Stat(fullPath)
if err != nil {
return fmt.Errorf("%w: %v", ErrStatingItem, err)
}
relPath, err := filepath.Rel(s.musicPath, fullPath)
if err != nil {
return fmt.Errorf("getting relative path: %w", err)
}
directory, filename := path.Split(relPath)
it := &item{
fullPath: fullPath,
relPath: relPath,
directory: directory,
filename: filename,
stat: stat,
}
isDir, err := info.IsDirOrSymlinkToDir()
if err != nil {
return fmt.Errorf("stating link to dir: %w", err)
}
if isDir {
return s.handleAlbum(it)
}
if isCover(filename) {
s.curCover = filename
return nil
}
ext := path.Ext(filename)
if ext == "" {
return nil
}
if _, ok := mime.FromExtension(ext[1:]); ok {
return s.handleTrack(it)
}
return nil
}
func (s *Scanner) callbackPost(fullPath string, info *godirwalk.Dirent) error {
defer func() {
s.curCover = ""
}()
if s.trTxOpen {
s.trTx.Commit()
s.trTxOpen = false
}
// begin taking the current album off the stack and add it's
// parent, cover that we found, etc.
album := s.curAlbums.Pop()
if album.Cover == s.curCover && album.ParentID != 0 {
return nil
}
album.ParentID = s.curAlbums.PeekID()
album.Cover = s.curCover
if err := s.db.Save(album).Error; err != nil {
return fmt.Errorf("writing albums table: %w", err)
}
// we only log changed albums
log.Printf("processed folder `%s`\n",
path.Join(album.LeftPath, album.RightPath))
return nil
}
// ## begin handlers
// ## begin handlers
// ## begin handlers
func (s *Scanner) itemUnchanged(statModTime, updatedInDB time.Time) bool {
if s.isFull {
func isCover(name string) bool {
switch path := strings.ToLower(name); path {
case
"cover.png", "cover.jpg", "cover.jpeg",
"folder.png", "folder.jpg", "folder.jpeg",
"album.png", "album.jpg", "album.jpeg",
"albumart.png", "albumart.jpg", "albumart.jpeg",
"front.png", "front.jpg", "front.jpeg":
return true
default:
return false
}
return statModTime.Before(updatedInDB)
}
func (s *Scanner) handleAlbum(it *item) error {
if s.trTxOpen {
// a transaction still being open when we handle an album can
// happen if there is a album that contains /both/ tracks and
// sub albums
s.trTx.Commit()
s.trTxOpen = false
// decoded converts a string to it's latin equivalent.
// it will be used by the model's *UDec fields, and is only set if it
// differs from the original. the fields are used for searching.
func decoded(in string) string {
if u := unidecode.Unidecode(in); u != in {
return u
}
album := &db.Album{}
defer func() {
// album's id will come from early return
// or save at the end
s.seenAlbums[album.ID] = struct{}{}
s.curAlbums.Push(album)
}()
err := s.db.
Where(db.Album{
LeftPath: it.directory,
RightPath: it.filename,
}).
First(album).
Error
if !gorm.IsRecordNotFoundError(err) &&
s.itemUnchanged(it.stat.ModTime(), album.UpdatedAt) {
// we found the record but it hasn't changed
return nil
}
album.LeftPath = it.directory
album.RightPath = it.filename
album.RightPathUDec = decoded(it.filename)
album.ModifiedAt = it.stat.ModTime()
if err := s.db.Save(album).Error; err != nil {
return fmt.Errorf("writing albums table: %w", err)
}
return nil
return ""
}
func (s *Scanner) handleTrack(it *item) error {
if !s.trTxOpen {
s.trTx = s.db.Begin()
s.trTxOpen = true
}
// init empty track and mark its ID (from lookup or save)
// for later cleanup later
var track db.Track
defer func() {
s.seenTracks[track.ID] = struct{}{}
}()
album := s.curAlbums.Peek()
err := s.trTx.
Select("id, updated_at").
Where(db.Track{
AlbumID: album.ID,
Filename: it.filename,
}).
First(&track).
Error
if !gorm.IsRecordNotFoundError(err) &&
s.itemUnchanged(it.stat.ModTime(), track.UpdatedAt) {
// we found the record but it hasn't changed
return nil
}
trags, err := tags.New(it.fullPath)
if err != nil {
return ErrReadingTags
}
genreIDs, err := s.populateGenres(&track, trags)
if err != nil {
return fmt.Errorf("populate genres: %w", err)
}
// create album and album artist records for first track in album
if album.TagTitle == "" {
albumArtist, err := s.populateAlbumArtist(trags)
if err != nil {
return fmt.Errorf("populate artist: %w", err)
}
albumName := trags.SomeAlbum()
album.TagTitle = albumName
album.TagTitleUDec = decoded(albumName)
album.TagBrainzID = trags.AlbumBrainzID()
album.TagYear = trags.Year()
album.TagArtistID = albumArtist.ID
if err := s.populateAlbumGenres(album, genreIDs); err != nil {
return fmt.Errorf("populate album genres: %w", err)
}
}
track.Filename = it.filename
track.FilenameUDec = decoded(it.filename)
track.Size = int(it.stat.Size())
track.AlbumID = album.ID
track.ArtistID = album.TagArtistID
track.TagTitle = trags.Title()
track.TagTitleUDec = decoded(trags.Title())
track.TagTrackArtist = trags.Artist()
track.TagTrackNumber = trags.TrackNumber()
track.TagDiscNumber = trags.DiscNumber()
track.TagBrainzID = trags.BrainzID()
track.Length = trags.Length() // these two should be calculated
track.Bitrate = trags.Bitrate() // ...from the file instead of tags
if err := s.trTx.Save(&track).Error; err != nil {
return fmt.Errorf("writing track table: %w", err)
}
s.seenTracksNew++
if err := s.populateTrackGenres(&track, genreIDs); err != nil {
return fmt.Errorf("populating track genres : %w", err)
}
return nil
func durSince(t time.Time) time.Duration {
return time.Since(t).Truncate(10 * time.Microsecond)
}
func (s *Scanner) populateAlbumArtist(trags *tags.Tags) (*db.Artist, error) {
var artist db.Artist
artistName := trags.SomeAlbumArtist()
err := s.trTx.
Where("name=?", artistName).
Assign(db.Artist{
Name: artistName,
NameUDec: decoded(artistName),
}).
FirstOrCreate(&artist).
Error
if err != nil {
return nil, fmt.Errorf("find or create artist: %w", err)
}
return &artist, nil
}
func (s *Scanner) populateGenres(track *db.Track, trags *tags.Tags) ([]int, error) {
var genreIDs []int
genreNames := strings.Split(trags.SomeGenre(), s.genreSplit)
for _, genreName := range genreNames {
genre := &db.Genre{}
q := s.trTx.FirstOrCreate(genre, db.Genre{
Name: genreName,
})
if err := q.Error; err != nil {
return nil, err
}
genreIDs = append(genreIDs, genre.ID)
}
return genreIDs, nil
}
func (s *Scanner) populateTrackGenres(track *db.Track, genreIDs []int) error {
err := s.trTx.
Where("track_id=?", track.ID).
Delete(db.TrackGenre{}).
Error
if err != nil {
return fmt.Errorf("delete old track genre records: %w", err)
}
err = s.trTx.InsertBulkLeftMany(
"track_genres",
[]string{"track_id", "genre_id"},
track.ID,
genreIDs,
)
if err != nil {
return fmt.Errorf("insert bulk track genres: %w", err)
}
return nil
}
func (s *Scanner) populateAlbumGenres(album *db.Album, genreIDs []int) error {
err := s.trTx.
Where("album_id=?", album.ID).
Delete(db.AlbumGenre{}).
Error
if err != nil {
return fmt.Errorf("delete old album genre records: %w", err)
}
err = s.trTx.InsertBulkLeftMany(
"album_genres",
[]string{"album_id", "genre_id"},
album.ID,
genreIDs,
)
if err != nil {
return fmt.Errorf("insert bulk album genres: %w", err)
}
return nil
type collected struct {
seenTracks map[int]struct{}
seenAlbums map[int]struct{}
seenTracksNew int
}

View File

@@ -1,4 +1,4 @@
package scanner
package scanner_test
import (
"io/ioutil"
@@ -6,62 +6,319 @@ import (
"os"
"testing"
"github.com/jinzhu/gorm"
_ "github.com/jinzhu/gorm/dialects/sqlite"
"github.com/matryer/is"
"go.senan.xyz/gonic/server/db"
"go.senan.xyz/gonic/server/mockfs"
)
var testScanner *Scanner
func resetTables(db *db.DB) {
tx := db.Begin()
defer tx.Commit()
tx.Exec("delete from tracks")
tx.Exec("delete from artists")
tx.Exec("delete from albums")
}
func resetTablesPause(db *db.DB, b *testing.B) {
b.StopTimer()
defer b.StartTimer()
resetTables(db)
}
func BenchmarkScanFresh(b *testing.B) {
for n := 0; n < b.N; n++ {
resetTablesPause(testScanner.db, b)
_ = testScanner.Start(ScanOptions{})
}
}
func BenchmarkScanIncremental(b *testing.B) {
// do a full scan and reset
_ = testScanner.Start(ScanOptions{})
b.ResetTimer()
// do the inc scans
for n := 0; n < b.N; n++ {
_ = testScanner.Start(ScanOptions{})
}
}
func TestMain(m *testing.M) {
db, err := db.NewMock()
if err != nil {
log.Fatalf("error opening database: %v\n", err)
}
// benchmarks aren't real code are they? >:)
// here is an absolute path to my music directory
testScanner = New("/home/senan/music", db, "\n")
log.SetOutput(ioutil.Discard)
os.Exit(m.Run())
}
// RESULTS fresh
// 20 times / 1.436
// 20 times / 1.39
func TestTableCounts(t *testing.T) {
t.Parallel()
is := is.NewRelaxed(t)
m := mockfs.New(t)
defer m.CleanUp()
// RESULTS inc
// 100 times / 1.86
// 100 times / 1.9
// 100 times / 1.5
// 100 times / 1.48
m.AddItems()
m.ScanAndClean()
var tracks int
is.NoErr(m.DB().Model(&db.Track{}).Count(&tracks).Error) // not all tracks
is.Equal(tracks, 3*3*3) // not all tracks
var albums int
is.NoErr(m.DB().Model(&db.Album{}).Count(&albums).Error) // not all albums
is.Equal(albums, 13) // not all albums
var artists int
is.NoErr(m.DB().Model(&db.Artist{}).Count(&artists).Error) // not all artists
is.Equal(artists, 3) // not all artists
}
func TestParentID(t *testing.T) {
t.Parallel()
is := is.New(t)
m := mockfs.New(t)
defer m.CleanUp()
m.AddItems()
m.ScanAndClean()
var nullParentAlbums []*db.Album
is.NoErr(m.DB().Where("parent_id IS NULL").Find(&nullParentAlbums).Error) // one parent_id=NULL which is root folder
is.Equal(len(nullParentAlbums), 1) // one parent_id=NULL which is root folder
is.Equal(nullParentAlbums[0].LeftPath, "")
is.Equal(nullParentAlbums[0].RightPath, ".")
is.Equal(m.DB().Where("id=parent_id").Find(&db.Album{}).Error, gorm.ErrRecordNotFound) // no self-referencing albums
var album db.Album
var parent db.Album
is.NoErr(m.DB().Find(&album, "left_path=? AND right_path=?", "artist-0/", "album-0").Error) // album has parent ID
is.NoErr(m.DB().Find(&parent, "right_path=?", "artist-0").Error) // album has parent ID
is.Equal(album.ParentID, parent.ID) // album has parent ID
}
func TestUpdatedCover(t *testing.T) {
t.Parallel()
is := is.NewRelaxed(t)
m := mockfs.New(t)
defer m.CleanUp()
m.AddItems()
m.ScanAndClean()
m.AddCover("artist-0/album-0/cover.jpg")
m.ScanAndClean()
var album db.Album
is.NoErr(m.DB().Where("left_path=? AND right_path=?", "artist-0/", "album-0").Find(&album).Error) // album has cover
is.Equal(album.Cover, "cover.jpg") // album has cover
}
func TestCoverBeforeTracks(t *testing.T) {
t.Parallel()
is := is.New(t)
m := mockfs.New(t)
defer m.CleanUp()
m.AddCover("artist-2/album-2/cover.jpg")
m.ScanAndClean()
m.AddItems()
m.ScanAndClean()
var album db.Album
is.NoErr(m.DB().Preload("TagArtist").Where("left_path=? AND right_path=?", "artist-2/", "album-2").Find(&album).Error) // album has cover
is.Equal(album.Cover, "cover.jpg") // album has cover
is.Equal(album.TagArtist.Name, "artist-2") // album artist
var tracks []*db.Track
is.NoErr(m.DB().Where("album_id=?", album.ID).Find(&tracks).Error) // album has tracks
is.Equal(len(tracks), 3) // album has tracks
}
func TestUpdatedTags(t *testing.T) {
t.Parallel()
is := is.New(t)
m := mockfs.New(t)
defer m.CleanUp()
m.AddTrack("artist-10/album-10/track-10.flac")
m.SetTags("artist-10/album-10/track-10.flac", func(tags *mockfs.Tags) {
tags.RawArtist = "artist"
tags.RawAlbumArtist = "album-artist"
tags.RawAlbum = "album"
tags.RawTitle = "title"
})
m.ScanAndClean()
var track db.Track
is.NoErr(m.DB().Preload("Album").Preload("Artist").Where("filename=?", "track-10.flac").Find(&track).Error) // track has tags
is.Equal(track.TagTrackArtist, "artist") // track has tags
is.Equal(track.Artist.Name, "album-artist") // track has tags
is.Equal(track.Album.TagTitle, "album") // track has tags
is.Equal(track.TagTitle, "title") // track has tags
m.SetTags("artist-10/album-10/track-10.flac", func(tags *mockfs.Tags) {
tags.RawArtist = "artist-upd"
tags.RawAlbumArtist = "album-artist-upd"
tags.RawAlbum = "album-upd"
tags.RawTitle = "title-upd"
})
m.ScanAndClean()
var updated db.Track
is.NoErr(m.DB().Preload("Album").Preload("Artist").Where("filename=?", "track-10.flac").Find(&updated).Error) // updated has tags
is.Equal(updated.ID, track.ID) // updated has tags
is.Equal(updated.TagTrackArtist, "artist-upd") // updated has tags
is.Equal(updated.Artist.Name, "album-artist-upd") // updated has tags
is.Equal(updated.Album.TagTitle, "album-upd") // updated has tags
is.Equal(updated.TagTitle, "title-upd") // updated has tags
}
func TestDelete(t *testing.T) {
t.Parallel()
is := is.NewRelaxed(t)
m := mockfs.New(t)
defer m.CleanUp()
m.AddItems()
m.ScanAndClean()
var album db.Album
is.NoErr(m.DB().Where("left_path=? AND right_path=?", "artist-2/", "album-2").Find(&album).Error) // album exists
m.RemoveAll("artist-2/album-2")
m.ScanAndClean()
is.Equal(m.DB().Where("left_path=? AND right_path=?", "artist-2/", "album-2").Find(&album).Error, gorm.ErrRecordNotFound) // album doesn't exist
}
func TestGenres(t *testing.T) {
t.Parallel()
is := is.New(t)
m := mockfs.New(t)
defer m.CleanUp()
albumGenre := func(artist, album, genre string) error {
return m.DB().
Where("albums.left_path=? AND albums.right_path=? AND genres.name=?", artist, album, genre).
Joins("JOIN albums ON albums.id=album_genres.album_id").
Joins("JOIN genres ON genres.id=album_genres.genre_id").
Find(&db.AlbumGenre{}).
Error
}
isAlbumGenre := func(artist, album, genreName string) {
is.Helper()
is.NoErr(albumGenre(artist, album, genreName))
}
isAlbumGenreMissing := func(artist, album, genreName string) {
is.Helper()
is.Equal(albumGenre(artist, album, genreName), gorm.ErrRecordNotFound)
}
trackGenre := func(artist, album, filename, genreName string) error {
return m.DB().
Where("albums.left_path=? AND albums.right_path=? AND tracks.filename=? AND genres.name=?", artist, album, filename, genreName).
Joins("JOIN tracks ON tracks.id=track_genres.track_id").
Joins("JOIN genres ON genres.id=track_genres.genre_id").
Joins("JOIN albums ON albums.id=tracks.album_id").
Find(&db.TrackGenre{}).
Error
}
isTrackGenre := func(artist, album, filename, genreName string) {
is.Helper()
is.NoErr(trackGenre(artist, album, filename, genreName))
}
isTrackGenreMissing := func(artist, album, filename, genreName string) {
is.Helper()
is.Equal(trackGenre(artist, album, filename, genreName), gorm.ErrRecordNotFound)
}
genre := func(genre string) error {
return m.DB().Where("name=?", genre).Find(&db.Genre{}).Error
}
isGenre := func(genreName string) {
is.Helper()
is.NoErr(genre(genreName))
}
isGenreMissing := func(genreName string) {
is.Helper()
is.Equal(genre(genreName), gorm.ErrRecordNotFound)
}
m.AddItems()
m.SetTags("artist-0/album-0/track-0.flac", func(tags *mockfs.Tags) { tags.RawGenre = "genre-a;genre-b" })
m.SetTags("artist-0/album-0/track-1.flac", func(tags *mockfs.Tags) { tags.RawGenre = "genre-c;genre-d" })
m.SetTags("artist-1/album-2/track-0.flac", func(tags *mockfs.Tags) { tags.RawGenre = "genre-e;genre-f" })
m.SetTags("artist-1/album-2/track-1.flac", func(tags *mockfs.Tags) { tags.RawGenre = "genre-g;genre-h" })
m.ScanAndClean()
isGenre("genre-a") // genre exists
isGenre("genre-b") // genre exists
isGenre("genre-c") // genre exists
isGenre("genre-d") // genre exists
isTrackGenre("artist-0/", "album-0", "track-0.flac", "genre-a") // track genre exists
isTrackGenre("artist-0/", "album-0", "track-0.flac", "genre-b") // track genre exists
isTrackGenre("artist-0/", "album-0", "track-1.flac", "genre-c") // track genre exists
isTrackGenre("artist-0/", "album-0", "track-1.flac", "genre-d") // track genre exists
isTrackGenre("artist-1/", "album-2", "track-0.flac", "genre-e") // track genre exists
isTrackGenre("artist-1/", "album-2", "track-0.flac", "genre-f") // track genre exists
isTrackGenre("artist-1/", "album-2", "track-1.flac", "genre-g") // track genre exists
isTrackGenre("artist-1/", "album-2", "track-1.flac", "genre-h") // track genre exists
isAlbumGenre("artist-0/", "album-0", "genre-a") // album genre exists
isAlbumGenre("artist-0/", "album-0", "genre-b") // album genre exists
m.SetTags("artist-0/album-0/track-0.flac", func(tags *mockfs.Tags) { tags.RawGenre = "genre-aa;genre-bb" })
m.ScanAndClean()
isTrackGenre("artist-0/", "album-0", "track-0.flac", "genre-aa") // updated track genre exists
isTrackGenre("artist-0/", "album-0", "track-0.flac", "genre-bb") // updated track genre exists
isTrackGenreMissing("artist-0/", "album-0", "track-0.flac", "genre-a") // old track genre missing
isTrackGenreMissing("artist-0/", "album-0", "track-0.flac", "genre-b") // old track genre missing
isAlbumGenreMissing("artist-0/", "album-0", "genre-a") // old album genre missing
isAlbumGenreMissing("artist-0/", "album-0", "genre-b") // old album genre missing
isGenreMissing("genre-a") // old genre missing
isGenreMissing("genre-b") // old genre missing
}
func TestMultiFolders(t *testing.T) {
t.Parallel()
is := is.New(t)
m := mockfs.NewWithDirs(t, []string{"m-1", "m-2", "m-3"})
defer m.CleanUp()
m.AddItemsPrefix("m-1")
m.AddItemsPrefix("m-2")
m.AddItemsPrefix("m-3")
m.ScanAndClean()
var rootDirs []*db.Album
is.NoErr(m.DB().Where("parent_id IS NULL").Find(&rootDirs).Error)
is.Equal(len(rootDirs), 3)
for i, r := range rootDirs {
is.Equal(r.RootDir, filepath.Join(m.TmpDir(), fmt.Sprintf("m-%d", i+1)))
is.Equal(r.ParentID, 0)
is.Equal(r.LeftPath, "")
is.Equal(r.RightPath, ".")
}
m.AddCover("m-3/artist-0/album-0/cover.jpg")
m.ScanAndClean()
m.LogItems()
checkCover := func(root string, q string) {
is.Helper()
is.NoErr(m.DB().Where(q, filepath.Join(m.TmpDir(), root)).Find(&db.Album{}).Error)
}
checkCover("m-1", "root_dir=? AND cover IS NULL") // mf 1 no cover
checkCover("m-2", "root_dir=? AND cover IS NULL") // mf 2 no cover
checkCover("m-3", "root_dir=? AND cover='cover.jpg'") // mf 3 has cover
}
func TestNewAlbumForExistingArtist(t *testing.T) {
t.Parallel()
is := is.New(t)
m := mockfs.New(t)
defer m.CleanUp()
m.AddItems()
m.ScanAndClean()
m.LogAlbums()
m.LogArtists()
var artist db.Artist
is.NoErr(m.DB().Where("name=?", "artist-2").Find(&artist).Error) // find orig artist
is.True(artist.ID > 0)
for tr := 0; tr < 3; tr++ {
m.AddTrack(fmt.Sprintf("artist-2/new-album/track-%d.mp3", tr))
m.SetTags(fmt.Sprintf("artist-2/new-album/track-%d.mp3", tr), func(tags *mockfs.Tags) {
tags.RawArtist = "artist-2"
tags.RawAlbumArtist = "artist-2"
tags.RawAlbum = "new-album"
tags.RawTitle = fmt.Sprintf("title-%d", tr)
})
}
var updated db.Artist
is.NoErr(m.DB().Where("name=?", "artist-2").Find(&updated).Error) // find updated artist
is.Equal(artist.ID, updated.ID) // find updated artist
var all []*db.Artist
is.NoErr(m.DB().Find(&all).Error) // still only 3?
is.Equal(len(all), 3) // still only 3?
}

View File

@@ -1,61 +0,0 @@
package stack
import (
"fmt"
"strings"
"go.senan.xyz/gonic/server/db"
)
type item struct {
value *db.Album
next *item
}
type Stack struct {
top *item
len uint
}
func (s *Stack) Push(v *db.Album) {
s.top = &item{
value: v,
next: s.top,
}
s.len++
}
func (s *Stack) Pop() *db.Album {
if s.len == 0 {
return nil
}
v := s.top.value
s.top = s.top.next
s.len--
return v
}
func (s *Stack) Peek() *db.Album {
if s.len == 0 {
return nil
}
return s.top.value
}
func (s *Stack) PeekID() int {
if s.len == 0 {
return 0
}
return s.top.value.ID
}
func (s *Stack) String() string {
var str strings.Builder
str.WriteString("[")
for i, f := uint(0), s.top; i < s.len; i++ {
str.WriteString(fmt.Sprintf("%d, ", f.value.ID))
f = f.next
}
str.WriteString("]")
return str.String()
}

View File

@@ -1,36 +0,0 @@
package stack
import (
"testing"
"go.senan.xyz/gonic/server/db"
)
func TestFolderStack(t *testing.T) {
sta := &Stack{}
sta.Push(&db.Album{ID: 3})
sta.Push(&db.Album{ID: 4})
sta.Push(&db.Album{ID: 5})
sta.Push(&db.Album{ID: 6})
expected := "[6, 5, 4, 3, ]"
actual := sta.String()
if expected != actual {
t.Errorf("first stack: expected string "+
"%q, got %q", expected, actual)
}
//
sta = &Stack{}
sta.Push(&db.Album{ID: 27})
sta.Push(&db.Album{ID: 4})
sta.Peek()
sta.Push(&db.Album{ID: 5})
sta.Push(&db.Album{ID: 6})
sta.Push(&db.Album{ID: 7})
sta.Pop()
expected = "[6, 5, 4, 27, ]"
actual = sta.String()
if expected != actual {
t.Errorf("second stack: expected string "+
"%q, got %q", expected, actual)
}
}

View File

@@ -7,6 +7,56 @@ import (
"github.com/nicksellen/audiotags"
)
type TagReader struct{}
func (*TagReader) Read(abspath string) (Parser, error) {
raw, props, err := audiotags.Read(abspath)
return &Tagger{raw, props}, err
}
type Tagger struct {
raw map[string]string
props *audiotags.AudioProperties
}
func (t *Tagger) first(keys ...string) string {
for _, key := range keys {
if val, ok := t.raw[key]; ok {
return val
}
}
return ""
}
func (t *Tagger) Title() string { return t.first("title") }
func (t *Tagger) BrainzID() string { return t.first("musicbrainz_trackid") }
func (t *Tagger) Artist() string { return t.first("artist") }
func (t *Tagger) Album() string { return t.first("album") }
func (t *Tagger) AlbumArtist() string { return t.first("albumartist", "album artist") }
func (t *Tagger) AlbumBrainzID() string { return t.first("musicbrainz_albumid") }
func (t *Tagger) Genre() string { return t.first("genre") }
func (t *Tagger) TrackNumber() int { return intSep(t.first("tracknumber"), "/") } // eg. 5/12
func (t *Tagger) DiscNumber() int { return intSep(t.first("discnumber"), "/") } // eg. 1/2
func (t *Tagger) Length() int { return t.props.Length }
func (t *Tagger) Bitrate() int { return t.props.Bitrate }
func (t *Tagger) Year() int { return intSep(t.first("originaldate", "date", "year"), "-") }
func (t *Tagger) SomeAlbum() string { return first("Unknown Album", t.Album()) }
func (t *Tagger) SomeArtist() string { return first("Unknown Artist", t.Artist()) }
func (t *Tagger) SomeAlbumArtist() string {
return first("Unknown Artist", t.AlbumArtist(), t.Artist())
}
func (t *Tagger) SomeGenre() string { return first("Unknown Genre", t.Genre()) }
func first(or string, strs ...string) string {
for _, str := range strs {
if str != "" {
return str
}
}
return or
}
func intSep(in, sep string) int {
if in == "" {
return 0
@@ -19,48 +69,26 @@ func intSep(in, sep string) int {
return out
}
type Tags struct {
raw map[string]string
props *audiotags.AudioProperties
type Reader interface {
Read(abspath string) (Parser, error)
}
func New(path string) (*Tags, error) {
raw, props, err := audiotags.Read(path)
return &Tags{raw, props}, err
}
func (t *Tags) firstTag(keys ...string) string {
for _, key := range keys {
if val, ok := t.raw[key]; ok {
return val
}
}
return ""
}
func (t *Tags) Title() string { return t.firstTag("title") }
func (t *Tags) BrainzID() string { return t.firstTag("musicbrainz_trackid") }
func (t *Tags) Artist() string { return t.firstTag("artist") }
func (t *Tags) Album() string { return t.firstTag("album") }
func (t *Tags) AlbumArtist() string { return t.firstTag("albumartist", "album artist") }
func (t *Tags) AlbumBrainzID() string { return t.firstTag("musicbrainz_albumid") }
func (t *Tags) Genre() string { return t.firstTag("genre") }
func (t *Tags) TrackNumber() int { return intSep(t.firstTag("tracknumber"), "/") } // eg. 5/12
func (t *Tags) DiscNumber() int { return intSep(t.firstTag("discnumber"), "/") } // eg. 1/2
func (t *Tags) Length() int { return t.props.Length }
func (t *Tags) Bitrate() int { return t.props.Bitrate }
func (t *Tags) Year() int { return intSep(t.firstTag("originaldate", "date", "year"), "-") }
func (t *Tags) SomeAlbum() string { return first("Unknown Album", t.Album()) }
func (t *Tags) SomeArtist() string { return first("Unknown Artist", t.Artist()) }
func (t *Tags) SomeAlbumArtist() string { return first("Unknown Artist", t.AlbumArtist(), t.Artist()) }
func (t *Tags) SomeGenre() string { return first("Unknown Genre", t.Genre()) }
func first(or string, strs ...string) string {
for _, str := range strs {
if str != "" {
return str
}
}
return or
type Parser interface {
Title() string
BrainzID() string
Artist() string
Album() string
AlbumArtist() string
AlbumBrainzID() string
Genre() string
TrackNumber() int
DiscNumber() int
Length() int
Bitrate() int
Year() int
SomeAlbum() string
SomeArtist() string
SomeAlbumArtist() string
SomeGenre() string
}