refactor: update scanner, scanner tests, mockfs

closes #165
closes #163
This commit is contained in:
sentriz
2021-11-06 21:55:46 +00:00
parent b07b9a8be6
commit fa587fc7de
64 changed files with 3468 additions and 2372 deletions
+364 -420
View File
@@ -5,8 +5,8 @@ import (
"fmt"
"log"
"os"
"path"
"path/filepath"
"sort"
"strconv"
"strings"
"sync/atomic"
@@ -16,9 +16,9 @@ import (
"github.com/karrick/godirwalk"
"github.com/rainycape/unidecode"
"go.senan.xyz/gonic/multierr"
"go.senan.xyz/gonic/server/db"
"go.senan.xyz/gonic/server/mime"
"go.senan.xyz/gonic/server/scanner/stack"
"go.senan.xyz/gonic/server/scanner/tags"
)
@@ -28,70 +28,347 @@ var (
ErrReadingTags = errors.New("could not read tags")
)
func durSince(t time.Time) time.Duration {
return time.Since(t).Truncate(10 * time.Microsecond)
}
// decoded converts a string to it's latin equivalent.
// it will be used by the model's *UDec fields, and is only set if it
// differs from the original. the fields are used for searching.
func decoded(in string) string {
if u := unidecode.Unidecode(in); u != in {
return u
}
return ""
}
// isScanning acts as an atomic boolean semaphore. we don't
// want to have more than one scan going on at a time
var isScanning int32 //nolint:gochecknoglobals
func IsScanning() bool {
return atomic.LoadInt32(&isScanning) == 1
}
func SetScanning() func() {
atomic.StoreInt32(&isScanning, 1)
return func() {
atomic.StoreInt32(&isScanning, 0)
}
}
type Scanner struct {
db *db.DB
musicPath string
isFull bool
musicPaths []string
sorted bool
genreSplit string
// these two are for the transaction we do for every album.
// the boolean is there so we dont begin or commit multiple
// times in the handle album or post children callback
trTx *db.DB
trTxOpen bool
// these two are for keeping state between noted in the tree.
// eg. keep track of a parents album or the path to a cover
// we just saw that we need to commit in the post children
// callback
curAlbums *stack.Stack
curCover string
// then the rest are for stats and cleanup at the very end
seenTracks map[int]struct{} // set of p keys
seenAlbums map[int]struct{} // set of p keys
seenTracksNew int // n tracks not seen before
tagger tags.Reader
scanning *int32
}
func New(musicPath string, db *db.DB, genreSplit string) *Scanner {
func New(musicPaths []string, sorted bool, db *db.DB, genreSplit string, tagger tags.Reader) *Scanner {
return &Scanner{
db: db,
musicPath: musicPath,
musicPaths: musicPaths,
sorted: sorted,
genreSplit: genreSplit,
tagger: tagger,
scanning: new(int32),
}
}
// ## begin clean funcs
// ## begin clean funcs
// ## begin clean funcs
type ScanOptions struct {
IsFull bool
// TODO https://github.com/sentriz/gonic/issues/64
Path string
}
func (s *Scanner) cleanTracks() error {
func (s *Scanner) IsScanning() bool {
return atomic.LoadInt32(s.scanning) == 1
}
type ScanOptions struct {
IsFull bool
}
func (s *Scanner) ScanAndClean(opts ScanOptions) error {
c := &collected{
seenTracks: map[int]struct{}{},
seenAlbums: map[int]struct{}{},
}
if err := s.scan(c, opts.IsFull); err != nil {
return err
}
if err := s.clean(c); err != nil {
return err
}
return nil
}
func (s *Scanner) scan(c *collected, isFull bool) error {
if s.IsScanning() {
return ErrAlreadyScanning
}
atomic.StoreInt32(s.scanning, 1)
defer atomic.StoreInt32(s.scanning, 0)
start := time.Now()
itemErrs := multierr.Err{}
log.Println("starting scan")
defer func() {
log.Printf("finished scan in %s, +%d/%d tracks (%d err)\n",
durSince(start), c.seenTracksNew, len(c.seenTracks), itemErrs.Len())
}()
for _, musicPath := range s.musicPaths {
err := godirwalk.Walk(musicPath, &godirwalk.Options{
Callback: func(_ string, _ *godirwalk.Dirent) error {
return nil
},
PostChildrenCallback: func(itemPath string, _ *godirwalk.Dirent) error {
return s.callback(c, isFull, musicPath, itemPath)
},
Unsorted: !s.sorted,
FollowSymbolicLinks: true,
ErrorCallback: func(path string, err error) godirwalk.ErrorAction {
itemErrs.Add(fmt.Errorf("%q: %w", path, err))
return godirwalk.SkipNode
},
})
if err != nil {
return fmt.Errorf("walking filesystem: %w", err)
}
}
if err := s.db.SetSetting("last_scan_time", strconv.FormatInt(time.Now().Unix(), 10)); err != nil {
return fmt.Errorf("set scan time: %w", err)
}
if itemErrs.Len() > 0 {
return itemErrs
}
return nil
}
func (s *Scanner) clean(c *collected) error {
if err := s.cleanTracks(c.seenTracks); err != nil {
return fmt.Errorf("clean tracks: %w", err)
}
if err := s.cleanAlbums(c.seenAlbums); err != nil {
return fmt.Errorf("clean albums: %w", err)
}
if err := s.cleanArtists(); err != nil {
return fmt.Errorf("clean artists: %w", err)
}
if err := s.cleanGenres(); err != nil {
return fmt.Errorf("clean genres: %w", err)
}
return nil
}
func (s *Scanner) callback(c *collected, isFull bool, rootAbsPath string, itemAbsPath string) error {
if rootAbsPath == itemAbsPath {
return nil
}
relpath, _ := filepath.Rel(rootAbsPath, itemAbsPath)
gs, err := godirwalk.NewScanner(itemAbsPath)
if err != nil {
return err
}
var tracks []string
var cover string
for gs.Scan() {
if isCover(gs.Name()) {
cover = gs.Name()
continue
}
if _, ok := mime.FromExtension(ext(gs.Name())); ok {
tracks = append(tracks, gs.Name())
continue
}
}
tx := s.db.Begin()
defer tx.Commit()
pdir, pbasename := filepath.Split(filepath.Dir(relpath))
parent := &db.Album{}
if err := tx.Where(db.Album{RootDir: rootAbsPath, LeftPath: pdir, RightPath: pbasename}).FirstOrCreate(parent).Error; err != nil {
return fmt.Errorf("first or create parent: %w", err)
}
c.seenAlbums[parent.ID] = struct{}{}
dir, basename := filepath.Split(relpath)
album := &db.Album{}
if err := tx.Where(db.Album{RootDir: rootAbsPath, LeftPath: dir, RightPath: basename}).First(album).Error; err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
return fmt.Errorf("find album: %w", err)
}
if err := populateAlbumBasics(tx, rootAbsPath, parent, album, dir, basename, cover); err != nil {
return fmt.Errorf("populate album basics: %w", err)
}
c.seenAlbums[album.ID] = struct{}{}
sort.Strings(tracks)
for i, basename := range tracks {
abspath := filepath.Join(itemAbsPath, basename)
if err := s.populateTrackAndAlbumArtists(tx, c, i, album, basename, abspath, isFull); err != nil {
return fmt.Errorf("process %q: %w", "", err)
}
}
return nil
}
func (s *Scanner) populateTrackAndAlbumArtists(tx *db.DB, c *collected, i int, album *db.Album, basename string, abspath string, isFull bool) error {
track := &db.Track{AlbumID: album.ID, Filename: filepath.Base(basename)}
if err := tx.Where(track).First(track).Error; err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
return fmt.Errorf("query track: %w", err)
}
c.seenTracks[track.ID] = struct{}{}
stat, err := os.Stat(abspath)
if err != nil {
return fmt.Errorf("stating %q: %w", basename, err)
}
if !isFull && stat.ModTime().Before(track.UpdatedAt) {
return nil
}
trags, err := s.tagger.Read(abspath)
if err != nil {
return fmt.Errorf("%v: %w", err, ErrReadingTags)
}
artistName := trags.SomeAlbumArtist()
albumArtist, err := s.populateAlbumArtist(tx, artistName)
if err != nil {
return fmt.Errorf("populate artist: %w", err)
}
if err := populateTrack(tx, album, albumArtist, track, trags, basename, int(stat.Size())); err != nil {
return fmt.Errorf("process %q: %w", basename, err)
}
c.seenTracks[track.ID] = struct{}{}
c.seenTracksNew++
genreNames := strings.Split(trags.SomeGenre(), s.genreSplit)
genreIDs, err := s.populateGenres(tx, track, genreNames)
if err != nil {
return fmt.Errorf("populate genres: %w", err)
}
if err := s.populateTrackGenres(tx, track, genreIDs); err != nil {
return fmt.Errorf("propulate track genres: %w", err)
}
// metadata for the album table comes only from the the first track's tags
if i > 0 {
return nil
}
if err := populateAlbum(tx, album, albumArtist, trags, stat.ModTime()); err != nil {
return fmt.Errorf("propulate album: %w", err)
}
if err := populateAlbumGenres(tx, album, genreIDs); err != nil {
return fmt.Errorf("populate album genres: %w", err)
}
return nil
}
func populateAlbum(tx *db.DB, album *db.Album, albumArtist *db.Artist, trags tags.Parser, modTime time.Time) error {
albumName := trags.SomeAlbum()
album.TagTitle = albumName
album.TagTitleUDec = decoded(albumName)
album.TagBrainzID = trags.AlbumBrainzID()
album.TagYear = trags.Year()
album.TagArtistID = albumArtist.ID
album.ModifiedAt = modTime
if err := tx.Save(&album).Error; err != nil {
return fmt.Errorf("saving album: %w", err)
}
return nil
}
func populateAlbumBasics(tx *db.DB, rootAbsPath string, parent, album *db.Album, dir, basename string, cover string) error {
album.RootDir = rootAbsPath
album.LeftPath = dir
album.RightPath = basename
album.Cover = cover
album.RightPathUDec = decoded(basename)
album.ParentID = parent.ID
if err := tx.Save(&album).Error; err != nil {
return fmt.Errorf("saving album: %w", err)
}
return nil
}
func populateTrack(tx *db.DB, album *db.Album, albumArtist *db.Artist, track *db.Track, trags tags.Parser, abspath string, size int) error {
basename := filepath.Base(abspath)
track.Filename = basename
track.FilenameUDec = decoded(basename)
track.Size = size
track.AlbumID = album.ID
track.ArtistID = albumArtist.ID
track.TagTitle = trags.Title()
track.TagTitleUDec = decoded(trags.Title())
track.TagTrackArtist = trags.Artist()
track.TagTrackNumber = trags.TrackNumber()
track.TagDiscNumber = trags.DiscNumber()
track.TagBrainzID = trags.BrainzID()
track.Length = trags.Length() // these two should be calculated
track.Bitrate = trags.Bitrate() // ...from the file instead of tags
if err := tx.Save(&track).Error; err != nil {
return fmt.Errorf("saving track: %w", err)
}
return nil
}
func (s *Scanner) populateAlbumArtist(tx *db.DB, artistName string) (*db.Artist, error) {
var artist db.Artist
update := db.Artist{
Name: artistName,
NameUDec: decoded(artistName),
}
if err := tx.Where("name=?", artistName).Assign(update).FirstOrCreate(&artist).Error; err != nil {
return nil, fmt.Errorf("find or create artist: %w", err)
}
return &artist, nil
}
func (s *Scanner) populateGenres(tx *db.DB, track *db.Track, names []string) ([]int, error) {
var filteredNames []string
for _, name := range names {
if clean := strings.TrimSpace(name); clean != "" {
filteredNames = append(filteredNames, clean)
}
}
if len(filteredNames) == 0 {
return []int{}, nil
}
var ids []int
for _, name := range filteredNames {
var genre db.Genre
if err := tx.FirstOrCreate(&genre, db.Genre{Name: name}).Error; err != nil {
return nil, fmt.Errorf("find or create genre: %w", err)
}
ids = append(ids, genre.ID)
}
return ids, nil
}
func (s *Scanner) populateTrackGenres(tx *db.DB, track *db.Track, genreIDs []int) error {
if err := tx.Where("track_id=?", track.ID).Delete(db.TrackGenre{}).Error; err != nil {
return fmt.Errorf("delete old track genre records: %w", err)
}
if err := tx.InsertBulkLeftMany("track_genres", []string{"track_id", "genre_id"}, track.ID, genreIDs); err != nil {
return fmt.Errorf("insert bulk track genres: %w", err)
}
return nil
}
func populateAlbumGenres(tx *db.DB, album *db.Album, genreIDs []int) error {
if err := tx.Where("album_id=?", album.ID).Delete(db.AlbumGenre{}).Error; err != nil {
return fmt.Errorf("delete old album genre records: %w", err)
}
if err := tx.InsertBulkLeftMany("album_genres", []string{"album_id", "genre_id"}, album.ID, genreIDs); err != nil {
return fmt.Errorf("insert bulk album genres: %w", err)
}
return nil
}
func (s *Scanner) cleanTracks(seenTracks map[int]struct{}) error {
start := time.Now()
var previous []int
var missing []int64
@@ -103,7 +380,7 @@ func (s *Scanner) cleanTracks() error {
return fmt.Errorf("plucking ids: %w", err)
}
for _, prev := range previous {
if _, ok := s.seenTracks[prev]; !ok {
if _, ok := seenTracks[prev]; !ok {
missing = append(missing, int64(prev))
}
}
@@ -117,7 +394,7 @@ func (s *Scanner) cleanTracks() error {
return nil
}
func (s *Scanner) cleanAlbums() error {
func (s *Scanner) cleanAlbums(seenAlbums map[int]struct{}) error {
start := time.Now()
var previous []int
var missing []int64
@@ -129,7 +406,7 @@ func (s *Scanner) cleanAlbums() error {
return fmt.Errorf("plucking ids: %w", err)
}
for _, prev := range previous {
if _, ok := s.seenAlbums[prev]; !ok {
if _, ok := seenAlbums[prev]; !ok {
missing = append(missing, int64(prev))
}
}
@@ -176,383 +453,50 @@ func (s *Scanner) cleanGenres() error {
Where("album_genres.genre_id IS NULL").
SubQuery()
q := s.db.
Where("genres.id IN ?", subTrack).
Or("genres.id IN ?", subAlbum).
Where("genres.id IN ? AND genres.id IN ?", subTrack, subAlbum).
Delete(&db.Genre{})
log.Printf("finished clean genres in %s, %d removed", durSince(start), q.RowsAffected)
return nil
}
// ## begin entries
// ## begin entries
// ## begin entries
type ScanOptions struct {
IsFull bool
// TODO https://github.com/sentriz/gonic/issues/64
Path string
func ext(name string) string {
ext := filepath.Ext(name)
if len(ext) == 0 {
return ""
}
return ext[1:]
}
func (s *Scanner) Start(opts ScanOptions) error {
if IsScanning() {
return ErrAlreadyScanning
}
unSet := SetScanning()
defer unSet()
// reset state vars for the new scan
s.isFull = opts.IsFull
s.seenTracks = map[int]struct{}{}
s.seenAlbums = map[int]struct{}{}
s.curAlbums = &stack.Stack{}
s.seenTracksNew = 0
// begin walking
log.Println("starting scan")
var errCount int
start := time.Now()
err := godirwalk.Walk(s.musicPath, &godirwalk.Options{
Callback: s.callbackItem,
PostChildrenCallback: s.callbackPost,
Unsorted: true,
FollowSymbolicLinks: true,
ErrorCallback: func(path string, err error) godirwalk.ErrorAction {
log.Printf("error processing `%s`: %v", path, err)
errCount++
return godirwalk.SkipNode
},
})
if err != nil {
return fmt.Errorf("walking filesystem: %w", err)
}
log.Printf("finished scan in %s, +%d/%d tracks (%d err)\n",
durSince(start),
s.seenTracksNew,
len(s.seenTracks),
errCount,
)
if err := s.cleanTracks(); err != nil {
return fmt.Errorf("clean tracks: %w", err)
}
if err := s.cleanAlbums(); err != nil {
return fmt.Errorf("clean albums: %w", err)
}
if err := s.cleanArtists(); err != nil {
return fmt.Errorf("clean artists: %w", err)
}
if err := s.cleanGenres(); err != nil {
return fmt.Errorf("clean genres: %w", err)
}
// finish up
strNow := strconv.FormatInt(time.Now().Unix(), 10)
s.db.SetSetting("last_scan_time", strNow)
return nil
}
// items are passed to the handle*() functions
type item struct {
fullPath string
relPath string
directory string
filename string
stat os.FileInfo
}
func isCover(filename string) bool {
filename = strings.ToLower(filename)
known := map[string]struct{}{
"cover.png": {},
"cover.jpg": {},
"cover.jpeg": {},
"folder.png": {},
"folder.jpg": {},
"folder.jpeg": {},
"album.png": {},
"album.jpg": {},
"album.jpeg": {},
"albumart.png": {},
"albumart.jpg": {},
"albumart.jpeg": {},
"front.png": {},
"front.jpg": {},
"front.jpeg": {},
}
_, ok := known[filename]
return ok
}
// ## begin callbacks
// ## begin callbacks
// ## begin callbacks
func (s *Scanner) callbackItem(fullPath string, info *godirwalk.Dirent) error {
stat, err := os.Stat(fullPath)
if err != nil {
return fmt.Errorf("%w: %v", ErrStatingItem, err)
}
relPath, err := filepath.Rel(s.musicPath, fullPath)
if err != nil {
return fmt.Errorf("getting relative path: %w", err)
}
directory, filename := path.Split(relPath)
it := &item{
fullPath: fullPath,
relPath: relPath,
directory: directory,
filename: filename,
stat: stat,
}
isDir, err := info.IsDirOrSymlinkToDir()
if err != nil {
return fmt.Errorf("stating link to dir: %w", err)
}
if isDir {
return s.handleAlbum(it)
}
if isCover(filename) {
s.curCover = filename
return nil
}
ext := path.Ext(filename)
if ext == "" {
return nil
}
if _, ok := mime.FromExtension(ext[1:]); ok {
return s.handleTrack(it)
}
return nil
}
func (s *Scanner) callbackPost(fullPath string, info *godirwalk.Dirent) error {
defer func() {
s.curCover = ""
}()
if s.trTxOpen {
s.trTx.Commit()
s.trTxOpen = false
}
// begin taking the current album off the stack and add it's
// parent, cover that we found, etc.
album := s.curAlbums.Pop()
if album.Cover == s.curCover && album.ParentID != 0 {
return nil
}
album.ParentID = s.curAlbums.PeekID()
album.Cover = s.curCover
if err := s.db.Save(album).Error; err != nil {
return fmt.Errorf("writing albums table: %w", err)
}
// we only log changed albums
log.Printf("processed folder `%s`\n",
path.Join(album.LeftPath, album.RightPath))
return nil
}
// ## begin handlers
// ## begin handlers
// ## begin handlers
func (s *Scanner) itemUnchanged(statModTime, updatedInDB time.Time) bool {
if s.isFull {
func isCover(name string) bool {
switch path := strings.ToLower(name); path {
case
"cover.png", "cover.jpg", "cover.jpeg",
"folder.png", "folder.jpg", "folder.jpeg",
"album.png", "album.jpg", "album.jpeg",
"albumart.png", "albumart.jpg", "albumart.jpeg",
"front.png", "front.jpg", "front.jpeg":
return true
default:
return false
}
return statModTime.Before(updatedInDB)
}
func (s *Scanner) handleAlbum(it *item) error {
if s.trTxOpen {
// a transaction still being open when we handle an album can
// happen if there is a album that contains /both/ tracks and
// sub albums
s.trTx.Commit()
s.trTxOpen = false
// decoded converts a string to it's latin equivalent.
// it will be used by the model's *UDec fields, and is only set if it
// differs from the original. the fields are used for searching.
func decoded(in string) string {
if u := unidecode.Unidecode(in); u != in {
return u
}
album := &db.Album{}
defer func() {
// album's id will come from early return
// or save at the end
s.seenAlbums[album.ID] = struct{}{}
s.curAlbums.Push(album)
}()
err := s.db.
Where(db.Album{
LeftPath: it.directory,
RightPath: it.filename,
}).
First(album).
Error
if !gorm.IsRecordNotFoundError(err) &&
s.itemUnchanged(it.stat.ModTime(), album.UpdatedAt) {
// we found the record but it hasn't changed
return nil
}
album.LeftPath = it.directory
album.RightPath = it.filename
album.RightPathUDec = decoded(it.filename)
album.ModifiedAt = it.stat.ModTime()
if err := s.db.Save(album).Error; err != nil {
return fmt.Errorf("writing albums table: %w", err)
}
return nil
return ""
}
func (s *Scanner) handleTrack(it *item) error {
if !s.trTxOpen {
s.trTx = s.db.Begin()
s.trTxOpen = true
}
// init empty track and mark its ID (from lookup or save)
// for later cleanup later
var track db.Track
defer func() {
s.seenTracks[track.ID] = struct{}{}
}()
album := s.curAlbums.Peek()
err := s.trTx.
Select("id, updated_at").
Where(db.Track{
AlbumID: album.ID,
Filename: it.filename,
}).
First(&track).
Error
if !gorm.IsRecordNotFoundError(err) &&
s.itemUnchanged(it.stat.ModTime(), track.UpdatedAt) {
// we found the record but it hasn't changed
return nil
}
trags, err := tags.New(it.fullPath)
if err != nil {
return ErrReadingTags
}
genreIDs, err := s.populateGenres(&track, trags)
if err != nil {
return fmt.Errorf("populate genres: %w", err)
}
// create album and album artist records for first track in album
if album.TagTitle == "" {
albumArtist, err := s.populateAlbumArtist(trags)
if err != nil {
return fmt.Errorf("populate artist: %w", err)
}
albumName := trags.SomeAlbum()
album.TagTitle = albumName
album.TagTitleUDec = decoded(albumName)
album.TagBrainzID = trags.AlbumBrainzID()
album.TagYear = trags.Year()
album.TagArtistID = albumArtist.ID
if err := s.populateAlbumGenres(album, genreIDs); err != nil {
return fmt.Errorf("populate album genres: %w", err)
}
}
track.Filename = it.filename
track.FilenameUDec = decoded(it.filename)
track.Size = int(it.stat.Size())
track.AlbumID = album.ID
track.ArtistID = album.TagArtistID
track.TagTitle = trags.Title()
track.TagTitleUDec = decoded(trags.Title())
track.TagTrackArtist = trags.Artist()
track.TagTrackNumber = trags.TrackNumber()
track.TagDiscNumber = trags.DiscNumber()
track.TagBrainzID = trags.BrainzID()
track.Length = trags.Length() // these two should be calculated
track.Bitrate = trags.Bitrate() // ...from the file instead of tags
if err := s.trTx.Save(&track).Error; err != nil {
return fmt.Errorf("writing track table: %w", err)
}
s.seenTracksNew++
if err := s.populateTrackGenres(&track, genreIDs); err != nil {
return fmt.Errorf("populating track genres : %w", err)
}
return nil
func durSince(t time.Time) time.Duration {
return time.Since(t).Truncate(10 * time.Microsecond)
}
func (s *Scanner) populateAlbumArtist(trags *tags.Tags) (*db.Artist, error) {
var artist db.Artist
artistName := trags.SomeAlbumArtist()
err := s.trTx.
Where("name=?", artistName).
Assign(db.Artist{
Name: artistName,
NameUDec: decoded(artistName),
}).
FirstOrCreate(&artist).
Error
if err != nil {
return nil, fmt.Errorf("find or create artist: %w", err)
}
return &artist, nil
}
func (s *Scanner) populateGenres(track *db.Track, trags *tags.Tags) ([]int, error) {
var genreIDs []int
genreNames := strings.Split(trags.SomeGenre(), s.genreSplit)
for _, genreName := range genreNames {
genre := &db.Genre{}
q := s.trTx.FirstOrCreate(genre, db.Genre{
Name: genreName,
})
if err := q.Error; err != nil {
return nil, err
}
genreIDs = append(genreIDs, genre.ID)
}
return genreIDs, nil
}
func (s *Scanner) populateTrackGenres(track *db.Track, genreIDs []int) error {
err := s.trTx.
Where("track_id=?", track.ID).
Delete(db.TrackGenre{}).
Error
if err != nil {
return fmt.Errorf("delete old track genre records: %w", err)
}
err = s.trTx.InsertBulkLeftMany(
"track_genres",
[]string{"track_id", "genre_id"},
track.ID,
genreIDs,
)
if err != nil {
return fmt.Errorf("insert bulk track genres: %w", err)
}
return nil
}
func (s *Scanner) populateAlbumGenres(album *db.Album, genreIDs []int) error {
err := s.trTx.
Where("album_id=?", album.ID).
Delete(db.AlbumGenre{}).
Error
if err != nil {
return fmt.Errorf("delete old album genre records: %w", err)
}
err = s.trTx.InsertBulkLeftMany(
"album_genres",
[]string{"album_id", "genre_id"},
album.ID,
genreIDs,
)
if err != nil {
return fmt.Errorf("insert bulk album genres: %w", err)
}
return nil
type collected struct {
seenTracks map[int]struct{}
seenAlbums map[int]struct{}
seenTracksNew int
}
+306 -49
View File
@@ -1,4 +1,4 @@
package scanner
package scanner_test
import (
"io/ioutil"
@@ -6,62 +6,319 @@ import (
"os"
"testing"
"github.com/jinzhu/gorm"
_ "github.com/jinzhu/gorm/dialects/sqlite"
"github.com/matryer/is"
"go.senan.xyz/gonic/server/db"
"go.senan.xyz/gonic/server/mockfs"
)
var testScanner *Scanner
func resetTables(db *db.DB) {
tx := db.Begin()
defer tx.Commit()
tx.Exec("delete from tracks")
tx.Exec("delete from artists")
tx.Exec("delete from albums")
}
func resetTablesPause(db *db.DB, b *testing.B) {
b.StopTimer()
defer b.StartTimer()
resetTables(db)
}
func BenchmarkScanFresh(b *testing.B) {
for n := 0; n < b.N; n++ {
resetTablesPause(testScanner.db, b)
_ = testScanner.Start(ScanOptions{})
}
}
func BenchmarkScanIncremental(b *testing.B) {
// do a full scan and reset
_ = testScanner.Start(ScanOptions{})
b.ResetTimer()
// do the inc scans
for n := 0; n < b.N; n++ {
_ = testScanner.Start(ScanOptions{})
}
}
func TestMain(m *testing.M) {
db, err := db.NewMock()
if err != nil {
log.Fatalf("error opening database: %v\n", err)
}
// benchmarks aren't real code are they? >:)
// here is an absolute path to my music directory
testScanner = New("/home/senan/music", db, "\n")
log.SetOutput(ioutil.Discard)
os.Exit(m.Run())
}
// RESULTS fresh
// 20 times / 1.436
// 20 times / 1.39
func TestTableCounts(t *testing.T) {
t.Parallel()
is := is.NewRelaxed(t)
m := mockfs.New(t)
defer m.CleanUp()
// RESULTS inc
// 100 times / 1.86
// 100 times / 1.9
// 100 times / 1.5
// 100 times / 1.48
m.AddItems()
m.ScanAndClean()
var tracks int
is.NoErr(m.DB().Model(&db.Track{}).Count(&tracks).Error) // not all tracks
is.Equal(tracks, 3*3*3) // not all tracks
var albums int
is.NoErr(m.DB().Model(&db.Album{}).Count(&albums).Error) // not all albums
is.Equal(albums, 13) // not all albums
var artists int
is.NoErr(m.DB().Model(&db.Artist{}).Count(&artists).Error) // not all artists
is.Equal(artists, 3) // not all artists
}
func TestParentID(t *testing.T) {
t.Parallel()
is := is.New(t)
m := mockfs.New(t)
defer m.CleanUp()
m.AddItems()
m.ScanAndClean()
var nullParentAlbums []*db.Album
is.NoErr(m.DB().Where("parent_id IS NULL").Find(&nullParentAlbums).Error) // one parent_id=NULL which is root folder
is.Equal(len(nullParentAlbums), 1) // one parent_id=NULL which is root folder
is.Equal(nullParentAlbums[0].LeftPath, "")
is.Equal(nullParentAlbums[0].RightPath, ".")
is.Equal(m.DB().Where("id=parent_id").Find(&db.Album{}).Error, gorm.ErrRecordNotFound) // no self-referencing albums
var album db.Album
var parent db.Album
is.NoErr(m.DB().Find(&album, "left_path=? AND right_path=?", "artist-0/", "album-0").Error) // album has parent ID
is.NoErr(m.DB().Find(&parent, "right_path=?", "artist-0").Error) // album has parent ID
is.Equal(album.ParentID, parent.ID) // album has parent ID
}
func TestUpdatedCover(t *testing.T) {
t.Parallel()
is := is.NewRelaxed(t)
m := mockfs.New(t)
defer m.CleanUp()
m.AddItems()
m.ScanAndClean()
m.AddCover("artist-0/album-0/cover.jpg")
m.ScanAndClean()
var album db.Album
is.NoErr(m.DB().Where("left_path=? AND right_path=?", "artist-0/", "album-0").Find(&album).Error) // album has cover
is.Equal(album.Cover, "cover.jpg") // album has cover
}
func TestCoverBeforeTracks(t *testing.T) {
t.Parallel()
is := is.New(t)
m := mockfs.New(t)
defer m.CleanUp()
m.AddCover("artist-2/album-2/cover.jpg")
m.ScanAndClean()
m.AddItems()
m.ScanAndClean()
var album db.Album
is.NoErr(m.DB().Preload("TagArtist").Where("left_path=? AND right_path=?", "artist-2/", "album-2").Find(&album).Error) // album has cover
is.Equal(album.Cover, "cover.jpg") // album has cover
is.Equal(album.TagArtist.Name, "artist-2") // album artist
var tracks []*db.Track
is.NoErr(m.DB().Where("album_id=?", album.ID).Find(&tracks).Error) // album has tracks
is.Equal(len(tracks), 3) // album has tracks
}
func TestUpdatedTags(t *testing.T) {
t.Parallel()
is := is.New(t)
m := mockfs.New(t)
defer m.CleanUp()
m.AddTrack("artist-10/album-10/track-10.flac")
m.SetTags("artist-10/album-10/track-10.flac", func(tags *mockfs.Tags) {
tags.RawArtist = "artist"
tags.RawAlbumArtist = "album-artist"
tags.RawAlbum = "album"
tags.RawTitle = "title"
})
m.ScanAndClean()
var track db.Track
is.NoErr(m.DB().Preload("Album").Preload("Artist").Where("filename=?", "track-10.flac").Find(&track).Error) // track has tags
is.Equal(track.TagTrackArtist, "artist") // track has tags
is.Equal(track.Artist.Name, "album-artist") // track has tags
is.Equal(track.Album.TagTitle, "album") // track has tags
is.Equal(track.TagTitle, "title") // track has tags
m.SetTags("artist-10/album-10/track-10.flac", func(tags *mockfs.Tags) {
tags.RawArtist = "artist-upd"
tags.RawAlbumArtist = "album-artist-upd"
tags.RawAlbum = "album-upd"
tags.RawTitle = "title-upd"
})
m.ScanAndClean()
var updated db.Track
is.NoErr(m.DB().Preload("Album").Preload("Artist").Where("filename=?", "track-10.flac").Find(&updated).Error) // updated has tags
is.Equal(updated.ID, track.ID) // updated has tags
is.Equal(updated.TagTrackArtist, "artist-upd") // updated has tags
is.Equal(updated.Artist.Name, "album-artist-upd") // updated has tags
is.Equal(updated.Album.TagTitle, "album-upd") // updated has tags
is.Equal(updated.TagTitle, "title-upd") // updated has tags
}
func TestDelete(t *testing.T) {
t.Parallel()
is := is.NewRelaxed(t)
m := mockfs.New(t)
defer m.CleanUp()
m.AddItems()
m.ScanAndClean()
var album db.Album
is.NoErr(m.DB().Where("left_path=? AND right_path=?", "artist-2/", "album-2").Find(&album).Error) // album exists
m.RemoveAll("artist-2/album-2")
m.ScanAndClean()
is.Equal(m.DB().Where("left_path=? AND right_path=?", "artist-2/", "album-2").Find(&album).Error, gorm.ErrRecordNotFound) // album doesn't exist
}
func TestGenres(t *testing.T) {
t.Parallel()
is := is.New(t)
m := mockfs.New(t)
defer m.CleanUp()
albumGenre := func(artist, album, genre string) error {
return m.DB().
Where("albums.left_path=? AND albums.right_path=? AND genres.name=?", artist, album, genre).
Joins("JOIN albums ON albums.id=album_genres.album_id").
Joins("JOIN genres ON genres.id=album_genres.genre_id").
Find(&db.AlbumGenre{}).
Error
}
isAlbumGenre := func(artist, album, genreName string) {
is.Helper()
is.NoErr(albumGenre(artist, album, genreName))
}
isAlbumGenreMissing := func(artist, album, genreName string) {
is.Helper()
is.Equal(albumGenre(artist, album, genreName), gorm.ErrRecordNotFound)
}
trackGenre := func(artist, album, filename, genreName string) error {
return m.DB().
Where("albums.left_path=? AND albums.right_path=? AND tracks.filename=? AND genres.name=?", artist, album, filename, genreName).
Joins("JOIN tracks ON tracks.id=track_genres.track_id").
Joins("JOIN genres ON genres.id=track_genres.genre_id").
Joins("JOIN albums ON albums.id=tracks.album_id").
Find(&db.TrackGenre{}).
Error
}
isTrackGenre := func(artist, album, filename, genreName string) {
is.Helper()
is.NoErr(trackGenre(artist, album, filename, genreName))
}
isTrackGenreMissing := func(artist, album, filename, genreName string) {
is.Helper()
is.Equal(trackGenre(artist, album, filename, genreName), gorm.ErrRecordNotFound)
}
genre := func(genre string) error {
return m.DB().Where("name=?", genre).Find(&db.Genre{}).Error
}
isGenre := func(genreName string) {
is.Helper()
is.NoErr(genre(genreName))
}
isGenreMissing := func(genreName string) {
is.Helper()
is.Equal(genre(genreName), gorm.ErrRecordNotFound)
}
m.AddItems()
m.SetTags("artist-0/album-0/track-0.flac", func(tags *mockfs.Tags) { tags.RawGenre = "genre-a;genre-b" })
m.SetTags("artist-0/album-0/track-1.flac", func(tags *mockfs.Tags) { tags.RawGenre = "genre-c;genre-d" })
m.SetTags("artist-1/album-2/track-0.flac", func(tags *mockfs.Tags) { tags.RawGenre = "genre-e;genre-f" })
m.SetTags("artist-1/album-2/track-1.flac", func(tags *mockfs.Tags) { tags.RawGenre = "genre-g;genre-h" })
m.ScanAndClean()
isGenre("genre-a") // genre exists
isGenre("genre-b") // genre exists
isGenre("genre-c") // genre exists
isGenre("genre-d") // genre exists
isTrackGenre("artist-0/", "album-0", "track-0.flac", "genre-a") // track genre exists
isTrackGenre("artist-0/", "album-0", "track-0.flac", "genre-b") // track genre exists
isTrackGenre("artist-0/", "album-0", "track-1.flac", "genre-c") // track genre exists
isTrackGenre("artist-0/", "album-0", "track-1.flac", "genre-d") // track genre exists
isTrackGenre("artist-1/", "album-2", "track-0.flac", "genre-e") // track genre exists
isTrackGenre("artist-1/", "album-2", "track-0.flac", "genre-f") // track genre exists
isTrackGenre("artist-1/", "album-2", "track-1.flac", "genre-g") // track genre exists
isTrackGenre("artist-1/", "album-2", "track-1.flac", "genre-h") // track genre exists
isAlbumGenre("artist-0/", "album-0", "genre-a") // album genre exists
isAlbumGenre("artist-0/", "album-0", "genre-b") // album genre exists
m.SetTags("artist-0/album-0/track-0.flac", func(tags *mockfs.Tags) { tags.RawGenre = "genre-aa;genre-bb" })
m.ScanAndClean()
isTrackGenre("artist-0/", "album-0", "track-0.flac", "genre-aa") // updated track genre exists
isTrackGenre("artist-0/", "album-0", "track-0.flac", "genre-bb") // updated track genre exists
isTrackGenreMissing("artist-0/", "album-0", "track-0.flac", "genre-a") // old track genre missing
isTrackGenreMissing("artist-0/", "album-0", "track-0.flac", "genre-b") // old track genre missing
isAlbumGenreMissing("artist-0/", "album-0", "genre-a") // old album genre missing
isAlbumGenreMissing("artist-0/", "album-0", "genre-b") // old album genre missing
isGenreMissing("genre-a") // old genre missing
isGenreMissing("genre-b") // old genre missing
}
func TestMultiFolders(t *testing.T) {
t.Parallel()
is := is.New(t)
m := mockfs.NewWithDirs(t, []string{"m-1", "m-2", "m-3"})
defer m.CleanUp()
m.AddItemsPrefix("m-1")
m.AddItemsPrefix("m-2")
m.AddItemsPrefix("m-3")
m.ScanAndClean()
var rootDirs []*db.Album
is.NoErr(m.DB().Where("parent_id IS NULL").Find(&rootDirs).Error)
is.Equal(len(rootDirs), 3)
for i, r := range rootDirs {
is.Equal(r.RootDir, filepath.Join(m.TmpDir(), fmt.Sprintf("m-%d", i+1)))
is.Equal(r.ParentID, 0)
is.Equal(r.LeftPath, "")
is.Equal(r.RightPath, ".")
}
m.AddCover("m-3/artist-0/album-0/cover.jpg")
m.ScanAndClean()
m.LogItems()
checkCover := func(root string, q string) {
is.Helper()
is.NoErr(m.DB().Where(q, filepath.Join(m.TmpDir(), root)).Find(&db.Album{}).Error)
}
checkCover("m-1", "root_dir=? AND cover IS NULL") // mf 1 no cover
checkCover("m-2", "root_dir=? AND cover IS NULL") // mf 2 no cover
checkCover("m-3", "root_dir=? AND cover='cover.jpg'") // mf 3 has cover
}
func TestNewAlbumForExistingArtist(t *testing.T) {
t.Parallel()
is := is.New(t)
m := mockfs.New(t)
defer m.CleanUp()
m.AddItems()
m.ScanAndClean()
m.LogAlbums()
m.LogArtists()
var artist db.Artist
is.NoErr(m.DB().Where("name=?", "artist-2").Find(&artist).Error) // find orig artist
is.True(artist.ID > 0)
for tr := 0; tr < 3; tr++ {
m.AddTrack(fmt.Sprintf("artist-2/new-album/track-%d.mp3", tr))
m.SetTags(fmt.Sprintf("artist-2/new-album/track-%d.mp3", tr), func(tags *mockfs.Tags) {
tags.RawArtist = "artist-2"
tags.RawAlbumArtist = "artist-2"
tags.RawAlbum = "new-album"
tags.RawTitle = fmt.Sprintf("title-%d", tr)
})
}
var updated db.Artist
is.NoErr(m.DB().Where("name=?", "artist-2").Find(&updated).Error) // find updated artist
is.Equal(artist.ID, updated.ID) // find updated artist
var all []*db.Artist
is.NoErr(m.DB().Find(&all).Error) // still only 3?
is.Equal(len(all), 3) // still only 3?
}
-61
View File
@@ -1,61 +0,0 @@
package stack
import (
"fmt"
"strings"
"go.senan.xyz/gonic/server/db"
)
type item struct {
value *db.Album
next *item
}
type Stack struct {
top *item
len uint
}
func (s *Stack) Push(v *db.Album) {
s.top = &item{
value: v,
next: s.top,
}
s.len++
}
func (s *Stack) Pop() *db.Album {
if s.len == 0 {
return nil
}
v := s.top.value
s.top = s.top.next
s.len--
return v
}
func (s *Stack) Peek() *db.Album {
if s.len == 0 {
return nil
}
return s.top.value
}
func (s *Stack) PeekID() int {
if s.len == 0 {
return 0
}
return s.top.value.ID
}
func (s *Stack) String() string {
var str strings.Builder
str.WriteString("[")
for i, f := uint(0), s.top; i < s.len; i++ {
str.WriteString(fmt.Sprintf("%d, ", f.value.ID))
f = f.next
}
str.WriteString("]")
return str.String()
}
-36
View File
@@ -1,36 +0,0 @@
package stack
import (
"testing"
"go.senan.xyz/gonic/server/db"
)
func TestFolderStack(t *testing.T) {
sta := &Stack{}
sta.Push(&db.Album{ID: 3})
sta.Push(&db.Album{ID: 4})
sta.Push(&db.Album{ID: 5})
sta.Push(&db.Album{ID: 6})
expected := "[6, 5, 4, 3, ]"
actual := sta.String()
if expected != actual {
t.Errorf("first stack: expected string "+
"%q, got %q", expected, actual)
}
//
sta = &Stack{}
sta.Push(&db.Album{ID: 27})
sta.Push(&db.Album{ID: 4})
sta.Peek()
sta.Push(&db.Album{ID: 5})
sta.Push(&db.Album{ID: 6})
sta.Push(&db.Album{ID: 7})
sta.Pop()
expected = "[6, 5, 4, 27, ]"
actual = sta.String()
if expected != actual {
t.Errorf("second stack: expected string "+
"%q, got %q", expected, actual)
}
}
+69 -41
View File
@@ -7,6 +7,56 @@ import (
"github.com/nicksellen/audiotags"
)
type TagReader struct{}
func (*TagReader) Read(abspath string) (Parser, error) {
raw, props, err := audiotags.Read(abspath)
return &Tagger{raw, props}, err
}
type Tagger struct {
raw map[string]string
props *audiotags.AudioProperties
}
func (t *Tagger) first(keys ...string) string {
for _, key := range keys {
if val, ok := t.raw[key]; ok {
return val
}
}
return ""
}
func (t *Tagger) Title() string { return t.first("title") }
func (t *Tagger) BrainzID() string { return t.first("musicbrainz_trackid") }
func (t *Tagger) Artist() string { return t.first("artist") }
func (t *Tagger) Album() string { return t.first("album") }
func (t *Tagger) AlbumArtist() string { return t.first("albumartist", "album artist") }
func (t *Tagger) AlbumBrainzID() string { return t.first("musicbrainz_albumid") }
func (t *Tagger) Genre() string { return t.first("genre") }
func (t *Tagger) TrackNumber() int { return intSep(t.first("tracknumber"), "/") } // eg. 5/12
func (t *Tagger) DiscNumber() int { return intSep(t.first("discnumber"), "/") } // eg. 1/2
func (t *Tagger) Length() int { return t.props.Length }
func (t *Tagger) Bitrate() int { return t.props.Bitrate }
func (t *Tagger) Year() int { return intSep(t.first("originaldate", "date", "year"), "-") }
func (t *Tagger) SomeAlbum() string { return first("Unknown Album", t.Album()) }
func (t *Tagger) SomeArtist() string { return first("Unknown Artist", t.Artist()) }
func (t *Tagger) SomeAlbumArtist() string {
return first("Unknown Artist", t.AlbumArtist(), t.Artist())
}
func (t *Tagger) SomeGenre() string { return first("Unknown Genre", t.Genre()) }
func first(or string, strs ...string) string {
for _, str := range strs {
if str != "" {
return str
}
}
return or
}
func intSep(in, sep string) int {
if in == "" {
return 0
@@ -19,48 +69,26 @@ func intSep(in, sep string) int {
return out
}
type Tags struct {
raw map[string]string
props *audiotags.AudioProperties
type Reader interface {
Read(abspath string) (Parser, error)
}
func New(path string) (*Tags, error) {
raw, props, err := audiotags.Read(path)
return &Tags{raw, props}, err
}
type Parser interface {
Title() string
BrainzID() string
Artist() string
Album() string
AlbumArtist() string
AlbumBrainzID() string
Genre() string
TrackNumber() int
DiscNumber() int
Length() int
Bitrate() int
Year() int
func (t *Tags) firstTag(keys ...string) string {
for _, key := range keys {
if val, ok := t.raw[key]; ok {
return val
}
}
return ""
}
func (t *Tags) Title() string { return t.firstTag("title") }
func (t *Tags) BrainzID() string { return t.firstTag("musicbrainz_trackid") }
func (t *Tags) Artist() string { return t.firstTag("artist") }
func (t *Tags) Album() string { return t.firstTag("album") }
func (t *Tags) AlbumArtist() string { return t.firstTag("albumartist", "album artist") }
func (t *Tags) AlbumBrainzID() string { return t.firstTag("musicbrainz_albumid") }
func (t *Tags) Genre() string { return t.firstTag("genre") }
func (t *Tags) TrackNumber() int { return intSep(t.firstTag("tracknumber"), "/") } // eg. 5/12
func (t *Tags) DiscNumber() int { return intSep(t.firstTag("discnumber"), "/") } // eg. 1/2
func (t *Tags) Length() int { return t.props.Length }
func (t *Tags) Bitrate() int { return t.props.Bitrate }
func (t *Tags) Year() int { return intSep(t.firstTag("originaldate", "date", "year"), "-") }
func (t *Tags) SomeAlbum() string { return first("Unknown Album", t.Album()) }
func (t *Tags) SomeArtist() string { return first("Unknown Artist", t.Artist()) }
func (t *Tags) SomeAlbumArtist() string { return first("Unknown Artist", t.AlbumArtist(), t.Artist()) }
func (t *Tags) SomeGenre() string { return first("Unknown Genre", t.Genre()) }
func first(or string, strs ...string) string {
for _, str := range strs {
if str != "" {
return str
}
}
return or
SomeAlbum() string
SomeArtist() string
SomeAlbumArtist() string
SomeGenre() string
}