merge album and folder models

go.mod

@@ -28,4 +28,7 @@ require (
 	golang.org/x/sys v0.0.0-20190422165155-953cdadca894 // indirect
 	golang.org/x/tools v0.0.0-20190422233926-fe54fb35175b // indirect
 	google.golang.org/appengine v1.5.0 // indirect
+	gopkg.in/axiomzen/null.v3 v3.2.4
+	gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce // indirect
+	gopkg.in/pg.v4 v4.9.5 // indirect
 )

go.sum

@@ -263,10 +263,16 @@ google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9M
 google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/axiomzen/null.v3 v3.2.4 h1:5VmJ9lSU0dBJjisXhuhRnGiIolhTjkmQQ0EBNE9Z5QY=
+gopkg.in/axiomzen/null.v3 v3.2.4/go.mod h1:Vq8/79AVvSZVg5PdN4kNROnTff7WtSh0HOiBHfOvVNU=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU=
+gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/pg.v4 v4.9.5 h1:bs21aaMPPPcUPhNtqGxN8EeYUFU10MsNrC7U9m/lJgU=
+gopkg.in/pg.v4 v4.9.5/go.mod h1:cSUPtzgofjgARAbFCE5u6WDHGPgbR1sjUYcWQlKvpec=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

model/model.go

@@ -1,6 +1,8 @@
 package model
 
-import "time"
+import (
+	"time"
+)
 
 // q:  what in tarnation are the `IsNew`s for?
 // a:  it's a bit of a hack - but we set a models IsNew to true if
@@ -9,108 +11,73 @@ import "time"
 //     that bool being true - since it won't be true if it was already
 //     in the db
 
-// Album represents the albums table
-type Album struct {
-	IDBase
-	CrudBase
-	Artist   Artist
-	ArtistID int    `gorm:"index" sql:"default: null; type:int REFERENCES artists(id) ON DELETE CASCADE"`
-	Title    string `gorm:"not null; index"`
-	// an Album having a `Path` is a little weird when browsing by tags
-	// (for the most part - the library's folder structure is treated as
-	// if it were flat), but this solves the "American Football problem"
-	// https://en.wikipedia.org/wiki/American_Football_(band)#Discography
-	Path    string `gorm:"not null; unique_index"`
-	CoverID int    `sql:"default: null; type:int REFERENCES covers(id)"`
-	Cover   Cover
-	Year    int
-	Tracks  []Track
-	IsNew   bool `gorm:"-"`
-}
-
-// Artist represents the Artists table
 type Artist struct {
 	IDBase
 	CrudBase
-	Name   string `gorm:"not null; unique_index"`
+	Name    string `gorm:"not null; unique_index"`
-	Albums []Album
+	Folders []Folder
 }
 
-// Track represents the tracks table
 type Track struct {
 	IDBase
 	CrudBase
-	Album       Album
+	Folder Folder
-	AlbumID     int `gorm:"index" sql:"default: null; type:int REFERENCES albums(id) ON DELETE CASCADE"`
+	// TODO: try removing idx_folder_basename_ext
-	Artist      Artist
+	FolderID       int    `gorm:"not null; unique_index:idx_folder_filename_ext" sql:"default: null; type:int REFERENCES folders(id) ON DELETE CASCADE"`
-	ArtistID    int `gorm:"index" sql:"default: null; type:int REFERENCES artists(id) ON DELETE CASCADE"`
+	Filename       string `gorm:"not null; unique_index:idx_folder_filename_ext" sql:"default: null"`
-	TrackArtist string
+	Ext            string `gorm:"not nill; unique_index:idx_folder_filename_ext" sql:"default: null"`
-	Bitrate     int
+	Artist         Artist
-	Codec       string
+	ArtistID       int    `gorm:"not null; index" sql:"default: null; type:int REFERENCES artists(id) ON DELETE CASCADE"`
-	DiscNumber  int
+	ContentType    string `gorm:"not null" sql:"default: null"`
-	Duration    int
+	Duration       int    `gorm:"not null" sql:"default: null"`
-	Title       string
+	Size           int    `gorm:"not null" sql:"default: null"`
-	TotalDiscs  int
+	Bitrate        int    `gorm:"not null" sql:"default: null"`
-	TotalTracks int
+	TagDiscNumber  int    `sql:"default: null"`
-	TrackNumber int
+	TagTitle       string `sql:"default: null"`
-	Year        int
+	TagTotalDiscs  int    `sql:"default: null"`
-	Suffix      string
+	TagTotalTracks int    `sql:"default: null"`
-	ContentType string
+	TagTrackArtist string `sql:"default: null"`
-	Size        int
+	TagTrackNumber int    `sql:"default: null"`
-	Folder      Folder
+	TagYear        int    `sql:"default: null"`
-	FolderID    int    `gorm:"not null; index" sql:"default: null; type:int REFERENCES folders(id) ON DELETE CASCADE"`
-	Path        string `gorm:"not null; unique_index"`
 }
 
-// Cover represents the covers table
-type Cover struct {
-	IDBase
-	CrudBase
-	Image []byte
-	Path  string `gorm:"not null; unique_index"`
-	IsNew bool   `gorm:"-"`
-}
-
-// User represents the users table
 type User struct {
 	IDBase
 	CrudBase
-	Name          string `gorm:"not null; unique_index"`
+	Name          string `gorm:"not null; unique_index" sql:"default: null"`
-	Password      string
+	Password      string `gorm:"not null" sql:"default: null"`
-	LastFMSession string
+	LastFMSession string `sql:"default: null"`
-	IsAdmin       bool
+	IsAdmin       bool   `sql:"default: null"`
 }
 
-// Setting represents the settings table
 type Setting struct {
 	CrudBase
-	Key   string `gorm:"primary_key; auto_increment:false"`
+	Key   string `gorm:"not null; primary_key; auto_increment:false" sql:"default: null"`
-	Value string
+	Value string `sql:"default: null"`
 }
 
-// Play represents the settings table
 type Play struct {
 	IDBase
 	User     User
 	UserID   int `gorm:"not null; index" sql:"default: null; type:int REFERENCES users(id) ON DELETE CASCADE"`
-	Album    Album
-	AlbumID  int `gorm:"not null; index" sql:"default: null; type:int REFERENCES albums(id) ON DELETE CASCADE"`
 	Folder   Folder
-	FolderID int `gorm:"not null; index" sql:"default: null; type:int REFERENCES folders(id) ON DELETE CASCADE"`
+	FolderID int       `gorm:"not null; index" sql:"default: null; type:int REFERENCES folders(id) ON DELETE CASCADE"`
-	Time     time.Time
+	Time     time.Time `sql:"default: null"`
 	Count    int
 }
 
-// Folder represents the settings table
 type Folder struct {
 	IDBase
 	CrudBase
-	Name      string
+	LeftPath      string `gorm:"unique_index:idx_left_path_right_path"`
-	Path      string `gorm:"not null; unique_index"`
+	RightPath     string `gorm:"not null; unique_index:idx_left_path_right_path" sql:"default: null"`
-	Parent    *Folder
+	Parent        *Folder
-	ParentID  int  `sql:"default: null; type:int REFERENCES folders(id) ON DELETE CASCADE"`
+	ParentID      int `sql:"default: null; type:int REFERENCES folders(id) ON DELETE CASCADE"`
-	CoverID   int  `sql:"default: null; type:int REFERENCES covers(id)"`
+	AlbumArtist   Artist
-	HasTracks bool `gorm:"not null; index"`
+	AlbumArtistID int    `gorm:"index" sql:"default: null; type:int REFERENCES artists(id) ON DELETE CASCADE"`
-	Cover     Cover
+	AlbumTitle    string `gorm:"index" sql:"default: null"`
-	IsNew     bool `gorm:"-"`
+	AlbumYear     int    `sql:"default: null"`
+	Cover         string `sql:"default: null"`
+	Tracks        []Track
+	IsNew         bool `gorm:"-"`
 }

scanner/folder_stack.go

@@ -1,35 +1,40 @@
 package scanner
 
-import "github.com/sentriz/gonic/model"
+import (
+	"fmt"
+	"strings"
 
-type folderStack []model.Folder
+	"github.com/sentriz/gonic/model"
+)
 
-func (s *folderStack) Push(v model.Folder) {
+type folderStack []*model.Folder
+
+func (s *folderStack) Push(v *model.Folder) {
 	*s = append(*s, v)
 }
 
-func (s *folderStack) Pop() model.Folder {
+func (s *folderStack) Pop() *model.Folder {
 	l := len(*s)
 	if l == 0 {
-		return model.Folder{}
+		return nil
 	}
 	r := (*s)[l-1]
 	*s = (*s)[:l-1]
 	return r
 }
 
-func (s *folderStack) Peek() model.Folder {
+func (s *folderStack) Peek() *model.Folder {
 	l := len(*s)
 	if l == 0 {
-		return model.Folder{}
+		return nil
 	}
 	return (*s)[l-1]
 }
 
-func (s *folderStack) PeekID() int {
+func (s *folderStack) String() string {
-	l := len(*s)
+	paths := make([]string, len(*s))
-	if l == 0 {
+	for i, folder := range *s {
-		return 0
+		paths[i] = folder.RightPath
 	}
-	return (*s)[l-1].ID
+	return fmt.Sprintf("[%s]", strings.Join(paths, " "))
 }

scanner/scanner.go

@@ -1,12 +1,5 @@
 package scanner
 
-// Album  -> needs a  CoverID
-//        -> needs a  FolderID (American Football)
-// Folder -> needs a  CoverID
-//        -> needs a  ParentID
-// Track  -> needs an AlbumID
-//        -> needs a  FolderID
-
 import (
 	"log"
 	"sync/atomic"
@@ -26,27 +19,32 @@ var (
 type Scanner struct {
 	db, tx     *gorm.DB
 	musicPath  string
-	seenTracks map[string]bool
+	seenTracks map[int]struct{}
 	curFolders folderStack
-	curTracks  []model.Track
+	curCover   string
-	curCover   model.Cover
-	curAlbum   model.Album
-	curAArtist model.Artist
 }
 
 func New(db *gorm.DB, musicPath string) *Scanner {
 	return &Scanner{
 		db:         db,
 		musicPath:  musicPath,
-		seenTracks: make(map[string]bool),
+		seenTracks: make(map[int]struct{}),
 		curFolders: make(folderStack, 0),
-		curTracks:  make([]model.Track, 0),
-		curCover:   model.Cover{},
-		curAlbum:   model.Album{},
-		curAArtist: model.Artist{},
 	}
 }
 
+func (s *Scanner) curFolder() *model.Folder {
+	return s.curFolders.Peek()
+}
+
+func (s *Scanner) curFolderID() int {
+	peek := s.curFolders.Peek()
+	if peek == nil {
+		return 0
+	}
+	return peek.ID
+}
+
 func (s *Scanner) Start() error {
 	if atomic.LoadInt32(&IsScanning) == 1 {
 		return errors.New("already scanning")
@@ -86,16 +84,17 @@ func (s *Scanner) startClean() error {
 	defer logElapsed(time.Now(), "cleaning database")
 	var tracks []model.Track
 	s.tx.
-		Select("id, path").
+		Select("id").
 		Find(&tracks)
+	var deleted int
 	for _, track := range tracks {
-		_, ok := s.seenTracks[track.Path]
+		_, ok := s.seenTracks[track.ID]
-		if ok {
+		if !ok {
-			continue
+			s.tx.Delete(&track)
+			deleted++
 		}
-		s.tx.Delete(&track)
-		log.Println("removed track", track.Path)
 	}
+	log.Printf("removed %d tracks\n", deleted)
 	return nil
 }
 
@@ -104,10 +103,8 @@ func (s *Scanner) MigrateDB() error {
 	s.tx = s.db.Begin()
 	defer s.tx.Commit()
 	s.tx.AutoMigrate(
-		model.Album{},
 		model.Artist{},
 		model.Track{},
-		model.Cover{},
 		model.User{},
 		model.Setting{},
 		model.Play{},

scanner/utilities.go

@@ -2,15 +2,12 @@ package scanner
 
 import (
 	"os"
-	"path"
-	"path/filepath"
-	"strings"
 
 	"github.com/dhowden/tag"
 	"github.com/pkg/errors"
 )
 
-var trackExtensions = map[string]string{
+var mimeTypes = map[string]string{
 	"mp3":  "audio/mpeg",
 	"flac": "audio/x-flac",
 	"aac":  "audio/x-aac",
@@ -18,15 +15,6 @@ var trackExtensions = map[string]string{
 	"ogg":  "audio/ogg",
 }
 
-func isTrack(fullPath string) (string, string, bool) {
-	ext := filepath.Ext(fullPath)[1:]
-	mine, ok := trackExtensions[ext]
-	if !ok {
-		return "", "", false
-	}
-	return mine, ext, true
-}
-
 var coverFilenames = map[string]struct{}{
 	"cover.png":   struct{}{},
 	"cover.jpg":   struct{}{},
@@ -42,12 +30,6 @@ var coverFilenames = map[string]struct{}{
 	"front.jpeg":  struct{}{},
 }
 
-func isCover(fullPath string) bool {
-	_, filename := path.Split(fullPath)
-	_, ok := coverFilenames[strings.ToLower(filename)]
-	return ok
-}
-
 func readTags(path string) (tag.Metadata, error) {
 	trackData, err := os.Open(path)
 	if err != nil {

scanner/walk.go

@@ -1,7 +1,6 @@
 package scanner
 
 import (
-	"io/ioutil"
 	"log"
 	"os"
 	"path"
@@ -14,182 +13,146 @@ import (
 	"github.com/sentriz/gonic/model"
 )
 
-type trackItem struct {
-	mime string
-	ext  string
-}
-
 type item struct {
-	path    string
+	//
-	relPath string
+	// common
-	stat    os.FileInfo
+	fullPath  string
-	track   *trackItem
+	relPath   string
+	directory string
+	filename  string
+	stat      os.FileInfo
+	//
+	// track only
+	ext  string
+	mime string
 }
 
-func (s *Scanner) callbackItem(path string, info *godirwalk.Dirent) error {
+func (s *Scanner) callbackItem(fullPath string, info *godirwalk.Dirent) error {
-	stat, err := os.Stat(path)
+	stat, err := os.Stat(fullPath)
 	if err != nil {
 		return errors.Wrap(err, "stating")
 	}
-	relPath, err := filepath.Rel(s.musicPath, path)
+	relPath, err := filepath.Rel(s.musicPath, fullPath)
 	if err != nil {
 		return errors.Wrap(err, "getting relative path")
 	}
+	directory, filename := path.Split(relPath)
 	it := &item{
-		path:    path,
+		fullPath:  fullPath,
-		relPath: relPath,
+		relPath:   relPath,
-		stat:    stat,
+		directory: directory,
+		filename:  filename,
+		stat:      stat,
 	}
 	if info.IsDir() {
 		return s.handleFolder(it)
 	}
-	if isCover(path) {
+	if _, ok := coverFilenames[filename]; ok {
-		return s.handleCover(it)
+		s.curCover = filename
+		return nil
 	}
-	if mime, ext, ok := isTrack(path); ok {
+	ext := path.Ext(filename)[1:]
-		s.seenTracks[relPath] = true
+	if mime, ok := mimeTypes[ext]; ok {
-		it.track = &trackItem{mime: mime, ext: ext}
+		it.ext = ext
+		it.mime = mime
 		return s.handleTrack(it)
 	}
 	return nil
 }
 
-func (s *Scanner) callbackPost(path string, info *godirwalk.Dirent) error {
+func (s *Scanner) callbackPost(fullPath string, info *godirwalk.Dirent) error {
-	// in general in this function - if a model is not nil, then it
-	// has at least been looked up. if it has a id of 0, then it is
-	// a new record and needs to be inserted
-	if s.curCover.IsNew {
-		s.tx.Save(&s.curCover)
-	}
-	if s.curAlbum.IsNew {
-		s.curAlbum.CoverID = s.curCover.ID
-		s.tx.Save(&s.curAlbum)
-	}
 	folder := s.curFolders.Pop()
 	if folder.IsNew {
-		folder.ParentID = s.curFolders.PeekID()
+		folder.ParentID = s.curFolderID()
-		folder.CoverID = s.curCover.ID
+		folder.Cover = s.curCover
-		folder.HasTracks = len(s.curTracks) > 1
 		s.tx.Save(&folder)
 	}
-	for _, t := range s.curTracks {
+	s.curCover = ""
-		t.FolderID = folder.ID
+	log.Printf("processed folder `%s`\n", fullPath)
-		t.AlbumID = s.curAlbum.ID
-		s.tx.Save(&t)
-	}
-	//
-	s.curTracks = make([]model.Track, 0)
-	s.curCover = model.Cover{}
-	s.curAlbum = model.Album{}
-	s.curAArtist = model.Artist{}
-	//
-	log.Printf("processed folder `%s`\n", path)
 	return nil
 }
 
 func (s *Scanner) handleFolder(it *item) error {
-	// TODO:
 	var folder model.Folder
 	err := s.tx.
-		Where("path = ?", it.relPath).
+		Where(model.Folder{
+			LeftPath:  it.directory,
+			RightPath: it.filename,
+		}).
 		First(&folder).
 		Error
 	if !gorm.IsRecordNotFoundError(err) &&
 		it.stat.ModTime().Before(folder.UpdatedAt) {
 		// we found the record but it hasn't changed
-		s.curFolders.Push(folder)
+		s.curFolders.Push(&folder)
 		return nil
 	}
-	folder.Path = it.relPath
+	folder.LeftPath = it.directory
-	folder.Name = it.stat.Name()
+	folder.RightPath = it.filename
 	s.tx.Save(&folder)
 	folder.IsNew = true
-	s.curFolders.Push(folder)
+	s.curFolders.Push(&folder)
-	return nil
-}
-
-func (s *Scanner) handleCover(it *item) error {
-	err := s.tx.
-		Where("path = ?", it.relPath).
-		First(&s.curCover).
-		Error
-	if !gorm.IsRecordNotFoundError(err) &&
-		it.stat.ModTime().Before(s.curCover.UpdatedAt) {
-		// we found the record but it hasn't changed
-		return nil
-	}
-	s.curCover.Path = it.relPath
-	image, err := ioutil.ReadFile(it.path)
-	if err != nil {
-		return errors.Wrap(err, "reading cover")
-	}
-	s.curCover.Image = image
-	s.curCover.IsNew = true
 	return nil
 }
 
 func (s *Scanner) handleTrack(it *item) error {
 	//
 	// set track basics
-	track := model.Track{}
+	var track model.Track
 	err := s.tx.
-		Where("path = ?", it.relPath).
+		Where(model.Track{
+			FolderID: s.curFolderID(),
+			Filename: it.filename,
+			Ext:      it.ext,
+		}).
 		First(&track).
 		Error
 	if !gorm.IsRecordNotFoundError(err) &&
 		it.stat.ModTime().Before(track.UpdatedAt) {
+		s.seenTracks[track.ID] = struct{}{}
 		// we found the record but it hasn't changed
 		return nil
 	}
-	tags, err := readTags(it.path)
+	track.Filename = it.filename
+	track.Ext = it.ext
+	track.ContentType = it.mime
+	track.Size = int(it.stat.Size())
+	track.FolderID = s.curFolderID()
+	track.Duration = -1
+	track.Bitrate = -1
+	tags, err := readTags(it.fullPath)
 	if err != nil {
 		return errors.Wrap(err, "reading tags")
 	}
 	trackNumber, totalTracks := tags.Track()
 	discNumber, totalDiscs := tags.Disc()
-	track.DiscNumber = discNumber
+	track.TagDiscNumber = discNumber
-	track.TotalDiscs = totalDiscs
+	track.TagTotalDiscs = totalDiscs
-	track.TotalTracks = totalTracks
+	track.TagTotalTracks = totalTracks
-	track.TrackNumber = trackNumber
+	track.TagTrackNumber = trackNumber
-	track.Path = it.relPath
+	track.TagTitle = tags.Title()
-	track.Suffix = it.track.ext
+	track.TagTrackArtist = tags.Artist()
-	track.ContentType = it.track.mime
+	track.TagYear = tags.Year()
-	track.Size = int(it.stat.Size())
-	track.Title = tags.Title()
-	track.TrackArtist = tags.Artist()
-	track.Year = tags.Year()
-	track.FolderID = s.curFolders.PeekID()
 	//
 	// set album artist basics
+	var artist model.Artist
 	err = s.tx.Where("name = ?", tags.AlbumArtist()).
-		First(&s.curAArtist).
+		First(&artist).
 		Error
 	if gorm.IsRecordNotFoundError(err) {
-		s.curAArtist.Name = tags.AlbumArtist()
+		artist.Name = tags.AlbumArtist()
-		s.tx.Save(&s.curAArtist)
+		s.tx.Save(&artist)
 	}
-	track.ArtistID = s.curAArtist.ID
+	track.ArtistID = artist.ID
+	s.tx.Save(&track)
+	s.seenTracks[track.ID] = struct{}{}
 	//
 	// set album if this is the first track in the folder
-	if len(s.curTracks) > 0 {
+	if !s.curFolder().IsNew {
-		s.curTracks = append(s.curTracks, track)
 		return nil
 	}
-	s.curTracks = append(s.curTracks, track)
+	s.curFolder().AlbumTitle = tags.Album()
-	//
+	s.curFolder().AlbumYear = tags.Year()
-	directory, _ := path.Split(it.relPath)
+	s.curFolder().AlbumArtistID = artist.ID
-	err = s.tx.
-		Where("path = ?", directory).
-		First(&s.curAlbum).
-		Error
-	if !gorm.IsRecordNotFoundError(err) {
-		// we found the record
-		return nil
-	}
-	s.curAlbum.Path = directory
-	s.curAlbum.Title = tags.Album()
-	s.curAlbum.Year = tags.Year()
-	s.curAlbum.ArtistID = s.curAArtist.ID
-	s.curAlbum.IsNew = true
 	return nil
 }