From 29702efd51128fb80262381844627f4074108476 Mon Sep 17 00:00:00 2001 From: sentriz Date: Fri, 24 May 2019 14:10:05 +0100 Subject: [PATCH] use rel paths --- model/model.go | 18 +- scanner/scanner.go | 287 +++++++++++++----------- scanner/utilities.go | 14 +- server/handler/handler_sub_by_folder.go | 24 +- server/handler/handler_sub_by_tags.go | 28 +-- server/handler/handler_sub_common.go | 8 +- server/handler/middleware_admin.go | 2 +- server/handler/middleware_sub.go | 2 +- 8 files changed, 204 insertions(+), 179 deletions(-) diff --git a/model/model.go b/model/model.go index b5bcca1..b694e48 100644 --- a/model/model.go +++ b/model/model.go @@ -3,9 +3,15 @@ package model import "time" // q: why in tarnation are all the foreign keys pointers to ints? -// // a: so they will be true sqlite null values instead of go zero // values when we save a row without that value +// +// q: what in tarnation are the `IsNew`s for? +// a: it's a bit of a hack - but we set a models IsNew to true if +// we just filled it in for the first time, so when it comes +// time to insert them (post children callback) we can check for +// that bool being true - since it won't be true if it was already +// in the db // Album represents the albums table type Album struct { @@ -90,11 +96,11 @@ type Setting struct { type Play struct { IDBase User User - UserID *int `gorm:"not null;index"` + UserID *int `gorm:"not null;index" sql:"type:int REFERENCES users(id) ON DELETE CASCADE"` Album Album - AlbumID *int `gorm:"not null;index"` + AlbumID *int `gorm:"not null;index" sql:"type:int REFERENCES albums(id) ON DELETE CASCADE"` Folder Folder - FolderID *int `gorm:"not null;index"` + FolderID *int `gorm:"not null;index" sql:"type:int REFERENCES folders(id) ON DELETE CASCADE"` Time time.Time Count int } @@ -106,8 +112,8 @@ type Folder struct { Name string Path string `gorm:"not null;unique_index"` Parent *Folder - ParentID *int - CoverID *int + ParentID *int `sql:"type:int REFERENCES folders(id) ON DELETE CASCADE"` + CoverID *int `sql:"type:int REFERENCES covers(id)"` HasTracks bool `gorm:"not null;index"` Cover Cover IsNew bool `gorm:"-"` diff --git a/scanner/scanner.go b/scanner/scanner.go index f89c56c..a4bdce6 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -14,11 +14,11 @@ package scanner // -> needs a FolderID import ( - "fmt" "io/ioutil" "log" "os" "path" + "path/filepath" "sync/atomic" "time" @@ -33,24 +33,34 @@ var ( IsScanning int32 ) +type trackItem struct { + mime string + ext string +} + +type item struct { + path string + relPath string + stat os.FileInfo + track *trackItem +} + type Scanner struct { - db *gorm.DB - tx *gorm.DB - musicPath string - seenPaths map[string]bool - folderCount uint - curFolders folderStack - curTracks []model.Track - curCover model.Cover - curAlbum model.Album - curAArtist model.AlbumArtist + db, tx *gorm.DB + musicPath string + seenTracks map[string]bool + curFolders folderStack + curTracks []model.Track + curCover model.Cover + curAlbum model.Album + curAArtist model.AlbumArtist } func New(db *gorm.DB, musicPath string) *Scanner { return &Scanner{ db: db, musicPath: musicPath, - seenPaths: make(map[string]bool), + seenTracks: make(map[string]bool), curFolders: make(folderStack, 0), curTracks: make([]model.Track, 0), curCover: model.Cover{}, @@ -59,18 +69,18 @@ func New(db *gorm.DB, musicPath string) *Scanner { } } -func (s *Scanner) handleCover(fullPath string, stat os.FileInfo) error { +func (s *Scanner) handleCover(it *item) error { err := s.tx. - Where("path = ?", fullPath). + Where("path = ?", it.relPath). First(&s.curCover). Error if !gorm.IsRecordNotFoundError(err) && - stat.ModTime().Before(s.curCover.UpdatedAt) { + it.stat.ModTime().Before(s.curCover.UpdatedAt) { // we found the record but it hasn't changed return nil } - s.curCover.Path = fullPath - image, err := ioutil.ReadFile(fullPath) + s.curCover.Path = it.relPath + image, err := ioutil.ReadFile(it.path) if err != nil { return errors.Wrap(err, "reading cover") } @@ -79,28 +89,94 @@ func (s *Scanner) handleCover(fullPath string, stat os.FileInfo) error { return nil } -func (s *Scanner) handleFolder(fullPath string, stat os.FileInfo) error { +func (s *Scanner) handleFolder(it *item) error { // TODO: var folder model.Folder err := s.tx. - Where("path = ?", fullPath). + Where("path = ?", it.relPath). First(&folder). Error if !gorm.IsRecordNotFoundError(err) && - stat.ModTime().Before(folder.UpdatedAt) { + it.stat.ModTime().Before(folder.UpdatedAt) { // we found the record but it hasn't changed s.curFolders.Push(folder) return nil } - folder.Path = fullPath - folder.Name = stat.Name() + folder.Path = it.relPath + folder.Name = it.stat.Name() s.tx.Save(&folder) folder.IsNew = true s.curFolders.Push(folder) return nil } -func (s *Scanner) handleFolderCompletion(fullPath string, info *godirwalk.Dirent) error { +func (s *Scanner) handleTrack(it *item) error { + // + // set track basics + track := model.Track{} + err := s.tx. + Where("path = ?", it.relPath). + First(&track). + Error + if !gorm.IsRecordNotFoundError(err) && + it.stat.ModTime().Before(track.UpdatedAt) { + // we found the record but it hasn't changed + return nil + } + tags, err := readTags(it.path) + if err != nil { + return errors.Wrap(err, "reading tags") + } + trackNumber, totalTracks := tags.Track() + discNumber, totalDiscs := tags.Disc() + track.DiscNumber = discNumber + track.TotalDiscs = totalDiscs + track.TotalTracks = totalTracks + track.TrackNumber = trackNumber + track.Path = it.relPath + track.Suffix = it.track.ext + track.ContentType = it.track.mime + track.Size = int(it.stat.Size()) + track.Title = tags.Title() + track.Artist = tags.Artist() + track.Year = tags.Year() + track.FolderID = s.curFolders.PeekID() + // + // set album artist basics + err = s.tx.Where("name = ?", tags.AlbumArtist()). + First(&s.curAArtist). + Error + if gorm.IsRecordNotFoundError(err) { + s.curAArtist.Name = tags.AlbumArtist() + s.tx.Save(&s.curAArtist) + } + track.AlbumArtistID = s.curAArtist.ID + // + // set album if this is the first track in the folder + if len(s.curTracks) > 0 { + s.curTracks = append(s.curTracks, track) + return nil + } + s.curTracks = append(s.curTracks, track) + // + directory, _ := path.Split(it.relPath) + err = s.tx. + Where("path = ?", directory). + First(&s.curAlbum). + Error + if !gorm.IsRecordNotFoundError(err) { + // we found the record + return nil + } + s.curAlbum.Path = directory + s.curAlbum.Title = tags.Album() + s.curAlbum.Year = tags.Year() + s.curAlbum.AlbumArtistID = s.curAArtist.ID + s.curAlbum.IsNew = true + return nil +} + +func (s *Scanner) handleFolderCompletion(path string, info *godirwalk.Dirent) error { // in general in this function - if a model is not nil, then it // has at least been looked up. if it has a id of 0, then it is // a new record and needs to be inserted @@ -129,91 +205,64 @@ func (s *Scanner) handleFolderCompletion(fullPath string, info *godirwalk.Dirent s.curAlbum = model.Album{} s.curAArtist = model.AlbumArtist{} // - log.Printf("processed folder `%s`\n", fullPath) + log.Printf("processed folder `%s`\n", path) return nil } -func (s *Scanner) handleTrack(fullPath string, stat os.FileInfo, mime, exten string) error { - // - // set track basics - track := model.Track{} - modTime := stat.ModTime() - err := s.tx. - Where("path = ?", fullPath). - First(&track). - Error - if !gorm.IsRecordNotFoundError(err) && - modTime.Before(track.UpdatedAt) { - // we found the record but it hasn't changed - return nil - } - tags, err := readTags(fullPath) +func (s *Scanner) handleItem(path string, info *godirwalk.Dirent) error { + stat, err := os.Stat(path) if err != nil { - return fmt.Errorf("when reading tags: %v", err) + return errors.Wrap(err, "stating") } - trackNumber, totalTracks := tags.Track() - discNumber, totalDiscs := tags.Disc() - track.Path = fullPath - track.Title = tags.Title() - track.Artist = tags.Artist() - track.DiscNumber = discNumber - track.TotalDiscs = totalDiscs - track.TotalTracks = totalTracks - track.TrackNumber = trackNumber - track.Year = tags.Year() - track.Suffix = exten - track.ContentType = mime - track.Size = int(stat.Size()) - track.FolderID = s.curFolders.PeekID() - // - // set album artist basics - err = s.tx.Where("name = ?", tags.AlbumArtist()). - First(&s.curAArtist). - Error - if gorm.IsRecordNotFoundError(err) { - s.curAArtist.Name = tags.AlbumArtist() - s.tx.Save(&s.curAArtist) - } - track.AlbumArtistID = s.curAArtist.ID - // - // set album if this is the first track in the folder - if len(s.curTracks) > 0 { - s.curTracks = append(s.curTracks, track) - return nil - } - s.curTracks = append(s.curTracks, track) - // - directory, _ := path.Split(fullPath) - err = s.tx. - Where("path = ?", directory). - First(&s.curAlbum). - Error - if !gorm.IsRecordNotFoundError(err) { - // we found the record - return nil - } - s.curAlbum.Path = directory - s.curAlbum.Title = tags.Album() - s.curAlbum.Year = tags.Year() - s.curAlbum.AlbumArtistID = s.curAArtist.ID - s.curAlbum.IsNew = true - return nil -} - -func (s *Scanner) handleItem(fullPath string, info *godirwalk.Dirent) error { - s.seenPaths[fullPath] = true - stat, err := os.Stat(fullPath) + relPath, err := filepath.Rel(s.musicPath, path) if err != nil { - return fmt.Errorf("error stating: %v", err) + return errors.Wrap(err, "getting relative path") + } + it := &item{ + path: path, + relPath: relPath, + stat: stat, } if info.IsDir() { - return s.handleFolder(fullPath, stat) + return s.handleFolder(it) } - if isCover(fullPath) { - return s.handleCover(fullPath, stat) + if isCover(path) { + return s.handleCover(it) } - if mime, exten, ok := isAudio(fullPath); ok { - return s.handleTrack(fullPath, stat, mime, exten) + if mime, ext, ok := isTrack(path); ok { + s.seenTracks[relPath] = true + it.track = &trackItem{mime: mime, ext: ext} + return s.handleTrack(it) + } + return nil +} + +func (s *Scanner) startScan() error { + defer logElapsed(time.Now(), "scanning") + err := godirwalk.Walk(s.musicPath, &godirwalk.Options{ + Callback: s.handleItem, + PostChildrenCallback: s.handleFolderCompletion, + Unsorted: true, + }) + if err != nil { + return errors.Wrap(err, "walking filesystem") + } + return nil +} + +func (s *Scanner) startClean() error { + defer logElapsed(time.Now(), "cleaning database") + var tracks []model.Track + s.tx. + Select("id, path"). + Find(&tracks) + for _, track := range tracks { + _, ok := s.seenTracks[track.Path] + if ok { + continue + } + s.tx.Delete(&track) + log.Println("removed track", track.Path) } return nil } @@ -247,46 +296,14 @@ func (s *Scanner) Start() error { } atomic.StoreInt32(&IsScanning, 1) defer atomic.StoreInt32(&IsScanning, 0) - defer logElapsed(time.Now(), "scanning") s.db.Exec("PRAGMA foreign_keys = ON") s.tx = s.db.Begin() defer s.tx.Commit() - // - // start scan logic - err := godirwalk.Walk(s.musicPath, &godirwalk.Options{ - Callback: s.handleItem, - PostChildrenCallback: s.handleFolderCompletion, - Unsorted: true, - }) - if err != nil { - return errors.Wrap(err, "walking filesystem") + if err := s.startScan(); err != nil { + return errors.Wrap(err, "start scan") + } + if err := s.startClean(); err != nil { + return errors.Wrap(err, "start clean") } - //// - //// start cleaning logic - //log.Println("cleaning database") - //var tracks []*model.Track - //s.tx.Select("id, path").Find(&tracks) - //for _, track := range tracks { - // _, ok := s.seenPaths[track.Path] - // if ok { - // continue - // } - // s.tx.Delete(&track) - // log.Println("removed", track.Path) - //} - //// delete albums without tracks - //s.tx.Exec(` - //DELETE FROM albums - //WHERE (SELECT count(id) - //FROM tracks - //WHERE album_id = albums.id) = 0; - //`) - //// delete artists without tracks - //s.tx.Exec(` - //DELETE FROM album_artists - //WHERE (SELECT count(id) - //FROM albums - //WHERE album_artist_id = album_artists.id) = 0; - //`) return nil } diff --git a/scanner/utilities.go b/scanner/utilities.go index e892c87..4318f9b 100644 --- a/scanner/utilities.go +++ b/scanner/utilities.go @@ -12,7 +12,7 @@ import ( "github.com/dhowden/tag" ) -var audioExtensions = map[string]string{ +var trackExtensions = map[string]string{ "mp3": "audio/mpeg", "flac": "audio/x-flac", "aac": "audio/x-aac", @@ -20,13 +20,13 @@ var audioExtensions = map[string]string{ "ogg": "audio/ogg", } -func isAudio(fullPath string) (string, string, bool) { - exten := filepath.Ext(fullPath)[1:] - mine, ok := audioExtensions[exten] +func isTrack(fullPath string) (string, string, bool) { + ext := filepath.Ext(fullPath)[1:] + mine, ok := trackExtensions[ext] if !ok { return "", "", false } - return mine, exten, true + return mine, ext, true } var coverFilenames = map[string]bool{ @@ -50,8 +50,8 @@ func isCover(fullPath string) bool { return ok } -func readTags(fullPath string) (tag.Metadata, error) { - trackData, err := os.Open(fullPath) +func readTags(path string) (tag.Metadata, error) { + trackData, err := os.Open(path) if err != nil { return nil, fmt.Errorf("when tags from disk: %v", err) } diff --git a/server/handler/handler_sub_by_folder.go b/server/handler/handler_sub_by_folder.go index c0c32cb..95a4dce 100644 --- a/server/handler/handler_sub_by_folder.go +++ b/server/handler/handler_sub_by_folder.go @@ -29,7 +29,7 @@ func (c *Controller) GetIndexes(w http.ResponseWriter, r *http.Request) { indexes = append(indexes, index) } index.Artists = append(index.Artists, &subsonic.Artist{ - ID: folder.ID, + ID: *folder.ID, Name: folder.Name, }) } @@ -58,11 +58,11 @@ func (c *Controller) GetMusicDirectory(w http.ResponseWriter, r *http.Request) { Find(&folders) for _, folder := range folders { childrenObj = append(childrenObj, &subsonic.Child{ - Parent: cFolder.ID, - ID: folder.ID, + Parent: *cFolder.ID, + ID: *folder.ID, Title: folder.Name, IsDir: true, - CoverID: folder.CoverID, + CoverID: *folder.CoverID, }) } // @@ -80,14 +80,14 @@ func (c *Controller) GetMusicDirectory(w http.ResponseWriter, r *http.Request) { track.Suffix = "mp3" } childrenObj = append(childrenObj, &subsonic.Child{ - ID: track.ID, + ID: *track.ID, Album: track.Album.Title, Artist: track.Artist, ContentType: track.ContentType, - CoverID: cFolder.CoverID, + CoverID: *cFolder.CoverID, Duration: 0, IsDir: false, - Parent: cFolder.ID, + Parent: *cFolder.ID, Path: track.Path, Size: track.Size, Suffix: track.Suffix, @@ -100,8 +100,8 @@ func (c *Controller) GetMusicDirectory(w http.ResponseWriter, r *http.Request) { // respond section sub := subsonic.NewResponse() sub.Directory = &subsonic.Directory{ - ID: cFolder.ID, - Parent: cFolder.ParentID, + ID: *cFolder.ID, + Parent: *cFolder.ParentID, Name: cFolder.Name, Children: childrenObj, } @@ -162,11 +162,11 @@ func (c *Controller) GetAlbumList(w http.ResponseWriter, r *http.Request) { listObj := []*subsonic.Album{} for _, folder := range folders { listObj = append(listObj, &subsonic.Album{ - ID: folder.ID, + ID: *folder.ID, Title: folder.Name, Album: folder.Name, - CoverID: folder.CoverID, - ParentID: folder.ParentID, + CoverID: *folder.CoverID, + ParentID: *folder.ParentID, IsDir: true, Artist: folder.Parent.Name, }) diff --git a/server/handler/handler_sub_by_tags.go b/server/handler/handler_sub_by_tags.go index 8a5599e..a34ee9e 100644 --- a/server/handler/handler_sub_by_tags.go +++ b/server/handler/handler_sub_by_tags.go @@ -27,7 +27,7 @@ func (c *Controller) GetArtists(w http.ResponseWriter, r *http.Request) { indexes.List = append(indexes.List, index) } index.Artists = append(index.Artists, &subsonic.Artist{ - ID: artist.ID, + ID: *artist.ID, Name: artist.Name, }) } @@ -49,17 +49,17 @@ func (c *Controller) GetArtist(w http.ResponseWriter, r *http.Request) { albumsObj := []*subsonic.Album{} for _, album := range artist.Albums { albumsObj = append(albumsObj, &subsonic.Album{ - ID: album.ID, + ID: *album.ID, Name: album.Title, Created: album.CreatedAt, Artist: artist.Name, - ArtistID: artist.ID, - CoverID: album.CoverID, + ArtistID: *artist.ID, + CoverID: *album.CoverID, }) } sub := subsonic.NewResponse() sub.Artist = &subsonic.Artist{ - ID: artist.ID, + ID: *artist.ID, Name: artist.Name, Albums: albumsObj, } @@ -80,7 +80,7 @@ func (c *Controller) GetAlbum(w http.ResponseWriter, r *http.Request) { tracksObj := []*subsonic.Track{} for _, track := range album.Tracks { tracksObj = append(tracksObj, &subsonic.Track{ - ID: track.ID, + ID: *track.ID, Title: track.Title, Artist: track.Artist, // track artist TrackNo: track.TrackNumber, @@ -90,17 +90,17 @@ func (c *Controller) GetAlbum(w http.ResponseWriter, r *http.Request) { Created: track.CreatedAt, Size: track.Size, Album: album.Title, - AlbumID: album.ID, - ArtistID: album.AlbumArtist.ID, // album artist - CoverID: album.CoverID, + AlbumID: *album.ID, + ArtistID: *album.AlbumArtist.ID, // album artist + CoverID: *album.CoverID, Type: "music", }) } sub := subsonic.NewResponse() sub.Album = &subsonic.Album{ - ID: album.ID, + ID: *album.ID, Name: album.Title, - CoverID: album.CoverID, + CoverID: *album.CoverID, Created: album.CreatedAt, Artist: album.AlbumArtist.Name, Tracks: tracksObj, @@ -164,12 +164,12 @@ func (c *Controller) GetAlbumListTwo(w http.ResponseWriter, r *http.Request) { listObj := []*subsonic.Album{} for _, album := range albums { listObj = append(listObj, &subsonic.Album{ - ID: album.ID, + ID: *album.ID, Name: album.Title, Created: album.CreatedAt, - CoverID: album.CoverID, + CoverID: *album.CoverID, Artist: album.AlbumArtist.Name, - ArtistID: album.AlbumArtist.ID, + ArtistID: *album.AlbumArtist.ID, }) } sub := subsonic.NewResponse() diff --git a/server/handler/handler_sub_common.go b/server/handler/handler_sub_common.go index 99c0a10..763fba1 100644 --- a/server/handler/handler_sub_common.go +++ b/server/handler/handler_sub_common.go @@ -4,6 +4,7 @@ import ( "fmt" "net/http" "os" + "path" "sync/atomic" "time" "unicode" @@ -40,13 +41,14 @@ func (c *Controller) Stream(w http.ResponseWriter, r *http.Request) { respondError(w, r, 70, fmt.Sprintf("media with id `%d` was not found", id)) return } - file, err := os.Open(track.Path) + absPath := path.Join(c.MusicPath, track.Path) + file, err := os.Open(absPath) if err != nil { respondError(w, r, 0, fmt.Sprintf("error while streaming media: %v", err)) return } stat, _ := file.Stat() - http.ServeContent(w, r, track.Path, stat.ModTime(), file) + http.ServeContent(w, r, absPath, stat.ModTime(), file) // // after we've served the file, mark the album as played user := r.Context().Value(contextUserKey).(*model.User) @@ -111,7 +113,7 @@ func (c *Controller) Scrobble(w http.ResponseWriter, r *http.Request) { &track, // clients will provide time in miliseconds, so use that or // instead convert UnixNano to miliseconds - getIntParamOr(r, "time", int(time.Now().UnixNano() / 1e6)), + getIntParamOr(r, "time", int(time.Now().UnixNano()/1e6)), getStrParamOr(r, "submission", "true") != "false", ) if err != nil { diff --git a/server/handler/middleware_admin.go b/server/handler/middleware_admin.go index 4b1fbcd..aebe808 100644 --- a/server/handler/middleware_admin.go +++ b/server/handler/middleware_admin.go @@ -30,7 +30,7 @@ func (c *Controller) WithUserSession(next http.HandlerFunc) http.HandlerFunc { } // take username from sesion and add the user row to the context user := c.GetUserFromName(username) - if user.ID == 0 { + if *user.ID == 0 { // the username in the client's session no longer relates to a // user in the database (maybe the user was deleted) session.Options.MaxAge = -1 diff --git a/server/handler/middleware_sub.go b/server/handler/middleware_sub.go index f579e2b..1bb5ac7 100644 --- a/server/handler/middleware_sub.go +++ b/server/handler/middleware_sub.go @@ -61,7 +61,7 @@ func (c *Controller) WithValidSubsonicArgs(next http.HandlerFunc) http.HandlerFu return } user := c.GetUserFromName(username) - if user.ID == 0 { + if *user.ID == 0 { // the user does not exist respondError(w, r, 40, "invalid username") return