From 92c17a46559357f816c23e0c1a825684b1af164d Mon Sep 17 00:00:00 2001 From: sentriz Date: Tue, 15 Dec 2020 23:26:13 +0000 Subject: [PATCH 01/18] Add inital multiple artist support --- cmd/gonic/main.go | 2 + server/ctrlsubsonic/handlers_by_tags.go | 14 +-- server/ctrlsubsonic/handlers_common.go | 8 +- server/ctrlsubsonic/spec/construct_by_tags.go | 6 +- server/ctrlsubsonic/specid/ids.go | 14 +-- server/db/db.go | 1 + server/db/migrations.go | 46 ++++++---- server/db/model.go | 72 +++++++++++----- server/scanner/scanner.go | 85 ++++++++++++------- server/server.go | 3 +- 10 files changed, 160 insertions(+), 91 deletions(-) diff --git a/cmd/gonic/main.go b/cmd/gonic/main.go index 50cf50d..a4386f8 100644 --- a/cmd/gonic/main.go +++ b/cmd/gonic/main.go @@ -34,6 +34,7 @@ func main() { confScanInterval := set.Int("scan-interval", 0, "interval (in minutes) to automatically scan music (optional)") confJukeboxEnabled := set.Bool("jukebox-enabled", false, "whether the subsonic jukebox api should be enabled (optional)") confProxyPrefix := set.String("proxy-prefix", "", "url path prefix to use if behind proxy. eg '/gonic' (optional)") + confGenreSplit := set.String("genre-split", "\n", "character or string to split genre tag data on (optional)") confShowVersion := set.Bool("version", false, "show gonic version") _ = set.String("config-path", "", "path to config (optional)") @@ -85,6 +86,7 @@ func main() { CachePath: *confCachePath, CoverCachePath: coverCachePath, ProxyPrefix: *confProxyPrefix, + GenreSplit: *confGenreSplit, }) var g run.Group diff --git a/server/ctrlsubsonic/handlers_by_tags.go b/server/ctrlsubsonic/handlers_by_tags.go index 8dfac2f..38bfd3b 100644 --- a/server/ctrlsubsonic/handlers_by_tags.go +++ b/server/ctrlsubsonic/handlers_by_tags.go @@ -82,7 +82,7 @@ func (c *Controller) ServeGetAlbum(r *http.Request) *spec.Response { Select("albums.*, count(tracks.id) child_count, sum(tracks.length) duration"). Joins("LEFT JOIN tracks ON tracks.album_id=albums.id"). Preload("TagArtist"). - Preload("TagGenre"). + Preload("Genres"). Preload("Tracks", func(db *gorm.DB) *gorm.DB { return db.Order("tracks.tag_disc_number, tracks.tag_track_number") }). @@ -123,8 +123,9 @@ func (c *Controller) ServeGetAlbumListTwo(r *http.Request) *spec.Response { params.GetOrInt("toYear", 2200)) q = q.Order("tag_year") case "byGenre": - q = q.Joins("JOIN genres ON albums.tag_genre_id=genres.id AND genres.name=?", - params.GetOr("genre", "Unknown Genre")) + genre, _ := params.Get("genre") + q = q.Joins("JOIN album_genres ON album_genres.album_id=albums.id") + q = q.Joins("JOIN genres ON genres.id=album_genres.genre_id AND genres.name=?", genre) case "frequent": user := r.Context().Value(CtxUser).(*db.User) q = q.Joins("JOIN plays ON albums.id=plays.album_id AND plays.user_id=?", @@ -291,8 +292,8 @@ func (c *Controller) ServeGetGenres(r *http.Request) *spec.Response { var genres []*db.Genre c.DB. Select(`*, - (SELECT count(id) FROM albums WHERE tag_genre_id=genres.id) album_count, - (SELECT count(id) FROM tracks WHERE tag_genre_id=genres.id) track_count`). + (SELECT count(1) FROM album_genres WHERE genre_id=genres.id) album_count, + (SELECT count(1) FROM track_genres WHERE genre_id=genres.id) track_count`). Group("genres.id"). Find(&genres) sub := spec.NewResponse() @@ -316,7 +317,8 @@ func (c *Controller) ServeGetSongsByGenre(r *http.Request) *spec.Response { var tracks []*db.Track c.DB. Joins("JOIN albums ON tracks.album_id=albums.id"). - Joins("JOIN genres ON tracks.tag_genre_id=genres.id AND genres.name=?", genre). + Joins("JOIN track_genres ON track_genres.track_id=tracks.id"). + Joins("JOIN genres ON track_genres.genre_id=genres.id AND genres.name=?", genre). Preload("Album"). Offset(params.GetOrInt("offset", 0)). Limit(params.GetOrInt("count", 10)). diff --git a/server/ctrlsubsonic/handlers_common.go b/server/ctrlsubsonic/handlers_common.go index b795bbb..187c8b8 100644 --- a/server/ctrlsubsonic/handlers_common.go +++ b/server/ctrlsubsonic/handlers_common.go @@ -197,9 +197,9 @@ func (c *Controller) ServeGetRandomSongs(r *http.Request) *spec.Response { params := r.Context().Value(CtxParams).(params.Params) var tracks []*db.Track q := c.DB.DB. - Joins("JOIN albums ON tracks.album_id=albums.id"). Limit(params.GetOrInt("size", 10)). Preload("Album"). + Joins("JOIN albums ON tracks.album_id=albums.id"). Order(gorm.Expr("random()")) if year, err := params.GetInt("fromYear"); err == nil { q = q.Where("albums.tag_year >= ?", year) @@ -208,10 +208,8 @@ func (c *Controller) ServeGetRandomSongs(r *http.Request) *spec.Response { q = q.Where("albums.tag_year <= ?", year) } if genre, err := params.Get("genre"); err == nil { - q = q.Joins( - "JOIN genres ON tracks.tag_genre_id=genres.id AND genres.name=?", - genre, - ) + q = q.Joins("JOIN track_genres ON track_genres.track_id=tracks.id") + q = q.Joins("JOIN genres ON genres.id=track_genres.genre_id AND genres.name=?", genre) } q.Find(&tracks) sub := spec.NewResponse() diff --git a/server/ctrlsubsonic/spec/construct_by_tags.go b/server/ctrlsubsonic/spec/construct_by_tags.go index 179acc1..c997b4c 100644 --- a/server/ctrlsubsonic/spec/construct_by_tags.go +++ b/server/ctrlsubsonic/spec/construct_by_tags.go @@ -2,6 +2,7 @@ package spec import ( "path" + "strings" "go.senan.xyz/gonic/server/db" ) @@ -13,11 +14,9 @@ func NewAlbumByTags(a *db.Album, artist *db.Artist) *Album { Name: a.TagTitle, Year: a.TagYear, TrackCount: a.ChildCount, + Genre: strings.Join(a.GenreStrings(), ", "), Duration: a.Duration, } - if a.TagGenre != nil { - ret.Genre = a.TagGenre.Name - } if a.Cover != "" { ret.CoverID = a.SID() } @@ -47,6 +46,7 @@ func NewTrackByTags(t *db.Track, album *db.Album) *TrackChild { ), Album: album.TagTitle, AlbumID: album.SID(), + Genre: strings.Join(t.GenreStrings(), ", "), Duration: t.Length, Bitrate: t.Bitrate, Type: "music", diff --git a/server/ctrlsubsonic/specid/ids.go b/server/ctrlsubsonic/specid/ids.go index e6cd71b..710187f 100644 --- a/server/ctrlsubsonic/specid/ids.go +++ b/server/ctrlsubsonic/specid/ids.go @@ -42,12 +42,16 @@ func New(in string) (ID, error) { if err != nil { return ID{}, fmt.Errorf("%q: %w", partValue, ErrNotAnInt) } - for _, acc := range []IDT{Artist, Album, Track} { - if partType == string(acc) { - return ID{Type: acc, Value: val}, nil - } + switch IDT(partType) { + case Artist: + return ID{Type: Artist, Value: val}, nil + case Album: + return ID{Type: Album, Value: val}, nil + case Track: + return ID{Type: Track, Value: val}, nil + default: + return ID{}, fmt.Errorf("%q: %w", partType, ErrBadPrefix) } - return ID{}, fmt.Errorf("%q: %w", partType, ErrBadPrefix) } func (i ID) String() string { diff --git a/server/db/db.go b/server/db/db.go index 6945125..41061ef 100644 --- a/server/db/db.go +++ b/server/db/db.go @@ -76,6 +76,7 @@ func New(path string) (*DB, error) { migrateAddGenre(), migrateUpdateTranscodePrefIDX(), migrateAddAlbumIDX(), + migrateMultiGenre(), )) if err = migr.Migrate(); err != nil { return nil, fmt.Errorf("migrating to latest version: %w", err) diff --git a/server/db/migrations.go b/server/db/migrations.go index 609762c..b7518ed 100644 --- a/server/db/migrations.go +++ b/server/db/migrations.go @@ -61,14 +61,14 @@ func migrateMergePlaylist() gormigrate.Migration { return nil } return tx.Exec(` - UPDATE playlists - SET items=( SELECT group_concat(track_id) FROM ( - SELECT track_id - FROM playlist_items - WHERE playlist_items.playlist_id=playlists.id - ORDER BY created_at - ) ); - DROP TABLE playlist_items;`, + UPDATE playlists + SET items=( SELECT group_concat(track_id) FROM ( + SELECT track_id + FROM playlist_items + WHERE playlist_items.playlist_id=playlists.id + ORDER BY created_at + ) ); + DROP TABLE playlist_items;`, ). Error }, @@ -117,8 +117,8 @@ func migrateUpdateTranscodePrefIDX() gormigrate.Migration { return nil } step := tx.Exec(` - ALTER TABLE transcode_preferences RENAME TO transcode_preferences_orig; - `) + ALTER TABLE transcode_preferences RENAME TO transcode_preferences_orig; + `) if err := step.Error; err != nil { return fmt.Errorf("step rename: %w", err) } @@ -129,11 +129,11 @@ func migrateUpdateTranscodePrefIDX() gormigrate.Migration { return fmt.Errorf("step create: %w", err) } step = tx.Exec(` - INSERT INTO transcode_preferences (user_id, client, profile) - SELECT user_id, client, profile - FROM transcode_preferences_orig; - DROP TABLE transcode_preferences_orig; - `) + INSERT INTO transcode_preferences (user_id, client, profile) + SELECT user_id, client, profile + FROM transcode_preferences_orig; + DROP TABLE transcode_preferences_orig; + `) if err := step.Error; err != nil { return fmt.Errorf("step copy: %w", err) } @@ -153,3 +153,19 @@ func migrateAddAlbumIDX() gormigrate.Migration { }, } } + +func migrateMultiGenre() gormigrate.Migration { + return gormigrate.Migration{ + ID: "202012151806", + Migrate: func(tx *gorm.DB) error { + return tx.AutoMigrate( + Track{}, + Album{}, + Genre{}, + TrackGenre{}, + AlbumGenre{}, + ). + Error + }, + } +} diff --git a/server/db/model.go b/server/db/model.go index 09e8d2e..3473b15 100644 --- a/server/db/model.go +++ b/server/db/model.go @@ -61,12 +61,10 @@ func (a *Artist) IndexName() string { } type Genre struct { - ID int `gorm:"primary_key"` - Name string `gorm:"not null; unique_index"` - Albums []*Album `gorm:"foreignkey:TagGenreID"` - AlbumCount int `sql:"-"` - Tracks []*Track `gorm:"foreignkey:TagGenreID"` - TrackCount int `sql:"-"` + ID int `gorm:"primary_key"` + Name string `gorm:"not null; unique_index"` + AlbumCount int `sql:"-"` + TrackCount int `sql:"-"` } type Track struct { @@ -78,18 +76,17 @@ type Track struct { Album *Album AlbumID int `gorm:"not null; unique_index:idx_folder_filename" sql:"default: null; type:int REFERENCES albums(id) ON DELETE CASCADE"` Artist *Artist - ArtistID int `gorm:"not null" sql:"default: null; type:int REFERENCES artists(id) ON DELETE CASCADE"` - Size int `gorm:"not null" sql:"default: null"` - Length int `sql:"default: null"` - Bitrate int `sql:"default: null"` - TagTitle string `sql:"default: null"` - TagTitleUDec string `sql:"default: null"` - TagTrackArtist string `sql:"default: null"` - TagTrackNumber int `sql:"default: null"` - TagDiscNumber int `sql:"default: null"` - TagGenre *Genre - TagGenreID int `sql:"default: null; type:int REFERENCES genres(id)"` - TagBrainzID string `sql:"default: null"` + ArtistID int `gorm:"not null" sql:"default: null; type:int REFERENCES artists(id) ON DELETE CASCADE"` + Genres []*Genre `gorm:"many2many:track_genres"` + Size int `gorm:"not null" sql:"default: null"` + Length int `sql:"default: null"` + Bitrate int `sql:"default: null"` + TagTitle string `sql:"default: null"` + TagTitleUDec string `sql:"default: null"` + TagTrackArtist string `sql:"default: null"` + TagTrackNumber int `sql:"default: null"` + TagDiscNumber int `sql:"default: null"` + TagBrainzID string `sql:"default: null"` } func (t *Track) SID() *specid.ID { @@ -128,6 +125,14 @@ func (t *Track) RelPath() string { ) } +func (t *Track) GenreStrings() []string { + var strs []string + for _, genre := range t.Genres { + strs = append(strs, genre.Name) + } + return strs +} + type User struct { ID int `gorm:"primary_key"` CreatedAt time.Time @@ -160,12 +165,11 @@ type Album struct { RightPath string `gorm:"not null; unique_index:idx_left_path_right_path" sql:"default: null"` RightPathUDec string `sql:"default: null"` Parent *Album - ParentID int `sql:"default: null; type:int REFERENCES albums(id) ON DELETE CASCADE"` - Cover string `sql:"default: null"` + ParentID int `sql:"default: null; type:int REFERENCES albums(id) ON DELETE CASCADE"` + Genres []*Genre `gorm:"many2many:album_genres"` + Cover string `sql:"default: null"` TagArtist *Artist - TagArtistID int `gorm:"index" sql:"default: null; type:int REFERENCES artists(id) ON DELETE CASCADE"` - TagGenre *Genre - TagGenreID int `sql:"default: null; type:int"` + TagArtistID int `gorm:"index" sql:"default: null; type:int REFERENCES artists(id) ON DELETE CASCADE"` TagTitle string `sql:"default: null"` TagTitleUDec string `sql:"default: null"` TagBrainzID string `sql:"default: null"` @@ -192,6 +196,14 @@ func (a *Album) IndexRightPath() string { return a.RightPath } +func (a *Album) GenreStrings() []string { + var strs []string + for _, genre := range a.Genres { + strs = append(strs, genre.Name) + } + return strs +} + type Playlist struct { ID int `gorm:"primary_key"` CreatedAt time.Time @@ -243,3 +255,17 @@ type TranscodePreference struct { Client string `gorm:"not null; unique_index:idx_user_id_client" sql:"default: null"` Profile string `gorm:"not null" sql:"default: null"` } + +type TrackGenre struct { + Track *Track + TrackID int `gorm:"not null; unique_index:idx_track_id_genre_id" sql:"default: null; type:int REFERENCES tracks(id) ON DELETE CASCADE"` + Genre *Genre + GenreID int `gorm:"not null; unique_index:idx_track_id_genre_id" sql:"default: null; type:int REFERENCES genres(id) ON DELETE CASCADE"` +} + +type AlbumGenre struct { + Album *Album + AlbumID int `gorm:"not null; unique_index:idx_album_id_genre_id" sql:"default: null; type:int REFERENCES albums(id) ON DELETE CASCADE"` + Genre *Genre + GenreID int `gorm:"not null; unique_index:idx_album_id_genre_id" sql:"default: null; type:int REFERENCES genres(id) ON DELETE CASCADE"` +} diff --git a/server/scanner/scanner.go b/server/scanner/scanner.go index ba86e9e..c082789 100644 --- a/server/scanner/scanner.go +++ b/server/scanner/scanner.go @@ -58,9 +58,10 @@ func SetScanning() func() { } type Scanner struct { - db *db.DB - musicPath string - isFull bool + db *db.DB + musicPath string + isFull bool + genreSplit string // these two are for the transaction we do for every folder. // the boolean is there so we dont begin or commit multiple // times in the handle folder or post children callback @@ -78,10 +79,11 @@ type Scanner struct { seenTracksNew int // n tracks not seen before } -func New(musicPath string, db *db.DB) *Scanner { +func New(musicPath string, db *db.DB, genreSplit string) *Scanner { return &Scanner{ - db: db, - musicPath: musicPath, + db: db, + musicPath: musicPath, + genreSplit: genreSplit, } } @@ -368,6 +370,7 @@ func (s *Scanner) handleTrack(it *item) error { s.trTx = s.db.Begin() s.trTxOpen = true } + // ** begin set track basics track := &db.Track{} defer func() { @@ -404,16 +407,9 @@ func (s *Scanner) handleTrack(it *item) error { track.TagBrainzID = trTags.BrainzID() track.Length = trTags.Length() // these two should be calculated track.Bitrate = trTags.Bitrate() // ...from the file instead of tags + // ** begin set album artist basics - artistName := func() string { - if r := trTags.AlbumArtist(); r != "" { - return r - } - if r := trTags.Artist(); r != "" { - return r - } - return "Unknown Artist" - }() + artistName := firstTag("Unknown Artist", trTags.AlbumArtist, trTags.Artist) artist := &db.Artist{} err = s.trTx. Select("id"). @@ -428,43 +424,66 @@ func (s *Scanner) handleTrack(it *item) error { } } track.ArtistID = artist.ID + // ** begin set genre - genreName := func() string { - if r := trTags.Genre(); r != "" { - return r - } - return "Unknown Genre" - }() - genre := &db.Genre{} - err = s.trTx. - Select("id"). - Where("name=?", genreName). - First(genre). - Error - if gorm.IsRecordNotFoundError(err) { - genre.Name = genreName - if err := s.trTx.Save(genre).Error; err != nil { - return fmt.Errorf("writing genres table: %w", err) + genreTag := firstTag("Unknown Genre", trTags.Genre) + genres := strings.Split(genreTag, s.genreSplit) + genreIDs := []int{} + for _, genreName := range genres { + // TODO insert or ignore + genre := &db.Genre{} + err = s.trTx. + Select("id"). + Where("name=?", genreName). + First(genre). + Error + if gorm.IsRecordNotFoundError(err) { + genre.Name = genreName + if err := s.trTx.Save(genre).Error; err != nil { + return fmt.Errorf("writing genres table: %w", err) + } } + genreIDs = append(genreIDs, genre.ID) } - track.TagGenreID = genre.ID + // ** begin save the track if err := s.trTx.Save(track).Error; err != nil { return fmt.Errorf("writing track table: %w", err) } + for _, genreID := range genreIDs { + trackGenre := &db.TrackGenre{TrackID: track.ID, GenreID: genreID} + if err := s.trTx.Save(trackGenre).Error; err != nil { + return fmt.Errorf("writing track table: %w", err) + } + } s.seenTracksNew++ + // ** begin set album if this is the first track in the folder folder := s.curFolders.Peek() if !folder.ReceivedPaths || folder.ReceivedTags { // the folder hasn't been modified or already has it's tags return nil } + for _, genreID := range genreIDs { + albumGenre := &db.AlbumGenre{AlbumID: folder.ID, GenreID: genreID} + if err := s.trTx.Save(albumGenre).Error; err != nil { + return fmt.Errorf("writing album table: %w", err) + } + } folder.TagTitle = trTags.Album() folder.TagTitleUDec = decoded(trTags.Album()) folder.TagBrainzID = trTags.AlbumBrainzID() folder.TagYear = trTags.Year() folder.TagArtistID = artist.ID - folder.TagGenreID = genre.ID folder.ReceivedTags = true return nil } + +func firstTag(fallback string, tags ...func() string) string { + for _, f := range tags { + if tag := f(); tag != "" { + return tag + } + } + return fallback +} diff --git a/server/server.go b/server/server.go index e720dce..22840b7 100644 --- a/server/server.go +++ b/server/server.go @@ -26,6 +26,7 @@ type Options struct { CachePath string CoverCachePath string ProxyPrefix string + GenreSplit string } type Server struct { @@ -40,7 +41,7 @@ func New(opts Options) *Server { opts.MusicPath = filepath.Clean(opts.MusicPath) opts.CachePath = filepath.Clean(opts.CachePath) // ** begin controllers - scanner := scanner.New(opts.MusicPath, opts.DB) + scanner := scanner.New(opts.MusicPath, opts.DB, opts.GenreSplit) jukebox := jukebox.New(opts.MusicPath) // the base controller, it's fields/middlewares are embedded/used by the // other two admin ui and subsonic controllers From a2f9deb3cc484dd36a1bba969d8549841d10c0b8 Mon Sep 17 00:00:00 2001 From: sentriz Date: Wed, 16 Dec 2020 21:09:14 +0000 Subject: [PATCH 02/18] add todo --- TODO | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 TODO diff --git a/TODO b/TODO new file mode 100644 index 0000000..9f4d19e --- /dev/null +++ b/TODO @@ -0,0 +1,7 @@ +- gorm v2 + - preload joins + - insert or ignore + - bulk inserts (eg. track_genres, album_genres) +- db package with queries encapsulated +- add newlines +- scanner tests and refactor From edb4d0512434489df08b5b57205f280b21eba48f Mon Sep 17 00:00:00 2001 From: sentriz Date: Wed, 16 Dec 2020 21:09:20 +0000 Subject: [PATCH 03/18] add genre split to readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9680d0b..b4ba970 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,7 @@ $ sudo systemctl enable --now gonic |`GONIC_PROXY_PREFIX`|`-proxy-prefix`|**optional** url path prefix to use if behind reverse proxy. eg `/gonic` (see example configs below)| |`GONIC_SCAN_INTERVAL`|`-scan-interval`|**optional** interval (in minutes) to check for new music (automatic scanning disabled if omitted)| |`GONIC_JUKEBOX_ENABLED`|`-jukebox-enabled`|**optional** whether the subsonic [jukebox api](https://airsonic.github.io/docs/jukebox/) should be enabled| +|`GONIC_GENRE_SPLIT`|`-genre-split`|**optional** a string or character to split genre tags on for multi-genre support (eg. `;`)| ## screenshots From 50a87667ff2e64665a8af6fbbe0b163abcb4311a Mon Sep 17 00:00:00 2001 From: sentriz Date: Wed, 16 Dec 2020 21:28:40 +0000 Subject: [PATCH 04/18] add a migration for multi genres --- server/db/migrations.go | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/server/db/migrations.go b/server/db/migrations.go index b7518ed..3bdd2c2 100644 --- a/server/db/migrations.go +++ b/server/db/migrations.go @@ -158,14 +158,37 @@ func migrateMultiGenre() gormigrate.Migration { return gormigrate.Migration{ ID: "202012151806", Migrate: func(tx *gorm.DB) error { - return tx.AutoMigrate( + step := tx.AutoMigrate( Track{}, Album{}, Genre{}, TrackGenre{}, AlbumGenre{}, - ). - Error + ) + if err := step.Error; err != nil { + return fmt.Errorf("step auto migrate: %w", err) + } + step = tx.Exec(` + INSERT INTO track_genres (track_id, genre_id) + SELECT id, tag_genre_id + FROM tracks + WHERE tag_genre_id IS NOT NULL; + UPDATE tracks SET tag_genre_id=NULL; + `) + if err := step.Error; err != nil { + return fmt.Errorf("step migrate track genres: %w", err) + } + step = tx.Exec(` + INSERT INTO album_genres (album_id, genre_id) + SELECT id, tag_genre_id + FROM albums + WHERE tag_genre_id IS NOT NULL; + UPDATE albums SET tag_genre_id=NULL; + `) + if err := step.Error; err != nil { + return fmt.Errorf("step migrate album genres: %w", err) + } + return nil }, } } From 964d49ef1463f6fc5ded5bd5051e53f39dd33633 Mon Sep 17 00:00:00 2001 From: sentriz Date: Wed, 16 Dec 2020 21:29:17 +0000 Subject: [PATCH 05/18] update todo --- TODO | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TODO b/TODO index 9f4d19e..cc28b88 100644 --- a/TODO +++ b/TODO @@ -1,7 +1,7 @@ - gorm v2 - preload joins - insert or ignore - - bulk inserts (eg. track_genres, album_genres) + - bulk inserts (eg. track_genres, album_genres, playlist stuff?) - db package with queries encapsulated - add newlines - scanner tests and refactor From 9295e603161d366ed669bf691dedd88488cd3e1b Mon Sep 17 00:00:00 2001 From: sentriz Date: Wed, 16 Dec 2020 21:42:47 +0000 Subject: [PATCH 06/18] dont run genre migration if there are no genres --- server/db/migrations.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/server/db/migrations.go b/server/db/migrations.go index 3bdd2c2..4c267c0 100644 --- a/server/db/migrations.go +++ b/server/db/migrations.go @@ -43,6 +43,7 @@ func migrateCreateInitUser() gormigrate.Migration { if !gorm.IsRecordNotFoundError(err) { return nil } + return tx.Create(&User{ Name: initUsername, Password: initPassword, @@ -60,6 +61,7 @@ func migrateMergePlaylist() gormigrate.Migration { if !tx.HasTable("playlist_items") { return nil } + return tx.Exec(` UPDATE playlists SET items=( SELECT group_concat(track_id) FROM ( @@ -116,18 +118,21 @@ func migrateUpdateTranscodePrefIDX() gormigrate.Migration { // index already exists return nil } + step := tx.Exec(` ALTER TABLE transcode_preferences RENAME TO transcode_preferences_orig; `) if err := step.Error; err != nil { return fmt.Errorf("step rename: %w", err) } + step = tx.AutoMigrate( TranscodePreference{}, ) if err := step.Error; err != nil { return fmt.Errorf("step create: %w", err) } + step = tx.Exec(` INSERT INTO transcode_preferences (user_id, client, profile) SELECT user_id, client, profile @@ -165,6 +170,14 @@ func migrateMultiGenre() gormigrate.Migration { TrackGenre{}, AlbumGenre{}, ) + var genreCount int + tx. + Model(Genre{}). + Count(&genreCount) + if genreCount == 0 { + return nil + } + if err := step.Error; err != nil { return fmt.Errorf("step auto migrate: %w", err) } @@ -175,6 +188,7 @@ func migrateMultiGenre() gormigrate.Migration { WHERE tag_genre_id IS NOT NULL; UPDATE tracks SET tag_genre_id=NULL; `) + if err := step.Error; err != nil { return fmt.Errorf("step migrate track genres: %w", err) } From 8c0473e7f6bf7d4aafab557afc08e710fba80d3c Mon Sep 17 00:00:00 2001 From: sentriz Date: Wed, 16 Dec 2020 21:53:38 +0000 Subject: [PATCH 07/18] fix scanner test --- server/scanner/scanner_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/scanner/scanner_test.go b/server/scanner/scanner_test.go index c7c7ad3..64d90af 100644 --- a/server/scanner/scanner_test.go +++ b/server/scanner/scanner_test.go @@ -51,7 +51,7 @@ func TestMain(m *testing.M) { } // benchmarks aren't real code are they? >:) // here is an absolute path to my music directory - testScanner = New("/home/senan/music", db) + testScanner = New("/home/senan/music", db, "\n") log.SetOutput(ioutil.Discard) os.Exit(m.Run()) } From 3e8884450f0089542a22264fdce41fb4249849ed Mon Sep 17 00:00:00 2001 From: sentriz Date: Wed, 30 Dec 2020 14:59:33 +0000 Subject: [PATCH 08/18] add bulk genre insert --- server/db/db.go | 20 ++++++++++++ server/scanner/scanner.go | 64 +++++++++++++++++---------------------- 2 files changed, 48 insertions(+), 36 deletions(-) diff --git a/server/db/db.go b/server/db/db.go index 41061ef..ebf32ae 100644 --- a/server/db/db.go +++ b/server/db/db.go @@ -5,6 +5,7 @@ import ( "log" "net/url" "os" + "strings" "github.com/gorilla/securecookie" "github.com/jinzhu/gorm" @@ -112,6 +113,21 @@ func (db *DB) GetOrCreateKey(key string) string { return value } +func (db *DB) InsertBulkLeftMany(table string, head []string, left int, col []int) error { + var rows []string + var values []interface{} + for _, c := range col { + rows = append(rows, "(?, ?)") + values = append(values, left, c) + } + q := fmt.Sprintf("INSERT OR IGNORE INTO %q (%s) VALUES %s", + table, + strings.Join(head, ", "), + strings.Join(rows, ", "), + ) + return db.Exec(q, values...).Error +} + func (db *DB) GetUserByID(id int) *User { user := &User{} err := db. @@ -136,6 +152,10 @@ func (db *DB) GetUserByName(name string) *User { return user } +func (db *DB) Begin() *DB { + return &DB{DB: db.DB.Begin()} +} + type ChunkFunc func(*gorm.DB, []int64) error func (db *DB) TransactionChunked(data []int64, cb ChunkFunc) error { diff --git a/server/scanner/scanner.go b/server/scanner/scanner.go index c082789..66c553b 100644 --- a/server/scanner/scanner.go +++ b/server/scanner/scanner.go @@ -65,7 +65,7 @@ type Scanner struct { // these two are for the transaction we do for every folder. // the boolean is there so we dont begin or commit multiple // times in the handle folder or post children callback - trTx *gorm.DB + trTx *db.DB trTxOpen bool // these two are for keeping state between noted in the tree. // eg. keep track of a parents folder or the path to a cover @@ -411,38 +411,24 @@ func (s *Scanner) handleTrack(it *item) error { // ** begin set album artist basics artistName := firstTag("Unknown Artist", trTags.AlbumArtist, trTags.Artist) artist := &db.Artist{} - err = s.trTx. - Select("id"). + s.trTx. Where("name=?", artistName). - First(artist). - Error - if gorm.IsRecordNotFoundError(err) { - artist.Name = artistName - artist.NameUDec = decoded(artistName) - if err := s.trTx.Save(artist).Error; err != nil { - return fmt.Errorf("writing artists table: %w", err) - } - } + Assign(db.Artist{ + Name: artistName, + NameUDec: decoded(artistName), + }). + FirstOrCreate(artist) track.ArtistID = artist.ID // ** begin set genre genreTag := firstTag("Unknown Genre", trTags.Genre) - genres := strings.Split(genreTag, s.genreSplit) + genreNames := strings.Split(genreTag, s.genreSplit) genreIDs := []int{} - for _, genreName := range genres { - // TODO insert or ignore + for _, genreName := range genreNames { genre := &db.Genre{} - err = s.trTx. - Select("id"). - Where("name=?", genreName). - First(genre). - Error - if gorm.IsRecordNotFoundError(err) { - genre.Name = genreName - if err := s.trTx.Save(genre).Error; err != nil { - return fmt.Errorf("writing genres table: %w", err) - } - } + s.trTx.FirstOrCreate(genre, db.Genre{ + Name: genreName, + }) genreIDs = append(genreIDs, genre.ID) } @@ -450,11 +436,14 @@ func (s *Scanner) handleTrack(it *item) error { if err := s.trTx.Save(track).Error; err != nil { return fmt.Errorf("writing track table: %w", err) } - for _, genreID := range genreIDs { - trackGenre := &db.TrackGenre{TrackID: track.ID, GenreID: genreID} - if err := s.trTx.Save(trackGenre).Error; err != nil { - return fmt.Errorf("writing track table: %w", err) - } + err = s.trTx.InsertBulkLeftMany( + "track_genres", + []string{"track_id", "genre_id"}, + track.ID, + genreIDs, + ) + if err != nil { + return fmt.Errorf("insert bulk track genres: %w", err) } s.seenTracksNew++ @@ -464,11 +453,14 @@ func (s *Scanner) handleTrack(it *item) error { // the folder hasn't been modified or already has it's tags return nil } - for _, genreID := range genreIDs { - albumGenre := &db.AlbumGenre{AlbumID: folder.ID, GenreID: genreID} - if err := s.trTx.Save(albumGenre).Error; err != nil { - return fmt.Errorf("writing album table: %w", err) - } + err = s.trTx.InsertBulkLeftMany( + "album_genres", + []string{"album_id", "genre_id"}, + folder.ID, + genreIDs, + ) + if err != nil { + return fmt.Errorf("insert bulk album genres: %w", err) } folder.TagTitle = trTags.Album() folder.TagTitleUDec = decoded(trTags.Album()) From 4562089da6e39a1476fb515fd40f7f0445ebb494 Mon Sep 17 00:00:00 2001 From: sentriz Date: Wed, 30 Dec 2020 18:26:46 +0000 Subject: [PATCH 09/18] bump docker version, go versions --- Dockerfile | 32 +++++++++++----------- Dockerfile.dev | 36 ++++++++++++------------- go.mod | 34 +++++++++++------------- go.sum | 72 ++++++++++++++++++++++++++++++-------------------- 4 files changed, 93 insertions(+), 81 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5c4cc1f..7778484 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,11 @@ -FROM golang:1.14-alpine AS builder +FROM golang:1.15-alpine AS builder RUN apk add -U --no-cache \ - build-base \ - ca-certificates \ - git \ - sqlite \ - taglib-dev \ - alsa-lib-dev + build-base \ + ca-certificates \ + git \ + sqlite \ + taglib-dev \ + alsa-lib-dev WORKDIR /src COPY go.mod . COPY go.sum . @@ -13,18 +13,18 @@ RUN go mod download COPY . . RUN ./_do_build_server -FROM alpine:3.9 +FROM alpine:3.12.3 RUN apk add -U --no-cache \ - ffmpeg \ - ca-certificates + ffmpeg \ + ca-certificates COPY --from=builder \ - /usr/lib/libgcc_s.so.1 \ - /usr/lib/libstdc++.so.6 \ - /usr/lib/libtag.so.1 \ - /usr/lib/ + /usr/lib/libgcc_s.so.1 \ + /usr/lib/libstdc++.so.6 \ + /usr/lib/libtag.so.1 \ + /usr/lib/ COPY --from=builder \ - /src/gonic \ - /bin/ + /src/gonic \ + /bin/ VOLUME ["/data", "/music", "/cache"] EXPOSE 80 ENV GONIC_DB_PATH /data/gonic.db diff --git a/Dockerfile.dev b/Dockerfile.dev index fc09439..8d71827 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -1,31 +1,31 @@ # syntax=docker/dockerfile:experimental -FROM golang:1.14-alpine AS builder +FROM golang:1.15-alpine AS builder RUN apk add -U --no-cache \ - build-base \ - ca-certificates \ - git \ - sqlite \ - taglib-dev \ - alsa-lib-dev + build-base \ + ca-certificates \ + git \ + sqlite \ + taglib-dev \ + alsa-lib-dev WORKDIR /src COPY . . RUN --mount=type=cache,target=/go/pkg/mod \ - --mount=type=cache,target=/root/.cache/go-build \ - ./_do_build_server + --mount=type=cache,target=/root/.cache/go-build \ + ./_do_build_server -FROM alpine:3.9 +FROM alpine:3.12.3 RUN apk add -U --no-cache \ - ffmpeg \ - ca-certificates + ffmpeg \ + ca-certificates COPY --from=builder \ - /usr/lib/libgcc_s.so.1 \ - /usr/lib/libstdc++.so.6 \ - /usr/lib/libtag.so.1 \ - /usr/lib/ + /usr/lib/libgcc_s.so.1 \ + /usr/lib/libstdc++.so.6 \ + /usr/lib/libtag.so.1 \ + /usr/lib/ COPY --from=builder \ - /src/gonic \ - /bin/ + /src/gonic \ + /bin/ VOLUME ["/data", "/music", "/cache"] EXPOSE 80 ENV GONIC_DB_PATH /data/gonic.db diff --git a/go.mod b/go.mod index 21cc728..bd355d2 100644 --- a/go.mod +++ b/go.mod @@ -8,20 +8,21 @@ require ( github.com/disintegration/imaging v1.6.2 github.com/dustin/go-humanize v1.0.0 github.com/faiface/beep v1.0.2 - github.com/google/uuid v1.1.1 // indirect - github.com/gorilla/mux v1.7.4 + github.com/google/uuid v1.1.2 // indirect + github.com/gorilla/mux v1.8.0 github.com/gorilla/securecookie v1.1.1 - github.com/gorilla/sessions v1.2.0 - github.com/hajimehoshi/go-mp3 v0.2.1 // indirect - github.com/hajimehoshi/oto v0.6.1 // indirect - github.com/huandu/xstrings v1.3.1 // indirect - github.com/imdario/mergo v0.3.9 // indirect - github.com/jinzhu/gorm v1.9.12 + github.com/gorilla/sessions v1.2.1 + github.com/hajimehoshi/go-mp3 v0.3.1 // indirect + github.com/hajimehoshi/oto v0.7.0 // indirect + github.com/huandu/xstrings v1.3.2 // indirect + github.com/imdario/mergo v0.3.11 // indirect + github.com/jinzhu/gorm v1.9.16 github.com/josephburnett/jd v0.0.0-20191228205456-aa1a7c66b42f - github.com/karrick/godirwalk v1.15.6 + github.com/karrick/godirwalk v1.16.1 github.com/kr/pretty v0.1.0 // indirect github.com/mewkiz/flac v1.0.6 // indirect - github.com/mewkiz/pkg v0.0.0-20200411195739-f6b5e26764c3 // indirect + github.com/mewkiz/pkg v0.0.0-20200702171441-dd47075182ea // indirect + github.com/mattn/go-sqlite3 v1.14.6 // indirect github.com/mitchellh/copystructure v1.0.0 // indirect github.com/mitchellh/reflectwalk v1.0.1 // indirect github.com/nicksellen/audiotags v0.0.0-20160226222119-94015fa599bd @@ -31,15 +32,12 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be github.com/wader/gormstore v0.0.0-20200328121358-65a111a20c23 - golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 // indirect - golang.org/x/exp v0.0.0-20200513190911-00229845015e // indirect - golang.org/x/image v0.0.0-20200430140353-33d19683fad8 // indirect - golang.org/x/mobile v0.0.0-20200329125638-4c31acba0007 // indirect - golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 // indirect + golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad // indirect + golang.org/x/exp v0.0.0-20201229011636-eab1b5eb1a03 // indirect + golang.org/x/image v0.0.0-20201208152932-35266b937fa6 // indirect + golang.org/x/sys v0.0.0-20201223074533-0d417f636930 // indirect gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/gormigrate.v1 v1.6.0 ) -replace github.com/golang/lint => golang.org/x/lint v0.0.0-20190409202823-959b441ac422 - -go 1.14 +go 1.15 diff --git a/go.sum b/go.sum index 5341869..349c403 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ cloud.google.com/go v0.33.1/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -11,6 +11,8 @@ github.com/Masterminds/sprig v2.22.0+incompatible h1:z4yfnGrZ7netVz+0EDJ0Wi+5VZC github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= +github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/d4l3k/messagediff v1.2.2-0.20190829033028-7e0a312ae40b/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkEQxENCrlLo= @@ -44,8 +46,8 @@ github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2V github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20180628210949-0892b62f0d9f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c h1:16eHWuMGvCjSfgRJKqIzapE78onvvTbdi1rMkU00lZw= github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -54,35 +56,39 @@ github.com/gopherjs/gopherwasm v1.0.0 h1:32nge/RlujS1Im4HNCJPp0NbBOAeBXFuT1KonUu github.com/gopherjs/gopherwasm v1.0.0/go.mod h1:SkZ8z7CWBz5VXbhJel8TxCmAcsQqzgWGR/8nMhyhZSI= github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= -github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= -github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.0 h1:S7P+1Hm5V/AT9cjEcUD5uDaQSX0OE577aCXgoaKpYbQ= github.com/gorilla/sessions v1.2.0/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= +github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7FsgI= +github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/hajimehoshi/go-mp3 v0.1.1 h1:Y33fAdTma70fkrxnc9u50Uq0lV6eZ+bkAlssdMmCwUc= github.com/hajimehoshi/go-mp3 v0.1.1/go.mod h1:4i+c5pDNKDrxl1iu9iG90/+fhP37lio6gNhjCx9WBJw= -github.com/hajimehoshi/go-mp3 v0.2.1 h1:DH4ns3cPv39n3cs8MPcAlWqPeAwLCK8iNgqvg0QBWI8= -github.com/hajimehoshi/go-mp3 v0.2.1/go.mod h1:Rr+2P46iH6PwTPVgSsEwBkon0CK5DxCAeX/Rp65DCTE= +github.com/hajimehoshi/go-mp3 v0.3.1 h1:pn/SKU1+/rfK8KaZXdGEC2G/KCB2aLRjbTCrwKcokao= +github.com/hajimehoshi/go-mp3 v0.3.1/go.mod h1:qMJj/CSDxx6CGHiZeCgbiq2DSUkbK0UbtXShQcnfyMM= github.com/hajimehoshi/oto v0.1.1/go.mod h1:hUiLWeBQnbDu4pZsAhOnGqMI1ZGibS6e2qhQdfpwz04= github.com/hajimehoshi/oto v0.3.1 h1:cpf/uIv4Q0oc5uf9loQn7PIehv+mZerh+0KKma6gzMk= github.com/hajimehoshi/oto v0.3.1/go.mod h1:e9eTLBB9iZto045HLbzfHJIc+jP3xaKrjZTghvb6fdM= -github.com/hajimehoshi/oto v0.3.4/go.mod h1:PgjqsBJff0efqL2nlMJidJgVJywLn6M4y8PI4TfeWfA= github.com/hajimehoshi/oto v0.6.1 h1:7cJz/zRQV4aJvMSSRqzN2TImoVVMpE0BCY4nrNJaDOM= github.com/hajimehoshi/oto v0.6.1/go.mod h1:0QXGEkbuJRohbJaxr7ZQSxnju7hEhseiPx2hrh6raOI= -github.com/huandu/xstrings v1.3.1 h1:4jgBlKK6tLKFvO8u5pmYjG91cqytmDCDvGh7ECVFfFs= -github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/hajimehoshi/oto v0.7.0 h1:4HbTRhNuHd4SdFfA4vhIgmwvVO3qWueHK+fF1cButpg= +github.com/hajimehoshi/oto v0.7.0/go.mod h1:wovJ8WWMfFKvP587mhHgot/MBr4DnNy9m6EepeVGnos= +github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw= +github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/icza/bitio v1.0.0 h1:squ/m1SHyFeCA6+6Gyol1AxV9nmPPlJFT8c2vKdj3U8= github.com/icza/bitio v1.0.0/go.mod h1:0jGnlLAx8MKMr9VGnn/4YrvZiprkvBelsVIbA9Jjr9A= github.com/icza/mighty v0.0.0-20180919140131-cfd07d671de6 h1:8UsGZ2rr2ksmEru6lToqnXgA8Mz1DP11X4zSJ159C3k= github.com/icza/mighty v0.0.0-20180919140131-cfd07d671de6/go.mod h1:xQig96I1VNBDIWGCdTt54nHt6EeI639SmHycLYL7FkA= -github.com/imdario/mergo v0.3.9 h1:UauaLniWCFHWd+Jp9oCEkTBj8VO/9DKg3PV3VCNMDIg= -github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA= +github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/jfreymuth/oggvorbis v1.0.0/go.mod h1:abe6F9QRjuU9l+2jek3gj46lu40N4qlYxh2grqkLEDM= github.com/jfreymuth/vorbis v1.0.0/go.mod h1:8zy3lUAm9K/rJJk223RKy6vjCZTWC61NA2QD06bfOE0= github.com/jinzhu/gorm v1.9.2/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo= -github.com/jinzhu/gorm v1.9.12 h1:Drgk1clyWT9t9ERbzHza6Mj/8FY/CqMyVzOiHviMo6Q= github.com/jinzhu/gorm v1.9.12/go.mod h1:vhTjlKSJUTWNtcbQtrMBFCxy7eXTzeCAzfL5fBZT/Qs= +github.com/jinzhu/gorm v1.9.16 h1:+IyIjPEABKRpsu/F8OvDPy9fyQlgsg2luMV2ZIH5i5o= +github.com/jinzhu/gorm v1.9.16/go.mod h1:G3LB3wezTOWM2ITLzPxEXgSkOXAntiLHS7UdBefADcs= github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= @@ -93,8 +99,8 @@ github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/josephburnett/jd v0.0.0-20191228205456-aa1a7c66b42f h1:ijUonnyvDekPD7lUF4oQ1LV+dKaTnchEzmenMFa6NL4= github.com/josephburnett/jd v0.0.0-20191228205456-aa1a7c66b42f/go.mod h1:aeV+6oc13ogwzcRNHBe4vbyLmoQxMfEDoqyqCU9oE30= -github.com/karrick/godirwalk v1.15.6 h1:Yf2mmR8TJy+8Fa0SuQVto5SYap6IF7lNVX4Jdl8G1qA= -github.com/karrick/godirwalk v1.15.6/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= +github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= +github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -108,6 +114,7 @@ github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lucasb-eyer/go-colorful v0.0.0-20181028223441-12d3b2882a08/go.mod h1:NXg0ArsFk0Y01623LgUqoqcouGDB+PwCCQlrwrG6xJ4= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= @@ -117,8 +124,8 @@ github.com/mewkiz/flac v1.0.6 h1:OnMwCWZPAnjDndjEzLynOZ71Y2U+/QYHoVI4JEKgKkk= github.com/mewkiz/flac v1.0.6/go.mod h1:yU74UH277dBUpqxPouHSQIar3G1X/QIclVbFahSd1pU= github.com/mewkiz/pkg v0.0.0-20190919212034-518ade7978e2 h1:EyTNMdePWaoWsRSGQnXiSoQu0r6RS1eA557AwJhlzHU= github.com/mewkiz/pkg v0.0.0-20190919212034-518ade7978e2/go.mod h1:3E2FUC/qYUfM8+r9zAwpeHJzqRVVMIYnpzD/clwWxyA= -github.com/mewkiz/pkg v0.0.0-20200411195739-f6b5e26764c3 h1:gVKRsSn2rXZ29y56Xrl+vtw8mpowYz9UmoPJM+V/g5s= -github.com/mewkiz/pkg v0.0.0-20200411195739-f6b5e26764c3/go.mod h1:3E2FUC/qYUfM8+r9zAwpeHJzqRVVMIYnpzD/clwWxyA= +github.com/mewkiz/pkg v0.0.0-20200702171441-dd47075182ea h1:LUvH2BZ4TRTtGAm7IPkIgdZ7rOobER2WQ/h5VUIwm9c= +github.com/mewkiz/pkg v0.0.0-20200702171441-dd47075182ea/go.mod h1:3E2FUC/qYUfM8+r9zAwpeHJzqRVVMIYnpzD/clwWxyA= github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= @@ -157,14 +164,14 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd h1:GGJVjV8waZKRHrgwvtH66z9ZGVurTD1MT0n1Bb+q4aM= golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 h1:cg5LA/zNPRzIXIWSCxQW10Rvpy94aQh3LT/ShoCpkHw= -golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad h1:DN0cp81fZ3njFcrLCytUHRSUkqBjfTo4Tx9RJTWs0EY= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/exp v0.0.0-20180710024300-14dda7b62fcd/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8 h1:idBdZTd9UioThJp8KpM/rTSinK/ChZFBE43/WtIy8zg= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= -golang.org/x/exp v0.0.0-20200513190911-00229845015e h1:rMqLP+9XLy+LdbCXHjJHAmTfXCr93W7oruWA6Hq1Alc= -golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= +golang.org/x/exp v0.0.0-20201229011636-eab1b5eb1a03 h1:XlAInxBYX5nBofPaY51uv/x9xmRgZGr/lDOsePd2AcE= +golang.org/x/exp v0.0.0-20201229011636-eab1b5eb1a03/go.mod h1:I6l2HNBLBZEcrOoCpyKLdY2lHoRZ8lI4x60KMCQDft4= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81 h1:00VmoueYNlNz/aHIilyyQz/MHSqGoWJzpFv/HW8xpzI= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190220214146-31aff87c08e9/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= @@ -172,26 +179,28 @@ golang.org/x/image v0.0.0-20190227222117-0694c2d4d067 h1:KYGJGHOQy8oSi1fDlSpcZF0 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.0.0-20200430140353-33d19683fad8 h1:6WW6V3x1P/jokJBpRQYUJnMHRP6isStQwCozxnU7XQw= -golang.org/x/image v0.0.0-20200430140353-33d19683fad8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.0.0-20201208152932-35266b937fa6 h1:nfeHNc1nAqecKCy2FCy4HY+soOOe5sDLJ/gZLbx6GYI= +golang.org/x/image v0.0.0-20201208152932-35266b937fa6/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/mobile v0.0.0-20180806140643-507816974b79 h1:t2JRgCWkY7Qaa1J2jal+wqC9OjbyHCHwIA9rVlRUSMo= golang.org/x/mobile v0.0.0-20180806140643-507816974b79/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190415191353-3e0bab5405d6 h1:vyLBGJPIl9ZYbcQFM2USFmJBK6KI+t+z6jL0lbwjrnc= golang.org/x/mobile v0.0.0-20190415191353-3e0bab5405d6/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mobile v0.0.0-20200329125638-4c31acba0007 h1:JxsyO7zPDWn1rBZW8FV5RFwCKqYeXnyaS/VQPLpXu6I= -golang.org/x/mobile v0.0.0-20200329125638-4c31acba0007/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4= +golang.org/x/mobile v0.0.0-20201217150744-e6ae53a27f4f h1:kgfVkAEEQXXQ0qc6dH7n6y37NAYmTFmz0YRwrRjgxKw= +golang.org/x/mobile v0.0.0-20201217150744-e6ae53a27f4f/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -201,8 +210,11 @@ golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201223074533-0d417f636930 h1:vRgIt+nup/B/BwIS0g2oC0haq0iqbV3ZA+u6+0TlNCo= +golang.org/x/sys v0.0.0-20201223074533-0d417f636930/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -221,3 +233,5 @@ gopkg.in/gormigrate.v1 v1.6.0 h1:XpYM6RHQPmzwY7Uyu+t+xxMXc86JYFJn4nEc9HzQjsI= gopkg.in/gormigrate.v1 v1.6.0/go.mod h1:Lf00lQrHqfSYWiTtPcyQabsDdM6ejZaMgV0OU6JMSlw= gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From 6a159555131546757977a6daf1e2fbf4ac65dada Mon Sep 17 00:00:00 2001 From: sentriz Date: Wed, 30 Dec 2020 18:27:02 +0000 Subject: [PATCH 10/18] add multigenre docs --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index b4ba970..084be93 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ - multiple users, each with their own transcoding preferences, playlists, top tracks, top artists, etc. - [last.fm](https://www.last.fm/) scrobbling - artist similarities and biographies from the last.fm api +- multiple genre support (see `GONIC_GENRE_SPLIT` to split tag strings on a character, eg. `;`, and browse them individually) - a web interface for configuration (set up last.fm, manage users, start scans, etc.) - support for the [album-artist](https://mkoby.com/2007/02/18/artist-versus-album-artist/) tag, to not clutter your artist list with compilation album appearances - written in [go](https://golang.org/), so lightweight and suitable for a raspberry pi, etc. From 8492561d2b209685011348feccc8733d110c8e29 Mon Sep 17 00:00:00 2001 From: sentriz Date: Wed, 30 Dec 2020 18:27:13 +0000 Subject: [PATCH 11/18] add genre clean --- server/db/migrations.go | 14 +++++++++----- server/db/model.go | 4 ++-- server/scanner/scanner.go | 23 +++++++++++++++++++++++ 3 files changed, 34 insertions(+), 7 deletions(-) diff --git a/server/db/migrations.go b/server/db/migrations.go index 4c267c0..1f2f680 100644 --- a/server/db/migrations.go +++ b/server/db/migrations.go @@ -14,8 +14,11 @@ func migrateInitSchema() gormigrate.Migration { ID: "202002192100", Migrate: func(tx *gorm.DB) error { return tx.AutoMigrate( - Artist{}, + Genre{}, + TrackGenre{}, + AlbumGenre{}, Track{}, + Artist{}, User{}, Setting{}, Play{}, @@ -170,6 +173,10 @@ func migrateMultiGenre() gormigrate.Migration { TrackGenre{}, AlbumGenre{}, ) + if err := step.Error; err != nil { + return fmt.Errorf("step auto migrate: %w", err) + } + var genreCount int tx. Model(Genre{}). @@ -178,9 +185,6 @@ func migrateMultiGenre() gormigrate.Migration { return nil } - if err := step.Error; err != nil { - return fmt.Errorf("step auto migrate: %w", err) - } step = tx.Exec(` INSERT INTO track_genres (track_id, genre_id) SELECT id, tag_genre_id @@ -188,10 +192,10 @@ func migrateMultiGenre() gormigrate.Migration { WHERE tag_genre_id IS NOT NULL; UPDATE tracks SET tag_genre_id=NULL; `) - if err := step.Error; err != nil { return fmt.Errorf("step migrate track genres: %w", err) } + step = tx.Exec(` INSERT INTO album_genres (album_id, genre_id) SELECT id, tag_genre_id diff --git a/server/db/model.go b/server/db/model.go index 3473b15..3948dbc 100644 --- a/server/db/model.go +++ b/server/db/model.go @@ -126,7 +126,7 @@ func (t *Track) RelPath() string { } func (t *Track) GenreStrings() []string { - var strs []string + strs := make([]string, 0, len(t.Genres)) for _, genre := range t.Genres { strs = append(strs, genre.Name) } @@ -197,7 +197,7 @@ func (a *Album) IndexRightPath() string { } func (a *Album) GenreStrings() []string { - var strs []string + strs := make([]string, 0, len(a.Genres)) for _, genre := range a.Genres { strs = append(strs, genre.Name) } diff --git a/server/scanner/scanner.go b/server/scanner/scanner.go index 66c553b..7bb2d59 100644 --- a/server/scanner/scanner.go +++ b/server/scanner/scanner.go @@ -145,6 +145,28 @@ func (s *Scanner) cleanArtists() (int, error) { return int(q.RowsAffected), q.Error } +func (s *Scanner) cleanGenres() (int, error) { + subTrack := s.db. + Select("genres.id"). + Model(&db.Genre{}). + Joins("JOIN track_genres ON track_genres.genre_id=genres.id"). + Joins("LEFT JOIN tracks ON tracks.id=track_genres.track_id"). + Where("tracks.id IS NULL"). + SubQuery() + subAlbum := s.db. + Select("genres.id"). + Model(&db.Genre{}). + Joins("JOIN album_genres ON album_genres.genre_id=genres.id"). + Joins("LEFT JOIN albums ON albums.id=album_genres.album_id"). + Where("albums.id IS NULL"). + SubQuery() + q := s.db. + Where("genres.id IN ?", subTrack). + Or("genres.id IN ?", subAlbum). + Delete(&db.Genre{}) + return int(q.RowsAffected), q.Error +} + // ## begin entries // ## begin entries // ## begin entries @@ -199,6 +221,7 @@ func (s *Scanner) Start(opts ScanOptions) error { {name: "tracks", f: s.cleanTracks}, {name: "folders", f: s.cleanFolders}, {name: "artists", f: s.cleanArtists}, + {name: "genres", f: s.cleanGenres}, } for _, clean := range cleanFuncs { start = time.Now() From a37993e24f1e3cb5a36be91e6a2f732e8c1945a9 Mon Sep 17 00:00:00 2001 From: sentriz Date: Wed, 30 Dec 2020 20:59:31 +0000 Subject: [PATCH 12/18] delete old track_genre and album_genre tags while scanning --- go.mod | 1 - go.sum | 1 + server/scanner/scanner.go | 14 ++++++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index bd355d2..8b807b3 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,6 @@ require ( github.com/kr/pretty v0.1.0 // indirect github.com/mewkiz/flac v1.0.6 // indirect github.com/mewkiz/pkg v0.0.0-20200702171441-dd47075182ea // indirect - github.com/mattn/go-sqlite3 v1.14.6 // indirect github.com/mitchellh/copystructure v1.0.0 // indirect github.com/mitchellh/reflectwalk v1.0.1 // indirect github.com/nicksellen/audiotags v0.0.0-20160226222119-94015fa599bd diff --git a/go.sum b/go.sum index 349c403..2a2f46b 100644 --- a/go.sum +++ b/go.sum @@ -115,6 +115,7 @@ github.com/lucasb-eyer/go-colorful v0.0.0-20181028223441-12d3b2882a08/go.mod h1: github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= +github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= diff --git a/server/scanner/scanner.go b/server/scanner/scanner.go index 7bb2d59..55f20c9 100644 --- a/server/scanner/scanner.go +++ b/server/scanner/scanner.go @@ -459,6 +459,13 @@ func (s *Scanner) handleTrack(it *item) error { if err := s.trTx.Save(track).Error; err != nil { return fmt.Errorf("writing track table: %w", err) } + err = s.trTx. + Where("track_id=?", track.ID). + Delete(db.TrackGenre{}). + Error + if err != nil { + return fmt.Errorf("delete old track genre records: %w", err) + } err = s.trTx.InsertBulkLeftMany( "track_genres", []string{"track_id", "genre_id"}, @@ -476,6 +483,13 @@ func (s *Scanner) handleTrack(it *item) error { // the folder hasn't been modified or already has it's tags return nil } + err = s.trTx. + Where("album_id=?", folder.ID). + Delete(db.AlbumGenre{}). + Error + if err != nil { + return fmt.Errorf("delete old album genre records: %w", err) + } err = s.trTx.InsertBulkLeftMany( "album_genres", []string{"album_id", "genre_id"}, From b1543fc5248f2eaf66e8c043ccd91c591e823028 Mon Sep 17 00:00:00 2001 From: sentriz Date: Wed, 30 Dec 2020 21:51:59 +0000 Subject: [PATCH 13/18] album update after tag changes with no folder update --- server/db/model.go | 3 +-- server/scanner/scanner.go | 8 +++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/server/db/model.go b/server/db/model.go index 3948dbc..92b92fc 100644 --- a/server/db/model.go +++ b/server/db/model.go @@ -177,8 +177,7 @@ type Album struct { Tracks []*Track ChildCount int `sql:"-"` Duration int `sql:"-"` - ReceivedPaths bool `gorm:"-"` - ReceivedTags bool `gorm:"-"` + ShouldSave bool `sql:"-"` } func (a *Album) SID() *specid.ID { diff --git a/server/scanner/scanner.go b/server/scanner/scanner.go index 55f20c9..c0a21e2 100644 --- a/server/scanner/scanner.go +++ b/server/scanner/scanner.go @@ -325,7 +325,7 @@ func (s *Scanner) callbackPost(fullPath string, info *godirwalk.Dirent) error { // begin taking the current folder off the stack and add it's // parent, cover that we found, etc. folder := s.curFolders.Pop() - if !folder.ReceivedPaths { + if !folder.ShouldSave { return nil } folder.ParentID = s.curFolders.PeekID() @@ -384,7 +384,6 @@ func (s *Scanner) handleFolder(it *item) error { if err := s.db.Save(folder).Error; err != nil { return fmt.Errorf("writing albums table: %w", err) } - folder.ReceivedPaths = true return nil } @@ -479,8 +478,7 @@ func (s *Scanner) handleTrack(it *item) error { // ** begin set album if this is the first track in the folder folder := s.curFolders.Peek() - if !folder.ReceivedPaths || folder.ReceivedTags { - // the folder hasn't been modified or already has it's tags + if folder.ShouldSave { return nil } err = s.trTx. @@ -504,7 +502,7 @@ func (s *Scanner) handleTrack(it *item) error { folder.TagBrainzID = trTags.AlbumBrainzID() folder.TagYear = trTags.Year() folder.TagArtistID = artist.ID - folder.ReceivedTags = true + folder.ShouldSave = true return nil } From 915cc9085381b2acd66d148f29b5283170cb9bce Mon Sep 17 00:00:00 2001 From: sentriz Date: Thu, 31 Dec 2020 00:14:27 +0000 Subject: [PATCH 14/18] make log quoting consistent --- server/ctrlsubsonic/handlers_raw.go | 6 +++--- server/scanner/scanner.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/server/ctrlsubsonic/handlers_raw.go b/server/ctrlsubsonic/handlers_raw.go index 88bc78e..42d2e9e 100644 --- a/server/ctrlsubsonic/handlers_raw.go +++ b/server/ctrlsubsonic/handlers_raw.go @@ -120,7 +120,7 @@ func (c *Controller) ServeGetCoverArt(w http.ResponseWriter, r *http.Request) *s case os.IsNotExist(err): coverPath, err := coverGetPath(c.DB, c.MusicPath, id.Value) if err != nil { - return spec.NewError(10, "couldn't find cover %q: %v", id, err) + return spec.NewError(10, "couldn't find cover `%s`: %v", id, err) } if err := coverScaleAndSave(coverPath, cachePath, size); err != nil { log.Printf("error scaling cover: %v", err) @@ -150,7 +150,7 @@ func (c *Controller) ServeStream(w http.ResponseWriter, r *http.Request) *spec.R trackPath := path.Join(c.MusicPath, track.RelPath()) // onInvalidProfile := func() error { - log.Printf("serving raw %q\n", track.Filename) + log.Printf("serving raw `%s`\n", track.Filename) w.Header().Set("Content-Type", track.MIME()) http.ServeFile(w, r, trackPath) return nil @@ -192,7 +192,7 @@ func (c *Controller) ServeDownload(w http.ResponseWriter, r *http.Request) *spec if err != nil { return spec.NewError(70, "media with id `%s` was not found", id) } - log.Printf("serving raw %q\n", track.Filename) + log.Printf("serving raw `%s`\n", track.Filename) w.Header().Set("Content-Type", track.MIME()) trackPath := path.Join(c.MusicPath, track.RelPath()) http.ServeFile(w, r, trackPath) diff --git a/server/scanner/scanner.go b/server/scanner/scanner.go index c0a21e2..25dbcf2 100644 --- a/server/scanner/scanner.go +++ b/server/scanner/scanner.go @@ -199,7 +199,7 @@ func (s *Scanner) Start(opts ScanOptions) error { Unsorted: true, FollowSymbolicLinks: true, ErrorCallback: func(path string, err error) godirwalk.ErrorAction { - log.Printf("error processing %q: %v", path, err) + log.Printf("error processing `%s`: %v", path, err) errCount++ return godirwalk.SkipNode }, From e5b2b3a681e759b0336b2958cabeb54f9053a3ea Mon Sep 17 00:00:00 2001 From: sentriz Date: Thu, 31 Dec 2020 00:24:02 +0000 Subject: [PATCH 15/18] clean genres by checking track_genres->genres null --- server/scanner/scanner.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/server/scanner/scanner.go b/server/scanner/scanner.go index 25dbcf2..68c75ff 100644 --- a/server/scanner/scanner.go +++ b/server/scanner/scanner.go @@ -149,16 +149,14 @@ func (s *Scanner) cleanGenres() (int, error) { subTrack := s.db. Select("genres.id"). Model(&db.Genre{}). - Joins("JOIN track_genres ON track_genres.genre_id=genres.id"). - Joins("LEFT JOIN tracks ON tracks.id=track_genres.track_id"). - Where("tracks.id IS NULL"). + Joins("LEFT JOIN track_genres ON track_genres.genre_id=genres.id"). + Where("track_genres.genre_id IS NULL"). SubQuery() subAlbum := s.db. Select("genres.id"). Model(&db.Genre{}). - Joins("JOIN album_genres ON album_genres.genre_id=genres.id"). - Joins("LEFT JOIN albums ON albums.id=album_genres.album_id"). - Where("albums.id IS NULL"). + Joins("LEFT JOIN album_genres ON album_genres.genre_id=genres.id"). + Where("album_genres.genre_id IS NULL"). SubQuery() q := s.db. Where("genres.id IN ?", subTrack). From f981ddef91c161b367c0a7c1eb535013c7a9318b Mon Sep 17 00:00:00 2001 From: sentriz Date: Thu, 31 Dec 2020 00:29:55 +0000 Subject: [PATCH 16/18] clean artists with join --- server/scanner/scanner.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/server/scanner/scanner.go b/server/scanner/scanner.go index 68c75ff..6ae0f44 100644 --- a/server/scanner/scanner.go +++ b/server/scanner/scanner.go @@ -135,12 +135,13 @@ func (s *Scanner) cleanFolders() (int, error) { func (s *Scanner) cleanArtists() (int, error) { sub := s.db. - Select("1"). - Model(&db.Album{}). - Where("albums.tag_artist_id=artists.id"). + Select("artists.id"). + Model(&db.Artist{}). + Joins("LEFT JOIN albums ON albums.tag_artist_id=artists.id"). + Where("albums.id IS NULL"). SubQuery() q := s.db. - Where("NOT EXISTS ?", sub). + Where("artists.id IN ?", sub). Delete(&db.Artist{}) return int(q.RowsAffected), q.Error } From 75fce9e214fd87548808d4915188efe25ab7002c Mon Sep 17 00:00:00 2001 From: sentriz Date: Sat, 2 Jan 2021 16:06:59 +0000 Subject: [PATCH 17/18] add dummy getPodcasts view fixes #98 --- server/ctrlsubsonic/handlers_unimplemented.go | 15 +++++++++++++++ server/ctrlsubsonic/spec/spec.go | 5 +++++ server/server.go | 1 + 3 files changed, 21 insertions(+) create mode 100644 server/ctrlsubsonic/handlers_unimplemented.go diff --git a/server/ctrlsubsonic/handlers_unimplemented.go b/server/ctrlsubsonic/handlers_unimplemented.go new file mode 100644 index 0000000..eca34f2 --- /dev/null +++ b/server/ctrlsubsonic/handlers_unimplemented.go @@ -0,0 +1,15 @@ +package ctrlsubsonic + +import ( + "net/http" + + "go.senan.xyz/gonic/server/ctrlsubsonic/spec" +) + +func (c *Controller) ServeGetPodcasts(r *http.Request) *spec.Response { + sub := spec.NewResponse() + sub.Podcasts = &spec.Podcasts{ + List: []struct{}{}, + } + return sub +} diff --git a/server/ctrlsubsonic/spec/spec.go b/server/ctrlsubsonic/spec/spec.go index 1263cd1..d3d5c9b 100644 --- a/server/ctrlsubsonic/spec/spec.go +++ b/server/ctrlsubsonic/spec/spec.go @@ -44,6 +44,7 @@ type Response struct { PlayQueue *PlayQueue `xml:"playQueue" json:"playQueue,omitempty"` JukeboxStatus *JukeboxStatus `xml:"jukeboxStatus" json:"jukeboxStatus,omitempty"` JukeboxPlaylist *JukeboxPlaylist `xml:"jukeboxPlaylist" json:"jukeboxPlaylist,omitempty"` + Podcasts *Podcasts `xml:"podcasts" json:"podcasts,omitempty"` } func NewResponse() *Response { @@ -285,3 +286,7 @@ type JukeboxPlaylist struct { List []*TrackChild `xml:"entry,omitempty" json:"entry,omitempty"` JukeboxStatus } + +type Podcasts struct { + List []struct{} `xml:"channel" json:"channel"` +} diff --git a/server/server.go b/server/server.go index 22840b7..78000c8 100644 --- a/server/server.go +++ b/server/server.go @@ -189,6 +189,7 @@ func setupSubsonic(r *mux.Router, ctrl *ctrlsubsonic.Controller) { r.Handle("/getGenres{_:(?:\\.view)?}", ctrl.H(ctrl.ServeGetGenres)) r.Handle("/getArtistInfo{_:(?:\\.view)?}", ctrl.H(ctrl.ServeGetArtistInfo)) // ** begin unimplemented + r.Handle("/getPodcasts{_:(?:\\.view)?}", ctrl.H(ctrl.ServeGetPodcasts)) // middlewares should be run for not found handler // https://github.com/gorilla/mux/issues/416 notFoundHandler := ctrl.H(ctrl.ServeNotFound) From 20f68ce26f52b83d2e7cf476171c73f0408949e4 Mon Sep 17 00:00:00 2001 From: sentriz Date: Sat, 2 Jan 2021 19:08:15 +0000 Subject: [PATCH 18/18] migrate TrackGenre before Track --- server/db/migrations.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/db/migrations.go b/server/db/migrations.go index 1f2f680..f2dfa25 100644 --- a/server/db/migrations.go +++ b/server/db/migrations.go @@ -167,11 +167,11 @@ func migrateMultiGenre() gormigrate.Migration { ID: "202012151806", Migrate: func(tx *gorm.DB) error { step := tx.AutoMigrate( - Track{}, - Album{}, Genre{}, TrackGenre{}, AlbumGenre{}, + Track{}, + Album{}, ) if err := step.Error; err != nil { return fmt.Errorf("step auto migrate: %w", err)