heimoshuiyu/gonic
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

merge album and folder models

Browse Source
This commit is contained in:
sentriz
2019-06-04 15:27:37 +01:00
parent 6d83072b91
commit 4d91bf2bda
7 changed files with 164 additions and 241 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
go.mod
View File

@@ -28,4 +28,7 @@ require (
golang.org/x/sys v0.0.0-20190422165155-953cdadca894 // indirect
golang.org/x/tools v0.0.0-20190422233926-fe54fb35175b // indirect
google.golang.org/appengine v1.5.0 // indirect
gopkg.in/axiomzen/null.v3 v3.2.4
gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce // indirect
gopkg.in/pg.v4 v4.9.5 // indirect
)

6
go.sum
View File

@@ -263,10 +263,16 @@ google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9M
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/axiomzen/null.v3 v3.2.4 h1:5VmJ9lSU0dBJjisXhuhRnGiIolhTjkmQQ0EBNE9Z5QY=
gopkg.in/axiomzen/null.v3 v3.2.4/go.mod h1:Vq8/79AVvSZVg5PdN4kNROnTff7WtSh0HOiBHfOvVNU=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU=
gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
gopkg.in/pg.v4 v4.9.5 h1:bs21aaMPPPcUPhNtqGxN8EeYUFU10MsNrC7U9m/lJgU=
gopkg.in/pg.v4 v4.9.5/go.mod h1:cSUPtzgofjgARAbFCE5u6WDHGPgbR1sjUYcWQlKvpec=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

105
model/model.go
View File

@@ -1,6 +1,8 @@
package model
import "time"
import (
"time"
)
// q: what in tarnation are the `IsNew`s for?
// a: it's a bit of a hack - but we set a models IsNew to true if
@@ -9,108 +11,73 @@ import "time"
// that bool being true - since it won't be true if it was already
// in the db
// Album represents the albums table
type Album struct {
IDBase
CrudBase
Artist Artist
ArtistID int `gorm:"index" sql:"default: null; type:int REFERENCES artists(id) ON DELETE CASCADE"`
Title string `gorm:"not null; index"`
// an Album having a `Path` is a little weird when browsing by tags
// (for the most part - the library's folder structure is treated as
// if it were flat), but this solves the "American Football problem"
// https://en.wikipedia.org/wiki/American_Football_(band)#Discography
Path string `gorm:"not null; unique_index"`
CoverID int `sql:"default: null; type:int REFERENCES covers(id)"`
Cover Cover
Year int
Tracks []Track
IsNew bool `gorm:"-"`
}
// Artist represents the Artists table
type Artist struct {
IDBase
CrudBase
Name string `gorm:"not null; unique_index"`
Albums []Album
Folders []Folder
}
// Track represents the tracks table
type Track struct {
IDBase
CrudBase
Album Album
AlbumID int `gorm:"index" sql:"default: null; type:int REFERENCES albums(id) ON DELETE CASCADE"`
Artist Artist
ArtistID int `gorm:"index" sql:"default: null; type:int REFERENCES artists(id) ON DELETE CASCADE"`
TrackArtist string
Bitrate int
Codec string
DiscNumber int
Duration int
Title string
TotalDiscs int
TotalTracks int
TrackNumber int
Year int
Suffix string
ContentType string
Size int
Folder Folder
FolderID int `gorm:"not null; index" sql:"default: null; type:int REFERENCES folders(id) ON DELETE CASCADE"`
Path string `gorm:"not null; unique_index"`
// TODO: try removing idx_folder_basename_ext
FolderID int `gorm:"not null; unique_index:idx_folder_filename_ext" sql:"default: null; type:int REFERENCES folders(id) ON DELETE CASCADE"`
Filename string `gorm:"not null; unique_index:idx_folder_filename_ext" sql:"default: null"`
Ext string `gorm:"not nill; unique_index:idx_folder_filename_ext" sql:"default: null"`
Artist Artist
ArtistID int `gorm:"not null; index" sql:"default: null; type:int REFERENCES artists(id) ON DELETE CASCADE"`
ContentType string `gorm:"not null" sql:"default: null"`
Duration int `gorm:"not null" sql:"default: null"`
Size int `gorm:"not null" sql:"default: null"`
Bitrate int `gorm:"not null" sql:"default: null"`
TagDiscNumber int `sql:"default: null"`
TagTitle string `sql:"default: null"`
TagTotalDiscs int `sql:"default: null"`
TagTotalTracks int `sql:"default: null"`
TagTrackArtist string `sql:"default: null"`
TagTrackNumber int `sql:"default: null"`
TagYear int `sql:"default: null"`
}
// Cover represents the covers table
type Cover struct {
IDBase
CrudBase
Image []byte
Path string `gorm:"not null; unique_index"`
IsNew bool `gorm:"-"`
}
// User represents the users table
type User struct {
IDBase
CrudBase
Name string `gorm:"not null; unique_index"`
Password string
LastFMSession string
IsAdmin bool
Name string `gorm:"not null; unique_index" sql:"default: null"`
Password string `gorm:"not null" sql:"default: null"`
LastFMSession string `sql:"default: null"`
IsAdmin bool `sql:"default: null"`
}
// Setting represents the settings table
type Setting struct {
CrudBase
Key string `gorm:"primary_key; auto_increment:false"`
Value string
Key string `gorm:"not null; primary_key; auto_increment:false" sql:"default: null"`
Value string `sql:"default: null"`
}
// Play represents the settings table
type Play struct {
IDBase
User User
UserID int `gorm:"not null; index" sql:"default: null; type:int REFERENCES users(id) ON DELETE CASCADE"`
Album Album
AlbumID int `gorm:"not null; index" sql:"default: null; type:int REFERENCES albums(id) ON DELETE CASCADE"`
Folder Folder
FolderID int `gorm:"not null; index" sql:"default: null; type:int REFERENCES folders(id) ON DELETE CASCADE"`
Time time.Time
Time time.Time `sql:"default: null"`
Count int
}
// Folder represents the settings table
type Folder struct {
IDBase
CrudBase
Name string
Path string `gorm:"not null; unique_index"`
LeftPath string `gorm:"unique_index:idx_left_path_right_path"`
RightPath string `gorm:"not null; unique_index:idx_left_path_right_path" sql:"default: null"`
Parent *Folder
ParentID int `sql:"default: null; type:int REFERENCES folders(id) ON DELETE CASCADE"`
CoverID int `sql:"default: null; type:int REFERENCES covers(id)"`
HasTracks bool `gorm:"not null; index"`
Cover Cover
AlbumArtist Artist
AlbumArtistID int `gorm:"index" sql:"default: null; type:int REFERENCES artists(id) ON DELETE CASCADE"`
AlbumTitle string `gorm:"index" sql:"default: null"`
AlbumYear int `sql:"default: null"`
Cover string `sql:"default: null"`
Tracks []Track
IsNew bool `gorm:"-"`
}

29
scanner/folder_stack.go
View File

@@ -1,35 +1,40 @@
package scanner
import "github.com/sentriz/gonic/model"
import (
"fmt"
"strings"
type folderStack []model.Folder
"github.com/sentriz/gonic/model"
)
func (s *folderStack) Push(v model.Folder) {
type folderStack []*model.Folder
func (s *folderStack) Push(v *model.Folder) {
*s = append(*s, v)
}
func (s *folderStack) Pop() model.Folder {
func (s *folderStack) Pop() *model.Folder {
l := len(*s)
if l == 0 {
return model.Folder{}
return nil
}
r := (*s)[l-1]
*s = (*s)[:l-1]
return r
}
func (s *folderStack) Peek() model.Folder {
func (s *folderStack) Peek() *model.Folder {
l := len(*s)
if l == 0 {
return model.Folder{}
return nil
}
return (*s)[l-1]
}
func (s *folderStack) PeekID() int {
l := len(*s)
if l == 0 {
return 0
func (s *folderStack) String() string {
paths := make([]string, len(*s))
for i, folder := range *s {
paths[i] = folder.RightPath
}
return (*s)[l-1].ID
return fmt.Sprintf("[%s]", strings.Join(paths, " "))
}

47
scanner/scanner.go
View File

@@ -1,12 +1,5 @@
package scanner
// Album -> needs a CoverID
// -> needs a FolderID (American Football)
// Folder -> needs a CoverID
// -> needs a ParentID
// Track -> needs an AlbumID
// -> needs a FolderID
import (
"log"
"sync/atomic"
@@ -26,27 +19,32 @@ var (
type Scanner struct {
db, tx *gorm.DB
musicPath string
seenTracks map[string]bool
seenTracks map[int]struct{}
curFolders folderStack
curTracks []model.Track
curCover model.Cover
curAlbum model.Album
curAArtist model.Artist
curCover string
}
func New(db *gorm.DB, musicPath string) *Scanner {
return &Scanner{
db: db,
musicPath: musicPath,
seenTracks: make(map[string]bool),
seenTracks: make(map[int]struct{}),
curFolders: make(folderStack, 0),
curTracks: make([]model.Track, 0),
curCover: model.Cover{},
curAlbum: model.Album{},
curAArtist: model.Artist{},
}
}
func (s *Scanner) curFolder() *model.Folder {
return s.curFolders.Peek()
}
func (s *Scanner) curFolderID() int {
peek := s.curFolders.Peek()
if peek == nil {
return 0
}
return peek.ID
}
func (s *Scanner) Start() error {
if atomic.LoadInt32(&IsScanning) == 1 {
return errors.New("already scanning")
@@ -86,16 +84,17 @@ func (s *Scanner) startClean() error {
defer logElapsed(time.Now(), "cleaning database")
var tracks []model.Track
s.tx.
Select("id, path").
Select("id").
Find(&tracks)
var deleted int
for _, track := range tracks {
_, ok := s.seenTracks[track.Path]
if ok {
continue
}
_, ok := s.seenTracks[track.ID]
if !ok {
s.tx.Delete(&track)
log.Println("removed track", track.Path)
deleted++
}
}
log.Printf("removed %d tracks\n", deleted)
return nil
}
@@ -104,10 +103,8 @@ func (s *Scanner) MigrateDB() error {
s.tx = s.db.Begin()
defer s.tx.Commit()
s.tx.AutoMigrate(
model.Album{},
model.Artist{},
model.Track{},
model.Cover{},
model.User{},
model.Setting{},
model.Play{},

20
scanner/utilities.go
View File

@@ -2,15 +2,12 @@ package scanner
import (
"os"
"path"
"path/filepath"
"strings"
"github.com/dhowden/tag"
"github.com/pkg/errors"
)
var trackExtensions = map[string]string{
var mimeTypes = map[string]string{
"mp3": "audio/mpeg",
"flac": "audio/x-flac",
"aac": "audio/x-aac",
@@ -18,15 +15,6 @@ var trackExtensions = map[string]string{
"ogg": "audio/ogg",
}
func isTrack(fullPath string) (string, string, bool) {
ext := filepath.Ext(fullPath)[1:]
mine, ok := trackExtensions[ext]
if !ok {
return "", "", false
}
return mine, ext, true
}
var coverFilenames = map[string]struct{}{
"cover.png": struct{}{},
"cover.jpg": struct{}{},
@@ -42,12 +30,6 @@ var coverFilenames = map[string]struct{}{
"front.jpeg": struct{}{},
}
func isCover(fullPath string) bool {
_, filename := path.Split(fullPath)
_, ok := coverFilenames[strings.ToLower(filename)]
return ok
}
func readTags(path string) (tag.Metadata, error) {
trackData, err := os.Open(path)
if err != nil {

175
scanner/walk.go
View File

@@ -1,7 +1,6 @@
package scanner
import (
"io/ioutil"
"log"
"os"
"path"
@@ -14,182 +13,146 @@ import (
"github.com/sentriz/gonic/model"
)
type trackItem struct {
mime string
ext string
}
type item struct {
path string
//
// common
fullPath string
relPath string
directory string
filename string
stat os.FileInfo
track *trackItem
//
// track only
ext string
mime string
}
func (s *Scanner) callbackItem(path string, info *godirwalk.Dirent) error {
stat, err := os.Stat(path)
func (s *Scanner) callbackItem(fullPath string, info *godirwalk.Dirent) error {
stat, err := os.Stat(fullPath)
if err != nil {
return errors.Wrap(err, "stating")
}
relPath, err := filepath.Rel(s.musicPath, path)
relPath, err := filepath.Rel(s.musicPath, fullPath)
if err != nil {
return errors.Wrap(err, "getting relative path")
}
directory, filename := path.Split(relPath)
it := &item{
path: path,
fullPath: fullPath,
relPath: relPath,
directory: directory,
filename: filename,
stat: stat,
}
if info.IsDir() {
return s.handleFolder(it)
}
if isCover(path) {
return s.handleCover(it)
if _, ok := coverFilenames[filename]; ok {
s.curCover = filename
return nil
}
if mime, ext, ok := isTrack(path); ok {
s.seenTracks[relPath] = true
it.track = &trackItem{mime: mime, ext: ext}
ext := path.Ext(filename)[1:]
if mime, ok := mimeTypes[ext]; ok {
it.ext = ext
it.mime = mime
return s.handleTrack(it)
}
return nil
}
func (s *Scanner) callbackPost(path string, info *godirwalk.Dirent) error {
// in general in this function - if a model is not nil, then it
// has at least been looked up. if it has a id of 0, then it is
// a new record and needs to be inserted
if s.curCover.IsNew {
s.tx.Save(&s.curCover)
}
if s.curAlbum.IsNew {
s.curAlbum.CoverID = s.curCover.ID
s.tx.Save(&s.curAlbum)
}
func (s *Scanner) callbackPost(fullPath string, info *godirwalk.Dirent) error {
folder := s.curFolders.Pop()
if folder.IsNew {
folder.ParentID = s.curFolders.PeekID()
folder.CoverID = s.curCover.ID
folder.HasTracks = len(s.curTracks) > 1
folder.ParentID = s.curFolderID()
folder.Cover = s.curCover
s.tx.Save(&folder)
}
for _, t := range s.curTracks {
t.FolderID = folder.ID
t.AlbumID = s.curAlbum.ID
s.tx.Save(&t)
}
//
s.curTracks = make([]model.Track, 0)
s.curCover = model.Cover{}
s.curAlbum = model.Album{}
s.curAArtist = model.Artist{}
//
log.Printf("processed folder `%s`\n", path)
s.curCover = ""
log.Printf("processed folder `%s`\n", fullPath)
return nil
}
func (s *Scanner) handleFolder(it *item) error {
// TODO:
var folder model.Folder
err := s.tx.
Where("path = ?", it.relPath).
Where(model.Folder{
LeftPath: it.directory,
RightPath: it.filename,
}).
First(&folder).
Error
if !gorm.IsRecordNotFoundError(err) &&
it.stat.ModTime().Before(folder.UpdatedAt) {
// we found the record but it hasn't changed
s.curFolders.Push(folder)
s.curFolders.Push(&folder)
return nil
}
folder.Path = it.relPath
folder.Name = it.stat.Name()
folder.LeftPath = it.directory
folder.RightPath = it.filename
s.tx.Save(&folder)
folder.IsNew = true
s.curFolders.Push(folder)
return nil
}
func (s *Scanner) handleCover(it *item) error {
err := s.tx.
Where("path = ?", it.relPath).
First(&s.curCover).
Error
if !gorm.IsRecordNotFoundError(err) &&
it.stat.ModTime().Before(s.curCover.UpdatedAt) {
// we found the record but it hasn't changed
return nil
}
s.curCover.Path = it.relPath
image, err := ioutil.ReadFile(it.path)
if err != nil {
return errors.Wrap(err, "reading cover")
}
s.curCover.Image = image
s.curCover.IsNew = true
s.curFolders.Push(&folder)
return nil
}
func (s *Scanner) handleTrack(it *item) error {
//
// set track basics
track := model.Track{}
var track model.Track
err := s.tx.
Where("path = ?", it.relPath).
Where(model.Track{
FolderID: s.curFolderID(),
Filename: it.filename,
Ext: it.ext,
}).
First(&track).
Error
if !gorm.IsRecordNotFoundError(err) &&
it.stat.ModTime().Before(track.UpdatedAt) {
s.seenTracks[track.ID] = struct{}{}
// we found the record but it hasn't changed
return nil
}
tags, err := readTags(it.path)
track.Filename = it.filename
track.Ext = it.ext
track.ContentType = it.mime
track.Size = int(it.stat.Size())
track.FolderID = s.curFolderID()
track.Duration = -1
track.Bitrate = -1
tags, err := readTags(it.fullPath)
if err != nil {
return errors.Wrap(err, "reading tags")
}
trackNumber, totalTracks := tags.Track()
discNumber, totalDiscs := tags.Disc()
track.DiscNumber = discNumber
track.TotalDiscs = totalDiscs
track.TotalTracks = totalTracks
track.TrackNumber = trackNumber
track.Path = it.relPath
track.Suffix = it.track.ext
track.ContentType = it.track.mime
track.Size = int(it.stat.Size())
track.Title = tags.Title()
track.TrackArtist = tags.Artist()
track.Year = tags.Year()
track.FolderID = s.curFolders.PeekID()
track.TagDiscNumber = discNumber
track.TagTotalDiscs = totalDiscs
track.TagTotalTracks = totalTracks
track.TagTrackNumber = trackNumber
track.TagTitle = tags.Title()
track.TagTrackArtist = tags.Artist()
track.TagYear = tags.Year()
//
// set album artist basics
var artist model.Artist
err = s.tx.Where("name = ?", tags.AlbumArtist()).
First(&s.curAArtist).
First(&artist).
Error
if gorm.IsRecordNotFoundError(err) {
s.curAArtist.Name = tags.AlbumArtist()
s.tx.Save(&s.curAArtist)
artist.Name = tags.AlbumArtist()
s.tx.Save(&artist)
}
track.ArtistID = s.curAArtist.ID
track.ArtistID = artist.ID
s.tx.Save(&track)
s.seenTracks[track.ID] = struct{}{}
//
// set album if this is the first track in the folder
if len(s.curTracks) > 0 {
s.curTracks = append(s.curTracks, track)
if !s.curFolder().IsNew {
return nil
}
s.curTracks = append(s.curTracks, track)
//
directory, _ := path.Split(it.relPath)
err = s.tx.
Where("path = ?", directory).
First(&s.curAlbum).
Error
if !gorm.IsRecordNotFoundError(err) {
// we found the record
return nil
}
s.curAlbum.Path = directory
s.curAlbum.Title = tags.Album()
s.curAlbum.Year = tags.Year()
s.curAlbum.ArtistID = s.curAArtist.ID
s.curAlbum.IsNew = true
s.curFolder().AlbumTitle = tags.Album()
s.curFolder().AlbumYear = tags.Year()
s.curFolder().AlbumArtistID = artist.ID
return nil
}