heimoshuiyu/gonic
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

refactor and bulk insert

Browse Source
This commit is contained in:
sentriz
2019-05-23 15:13:06 +01:00
parent f8c5c41509
commit ff009b8851
5 changed files with 65 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
go.mod
View File

@@ -21,6 +21,7 @@ require (
github.com/peterbourgon/ff v1.2.0
github.com/pkg/errors v0.8.1
github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be
github.com/t-tiger/gorm-bulk-insert v0.0.0-20190401142620-ba33202b110e
github.com/wader/gormstore v0.0.0-20190302154359-acb787ba3755
golang.org/x/crypto v0.0.0-20190422183909-d864b10871cd // indirect
golang.org/x/sync v0.0.0-20190423024810-112230192c58 // indirect

2
go.sum
View File

@@ -169,6 +169,8 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/t-tiger/gorm-bulk-insert v0.0.0-20190401142620-ba33202b110e h1:mOgAh77WyFUaHUCziKMExurMgVJobHeEZCnwIBdActY=
github.com/t-tiger/gorm-bulk-insert v0.0.0-20190401142620-ba33202b110e/go.mod h1:SK1RZT4TR1aMUNGtbk6YxTPgx2D/gfbxB571QGnAV+c=
github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
github.com/wader/gormstore v0.0.0-20190302154359-acb787ba3755 h1:pNaEDfvqe9W2h4D+xm5f+lnZdao3Rob6O0b8SovpGbE=
github.com/wader/gormstore v0.0.0-20190302154359-acb787ba3755/go.mod h1:PbEnTGtqU8NGCALR62gu2+eQYO8zQDEvaMJiPaj5Hic=

3
model/model.go
View File

@@ -18,6 +18,7 @@ type Album struct {
Cover Cover
Year int
Tracks []Track
IsNew bool `gorm:"-"`
}
// AlbumArtist represents the AlbumArtists table
@@ -60,6 +61,7 @@ type Cover struct {
CrudBase
Image []byte
Path string `gorm:"not null;unique_index"`
IsNew bool `gorm:"-"`
}
// User represents the users table
@@ -103,4 +105,5 @@ type Folder struct {
CoverID int
HasTracks bool `gorm:"not null;index"`
Cover Cover
IsNew bool `gorm:"-"`
}

12
scanner/folder_stack.go
View File

@@ -2,26 +2,26 @@ package scanner
import "github.com/sentriz/gonic/model"
type folderStack []*model.Folder
type folderStack []model.Folder
func (s *folderStack) Push(v *model.Folder) {
func (s *folderStack) Push(v model.Folder) {
*s = append(*s, v)
}
func (s *folderStack) Pop() *model.Folder {
func (s *folderStack) Pop() model.Folder {
l := len(*s)
if l == 0 {
return nil
return model.Folder{}
}
r := (*s)[l-1]
*s = (*s)[:l-1]
return r
}
func (s *folderStack) Peek() *model.Folder {
func (s *folderStack) Peek() model.Folder {
l := len(*s)
if l == 0 {
return nil
return model.Folder{}
}
return (*s)[l-1]
}

89
scanner/scanner.go
View File

@@ -25,6 +25,7 @@ import (
"github.com/jinzhu/gorm"
"github.com/karrick/godirwalk"
"github.com/pkg/errors"
gormbulk "github.com/t-tiger/gorm-bulk-insert"
"github.com/sentriz/gonic/model"
)
@@ -40,10 +41,10 @@ type Scanner struct {
seenPaths map[string]bool
folderCount uint
curFolders folderStack
curTracks []*model.Track
curCover *model.Cover
curAlbum *model.Album
curAArtist *model.AlbumArtist
curTracks []model.Track
curCover model.Cover
curAlbum model.Album
curAArtist model.AlbumArtist
}
func New(db *gorm.DB, musicPath string) *Scanner {
@@ -52,12 +53,18 @@ func New(db *gorm.DB, musicPath string) *Scanner {
musicPath: musicPath,
seenPaths: make(map[string]bool),
curFolders: make(folderStack, 0),
curTracks: make([]model.Track, 0),
curCover: model.Cover{},
curAlbum: model.Album{},
curAArtist: model.AlbumArtist{},
}
}
func (s *Scanner) handleCover(fullPath string, stat os.FileInfo) error {
s.curCover = &model.Cover{}
err := s.tx.Where("path = ?", fullPath).First(s.curCover).Error
err := s.tx.
Where("path = ?", fullPath).
First(&s.curCover).
Error
if !gorm.IsRecordNotFoundError(err) &&
stat.ModTime().Before(s.curCover.UpdatedAt) {
// we found the record but it hasn't changed
@@ -69,21 +76,28 @@ func (s *Scanner) handleCover(fullPath string, stat os.FileInfo) error {
return errors.Wrap(err, "reading cover")
}
s.curCover.Image = image
s.curCover.IsNew = true
return nil
}
func (s *Scanner) handleFolder(fullPath string, stat os.FileInfo) error {
folder := &model.Folder{}
defer s.curFolders.Push(folder)
err := s.tx.Where("path = ?", fullPath).First(folder).Error
// TODO:
var folder model.Folder
err := s.tx.
Where("path = ?", fullPath).
First(&folder).
Error
if !gorm.IsRecordNotFoundError(err) &&
stat.ModTime().Before(folder.UpdatedAt) {
// we found the record but it hasn't changed
s.curFolders.Push(folder)
return nil
}
folder.Path = fullPath
folder.Name = stat.Name()
s.tx.Save(folder)
s.tx.Save(&folder)
folder.IsNew = true
s.curFolders.Push(folder)
return nil
}
@@ -91,36 +105,37 @@ func (s *Scanner) handleFolderCompletion(fullPath string, info *godirwalk.Dirent
// in general in this function - if a model is not nil, then it
// has at least been looked up. if it has a id of 0, then it is
// a new record and needs to be inserted
//
var newCover bool
if s.curCover != nil && s.curCover.ID == 0 {
s.tx.Save(s.curCover)
newCover = true
if s.curCover.IsNew {
s.tx.Save(&s.curCover)
}
if s.curAlbum != nil {
if s.curAlbum.IsNew {
s.curAlbum.CoverID = s.curCover.ID
if newCover || s.curAlbum.ID == 0 {
s.tx.Save(s.curAlbum)
}
s.tx.Save(&s.curAlbum)
}
folder := s.curFolders.Pop()
if folder.ID == 0 || newCover {
if folder.IsNew {
folder.ParentID = s.curFolders.PeekID()
folder.CoverID = s.curCover.ID
folder.HasTracks = true
s.tx.Save(folder)
folder.HasTracks = len(s.curTracks) > 1
s.tx.Save(&folder)
}
for _, track := range s.curTracks {
// not checking for a nil album here because if there are
// tracks, then we at least lookup up the album
track.AlbumID = s.curAlbum.ID
track.FolderID = folder.ID
s.tx.Save(track)
}
s.curTracks = nil
s.curCover = nil
s.curAlbum = nil
s.curAArtist = nil
toInsert := make([]interface{}, len(s.curTracks))
for i, t := range s.curTracks {
t.FolderID = folder.ID
t.AlbumID = s.curAlbum.ID
toInsert[i] = t
}
gormbulk.BulkInsert(s.tx, toInsert, 3000)
//
s.curTracks = make([]model.Track, 0)
s.curCover = model.Cover{}
s.curAlbum = model.Album{}
s.curAArtist = model.AlbumArtist{}
//
log.Printf("processed folder `%s`\n", fullPath)
return nil
}
@@ -128,9 +143,12 @@ func (s *Scanner) handleFolderCompletion(fullPath string, info *godirwalk.Dirent
func (s *Scanner) handleTrack(fullPath string, stat os.FileInfo, mime, exten string) error {
//
// set track basics
track := &model.Track{}
track := model.Track{}
modTime := stat.ModTime()
err := s.tx.Where("path = ?", fullPath).First(track).Error
err := s.tx.
Where("path = ?", fullPath).
First(&track).
Error
if !gorm.IsRecordNotFoundError(err) &&
modTime.Before(track.UpdatedAt) {
// we found the record but it hasn't changed
@@ -156,13 +174,12 @@ func (s *Scanner) handleTrack(fullPath string, stat os.FileInfo, mime, exten str
track.FolderID = s.curFolders.PeekID()
//
// set album artist basics
s.curAArtist = &model.AlbumArtist{}
err = s.tx.Where("name = ?", tags.AlbumArtist()).
First(s.curAArtist).
First(&s.curAArtist).
Error
if gorm.IsRecordNotFoundError(err) {
s.curAArtist.Name = tags.AlbumArtist()
s.tx.Save(s.curAArtist)
s.tx.Save(&s.curAArtist)
}
track.AlbumArtistID = s.curAArtist.ID
//
@@ -173,11 +190,10 @@ func (s *Scanner) handleTrack(fullPath string, stat os.FileInfo, mime, exten str
}
s.curTracks = append(s.curTracks, track)
//
s.curAlbum = &model.Album{}
directory, _ := path.Split(fullPath)
err = s.tx.
Where("path = ?", directory).
First(s.curAlbum).
First(&s.curAlbum).
Error
if !gorm.IsRecordNotFoundError(err) {
// we found the record
@@ -187,6 +203,7 @@ func (s *Scanner) handleTrack(fullPath string, stat os.FileInfo, mime, exten str
s.curAlbum.Title = tags.Album()
s.curAlbum.Year = tags.Year()
s.curAlbum.AlbumArtistID = s.curAArtist.ID
s.curAlbum.IsNew = true
return nil
}