From ff009b88516a9289fe9cbd8fd2bc55a945d62923 Mon Sep 17 00:00:00 2001 From: sentriz Date: Thu, 23 May 2019 15:13:06 +0100 Subject: [PATCH] refactor and bulk insert --- go.mod | 1 + go.sum | 2 + model/model.go | 3 ++ scanner/folder_stack.go | 12 +++--- scanner/scanner.go | 89 ++++++++++++++++++++++++----------------- 5 files changed, 65 insertions(+), 42 deletions(-) diff --git a/go.mod b/go.mod index 7b21aa2..b01856b 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( github.com/peterbourgon/ff v1.2.0 github.com/pkg/errors v0.8.1 github.com/rainycape/unidecode v0.0.0-20150907023854-cb7f23ec59be + github.com/t-tiger/gorm-bulk-insert v0.0.0-20190401142620-ba33202b110e github.com/wader/gormstore v0.0.0-20190302154359-acb787ba3755 golang.org/x/crypto v0.0.0-20190422183909-d864b10871cd // indirect golang.org/x/sync v0.0.0-20190423024810-112230192c58 // indirect diff --git a/go.sum b/go.sum index aa34f0d..4423bb8 100644 --- a/go.sum +++ b/go.sum @@ -169,6 +169,8 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/t-tiger/gorm-bulk-insert v0.0.0-20190401142620-ba33202b110e h1:mOgAh77WyFUaHUCziKMExurMgVJobHeEZCnwIBdActY= +github.com/t-tiger/gorm-bulk-insert v0.0.0-20190401142620-ba33202b110e/go.mod h1:SK1RZT4TR1aMUNGtbk6YxTPgx2D/gfbxB571QGnAV+c= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/wader/gormstore v0.0.0-20190302154359-acb787ba3755 h1:pNaEDfvqe9W2h4D+xm5f+lnZdao3Rob6O0b8SovpGbE= github.com/wader/gormstore v0.0.0-20190302154359-acb787ba3755/go.mod h1:PbEnTGtqU8NGCALR62gu2+eQYO8zQDEvaMJiPaj5Hic= diff --git a/model/model.go b/model/model.go index 5200352..c36e83f 100644 --- a/model/model.go +++ b/model/model.go @@ -18,6 +18,7 @@ type Album struct { Cover Cover Year int Tracks []Track + IsNew bool `gorm:"-"` } // AlbumArtist represents the AlbumArtists table @@ -60,6 +61,7 @@ type Cover struct { CrudBase Image []byte Path string `gorm:"not null;unique_index"` + IsNew bool `gorm:"-"` } // User represents the users table @@ -103,4 +105,5 @@ type Folder struct { CoverID int HasTracks bool `gorm:"not null;index"` Cover Cover + IsNew bool `gorm:"-"` } diff --git a/scanner/folder_stack.go b/scanner/folder_stack.go index faebc92..d58585d 100644 --- a/scanner/folder_stack.go +++ b/scanner/folder_stack.go @@ -2,26 +2,26 @@ package scanner import "github.com/sentriz/gonic/model" -type folderStack []*model.Folder +type folderStack []model.Folder -func (s *folderStack) Push(v *model.Folder) { +func (s *folderStack) Push(v model.Folder) { *s = append(*s, v) } -func (s *folderStack) Pop() *model.Folder { +func (s *folderStack) Pop() model.Folder { l := len(*s) if l == 0 { - return nil + return model.Folder{} } r := (*s)[l-1] *s = (*s)[:l-1] return r } -func (s *folderStack) Peek() *model.Folder { +func (s *folderStack) Peek() model.Folder { l := len(*s) if l == 0 { - return nil + return model.Folder{} } return (*s)[l-1] } diff --git a/scanner/scanner.go b/scanner/scanner.go index 1c05437..19a1073 100644 --- a/scanner/scanner.go +++ b/scanner/scanner.go @@ -25,6 +25,7 @@ import ( "github.com/jinzhu/gorm" "github.com/karrick/godirwalk" "github.com/pkg/errors" + gormbulk "github.com/t-tiger/gorm-bulk-insert" "github.com/sentriz/gonic/model" ) @@ -40,10 +41,10 @@ type Scanner struct { seenPaths map[string]bool folderCount uint curFolders folderStack - curTracks []*model.Track - curCover *model.Cover - curAlbum *model.Album - curAArtist *model.AlbumArtist + curTracks []model.Track + curCover model.Cover + curAlbum model.Album + curAArtist model.AlbumArtist } func New(db *gorm.DB, musicPath string) *Scanner { @@ -52,12 +53,18 @@ func New(db *gorm.DB, musicPath string) *Scanner { musicPath: musicPath, seenPaths: make(map[string]bool), curFolders: make(folderStack, 0), + curTracks: make([]model.Track, 0), + curCover: model.Cover{}, + curAlbum: model.Album{}, + curAArtist: model.AlbumArtist{}, } } func (s *Scanner) handleCover(fullPath string, stat os.FileInfo) error { - s.curCover = &model.Cover{} - err := s.tx.Where("path = ?", fullPath).First(s.curCover).Error + err := s.tx. + Where("path = ?", fullPath). + First(&s.curCover). + Error if !gorm.IsRecordNotFoundError(err) && stat.ModTime().Before(s.curCover.UpdatedAt) { // we found the record but it hasn't changed @@ -69,21 +76,28 @@ func (s *Scanner) handleCover(fullPath string, stat os.FileInfo) error { return errors.Wrap(err, "reading cover") } s.curCover.Image = image + s.curCover.IsNew = true return nil } func (s *Scanner) handleFolder(fullPath string, stat os.FileInfo) error { - folder := &model.Folder{} - defer s.curFolders.Push(folder) - err := s.tx.Where("path = ?", fullPath).First(folder).Error + // TODO: + var folder model.Folder + err := s.tx. + Where("path = ?", fullPath). + First(&folder). + Error if !gorm.IsRecordNotFoundError(err) && stat.ModTime().Before(folder.UpdatedAt) { // we found the record but it hasn't changed + s.curFolders.Push(folder) return nil } folder.Path = fullPath folder.Name = stat.Name() - s.tx.Save(folder) + s.tx.Save(&folder) + folder.IsNew = true + s.curFolders.Push(folder) return nil } @@ -91,36 +105,37 @@ func (s *Scanner) handleFolderCompletion(fullPath string, info *godirwalk.Dirent // in general in this function - if a model is not nil, then it // has at least been looked up. if it has a id of 0, then it is // a new record and needs to be inserted - // - var newCover bool - if s.curCover != nil && s.curCover.ID == 0 { - s.tx.Save(s.curCover) - newCover = true + if s.curCover.IsNew { + s.tx.Save(&s.curCover) } - if s.curAlbum != nil { + if s.curAlbum.IsNew { s.curAlbum.CoverID = s.curCover.ID - if newCover || s.curAlbum.ID == 0 { - s.tx.Save(s.curAlbum) - } + s.tx.Save(&s.curAlbum) } folder := s.curFolders.Pop() - if folder.ID == 0 || newCover { + if folder.IsNew { folder.ParentID = s.curFolders.PeekID() folder.CoverID = s.curCover.ID - folder.HasTracks = true - s.tx.Save(folder) + folder.HasTracks = len(s.curTracks) > 1 + s.tx.Save(&folder) } for _, track := range s.curTracks { - // not checking for a nil album here because if there are - // tracks, then we at least lookup up the album track.AlbumID = s.curAlbum.ID track.FolderID = folder.ID - s.tx.Save(track) } - s.curTracks = nil - s.curCover = nil - s.curAlbum = nil - s.curAArtist = nil + toInsert := make([]interface{}, len(s.curTracks)) + for i, t := range s.curTracks { + t.FolderID = folder.ID + t.AlbumID = s.curAlbum.ID + toInsert[i] = t + } + gormbulk.BulkInsert(s.tx, toInsert, 3000) + // + s.curTracks = make([]model.Track, 0) + s.curCover = model.Cover{} + s.curAlbum = model.Album{} + s.curAArtist = model.AlbumArtist{} + // log.Printf("processed folder `%s`\n", fullPath) return nil } @@ -128,9 +143,12 @@ func (s *Scanner) handleFolderCompletion(fullPath string, info *godirwalk.Dirent func (s *Scanner) handleTrack(fullPath string, stat os.FileInfo, mime, exten string) error { // // set track basics - track := &model.Track{} + track := model.Track{} modTime := stat.ModTime() - err := s.tx.Where("path = ?", fullPath).First(track).Error + err := s.tx. + Where("path = ?", fullPath). + First(&track). + Error if !gorm.IsRecordNotFoundError(err) && modTime.Before(track.UpdatedAt) { // we found the record but it hasn't changed @@ -156,13 +174,12 @@ func (s *Scanner) handleTrack(fullPath string, stat os.FileInfo, mime, exten str track.FolderID = s.curFolders.PeekID() // // set album artist basics - s.curAArtist = &model.AlbumArtist{} err = s.tx.Where("name = ?", tags.AlbumArtist()). - First(s.curAArtist). + First(&s.curAArtist). Error if gorm.IsRecordNotFoundError(err) { s.curAArtist.Name = tags.AlbumArtist() - s.tx.Save(s.curAArtist) + s.tx.Save(&s.curAArtist) } track.AlbumArtistID = s.curAArtist.ID // @@ -173,11 +190,10 @@ func (s *Scanner) handleTrack(fullPath string, stat os.FileInfo, mime, exten str } s.curTracks = append(s.curTracks, track) // - s.curAlbum = &model.Album{} directory, _ := path.Split(fullPath) err = s.tx. Where("path = ?", directory). - First(s.curAlbum). + First(&s.curAlbum). Error if !gorm.IsRecordNotFoundError(err) { // we found the record @@ -187,6 +203,7 @@ func (s *Scanner) handleTrack(fullPath string, stat os.FileInfo, mime, exten str s.curAlbum.Title = tags.Album() s.curAlbum.Year = tags.Year() s.curAlbum.AlbumArtistID = s.curAArtist.ID + s.curAlbum.IsNew = true return nil }