heimoshuiyu/gonic
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

add browse by folder to scanner

Browse Source
This commit is contained in:
sentriz
2019-05-08 14:53:35 +01:00
parent 342928d608
commit 9992113e25
3 changed files with 155 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
TODO Normal file
View File

@@ -0,0 +1,72 @@
2019/05/07 14:34:55 entering folder `/home/senan/music`
2019/05/07 14:34:55 entering folder `/home/senan/music/A Certain Ratio`
2019/05/07 14:34:55 entering folder `/home/senan/music/A Certain Ratio/(1994) The Graveyard and the Ballroom`
2019/05/07 14:34:55 ___ processed folder `/home/senan/music/A Certain Ratio/(1994) The Graveyard and the Ballroom`
2019/05/07 14:34:55 entering folder `/home/senan/music/A Certain Ratio/(1981) To Each.`
2019/05/07 14:34:55 ___ processed folder `/home/senan/music/A Certain Ratio/(1981) To Each.`
2019/05/07 14:34:55 ___ processed folder `/home/senan/music/A Certain Ratio`
2019/05/07 14:34:55 entering folder `/home/senan/music/13th Floor Elevators`
2019/05/07 14:34:55 entering folder `/home/senan/music/13th Floor Elevators/(1966) The Psychedelic Sounds of the 13th Floor Elevators`
2019/05/07 14:34:55 ___ processed folder `/home/senan/music/13th Floor Elevators/(1966) The Psychedelic Sounds of the 13th Floor Elevators`
2019/05/07 14:34:55 entering folder `/home/senan/music/13th Floor Elevators/(1967) Easter Everywhere`
2019/05/07 14:34:55 ___ processed folder `/home/senan/music/13th Floor Elevators/(1967) Easter Everywhere`
2019/05/07 14:34:55 ___ processed folder `/home/senan/music/13th Floor Elevators`
2019/05/07 14:34:55 entering folder `/home/senan/music/Anika`
2019/05/07 14:34:55 entering folder `/home/senan/music/Anika/Hello`
2019/05/07 14:34:55 entering folder `/home/senan/music/Anika/Hello/There`
2019/05/07 14:34:55 entering folder `/home/senan/music/Anika/Hello/There/(2010) Anika`
2019/05/07 14:34:55 ___ processed folder `/home/senan/music/Anika/Hello/There/(2010) Anika`
2019/05/07 14:34:55 ___ processed folder `/home/senan/music/Anika/Hello/There`
2019/05/07 14:34:55 ___ processed folder `/home/senan/music/Anika/Hello`
2019/05/07 14:34:55 entering folder `/home/senan/music/Anika/No Music Here`
2019/05/07 14:34:55 ___ processed folder `/home/senan/music/Anika/No Music Here`
2019/05/07 14:34:55 ___ processed folder `/home/senan/music/Anika`
2019/05/07 14:34:55 ___ processed folder `/home/senan/music`
2019/05/07 14:34:55 scanned in 364.785µs
2019/05/07 14:34:55 cleaned in 106.441µs
2019/05/07 14:34:55 ___ processed folder `/home/senan/music/A Certain Ratio/(1994) The Graveyard and the Ballroom`
2019/05/07 14:34:55 ___ processed folder `/home/senan/music/A Certain Ratio/(1981) To Each.`
2019/05/07 14:34:55 ___ processed folder `/home/senan/music/A Certain Ratio`
2019/05/07 14:34:55 ___ processed folder `/home/senan/music/13th Floor Elevators/(1966) The Psychedelic Sounds of the 13th Floor Elevators`
2019/05/07 14:34:55 ___ processed folder `/home/senan/music/13th Floor Elevators/(1967) Easter Everywhere`
2019/05/07 14:34:55 ___ processed folder `/home/senan/music/13th Floor Elevators`
2019/05/07 14:34:55 ___ processed folder `/home/senan/music/Anika/Hello/There/(2010) Anika`
2019/05/07 14:34:55 ___ processed folder `/home/senan/music/Anika/Hello/There`
2019/05/07 14:34:55 ___ processed folder `/home/senan/music/Anika/Hello`
2019/05/07 14:34:55 ___ processed folder `/home/senan/music/Anika/No Music Here`
2019/05/07 14:34:55 ___ processed folder `/home/senan/music/Anika`
2019/05/07 14:34:55 ___ processed folder `/home/senan/music`
2019/05/07 14:34:55 scanned in 364.785µs
2019/05/07 14:34:55 cleaned in 106.441µs
// handleFolder is for browse by folders, while handleTrack is for both
func handleFolder(fullPath string, stat os.FileInfo) error {
log.Printf("entering folder `%s`", fullPath)
return nil
// this must be run before any tracks so that seenDirs is
// correct for the coming tracks
modTime := stat.ModTime()
folder := db.Folder{
Path: fullPath,
}
// skip if the record exists and hasn't been modified since
// the last scan
err := tx.Where(folder).First(&folder).Error
if !gorm.IsRecordNotFoundError(err) &&
modTime.Before(folder.UpdatedAt) {
// even though we don't want to update this record,
// add it to seenDirs now that we have the id
seenDirs.Push(folder.ID)
return nil
}
_, folderName := path.Split(fullPath)
folder.ParentID = seenDirs.Peek()
folder.Name = folderName
// save the record with new parent id, then add the new
// current id to seenDirs
tx.Save(&folder)
seenDirs.Push(folder.ID)
return nil
}

198
cmd/scanner/main.go
View File

@@ -2,12 +2,14 @@
// directory - which means you can come across the cover of an album/folder // directory - which means you can come across the cover of an album/folder
// before the tracks (and therefore the album) which is an issue because // before the tracks (and therefore the album) which is an issue because
// when inserting into the album table, we need a reference to the cover. // when inserting into the album table, we need a reference to the cover.
// to solve this we're using godirwalk's PostChildrenCallback and some // to solve this we're using godirwalk's PostChildrenCallback and some globals
// globals.
// //
// Album -> needs a CoverID // Album -> needs a CoverID
// Folder -> needs a CoverID // -> needs a FolderID (American Football)
// -> needs a ParentID // Folder -> needs a CoverID
// -> needs a ParentID
// Track -> needs an AlbumID
// -> needs a FolderID
package main package main
@@ -29,13 +31,20 @@ import (
var ( var (
orm *gorm.DB orm *gorm.DB
tx *gorm.DB tx *gorm.DB
// seenTracks is used to keep every track we've seen so that // seenPaths is used to keep every path we've seen so that
// we can remove old tracks in the clean up stage // we can remove old tracks, folders, and covers by path when we
seenTracks = make(map[string]bool) // are in the cleanDatabase stage
// seenDirs is used for inserting to the folders table (for browsing seenPaths = make(map[string]bool)
// by folders instead of tags) which helps us work out a folder's // currentDirStack is used for inserting to the folders (subsonic browse
// parent folder id // by folder) which helps us work out a folder's parent
seenDirs = make(dirStack, 0) currentDirStack = make(dirStack, 0)
// currentCover because we find a cover anywhere among the tracks during the
// walk and need a reference to it when we update folder and album records
// when we exit a folder
currentCover = db.Cover{}
// currentAlbum because we update this record when we exit a folder with
// our new reference to it's cover
currentAlbum = db.Album{}
) )
func readTags(fullPath string) (tag.Metadata, error) { func readTags(fullPath string) (tag.Metadata, error) {
@@ -53,62 +62,67 @@ func readTags(fullPath string) (tag.Metadata, error) {
func handleCover(fullPath string, stat os.FileInfo) error { func handleCover(fullPath string, stat os.FileInfo) error {
modTime := stat.ModTime() modTime := stat.ModTime()
cover := db.Cover{ err := tx.Where("path = ?", fullPath).First(&currentCover).Error
Path: fullPath,
}
err := tx.Where(cover).First(&cover).Error
if !gorm.IsRecordNotFoundError(err) && if !gorm.IsRecordNotFoundError(err) &&
modTime.Before(cover.UpdatedAt) { modTime.Before(currentCover.UpdatedAt) {
return nil return nil
} }
cover.AlbumID = 0 currentCover = db.Cover{Path: fullPath}
cover.FolderID = seenDirs.Peek() tx.Save(&currentCover)
tx.Save(&cover)
return nil return nil
} }
// handleFolder is for browse by folders, while handleTrack is for both
func handleFolder(fullPath string, stat os.FileInfo) error { func handleFolder(fullPath string, stat os.FileInfo) error {
// this must be run before any tracks so that seenDirs is
// correct for the coming tracks
modTime := stat.ModTime() modTime := stat.ModTime()
folder := db.Folder{ //
Path: fullPath, // update folder table for browsing by folder
} var folder db.Folder
// skip if the record exists and hasn't been modified since err := tx.Where("path = ?", fullPath).First(&folder).Error
// the last scan
err := tx.Where(folder).First(&folder).Error
if !gorm.IsRecordNotFoundError(err) && if !gorm.IsRecordNotFoundError(err) &&
modTime.Before(folder.UpdatedAt) { modTime.Before(folder.UpdatedAt) {
// even though we don't want to update this record, // we found the record but it hasn't changed
// add it to seenDirs now that we have the id currentDirStack.Push(&folder)
seenDirs.Push(folder.ID)
return nil return nil
} }
_, folderName := path.Split(fullPath) _, folderName := path.Split(fullPath)
folder.ParentID = seenDirs.Peek() folder.Path = fullPath
folder.ParentID = currentDirStack.PeekID()
folder.Name = folderName folder.Name = folderName
// save the record with new parent id, then add the new
// current id to seenDirs
tx.Save(&folder) tx.Save(&folder)
seenDirs.Push(folder.ID) currentDirStack.Push(&folder)
//
// update album table (the currentAlbum record will be updated when
// we exit this folder)
err = tx.Where("path = ?", fullPath).First(&currentAlbum).Error
if gorm.IsRecordNotFoundError(err) {
currentAlbum = db.Album{Path: fullPath}
tx.Save(&currentAlbum)
}
return nil
}
func handleFolderCompletion(fullPath string, info *godirwalk.Dirent) error {
if currentCover.ID != 0 {
currentDir := currentDirStack.Peek()
currentDir.CoverID = currentCover.ID
tx.Save(currentDir)
currentAlbum.CoverID = currentCover.ID
}
tx.Save(&currentAlbum)
currentCover = db.Cover{}
currentDirStack.Pop()
log.Printf("processed folder `%s`\n", fullPath)
return nil return nil
} }
func handleTrack(fullPath string, stat os.FileInfo, mime, exten string) error { func handleTrack(fullPath string, stat os.FileInfo, mime, exten string) error {
// add the full path to the seen set. see the comment above
// seenTracks for more
seenTracks[fullPath] = true
// set track basics // set track basics
track := db.Track{ var track db.Track
Path: fullPath,
}
modTime := stat.ModTime() modTime := stat.ModTime()
// skip if the record exists and hasn't been modified since err := tx.Where("path = ?", fullPath).First(&track).Error
// the last scan
err := tx.Where(track).First(&track).Error
if !gorm.IsRecordNotFoundError(err) && if !gorm.IsRecordNotFoundError(err) &&
modTime.Before(track.UpdatedAt) { modTime.Before(track.UpdatedAt) {
// we found the record but it hasn't changed
return nil return nil
} }
tags, err := readTags(fullPath) tags, err := readTags(fullPath)
@@ -128,37 +142,28 @@ func handleTrack(fullPath string, stat os.FileInfo, mime, exten string) error {
track.Suffix = exten track.Suffix = exten
track.ContentType = mime track.ContentType = mime
track.Size = int(stat.Size()) track.Size = int(stat.Size())
track.FolderID = seenDirs.Peek() track.FolderID = currentDirStack.PeekID()
// // set album artist basics
albumArtist := db.AlbumArtist{ var albumArtist db.AlbumArtist
Name: tags.AlbumArtist(), err = tx.Where("name = ?", tags.AlbumArtist()).
} First(&albumArtist).
err = tx.Where(albumArtist).First(&albumArtist).Error Error
if gorm.IsRecordNotFoundError(err) { if gorm.IsRecordNotFoundError(err) {
albumArtist.Name = tags.AlbumArtist() albumArtist.Name = tags.AlbumArtist()
tx.Save(&albumArtist) tx.Save(&albumArtist)
} }
track.AlbumArtistID = albumArtist.ID track.AlbumArtistID = albumArtist.ID
// track.AlbumID = currentAlbum.ID
album := db.Album{
AlbumArtistID: albumArtist.ID,
Title: tags.Album(),
}
err = tx.Where(album).First(&album).Error
if gorm.IsRecordNotFoundError(err) {
album.Title = tags.Album()
album.AlbumArtistID = albumArtist.ID
tx.Save(&album)
}
track.AlbumID = album.ID
//
tx.Save(&track) tx.Save(&track)
// update the current album's metadata - it will be
// inserted when we exit the folder
currentAlbum.AlbumArtistID = albumArtist.ID
currentAlbum.Title = tags.Album()
return nil return nil
} }
func handleItem(fullPath string, info *godirwalk.Dirent) error { func handleItem(fullPath string, info *godirwalk.Dirent) error {
fmt.Println(fullPath) seenPaths[fullPath] = true
return nil
stat, err := os.Stat(fullPath) stat, err := os.Stat(fullPath)
if err != nil { if err != nil {
return fmt.Errorf("error stating: %v", err) return fmt.Errorf("error stating: %v", err)
@@ -175,67 +180,6 @@ func handleItem(fullPath string, info *godirwalk.Dirent) error {
return nil return nil
} }
func handleFolderCompletion(fullPath string, info *godirwalk.Dirent) error {
seenDirs.Pop()
log.Printf("processed folder `%s`\n", fullPath)
return nil
}
func createDatabase() {
tx.AutoMigrate(
&db.Album{},
&db.AlbumArtist{},
&db.Track{},
&db.Cover{},
&db.User{},
&db.Setting{},
&db.Play{},
&db.Folder{},
)
// set starting value for `albums` table's
// auto increment
tx.Exec(`
INSERT INTO sqlite_sequence(name, seq)
SELECT 'albums', 500000
WHERE NOT EXISTS (SELECT *
FROM sqlite_sequence);
`)
// create the first user if there is none
tx.FirstOrCreate(&db.User{}, db.User{
Name: "admin",
Password: "admin",
IsAdmin: true,
})
}
func cleanDatabase() {
// delete tracks not on filesystem
var tracks []*db.Track
tx.Select("id, path").Find(&tracks)
for _, track := range tracks {
_, ok := seenTracks[track.Path]
if ok {
continue
}
tx.Delete(&track)
log.Println("removed", track.Path)
}
// delete albums without tracks
tx.Exec(`
DELETE FROM albums
WHERE (SELECT count(id)
FROM tracks
WHERE album_id = albums.id) = 0;
`)
// delete artists without tracks
tx.Exec(`
DELETE FROM album_artists
WHERE (SELECT count(id)
FROM albums
WHERE album_artist_id = album_artists.id) = 0;
`)
}
func main() { func main() {
if len(os.Args) != 2 { if len(os.Args) != 2 {
log.Fatalf("usage: %s <path to music>", os.Args[0]) log.Fatalf("usage: %s <path to music>", os.Args[0])

19
db/model.go
View File

@@ -9,7 +9,14 @@ type Album struct {
AlbumArtist AlbumArtist AlbumArtist AlbumArtist
AlbumArtistID int `gorm:"index"` AlbumArtistID int `gorm:"index"`
Title string `gorm:"not null;index"` Title string `gorm:"not null;index"`
Tracks []Track // an Album having a `Path` is a little weird when browsing by tags
// (for the most part - the library's folder structure is treated as
// if it were flat), but this solves the "American Football problem"
// https://en.wikipedia.org/wiki/American_Football_(band)#Discography
Path string `gorm:"not null;unique_index"`
CoverID int
Cover Cover
Tracks []Track
} }
// AlbumArtist represents the AlbumArtists table // AlbumArtist represents the AlbumArtists table
@@ -49,12 +56,8 @@ type Track struct {
type Cover struct { type Cover struct {
IDBase IDBase
CrudBase CrudBase
AlbumID int `gorm:"index"` Image []byte
Album Album Path string `gorm:"not null;unique_index"`
FolderID int `gorm:"index"`
Folder Folder
Image []byte
Path string `gorm:"not null;unique_index"`
} }
// User represents the users table // User represents the users table
@@ -92,4 +95,6 @@ type Folder struct {
Path string `gorm:"not null;unique_index"` Path string `gorm:"not null;unique_index"`
Parent *Folder `gorm:"foreignkey:ParentID"` Parent *Folder `gorm:"foreignkey:ParentID"`
ParentID int ParentID int
CoverID int
Cover Cover
} }