upgrade sqlite3

server/ctrladmin/ctrl.go

@@ -18,7 +18,7 @@ import (
 	"github.com/oxtoacart/bpool"
 	"github.com/wader/gormstore"
 
-	"go.senan.xyz/gonic/db"
+	"go.senan.xyz/gonic/server/db"
 	"go.senan.xyz/gonic/server/assets"
 	"go.senan.xyz/gonic/server/ctrlbase"
 	"go.senan.xyz/gonic/version"

server/ctrladmin/handlers.go

@@ -7,8 +7,8 @@ import (
 	"strconv"
 	"time"
 
-	"go.senan.xyz/gonic/db"
-	"go.senan.xyz/gonic/scanner"
+	"go.senan.xyz/gonic/server/db"
+	"go.senan.xyz/gonic/server/scanner"
 	"go.senan.xyz/gonic/server/encode"
 	"go.senan.xyz/gonic/server/lastfm"
 )

server/ctrladmin/handlers_playlist.go

@@ -10,7 +10,7 @@ import (
 	"github.com/jinzhu/gorm"
 	"github.com/pkg/errors"
 
-	"go.senan.xyz/gonic/db"
+	"go.senan.xyz/gonic/server/db"
 )
 
 func playlistParseLine(c *Controller, path string) (int, error) {

server/ctrladmin/middleware.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/gorilla/sessions"
 
-	"go.senan.xyz/gonic/db"
+	"go.senan.xyz/gonic/server/db"
 )
 
 func (c *Controller) WithSession(next http.Handler) http.Handler {

server/ctrlbase/ctrl.go

@@ -6,8 +6,8 @@ import (
 	"net/http"
 	"path"
 
-	"go.senan.xyz/gonic/db"
-	"go.senan.xyz/gonic/scanner"
+	"go.senan.xyz/gonic/server/db"
+	"go.senan.xyz/gonic/server/scanner"
 )
 
 type statusWriter struct {

server/ctrlsubsonic/ctrl_test.go

@@ -13,7 +13,7 @@ import (
 
 	jd "github.com/josephburnett/jd/lib"
 
-	"go.senan.xyz/gonic/db"
+	"go.senan.xyz/gonic/server/db"
 	"go.senan.xyz/gonic/server/ctrlbase"
 	"go.senan.xyz/gonic/server/ctrlsubsonic/params"
 )

server/ctrlsubsonic/handlers_by_folder.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/jinzhu/gorm"
 
-	"go.senan.xyz/gonic/db"
+	"go.senan.xyz/gonic/server/db"
 	"go.senan.xyz/gonic/server/ctrlsubsonic/params"
 	"go.senan.xyz/gonic/server/ctrlsubsonic/spec"
 )

server/ctrlsubsonic/handlers_by_tags.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/jinzhu/gorm"
 
-	"go.senan.xyz/gonic/db"
+	"go.senan.xyz/gonic/server/db"
 	"go.senan.xyz/gonic/server/ctrlsubsonic/params"
 	"go.senan.xyz/gonic/server/ctrlsubsonic/spec"
 	"go.senan.xyz/gonic/server/lastfm"

server/ctrlsubsonic/handlers_common.go

@@ -10,8 +10,8 @@ import (
 
 	"github.com/jinzhu/gorm"
 
-	"go.senan.xyz/gonic/db"
-	"go.senan.xyz/gonic/scanner"
+	"go.senan.xyz/gonic/server/db"
+	"go.senan.xyz/gonic/server/scanner"
 	"go.senan.xyz/gonic/server/ctrlsubsonic/params"
 	"go.senan.xyz/gonic/server/ctrlsubsonic/spec"
 	"go.senan.xyz/gonic/server/lastfm"

server/ctrlsubsonic/handlers_raw.go

@@ -9,7 +9,7 @@ import (
 
 	"github.com/jinzhu/gorm"
 
-	"go.senan.xyz/gonic/db"
+	"go.senan.xyz/gonic/server/db"
 	"go.senan.xyz/gonic/server/ctrlsubsonic/params"
 	"go.senan.xyz/gonic/server/ctrlsubsonic/spec"
 	"go.senan.xyz/gonic/server/encode"

server/ctrlsubsonic/spec/construct.go

@@ -1,6 +1,6 @@
 package spec
 
-import "go.senan.xyz/gonic/db"
+import "go.senan.xyz/gonic/server/db"
 
 func NewPlaylist(p *db.Playlist) *Playlist {
 	return &Playlist{

server/ctrlsubsonic/spec/construct_by_folder.go

@@ -3,7 +3,7 @@ package spec
 import (
 	"path"
 
-	"go.senan.xyz/gonic/db"
+	"go.senan.xyz/gonic/server/db"
 )
 
 func NewAlbumByFolder(f *db.Album) *Album {

server/ctrlsubsonic/spec/construct_by_tags.go

@@ -3,7 +3,7 @@ package spec
 import (
 	"path"
 
-	"go.senan.xyz/gonic/db"
+	"go.senan.xyz/gonic/server/db"
 )
 
 func NewAlbumByTags(a *db.Album, artist *db.Artist) *Album {

server/db/db.go

@@ -0,0 +1,89 @@
+package db
+
+import (
+	"log"
+	"net/url"
+	"os"
+
+	"github.com/jinzhu/gorm"
+	"github.com/pkg/errors"
+	"gopkg.in/gormigrate.v1"
+)
+
+var (
+	dbMaxOpenConns = 1
+	dbOptions      = url.Values{
+		// with this, multiple connections share a single data and schema cache.
+		// see https://www.sqlite.org/sharedcache.html
+		"cache": {"shared"},
+		// with this, the db sleeps for a little while when locked. can prevent
+		// a SQLITE_BUSY. see https://www.sqlite.org/c3ref/busy_timeout.html
+		"_busy_timeout": {"30000"},
+		"_journal_mode": {"WAL"},
+		"_foreign_keys": {"true"},
+	}
+)
+
+type DB struct {
+	*gorm.DB
+}
+
+func New(path string) (*DB, error) {
+	url := url.URL{Path: path}
+	url.RawQuery = dbOptions.Encode()
+	db, err := gorm.Open("sqlite3", url.String())
+	if err != nil {
+		return nil, errors.Wrap(err, "with gorm")
+	}
+	db.SetLogger(log.New(os.Stdout, "gorm ", 0))
+	db.DB().SetMaxOpenConns(dbMaxOpenConns)
+	migr := gormigrate.New(db, gormigrate.DefaultOptions, []*gormigrate.Migration{
+		&migrationInitSchema,
+		&migrationCreateInitUser,
+		&migrationMergePlaylist,
+		&migrationCreateTranscode,
+		&migrationAddGenre,
+		&migrationUpdateTranscodePrefIDX,
+	})
+	if err = migr.Migrate(); err != nil {
+		return nil, errors.Wrap(err, "migrating to latest version")
+	}
+	return &DB{DB: db}, nil
+}
+
+func NewMock() (*DB, error) {
+	return New(":memory:")
+}
+
+func (db *DB) GetSetting(key string) string {
+	setting := &Setting{}
+	db.
+		Where("key=?", key).
+		First(setting)
+	return setting.Value
+}
+
+func (db *DB) SetSetting(key, value string) {
+	db.
+		Where(Setting{Key: key}).
+		Assign(Setting{Value: value}).
+		FirstOrCreate(&Setting{})
+}
+
+func (db *DB) GetUserFromName(name string) *User {
+	user := &User{}
+	err := db.
+		Where("name=?", name).
+		First(user).
+		Error
+	if gorm.IsRecordNotFoundError(err) {
+		return nil
+	}
+	return user
+}
+
+func (db *DB) WithTx(cb func(tx *gorm.DB)) {
+	tx := db.Begin()
+	defer tx.Commit()
+	cb(tx)
+}

server/db/db_test.go

@@ -0,0 +1,46 @@
+package db
+
+import (
+	"log"
+	"math/rand"
+	"testing"
+
+	_ "github.com/jinzhu/gorm/dialects/sqlite"
+)
+
+var testDB *DB
+
+func init() {
+	var err error
+	testDB, err = NewMock()
+	if err != nil {
+		log.Fatalf("error opening database: %v\n", err)
+	}
+}
+
+func randKey() string {
+	letters := []rune("abcdef0123456789")
+	b := make([]rune, 16)
+	for i := range b {
+		b[i] = letters[rand.Intn(len(letters))]
+	}
+	return string(b)
+}
+
+func TestGetSetting(t *testing.T) {
+	key := randKey()
+	// new key
+	expected := "hello"
+	testDB.SetSetting(key, expected)
+	actual := testDB.GetSetting(key)
+	if actual != expected {
+		t.Errorf("expected %q, got %q", expected, actual)
+	}
+	// existing key
+	expected = "howdy"
+	testDB.SetSetting(key, expected)
+	actual = testDB.GetSetting(key)
+	if actual != expected {
+		t.Errorf("expected %q, got %q", expected, actual)
+	}
+}

server/db/migrations.go

@@ -0,0 +1,134 @@
+//nolint:deadcode,varcheck
+package db
+
+import (
+	"fmt"
+
+	"github.com/jinzhu/gorm"
+	"gopkg.in/gormigrate.v1"
+)
+
+// $ date '+%Y%m%d%H%M'
+
+// not really a migration
+var migrationInitSchema = gormigrate.Migration{
+	ID: "202002192100",
+	Migrate: func(tx *gorm.DB) error {
+		return tx.AutoMigrate(
+			Artist{},
+			Track{},
+			User{},
+			Setting{},
+			Play{},
+			Album{},
+			Playlist{},
+			PlayQueue{},
+		).
+			Error
+	},
+}
+
+// not really a migration
+var migrationCreateInitUser = gormigrate.Migration{
+	ID: "202002192019",
+	Migrate: func(tx *gorm.DB) error {
+		const (
+			initUsername = "admin"
+			initPassword = "admin"
+		)
+		err := tx.
+			Where("name=?", initUsername).
+			First(&User{}).
+			Error
+		if !gorm.IsRecordNotFoundError(err) {
+			return nil
+		}
+		return tx.Create(&User{
+			Name:     initUsername,
+			Password: initPassword,
+			IsAdmin:  true,
+		}).
+			Error
+	},
+}
+
+var migrationMergePlaylist = gormigrate.Migration{
+	ID: "202002192222",
+	Migrate: func(tx *gorm.DB) error {
+		if !tx.HasTable("playlist_items") {
+			return nil
+		}
+		return tx.Exec(`
+			UPDATE playlists
+			SET items=( SELECT group_concat(track_id) FROM (
+				SELECT track_id
+				FROM playlist_items
+				WHERE playlist_items.playlist_id=playlists.id
+				ORDER BY created_at
+			) );
+			DROP TABLE playlist_items;`,
+		).
+			Error
+	},
+}
+
+var migrationCreateTranscode = gormigrate.Migration{
+	ID: "202003111222",
+	Migrate: func(tx *gorm.DB) error {
+		return tx.AutoMigrate(
+			TranscodePreference{},
+		).
+			Error
+	},
+}
+
+var migrationAddGenre = gormigrate.Migration{
+	ID: "202003121330",
+	Migrate: func(tx *gorm.DB) error {
+		return tx.AutoMigrate(
+			Genre{},
+			Album{},
+			Track{},
+		).
+			Error
+	},
+}
+
+var migrationUpdateTranscodePrefIDX = gormigrate.Migration{
+	ID: "202003241509",
+	Migrate: func(tx *gorm.DB) error {
+		var hasIDX int
+		tx.
+			Select("1").
+			Table("sqlite_master").
+			Where("type = ?", "index").
+			Where("name = ?", "idx_user_id_client").
+			Count(&hasIDX)
+		if hasIDX == 1 {
+			// index already exists
+			return nil
+		}
+		step := tx.Exec(`
+			ALTER TABLE transcode_preferences RENAME TO transcode_preferences_orig;
+		`)
+		if err := step.Error; err != nil {
+			return fmt.Errorf("step rename: %w", err)
+		}
+		step = tx.AutoMigrate(
+			TranscodePreference{},
+		)
+		if err := step.Error; err != nil {
+			return fmt.Errorf("step create: %w", err)
+		}
+		step = tx.Exec(`
+			INSERT INTO transcode_preferences (user_id, client, profile)
+				SELECT user_id, client, profile
+				FROM transcode_preferences_orig;
+			DROP TABLE transcode_preferences_orig;
+		`)
+		if err := step.Error; err != nil {
+			return fmt.Errorf("step copy: %w", err)
+		}
+		return nil
+	},
+}

server/db/model.go

@@ -0,0 +1,209 @@
+//nolint:lll
+package db
+
+import (
+	"path"
+	"strconv"
+	"strings"
+	"time"
+
+	"go.senan.xyz/gonic/server/mime"
+)
+
+func splitInt(in, sep string) []int {
+	if len(in) == 0 {
+		return []int{}
+	}
+	parts := strings.Split(in, sep)
+	ret := make([]int, 0, len(parts))
+	for _, p := range parts {
+		i, _ := strconv.Atoi(p)
+		ret = append(ret, i)
+	}
+	return ret
+}
+
+func joinInt(in []int, sep string) string {
+	if in == nil {
+		return ""
+	}
+	strs := make([]string, 0, len(in))
+	for _, i := range in {
+		strs = append(strs, strconv.Itoa(i))
+	}
+	return strings.Join(strs, sep)
+}
+
+type Artist struct {
+	ID         int      `gorm:"primary_key"`
+	Name       string   `gorm:"not null; unique_index"`
+	NameUDec   string   `sql:"default: null"`
+	Albums     []*Album `gorm:"foreignkey:TagArtistID"`
+	AlbumCount int      `sql:"-"`
+}
+
+func (a *Artist) IndexName() string {
+	if len(a.NameUDec) > 0 {
+		return a.NameUDec
+	}
+	return a.Name
+}
+
+type Genre struct {
+	ID         int      `gorm:"primary_ket"`
+	Name       string   `gorm:"not null; unique_index"`
+	Albums     []*Album `gorm:"foreignkey:TagGenreID"`
+	AlbumCount int      `sql:"-"`
+	Tracks     []*Track `gorm:"foreignkey:TagGenreID"`
+	TrackCount int      `sql:"-"`
+}
+
+type Track struct {
+	ID             int `gorm:"primary_key"`
+	CreatedAt      time.Time
+	UpdatedAt      time.Time
+	Filename       string `gorm:"not null; unique_index:idx_folder_filename" sql:"default: null"`
+	FilenameUDec   string `sql:"default: null"`
+	Album          *Album
+	AlbumID        int `gorm:"not null; unique_index:idx_folder_filename" sql:"default: null; type:int REFERENCES albums(id) ON DELETE CASCADE"`
+	Artist         *Artist
+	ArtistID       int    `gorm:"not null" sql:"default: null; type:int REFERENCES artists(id) ON DELETE CASCADE"`
+	Size           int    `gorm:"not null" sql:"default: null"`
+	Length         int    `sql:"default: null"`
+	Bitrate        int    `sql:"default: null"`
+	TagTitle       string `sql:"default: null"`
+	TagTitleUDec   string `sql:"default: null"`
+	TagTrackArtist string `sql:"default: null"`
+	TagTrackNumber int    `sql:"default: null"`
+	TagDiscNumber  int    `sql:"default: null"`
+	TagGenre       *Genre
+	TagGenreID     int    `sql:"default: null; type:int REFERENCES genres(id) ON DELETE CASCADE"`
+	TagBrainzID    string `sql:"default: null"`
+}
+
+func (t *Track) Ext() string {
+	longExt := path.Ext(t.Filename)
+	if len(longExt) < 1 {
+		return ""
+	}
+	return longExt[1:]
+}
+
+func (t *Track) MIME() string {
+	ext := t.Ext()
+	return mime.Types[ext]
+}
+
+func (t *Track) RelPath() string {
+	if t.Album == nil {
+		return ""
+	}
+	return path.Join(
+		t.Album.LeftPath,
+		t.Album.RightPath,
+		t.Filename,
+	)
+}
+
+type User struct {
+	ID            int `gorm:"primary_key"`
+	CreatedAt     time.Time
+	Name          string `gorm:"not null; unique_index" sql:"default: null"`
+	Password      string `gorm:"not null" sql:"default: null"`
+	LastFMSession string `sql:"default: null"`
+	IsAdmin       bool   `sql:"default: null"`
+}
+
+type Setting struct {
+	Key   string `gorm:"not null; primary_key; auto_increment:false" sql:"default: null"`
+	Value string `sql:"default: null"`
+}
+
+type Play struct {
+	ID      int `gorm:"primary_key"`
+	User    *User
+	UserID  int `gorm:"not null; index" sql:"default: null; type:int REFERENCES users(id) ON DELETE CASCADE"`
+	Album   *Album
+	AlbumID int       `gorm:"not null; index" sql:"default: null; type:int REFERENCES albums(id) ON DELETE CASCADE"`
+	Time    time.Time `sql:"default: null"`
+	Count   int
+}
+
+type Album struct {
+	ID            int `gorm:"primary_key"`
+	UpdatedAt     time.Time
+	ModifiedAt    time.Time
+	LeftPath      string `gorm:"unique_index:idx_left_path_right_path"`
+	RightPath     string `gorm:"not null; unique_index:idx_left_path_right_path" sql:"default: null"`
+	RightPathUDec string `sql:"default: null"`
+	Parent        *Album
+	ParentID      int    `sql:"default: null; type:int REFERENCES albums(id) ON DELETE CASCADE"`
+	Cover         string `sql:"default: null"`
+	TagArtist     *Artist
+	TagArtistID   int `sql:"default: null; type:int REFERENCES artists(id) ON DELETE CASCADE"`
+	TagGenre      *Genre
+	TagGenreID    int    `sql:"default: null; type:int REFERENCES genres(id) ON DELETE CASCADE"`
+	TagTitle      string `sql:"default: null"`
+	TagTitleUDec  string `sql:"default: null"`
+	TagBrainzID   string `sql:"default: null"`
+	TagYear       int    `sql:"default: null"`
+	Tracks        []*Track
+	ChildCount    int  `sql:"-"`
+	ReceivedPaths bool `gorm:"-"`
+	ReceivedTags  bool `gorm:"-"`
+}
+
+func (a *Album) IndexRightPath() string {
+	if len(a.RightPathUDec) > 0 {
+		return a.RightPathUDec
+	}
+	return a.RightPath
+}
+
+type Playlist struct {
+	ID         int `gorm:"primary_key"`
+	CreatedAt  time.Time
+	UpdatedAt  time.Time
+	User       *User
+	UserID     int `sql:"default: null; type:int REFERENCES users(id) ON DELETE CASCADE"`
+	Name       string
+	Comment    string
+	TrackCount int
+	Items      string
+}
+
+func (p *Playlist) GetItems() []int {
+	return splitInt(p.Items, ",")
+}
+
+func (p *Playlist) SetItems(items []int) {
+	p.Items = joinInt(items, ",")
+	p.TrackCount = len(items)
+}
+
+type PlayQueue struct {
+	ID        int `gorm:"primary_key"`
+	CreatedAt time.Time
+	UpdatedAt time.Time
+	User      *User
+	UserID    int `sql:"default: null; type:int REFERENCES users(id) ON DELETE CASCADE"`
+	Current   int
+	Position  int
+	ChangedBy string
+	Items     string
+}
+
+func (p *PlayQueue) GetItems() []int {
+	return splitInt(p.Items, ",")
+}
+
+func (p *PlayQueue) SetItems(items []int) {
+	p.Items = joinInt(items, ",")
+}
+
+type TranscodePreference struct {
+	User    *User
+	UserID  int    `gorm:"not null; unique_index:idx_user_id_client" sql:"default: null; type:int REFERENCES users(id) ON DELETE CASCADE"`
+	Client  string `gorm:"not null; unique_index:idx_user_id_client" sql:"default: null"`
+	Profile string `gorm:"not null" sql:"default: null"`
+}

server/jukebox/jukebox.go

@@ -14,7 +14,7 @@ import (
 	"github.com/faiface/beep/mp3"
 	"github.com/faiface/beep/speaker"
 
-	"go.senan.xyz/gonic/db"
+	"go.senan.xyz/gonic/server/db"
 	"go.senan.xyz/gonic/server/ctrlsubsonic/spec"
 )
 

server/lastfm/lastfm.go

@@ -13,7 +13,7 @@ import (
 
 	"github.com/pkg/errors"
 
-	"go.senan.xyz/gonic/db"
+	"go.senan.xyz/gonic/server/db"
 )
 
 var (

server/mime/mime.go

@@ -0,0 +1,11 @@
+package mime
+
+var Types = map[string]string{
+	"mp3":  "audio/mpeg",
+	"flac": "audio/x-flac",
+	"aac":  "audio/x-aac",
+	"m4a":  "audio/m4a",
+	"m4b":  "audio/m4b",
+	"ogg":  "audio/ogg",
+	"opus": "audio/ogg",
+}

server/scanner/scanner.go

@@ -0,0 +1,401 @@
+package scanner
+
+import (
+	"log"
+	"os"
+	"path"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	"github.com/jinzhu/gorm"
+	"github.com/karrick/godirwalk"
+	"github.com/pkg/errors"
+	"github.com/rainycape/unidecode"
+
+	"go.senan.xyz/gonic/server/db"
+	"go.senan.xyz/gonic/server/mime"
+	"go.senan.xyz/gonic/server/scanner/stack"
+	"go.senan.xyz/gonic/server/scanner/tags"
+)
+
+// isScanning acts as an atomic boolean semaphore. we don't
+// want to have more than one scan going on at a time
+var isScanning int32
+
+func IsScanning() bool {
+	return atomic.LoadInt32(&isScanning) == 1
+}
+
+func SetScanning() func() {
+	atomic.StoreInt32(&isScanning, 1)
+	return func() {
+		atomic.StoreInt32(&isScanning, 0)
+	}
+}
+
+type Scanner struct {
+	db        *db.DB
+	musicPath string
+	isFull    bool
+	// these two are for the transaction we do for every folder.
+	// the boolean is there so we dont begin or commit multiple
+	// times in the handle folder or post children callback
+	trTx     *gorm.DB
+	trTxOpen bool
+	// these two are for keeping state between noted in the tree.
+	// eg. keep track of a parents folder or the path to a cover
+	// we just saw that we need to commit in the post children
+	// callback
+	curFolders *stack.Stack
+	curCover   string
+	// then the rest are for stats and cleanup at the very end
+	seenTracks    map[int]struct{} // set of p keys
+	seenFolders   map[int]struct{} // set of p keys
+	seenTracksNew int              // n tracks not seen before
+	seenTracksErr int              // n tracks we we couldn't scan
+}
+
+func New(musicPath string, db *db.DB) *Scanner {
+	return &Scanner{
+		db:          db,
+		musicPath:   musicPath,
+		seenTracks:  make(map[int]struct{}),
+		seenFolders: make(map[int]struct{}),
+		curFolders:  &stack.Stack{},
+	}
+}
+
+func (s *Scanner) Start(isFull bool) error {
+	if IsScanning() {
+		return errors.New("already scanning")
+	}
+	unSet := SetScanning()
+	defer unSet()
+	// reset tracking variables when finished
+	defer func() {
+		s.seenTracks = make(map[int]struct{})
+		s.seenFolders = make(map[int]struct{})
+		s.curFolders = &stack.Stack{}
+		s.seenTracksNew = 0
+		s.seenTracksErr = 0
+	}()
+	// ** begin being walking
+	s.isFull = isFull
+	start := time.Now()
+	err := godirwalk.Walk(s.musicPath, &godirwalk.Options{
+		Callback:             s.callbackItem,
+		PostChildrenCallback: s.callbackPost,
+		Unsorted:             true,
+		FollowSymbolicLinks:  true,
+	})
+	if err != nil {
+		return errors.Wrap(err, "walking filesystem")
+	}
+	log.Printf("finished scan in %s, +%d/%d tracks (%d err)\n",
+		time.Since(start),
+		s.seenTracksNew,
+		len(s.seenTracks),
+		s.seenTracksErr,
+	)
+	// ** begin cleaning
+	start = time.Now()
+	var deleted uint
+	// delete tracks not on filesystem
+	s.db.WithTx(func(tx *gorm.DB) {
+		var tracks []*db.Track
+		tx.Select("id").Find(&tracks)
+		for _, track := range tracks {
+			if _, ok := s.seenTracks[track.ID]; !ok {
+				tx.Delete(track)
+				deleted++
+			}
+		}
+	})
+	// delete folders not on filesystem
+	s.db.WithTx(func(tx *gorm.DB) {
+		var folders []*db.Album
+		tx.Select("id").Find(&folders)
+		for _, folder := range folders {
+			if _, ok := s.seenFolders[folder.ID]; !ok {
+				tx.Delete(folder)
+			}
+		}
+	})
+	// delete albums without tracks
+	s.db.Exec(`
+		DELETE FROM albums
+		WHERE tag_artist_id NOT NULL
+		AND NOT EXISTS ( SELECT 1 FROM tracks
+		                 WHERE tracks.album_id=albums.id
+		)`)
+	// delete artists without albums
+	s.db.Exec(`
+		DELETE FROM artists
+		WHERE NOT EXISTS ( SELECT 1 from albums
+		                   WHERE albums.tag_artist_id=artists.id
+		)`)
+	// finish up
+	strNow := strconv.FormatInt(time.Now().Unix(), 10)
+	s.db.SetSetting("last_scan_time", strNow)
+	//
+	log.Printf("finished clean in %s, -%d tracks\n",
+		time.Since(start),
+		deleted,
+	)
+	return nil
+}
+
+func (s *Scanner) StartInc() error {
+	return s.Start(false)
+}
+
+func (s *Scanner) StartFull() error {
+	return s.Start(true)
+}
+
+// items are passed to the handle*() functions
+type item struct {
+	fullPath  string
+	relPath   string
+	directory string
+	filename  string
+	stat      os.FileInfo
+}
+
+var coverFilenames = map[string]struct{}{
+	"cover.png":   {},
+	"cover.jpg":   {},
+	"cover.jpeg":  {},
+	"folder.png":  {},
+	"folder.jpg":  {},
+	"folder.jpeg": {},
+	"album.png":   {},
+	"album.jpg":   {},
+	"album.jpeg":  {},
+	"front.png":   {},
+	"front.jpg":   {},
+	"front.jpeg":  {},
+}
+
+// ## begin callbacks
+// ## begin callbacks
+// ## begin callbacks
+
+func (s *Scanner) callbackItem(fullPath string, info *godirwalk.Dirent) error {
+	stat, err := os.Stat(fullPath)
+	if err != nil {
+		return errors.Wrap(err, "stating")
+	}
+	relPath, err := filepath.Rel(s.musicPath, fullPath)
+	if err != nil {
+		return errors.Wrap(err, "getting relative path")
+	}
+	directory, filename := path.Split(relPath)
+	it := &item{
+		fullPath:  fullPath,
+		relPath:   relPath,
+		directory: directory,
+		filename:  filename,
+		stat:      stat,
+	}
+	isDir, err := info.IsDirOrSymlinkToDir()
+	if err != nil {
+		return errors.Wrap(err, "stating link to dir")
+	}
+	if isDir {
+		return s.handleFolder(it)
+	}
+	lowerFilename := strings.ToLower(filename)
+	if _, ok := coverFilenames[lowerFilename]; ok {
+		s.curCover = filename
+		return nil
+	}
+	ext := path.Ext(filename)
+	if ext == "" {
+		return nil
+	}
+	if _, ok := mime.Types[ext[1:]]; ok {
+		return s.handleTrack(it)
+	}
+	return nil
+}
+
+func (s *Scanner) callbackPost(fullPath string, info *godirwalk.Dirent) error {
+	defer func() {
+		s.curCover = ""
+	}()
+	if s.trTxOpen {
+		s.trTx.Commit()
+		s.trTxOpen = false
+	}
+	// begin taking the current folder off the stack and add it's
+	// parent, cover that we found, etc.
+	folder := s.curFolders.Pop()
+	if !folder.ReceivedPaths {
+		return nil
+	}
+	folder.ParentID = s.curFolders.PeekID()
+	folder.Cover = s.curCover
+	s.db.Save(folder)
+	// we only log changed folders
+	log.Printf("processed folder `%s`\n",
+		path.Join(folder.LeftPath, folder.RightPath))
+	return nil
+}
+
+// decoded converts a string to it's latin equivalent. it will
+// be used by the model's *UDec fields, and is only set if it
+// differs from the original. the fields are used for searching
+func decoded(in string) string {
+	result := unidecode.Unidecode(in)
+	if result == in {
+		return ""
+	}
+	return result
+}
+
+// ## begin handlers
+// ## begin handlers
+// ## begin handlers
+
+func (s *Scanner) handleFolder(it *item) error {
+	if s.trTxOpen {
+		// a transaction still being open when we handle a folder can
+		// happen if there is a folder that contains /both/ tracks and
+		// sub folders
+		s.trTx.Commit()
+		s.trTxOpen = false
+	}
+	folder := &db.Album{}
+	defer func() {
+		// folder's id will come from early return
+		// or save at the end
+		s.seenFolders[folder.ID] = struct{}{}
+		s.curFolders.Push(folder)
+	}()
+	err := s.db.
+		Select("id, updated_at").
+		Where(db.Album{
+			LeftPath:  it.directory,
+			RightPath: it.filename,
+		}).
+		First(folder).
+		Error
+	if !s.isFull && (!gorm.IsRecordNotFoundError(err) &&
+		it.stat.ModTime().Before(folder.UpdatedAt)) {
+		// we found the record but it hasn't changed
+		return nil
+	}
+	folder.LeftPath = it.directory
+	folder.RightPath = it.filename
+	folder.RightPathUDec = decoded(it.filename)
+	folder.ModifiedAt = it.stat.ModTime()
+	s.db.Save(folder)
+	folder.ReceivedPaths = true
+	return nil
+}
+
+func (s *Scanner) handleTrack(it *item) error {
+	if !s.trTxOpen {
+		s.trTx = s.db.Begin()
+		s.trTxOpen = true
+	}
+	// ** begin set track basics
+	track := &db.Track{}
+	err := s.trTx.
+		Select("id, updated_at").
+		Where(db.Track{
+			AlbumID:  s.curFolders.PeekID(),
+			Filename: it.filename,
+		}).
+		First(track).
+		Error
+	if !s.isFull && !gorm.IsRecordNotFoundError(err) &&
+		it.stat.ModTime().Before(track.UpdatedAt) {
+		// we found the record but it hasn't changed
+		s.seenTracks[track.ID] = struct{}{}
+		return nil
+	}
+	track.Filename = it.filename
+	track.FilenameUDec = decoded(it.filename)
+	track.Size = int(it.stat.Size())
+	track.AlbumID = s.curFolders.PeekID()
+	trTags, err := tags.New(it.fullPath)
+	if err != nil {
+		// not returning the error here because we don't want
+		// the entire walk to stop if we can't read the tags
+		// of a single file
+		log.Printf("error reading tags `%s`: %v", it.relPath, err)
+		s.seenTracksErr++
+		return nil
+	}
+	track.TagTitle = trTags.Title()
+	track.TagTitleUDec = decoded(trTags.Title())
+	track.TagTrackArtist = trTags.Artist()
+	track.TagTrackNumber = trTags.TrackNumber()
+	track.TagDiscNumber = trTags.DiscNumber()
+	track.TagBrainzID = trTags.BrainzID()
+	track.Length = trTags.Length()   // these two should be calculated
+	track.Bitrate = trTags.Bitrate() // ...from the file instead of tags
+	// ** begin set album artist basics
+	artistName := func() string {
+		if r := trTags.AlbumArtist(); r != "" {
+			return r
+		}
+		if r := trTags.Artist(); r != "" {
+			return r
+		}
+		return "Unknown Artist"
+	}()
+	artist := &db.Artist{}
+	err = s.trTx.
+		Select("id").
+		Where("name=?", artistName).
+		First(artist).
+		Error
+	if gorm.IsRecordNotFoundError(err) {
+		artist.Name = artistName
+		artist.NameUDec = decoded(artistName)
+		s.trTx.Save(artist)
+	}
+	track.ArtistID = artist.ID
+	// ** begin set genre
+	genreName := func() string {
+		if r := trTags.Genre(); r != "" {
+			return r
+		}
+		return "Unknown Genre"
+	}()
+	genre := &db.Genre{}
+	err = s.trTx.
+		Select("id").
+		Where("name=?", genreName).
+		First(genre).
+		Error
+	if gorm.IsRecordNotFoundError(err) {
+		genre.Name = genreName
+		s.trTx.Save(genre)
+	}
+	track.TagGenreID = genre.ID
+	// ** begin save the track
+	s.trTx.Save(track)
+	s.seenTracks[track.ID] = struct{}{}
+	s.seenTracksNew++
+	// ** begin set album if this is the first track in the folder
+	folder := s.curFolders.Peek()
+	if !folder.ReceivedPaths || folder.ReceivedTags {
+		// the folder hasn't been modified or already has it's tags
+		return nil
+	}
+	folder.TagTitle = trTags.Album()
+	folder.TagTitleUDec = decoded(trTags.Album())
+	folder.TagBrainzID = trTags.AlbumBrainzID()
+	folder.TagYear = trTags.Year()
+	folder.TagArtistID = artist.ID
+	folder.TagGenreID = genre.ID
+	folder.ReceivedTags = true
+	return nil
+}

server/scanner/scanner_test.go

@@ -0,0 +1,65 @@
+package scanner
+
+import (
+	"io/ioutil"
+	"log"
+	"testing"
+
+	_ "github.com/jinzhu/gorm/dialects/sqlite"
+
+	"go.senan.xyz/gonic/server/db"
+)
+
+var testScanner *Scanner
+
+func init() {
+	db, err := db.NewMock()
+	if err != nil {
+		log.Fatalf("error opening database: %v\n", err)
+	}
+	// benchmarks aren't real code are they? >:)
+	// here is an absolute path to my music directory
+	testScanner = New("/home/senan/music", db)
+	log.SetOutput(ioutil.Discard)
+}
+
+func resetTables(db *db.DB) {
+	tx := db.Begin()
+	defer tx.Commit()
+	tx.Exec("delete from tracks")
+	tx.Exec("delete from artists")
+	tx.Exec("delete from albums")
+}
+
+func resetTablesPause(db *db.DB, b *testing.B) {
+	b.StopTimer()
+	defer b.StartTimer()
+	resetTables(db)
+}
+
+func BenchmarkScanFresh(b *testing.B) {
+	for n := 0; n < b.N; n++ {
+		resetTablesPause(testScanner.db, b)
+		testScanner.StartInc()
+	}
+}
+
+func BenchmarkScanIncremental(b *testing.B) {
+	// do a full scan and reset
+	testScanner.StartInc()
+	b.ResetTimer()
+	// do the inc scans
+	for n := 0; n < b.N; n++ {
+		testScanner.StartInc()
+	}
+}
+
+// RESULTS fresh
+// 20 times / 1.436
+// 20 times / 1.39
+
+// RESULTS inc
+// 100 times / 1.86
+// 100 times / 1.9
+// 100 times / 1.5
+// 100 times / 1.48

server/scanner/stack/stack.go

@@ -0,0 +1,61 @@
+package stack
+
+import (
+	"fmt"
+	"strings"
+
+	"go.senan.xyz/gonic/server/db"
+)
+
+type item struct {
+	value *db.Album
+	next  *item
+}
+
+type Stack struct {
+	top *item
+	len uint
+}
+
+func (s *Stack) Push(v *db.Album) {
+	s.top = &item{
+		value: v,
+		next:  s.top,
+	}
+	s.len++
+}
+
+func (s *Stack) Pop() *db.Album {
+	if s.len == 0 {
+		return nil
+	}
+	v := s.top.value
+	s.top = s.top.next
+	s.len--
+	return v
+}
+
+func (s *Stack) Peek() *db.Album {
+	if s.len == 0 {
+		return nil
+	}
+	return s.top.value
+}
+
+func (s *Stack) PeekID() int {
+	if s.len == 0 {
+		return 0
+	}
+	return s.top.value.ID
+}
+
+func (s *Stack) String() string {
+	var str strings.Builder
+	str.WriteString("[")
+	for i, f := uint(0), s.top; i < s.len; i++ {
+		str.WriteString(fmt.Sprintf("%d, ", f.value.ID))
+		f = f.next
+	}
+	str.WriteString("]")
+	return str.String()
+}

server/scanner/stack/stack_test.go

@@ -0,0 +1,36 @@
+package stack
+
+import (
+	"testing"
+
+	"go.senan.xyz/gonic/server/db"
+)
+
+func TestFolderStack(t *testing.T) {
+	sta := &Stack{}
+	sta.Push(&db.Album{ID: 3})
+	sta.Push(&db.Album{ID: 4})
+	sta.Push(&db.Album{ID: 5})
+	sta.Push(&db.Album{ID: 6})
+	expected := "[6, 5, 4, 3, ]"
+	actual := sta.String()
+	if expected != actual {
+		t.Errorf("first stack: expected string "+
+			"%q, got %q", expected, actual)
+	}
+	//
+	sta = &Stack{}
+	sta.Push(&db.Album{ID: 27})
+	sta.Push(&db.Album{ID: 4})
+	sta.Peek()
+	sta.Push(&db.Album{ID: 5})
+	sta.Push(&db.Album{ID: 6})
+	sta.Push(&db.Album{ID: 7})
+	sta.Pop()
+	expected = "[6, 5, 4, 27, ]"
+	actual = sta.String()
+	if expected != actual {
+		t.Errorf("second stack: expected string "+
+			"%q, got %q", expected, actual)
+	}
+}

server/scanner/tags/tags.go

@@ -0,0 +1,59 @@
+package tags
+
+import (
+	"strconv"
+	"strings"
+
+	"github.com/nicksellen/audiotags"
+	"github.com/pkg/errors"
+)
+
+type Tags struct {
+	raw   map[string]string
+	props *audiotags.AudioProperties
+}
+
+func New(path string) (*Tags, error) {
+	raw, props, err := audiotags.Read(path)
+	if err != nil {
+		return nil, errors.Wrap(err, "audiotags module")
+	}
+	return &Tags{
+		raw:   raw,
+		props: props,
+	}, nil
+}
+
+func (t *Tags) firstTag(keys ...string) string {
+	for _, key := range keys {
+		if val, ok := t.raw[key]; ok {
+			return val
+		}
+	}
+	return ""
+}
+
+func (t *Tags) Title() string         { return t.firstTag("title") }
+func (t *Tags) BrainzID() string      { return t.firstTag("musicbrainz_trackid") }
+func (t *Tags) Artist() string        { return t.firstTag("artist") }
+func (t *Tags) Album() string         { return t.firstTag("album") }
+func (t *Tags) AlbumArtist() string   { return t.firstTag("albumartist", "album artist") }
+func (t *Tags) AlbumBrainzID() string { return t.firstTag("musicbrainz_albumid") }
+func (t *Tags) Genre() string         { return t.firstTag("genre") }
+func (t *Tags) Year() int             { return intSep(t.firstTag("date", "year"), "-") } // eg. 2019-6-11
+func (t *Tags) TrackNumber() int      { return intSep(t.firstTag("tracknumber"), "/") }  // eg. 5/12
+func (t *Tags) DiscNumber() int       { return intSep(t.firstTag("discnumber"), "/") }   // eg. 1/2
+func (t *Tags) Length() int           { return t.props.Length }
+func (t *Tags) Bitrate() int          { return t.props.Bitrate }
+
+func intSep(in, sep string) int {
+	if in == "" {
+		return 0
+	}
+	start := strings.SplitN(in, sep, 2)[0]
+	out, err := strconv.Atoi(start)
+	if err != nil {
+		return 0
+	}
+	return out
+}

server/server.go

@@ -10,8 +10,8 @@ import (
 
 	"github.com/gorilla/mux"
 
-	"go.senan.xyz/gonic/db"
-	"go.senan.xyz/gonic/scanner"
+	"go.senan.xyz/gonic/server/db"
+	"go.senan.xyz/gonic/server/scanner"
 	"go.senan.xyz/gonic/server/assets"
 	"go.senan.xyz/gonic/server/ctrladmin"
 	"go.senan.xyz/gonic/server/ctrlbase"