Files
gonic/db/migrations.go

816 lines
21 KiB
Go

//nolint:goerr113
package db
import (
"context"
"errors"
"fmt"
"log"
"os"
"path/filepath"
"strings"
"time"
"github.com/jinzhu/gorm"
"go.senan.xyz/gonic/fileutil"
"go.senan.xyz/gonic/playlist"
"go.senan.xyz/gonic/server/ctrlsubsonic/specid"
"gopkg.in/gormigrate.v1"
)
type MigrationContext struct {
Production bool
DBPath string
OriginalMusicPath string
PlaylistsPath string
PodcastsPath string
}
func (db *DB) Migrate(ctx MigrationContext) error {
options := &gormigrate.Options{
TableName: "migrations",
IDColumnName: "id",
IDColumnSize: 255,
UseTransaction: false,
}
// $ date '+%Y%m%d%H%M'
migrations := []*gormigrate.Migration{
construct(ctx, "202002192100", migrateInitSchema),
construct(ctx, "202002192019", migrateCreateInitUser),
construct(ctx, "202002192222", migrateMergePlaylist),
construct(ctx, "202003111222", migrateCreateTranscode),
construct(ctx, "202003121330", migrateAddGenre),
construct(ctx, "202003241509", migrateUpdateTranscodePrefIDX),
construct(ctx, "202004302006", migrateAddAlbumIDX),
construct(ctx, "202012151806", migrateMultiGenre),
construct(ctx, "202101081149", migrateListenBrainz),
construct(ctx, "202101111537", migratePodcast),
construct(ctx, "202102032210", migrateBookmarks),
construct(ctx, "202102191448", migratePodcastAutoDownload),
construct(ctx, "202110041330", migrateAlbumCreatedAt),
construct(ctx, "202111021951", migrateAlbumRootDir),
construct(ctx, "202201042236", migrateArtistGuessedFolder),
construct(ctx, "202202092013", migrateArtistCover),
construct(ctx, "202202121809", migrateAlbumRootDirAgain),
construct(ctx, "202202241218", migratePublicPlaylist),
construct(ctx, "202204270903", migratePodcastDropUserID),
construct(ctx, "202206011628", migrateInternetRadioStations),
construct(ctx, "202206101425", migrateUser),
construct(ctx, "202207251148", migrateStarRating),
construct(ctx, "202211111057", migratePlaylistsQueuesToFullID),
constructNoTx(ctx, "202212272312", backupDBPre016),
construct(ctx, "202304221528", migratePlaylistsToM3U),
construct(ctx, "202305301718", migratePlayCountToLength),
construct(ctx, "202307281628", migrateAlbumArtistsMany2Many),
construct(ctx, "202309070009", migrateDeleteArtistCoverField),
construct(ctx, "202309131743", migrateArtistInfo),
construct(ctx, "202309161411", migratePlaylistsPaths),
construct(ctx, "202310252205", migrateAlbumTagArtistString),
construct(ctx, "202310281803", migrateTrackArtists),
construct(ctx, "202311062259", migrateArtistAppearances),
construct(ctx, "202311072309", migrateAlbumInfo),
construct(ctx, "202311082304", migrateTemporaryDisplayAlbumArtist),
construct(ctx, "202312110003", migrateAddExtraIndexes),
}
return gormigrate.
New(db.DB, options, migrations).
Migrate()
}
func construct(ctx MigrationContext, id string, f func(*gorm.DB, MigrationContext) error) *gormigrate.Migration {
return constructNoTx(ctx, id, func(db *gorm.DB, ctx MigrationContext) error {
return db.Transaction(func(tx *gorm.DB) error {
return f(tx, ctx)
})
})
}
func constructNoTx(ctx MigrationContext, id string, f func(*gorm.DB, MigrationContext) error) *gormigrate.Migration {
return &gormigrate.Migration{
ID: id,
Migrate: func(db *gorm.DB) error {
if err := f(db, ctx); err != nil {
return fmt.Errorf("%q: %w", id, err)
}
log.Printf("migration '%s' finished", id)
return nil
},
Rollback: func(*gorm.DB) error {
return nil
},
}
}
func migrateInitSchema(tx *gorm.DB, _ MigrationContext) error {
return tx.AutoMigrate(
Genre{},
TrackGenre{},
AlbumGenre{},
Track{},
Artist{},
User{},
Setting{},
Play{},
Album{},
PlayQueue{},
).
Error
}
func migrateCreateInitUser(tx *gorm.DB, _ MigrationContext) error {
const (
initUsername = "admin"
initPassword = "admin"
)
err := tx.
Where("name=?", initUsername).
First(&User{}).
Error
if !errors.Is(err, gorm.ErrRecordNotFound) {
return nil
}
return tx.Create(&User{
Name: initUsername,
Password: initPassword,
IsAdmin: true,
}).
Error
}
func migrateMergePlaylist(tx *gorm.DB, _ MigrationContext) error {
if !tx.HasTable("playlists") {
return nil
}
if !tx.HasTable("playlist_items") {
return nil
}
return tx.Exec(`
UPDATE playlists
SET items=( SELECT group_concat(track_id) FROM (
SELECT track_id
FROM playlist_items
WHERE playlist_items.playlist_id=playlists.id
ORDER BY created_at
) );
DROP TABLE playlist_items;`,
).
Error
}
func migrateCreateTranscode(tx *gorm.DB, _ MigrationContext) error {
return tx.AutoMigrate(
TranscodePreference{},
).
Error
}
func migrateAddGenre(tx *gorm.DB, _ MigrationContext) error {
return tx.AutoMigrate(
Genre{},
Album{},
Track{},
).
Error
}
func migrateUpdateTranscodePrefIDX(tx *gorm.DB, _ MigrationContext) error {
var hasIDX int
tx.
Select("1").
Table("sqlite_master").
Where("type = ?", "index").
Where("name = ?", "idx_user_id_client").
Count(&hasIDX)
if hasIDX == 1 {
// index already exists
return nil
}
step := tx.Exec(`
ALTER TABLE transcode_preferences RENAME TO transcode_preferences_orig;
`)
if err := step.Error; err != nil {
return fmt.Errorf("step rename: %w", err)
}
step = tx.AutoMigrate(
TranscodePreference{},
)
if err := step.Error; err != nil {
return fmt.Errorf("step create: %w", err)
}
step = tx.Exec(`
INSERT INTO transcode_preferences (user_id, client, profile)
SELECT user_id, client, profile
FROM transcode_preferences_orig;
DROP TABLE transcode_preferences_orig;
`)
if err := step.Error; err != nil {
return fmt.Errorf("step copy: %w", err)
}
return nil
}
func migrateAddAlbumIDX(tx *gorm.DB, _ MigrationContext) error {
return tx.AutoMigrate(
Album{},
).
Error
}
func migrateMultiGenre(tx *gorm.DB, _ MigrationContext) error {
step := tx.AutoMigrate(
Genre{},
TrackGenre{},
AlbumGenre{},
Track{},
Album{},
)
if err := step.Error; err != nil {
return fmt.Errorf("step auto migrate: %w", err)
}
var genreCount int
tx.
Model(Genre{}).
Count(&genreCount)
if genreCount == 0 {
return nil
}
step = tx.Exec(`
INSERT INTO track_genres (track_id, genre_id)
SELECT id, tag_genre_id
FROM tracks
WHERE tag_genre_id IS NOT NULL;
UPDATE tracks SET tag_genre_id=NULL;
`)
if err := step.Error; err != nil {
return fmt.Errorf("step migrate track genres: %w", err)
}
step = tx.Exec(`
INSERT INTO album_genres (album_id, genre_id)
SELECT id, tag_genre_id
FROM albums
WHERE tag_genre_id IS NOT NULL;
UPDATE albums SET tag_genre_id=NULL;
`)
if err := step.Error; err != nil {
return fmt.Errorf("step migrate album genres: %w", err)
}
return nil
}
func migrateListenBrainz(tx *gorm.DB, _ MigrationContext) error {
step := tx.AutoMigrate(
User{},
)
if err := step.Error; err != nil {
return fmt.Errorf("step auto migrate: %w", err)
}
return nil
}
func migratePodcast(tx *gorm.DB, _ MigrationContext) error {
return tx.AutoMigrate(
Podcast{},
PodcastEpisode{},
).
Error
}
func migrateBookmarks(tx *gorm.DB, _ MigrationContext) error {
return tx.AutoMigrate(
Bookmark{},
).
Error
}
func migratePodcastAutoDownload(tx *gorm.DB, _ MigrationContext) error {
return tx.AutoMigrate(
Podcast{},
).
Error
}
func migrateAlbumCreatedAt(tx *gorm.DB, _ MigrationContext) error {
step := tx.AutoMigrate(
Album{},
)
if err := step.Error; err != nil {
return fmt.Errorf("step auto migrate: %w", err)
}
step = tx.Exec(`
UPDATE albums SET created_at=modified_at;
`)
if err := step.Error; err != nil {
return fmt.Errorf("step migrate album created_at: %w", err)
}
return nil
}
func migrateAlbumRootDir(tx *gorm.DB, ctx MigrationContext) error {
step := tx.AutoMigrate(
Album{},
)
if err := step.Error; err != nil {
return fmt.Errorf("step auto migrate: %w", err)
}
step = tx.Exec(`
DROP INDEX IF EXISTS idx_left_path_right_path;
`)
if err := step.Error; err != nil {
return fmt.Errorf("step drop idx: %w", err)
}
step = tx.Exec(`
UPDATE albums SET root_dir=? WHERE root_dir IS NULL
`, ctx.OriginalMusicPath)
if err := step.Error; err != nil {
return fmt.Errorf("step drop idx: %w", err)
}
return nil
}
func migrateArtistGuessedFolder(tx *gorm.DB, _ MigrationContext) error {
return tx.AutoMigrate(Artist{}).Error
}
func migrateArtistCover(tx *gorm.DB, _ MigrationContext) error {
step := tx.AutoMigrate(
Artist{},
)
if err := step.Error; err != nil {
return fmt.Errorf("step auto migrate: %w", err)
}
if !tx.Dialect().HasColumn("artists", "guessed_folder_id") {
return nil
}
step = tx.Exec(`
ALTER TABLE artists DROP COLUMN guessed_folder_id
`)
if err := step.Error; err != nil {
return fmt.Errorf("step drop column: %w", err)
}
return nil
}
// there was an issue with that migration, try it again since it's updated
func migrateAlbumRootDirAgain(tx *gorm.DB, ctx MigrationContext) error {
return migrateAlbumRootDir(tx, ctx)
}
func migratePublicPlaylist(tx *gorm.DB, _ MigrationContext) error {
if !tx.HasTable("playlists") {
return nil
}
return tx.AutoMigrate(__OldPlaylist{}).Error
}
func migratePodcastDropUserID(tx *gorm.DB, _ MigrationContext) error {
step := tx.AutoMigrate(
Podcast{},
)
if err := step.Error; err != nil {
return fmt.Errorf("step auto migrate: %w", err)
}
if !tx.Dialect().HasColumn("podcasts", "user_id") {
return nil
}
step = tx.Exec(`
ALTER TABLE podcasts DROP COLUMN user_id;
`)
if err := step.Error; err != nil {
return fmt.Errorf("step migrate podcasts drop user_id: %w", err)
}
return nil
}
func migrateInternetRadioStations(tx *gorm.DB, _ MigrationContext) error {
return tx.AutoMigrate(
InternetRadioStation{},
).
Error
}
func migrateUser(tx *gorm.DB, _ MigrationContext) error {
return tx.AutoMigrate(
User{},
).
Error
}
func migrateStarRating(tx *gorm.DB, _ MigrationContext) error {
return tx.AutoMigrate(
Album{},
AlbumStar{},
AlbumRating{},
Artist{},
ArtistStar{},
ArtistRating{},
Track{},
TrackStar{},
TrackRating{},
).
Error
}
func migratePlaylistsQueuesToFullID(tx *gorm.DB, _ MigrationContext) error {
if !tx.HasTable("playlists") {
return nil
}
step := tx.Exec(`
UPDATE playlists SET items=('tr-' || items) WHERE items IS NOT NULL;
`)
if err := step.Error; err != nil {
return fmt.Errorf("step migrate playlists to full id: %w", err)
}
step = tx.Exec(`
UPDATE playlists SET items=REPLACE(items,',',',tr-') WHERE items IS NOT NULL;
`)
if err := step.Error; err != nil {
return fmt.Errorf("step migrate playlists to full id: %w", err)
}
step = tx.Exec(`
UPDATE play_queues SET items=('tr-' || items) WHERE items IS NOT NULL;
`)
if err := step.Error; err != nil {
return fmt.Errorf("step migrate play_queues to full id: %w", err)
}
step = tx.Exec(`
UPDATE play_queues SET items=REPLACE(items,',',',tr-') WHERE items IS NOT NULL;
`)
if err := step.Error; err != nil {
return fmt.Errorf("step migrate play_queues to full id: %w", err)
}
step = tx.Exec(`
ALTER TABLE play_queues ADD COLUMN newcurrent varchar[255];
`)
if err := step.Error; err != nil {
return fmt.Errorf("step migrate play_queues to full id: %w", err)
}
step = tx.Exec(`
UPDATE play_queues SET newcurrent=('tr-' || CAST(current AS varchar(10)));
`)
if err := step.Error; err != nil {
return fmt.Errorf("step migrate play_queues to full id: %w", err)
}
step = tx.Exec(`
ALTER TABLE play_queues DROP COLUMN current;
`)
if err := step.Error; err != nil {
return fmt.Errorf("step migrate play_queues to full id: %w", err)
}
step = tx.Exec(`
ALTER TABLE play_queues RENAME COLUMN newcurrent TO "current";
`)
if err := step.Error; err != nil {
return fmt.Errorf("step migrate play_queues to full id: %w", err)
}
return nil
}
func migratePlaylistsToM3U(tx *gorm.DB, ctx MigrationContext) error {
if ctx.PlaylistsPath == "" || !tx.HasTable("playlists") {
return nil
}
// local copy of specidpaths.Locate to avoid circular dep
locate := func(id specid.ID) string {
switch id.Type {
case specid.Track:
var track Track
tx.Preload("Album").Where("id=?", id.Value).Find(&track)
return track.AbsPath()
case specid.PodcastEpisode:
var pe PodcastEpisode
tx.Where("id=?", id.Value).Preload("Podcast").Find(&pe)
if pe.Filename == "" {
return ""
}
return filepath.Join(ctx.PodcastsPath, fileutil.Safe(pe.Podcast.Title), pe.Filename)
}
return ""
}
store, err := playlist.NewStore(ctx.PlaylistsPath)
if err != nil {
return fmt.Errorf("create playlists store: %w", err)
}
var prevs []*__OldPlaylist
if err := tx.Find(&prevs).Error; err != nil {
return fmt.Errorf("fetch old playlists: %w", err)
}
for _, prev := range prevs {
var pl playlist.Playlist
pl.UpdatedAt = time.Now()
pl.UserID = prev.UserID
pl.Name = prev.Name
pl.Comment = prev.Comment
pl.IsPublic = prev.IsPublic
for _, id := range splitIDs(prev.Items, ",") {
path := locate(id)
if path == "" {
log.Printf("migrating: can't find item %s from playlist %q on filesystem", id, prev.Name)
continue
}
pl.Items = append(pl.Items, path)
}
if err := store.Write(playlist.NewPath(prev.UserID, prev.Name), &pl); err != nil {
return fmt.Errorf("write playlist: %w", err)
}
}
return nil
}
func migratePlayCountToLength(tx *gorm.DB, _ MigrationContext) error {
step := tx.AutoMigrate(
Play{},
)
if err := step.Error; err != nil {
return fmt.Errorf("step auto migrate: %w", err)
}
// As a best guess, we set length played so far as length of album * current count / number of tracks in album
step = tx.Exec(`
UPDATE plays SET length=
((SELECT SUM(length) FROM tracks WHERE tracks.album_id=plays.album_id)*plays.count/
(SELECT COUNT(*) FROM tracks WHERE tracks.album_id=plays.album_id));
`)
if err := step.Error; err != nil {
return fmt.Errorf("calculate length: %w", err)
}
return nil
}
func migrateAlbumArtistsMany2Many(tx *gorm.DB, _ MigrationContext) error {
// gorms seems to want to create the table automatically without ON DELETE rules
step := tx.DropTableIfExists(AlbumArtist{})
if err := step.Error; err != nil {
return fmt.Errorf("step drop prev: %w", err)
}
step = tx.AutoMigrate(
AlbumArtist{},
Album{},
Artist{},
)
if err := step.Error; err != nil {
return fmt.Errorf("step auto migrate: %w", err)
}
if tx.Dialect().HasColumn("albums", "tag_artist_id") {
tx = tx.LogMode(false)
step = tx.Exec(`
INSERT INTO album_artists (album_id, artist_id)
SELECT id album_id, tag_artist_id artist_id
FROM albums
WHERE tag_artist_id IS NOT NULL;
`)
if err := step.Error; err != nil && !strings.Contains(err.Error(), "no such column") {
return fmt.Errorf("step insert from albums: %w", err)
}
step = tx.Exec(`DROP INDEX idx_albums_tag_artist_id`)
if err := step.Error; err != nil {
return fmt.Errorf("step drop index: %w", err)
}
step = tx.Exec(`ALTER TABLE albums DROP COLUMN tag_artist_id;`)
if err := step.Error; err != nil {
return fmt.Errorf("step drop albums tag artist id: %w", err)
}
}
if tx.Dialect().HasColumn("tracks", "artist_id") {
step = tx.Exec(`ALTER TABLE tracks DROP COLUMN artist_id;`)
if err := step.Error; err != nil {
return fmt.Errorf("step drop track tag artist: %w", err)
}
}
return nil
}
func migrateDeleteArtistCoverField(tx *gorm.DB, _ MigrationContext) error {
if !tx.Dialect().HasColumn("artists", "cover") {
return nil
}
step := tx.Exec(`
ALTER TABLE artists DROP COLUMN cover;
`)
if err := step.Error; err != nil {
return fmt.Errorf("step drop: %w", err)
}
return nil
}
func migrateArtistInfo(tx *gorm.DB, _ MigrationContext) error {
return tx.AutoMigrate(
ArtistInfo{},
).
Error
}
func migratePlaylistsPaths(tx *gorm.DB, ctx MigrationContext) error {
if !tx.Dialect().HasColumn("podcast_episodes", "path") {
return nil
}
if !tx.Dialect().HasColumn("podcasts", "image_path") {
return nil
}
step := tx.Exec(`
ALTER TABLE podcasts RENAME COLUMN image_path TO image
`)
if err := step.Error; err != nil {
return fmt.Errorf("step drop podcast_episodes path: %w", err)
}
step = tx.AutoMigrate(
Podcast{},
PodcastEpisode{},
)
if err := step.Error; err != nil {
return fmt.Errorf("step auto migrate: %w", err)
}
var podcasts []*Podcast
if err := tx.Find(&podcasts).Error; err != nil {
return fmt.Errorf("step load: %w", err)
}
for _, p := range podcasts {
p.Image = filepath.Base(p.Image)
if err := tx.Save(p).Error; err != nil {
return fmt.Errorf("saving podcast for cover %d: %w", p.ID, err)
}
oldPath, err := fileutil.First(
filepath.Join(ctx.PodcastsPath, fileutil.Safe(p.Title)),
filepath.Join(ctx.PodcastsPath, strings.ReplaceAll(p.Title, string(filepath.Separator), "_")), // old safe func
filepath.Join(ctx.PodcastsPath, p.Title),
)
if err != nil {
log.Printf("error finding old podcast path: %v. ignoring", err)
continue
}
newPath := filepath.Join(ctx.PodcastsPath, fileutil.Safe(p.Title))
p.RootDir = newPath
if err := tx.Save(p).Error; err != nil {
return fmt.Errorf("saving podcast %d: %w", p.ID, err)
}
if oldPath == newPath {
continue
}
if err := os.Rename(oldPath, newPath); err != nil {
return fmt.Errorf("rename podcast path: %w", err)
}
}
var podcastEpisodes []*PodcastEpisode
if err := tx.Preload("Podcast").Find(&podcastEpisodes, "status=? OR status=?", PodcastEpisodeStatusCompleted, PodcastEpisodeStatusDownloading).Error; err != nil {
return fmt.Errorf("step load: %w", err)
}
for _, pe := range podcastEpisodes {
if pe.Filename == "" {
continue
}
oldPath, err := fileutil.First(
filepath.Join(pe.Podcast.RootDir, fileutil.Safe(pe.Filename)),
filepath.Join(pe.Podcast.RootDir, strings.ReplaceAll(pe.Filename, string(filepath.Separator), "_")), // old safe func
filepath.Join(pe.Podcast.RootDir, pe.Filename),
)
if err != nil {
log.Printf("error finding old podcast episode path: %v. ignoring", err)
continue
}
newName := fileutil.Safe(filepath.Base(oldPath))
pe.Filename = newName
if err := tx.Save(pe).Error; err != nil {
return fmt.Errorf("saving podcast episode %d: %w", pe.ID, err)
}
newPath := filepath.Join(pe.Podcast.RootDir, newName)
if oldPath == newPath {
continue
}
if err := os.Rename(oldPath, newPath); err != nil {
return fmt.Errorf("rename podcast episode path: %w", err)
}
}
step = tx.Exec(`
ALTER TABLE podcast_episodes DROP COLUMN path
`)
if err := step.Error; err != nil {
return fmt.Errorf("step drop podcast_episodes path: %w", err)
}
return nil
}
func backupDBPre016(tx *gorm.DB, ctx MigrationContext) error {
if !ctx.Production {
return nil
}
return Dump(context.Background(), tx, fmt.Sprintf("%s.%d.bak", ctx.DBPath, time.Now().Unix()))
}
func migrateAlbumTagArtistString(tx *gorm.DB, _ MigrationContext) error {
return tx.AutoMigrate(Album{}).Error
}
func migrateTrackArtists(tx *gorm.DB, _ MigrationContext) error {
// gorms seems to want to create the table automatically without ON DELETE rules
step := tx.DropTableIfExists(TrackArtist{})
if err := step.Error; err != nil {
return fmt.Errorf("step drop prev: %w", err)
}
return tx.AutoMigrate(TrackArtist{}).Error
}
func migrateArtistAppearances(tx *gorm.DB, _ MigrationContext) error {
// gorms seems to want to create the table automatically without ON DELETE rules
step := tx.DropTableIfExists(ArtistAppearances{})
if err := step.Error; err != nil {
return fmt.Errorf("step drop prev: %w", err)
}
step = tx.AutoMigrate(ArtistAppearances{})
if err := step.Error; err != nil {
return fmt.Errorf("step auto migrate: %w", err)
}
step = tx.Exec(`
INSERT INTO artist_appearances (artist_id, album_id)
SELECT artist_id, album_id
FROM album_artists
`)
if err := step.Error; err != nil {
return fmt.Errorf("step transfer album artists: %w", err)
}
step = tx.Exec(`
INSERT OR IGNORE INTO artist_appearances (artist_id, album_id)
SELECT track_artists.artist_id, tracks.album_id
FROM track_artists
JOIN tracks ON tracks.id=track_artists.track_id
`)
if err := step.Error; err != nil {
return fmt.Errorf("step transfer album artists: %w", err)
}
return nil
}
func migrateAlbumInfo(tx *gorm.DB, _ MigrationContext) error {
return tx.AutoMigrate(
AlbumInfo{},
).
Error
}
func migrateTemporaryDisplayAlbumArtist(tx *gorm.DB, _ MigrationContext) error {
// keep some things working so that people have an album.tag_artist_id until their next full scan
return tx.Exec(`
UPDATE albums
SET tag_album_artist=(
SELECT group_concat(artists.name, ', ')
FROM artists
JOIN album_artists ON album_artists.artist_id=artists.id AND album_artists.album_id=albums.id
GROUP BY album_artists.album_id
)
WHERE tag_album_artist=''
`).Error
}
func migrateAddExtraIndexes(tx *gorm.DB, _ MigrationContext) error {
return tx.Exec(`
CREATE INDEX idx_track_genres_genre_id ON "track_genres" (genre_id);
CREATE INDEX idx_album_genres_genre_id ON "album_genres" (genre_id);
CREATE INDEX idx_album_artists_artist_id ON "album_artists" (artist_id);
CREATE INDEX idx_track_artists_artist_id ON "track_artists" (artist_id);
CREATE INDEX idx_artist_appearances_album_id ON "artist_appearances" (album_id);
`).Error
}