heimoshuiyu/gonic
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

delete guess expected size feature

Browse Source 
it it doing some really bad guesses for opus files
This commit is contained in:
sentriz
2022-04-20 23:33:10 +01:00
parent 4658d07273
commit 6bebceccd9
20 changed files with 44 additions and 526 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
iout/copy_range.go
View File

@@ -1,23 +0,0 @@
package iout
import (
"errors"
"fmt"
"io"
)
func CopyRange(w io.Writer, r io.Reader, start, length int64) error {
if _, err := io.CopyN(io.Discard, r, start); err != nil && !errors.Is(err, io.EOF) {
return fmt.Errorf("discard %d: %w", start, err)
}
if length == 0 {
if _, err := io.Copy(w, r); err != nil && !errors.Is(err, io.EOF) {
return fmt.Errorf("direct copy: %w", err)
}
return nil
}
if _, err := io.CopyN(w, io.MultiReader(r, NewNullReader()), length); err != nil && !errors.Is(err, io.EOF) {
return fmt.Errorf("copy %d: %w", length, err)
}
return nil
}

42
iout/copy_range_test.go
View File

@@ -1,42 +0,0 @@
package iout_test
import (
"bytes"
"testing"
"github.com/matryer/is"
"go.senan.xyz/gonic/iout"
)
func TestCopyRange(t *testing.T) {
t.Parallel()
is := is.New(t)
realLength := 50
cr := func(start, length int64) []byte {
is.Helper()
var data []byte
for i := 0; i < realLength; i++ {
data = append(data, byte(i%10))
}
var buff bytes.Buffer
is.NoErr(iout.CopyRange(&buff, bytes.NewReader(data), start, length))
return buff.Bytes()
}
// range
is.Equal(len(cr(0, 50)), 50)
is.Equal(len(cr(10, 10)), 10)
is.Equal(cr(10, 10)[0], byte(0))
is.Equal(cr(10, 10)[5], byte(5))
is.Equal(cr(25, 35)[0], byte(5))
is.Equal(cr(25, 35)[5], byte(0))
// 0 padding
is.Equal(len(cr(0, 5000)), 5000)
is.Equal(cr(0, 5000)[50:], make([]byte, 5000-50))
// no bound
is.Equal(len(cr(0, 0)), 50)
is.Equal(len(cr(50, 0)), 0)
}

46
iout/countrw.go
View File

@@ -1,46 +0,0 @@
package iout
import (
"io"
"sync/atomic"
)
type CountReader struct {
r io.Reader
c *uint64
}
func NewCountReader(r io.Reader) *CountReader {
return &CountReader{r: r, c: new(uint64)}
}
func (c *CountReader) Reset() { atomic.StoreUint64(c.c, 0) }
func (c *CountReader) Count() uint64 { return atomic.LoadUint64(c.c) }
func (c *CountReader) Read(p []byte) (int, error) {
n, err := c.r.Read(p)
atomic.AddUint64(c.c, uint64(n))
return n, err
}
var _ io.Reader = (*CountReader)(nil)
type CountWriter struct {
r io.Writer
c *uint64
}
func NewCountWriter(r io.Writer) *CountWriter {
return &CountWriter{r: r, c: new(uint64)}
}
func (c *CountWriter) Reset() { atomic.StoreUint64(c.c, 0) }
func (c *CountWriter) Count() uint64 { return atomic.LoadUint64(c.c) }
func (c *CountWriter) Write(p []byte) (int, error) {
n, err := c.r.Write(p)
atomic.AddUint64(c.c, uint64(n))
return n, err
}
var _ io.Writer = (*CountWriter)(nil)

18
iout/null_reader.go
View File

@@ -1,18 +0,0 @@
package iout
import "io"
type nullReader struct{}
func NewNullReader() io.Reader {
return &nullReader{}
}
func (*nullReader) Read(p []byte) (n int, err error) {
for b := range p {
p[b] = 0
}
return len(p), nil
}
var _ io.Reader = (*nullReader)(nil)

28
iout/tee_closer.go
View File

@@ -1,28 +0,0 @@
package iout
import "io"
type teeCloser struct {
r io.ReadCloser
w io.WriteCloser
}
func NewTeeCloser(r io.ReadCloser, w io.WriteCloser) io.ReadCloser {
return &teeCloser{r, w}
}
func (t *teeCloser) Read(p []byte) (int, error) {
n, err := t.r.Read(p)
if n > 0 {
if n, err := t.w.Write(p[:n]); err != nil {
return n, err
}
}
return n, err
}
func (t *teeCloser) Close() error {
t.r.Close()
t.w.Close()
return nil
}

5
server/ctrlsubsonic/ctrl_test.go
View File

@@ -1,3 +1,4 @@
//nolint:deadcode
package ctrlsubsonic
import (
@@ -16,10 +17,10 @@ import (
jd "github.com/josephburnett/jd/lib"
"go.senan.xyz/gonic/server/ctrlbase"
"go.senan.xyz/gonic/server/ctrlsubsonic/params"
"go.senan.xyz/gonic/db"
"go.senan.xyz/gonic/mockfs"
"go.senan.xyz/gonic/server/ctrlbase"
"go.senan.xyz/gonic/server/ctrlsubsonic/params"
"go.senan.xyz/gonic/transcode"
)

27
server/ctrlsubsonic/handlers_raw.go
View File

@@ -12,12 +12,10 @@ import (
"github.com/disintegration/imaging"
"github.com/jinzhu/gorm"
"go.senan.xyz/gonic/iout"
"go.senan.xyz/gonic/server/ctrlsubsonic/httprange"
"go.senan.xyz/gonic/db"
"go.senan.xyz/gonic/server/ctrlsubsonic/params"
"go.senan.xyz/gonic/server/ctrlsubsonic/spec"
"go.senan.xyz/gonic/server/ctrlsubsonic/specid"
"go.senan.xyz/gonic/db"
"go.senan.xyz/gonic/transcode"
)
@@ -296,30 +294,11 @@ func (c *Controller) ServeStream(w http.ResponseWriter, r *http.Request) *spec.R
log.Printf("trancoding to %q with max bitrate %dk", profile.MIME(), profile.BitRate())
transcodeReader, err := c.Transcoder.Transcode(r.Context(), profile, audioPath)
if err != nil {
w.Header().Set("Content-Type", profile.MIME())
if err := c.Transcoder.Transcode(r.Context(), profile, audioPath, w); err != nil {
return spec.NewError(0, "error transcoding: %v", err)
}
defer transcodeReader.Close()
length := transcode.GuessExpectedSize(profile, time.Duration(file.AudioLength())*time.Second) // TODO: if there's no duration?
rreq, err := httprange.Parse(r.Header.Get("Range"), length)
if err != nil {
return spec.NewError(0, "error parsing range: %v", err)
}
w.Header().Set("Content-Type", profile.MIME())
w.Header().Set("Content-Length", fmt.Sprintf("%d", rreq.Length))
w.Header().Set("Accept-Ranges", string(httprange.UnitBytes))
if rreq.Partial {
w.WriteHeader(http.StatusPartialContent)
w.Header().Set("Content-Range", fmt.Sprintf("%s %d-%d/%d", httprange.UnitBytes, rreq.Start, rreq.End, length))
}
if err := iout.CopyRange(w, transcodeReader, int64(rreq.Start), int64(rreq.Length)); err != nil {
log.Printf("error writing transcoded data: %v", err)
}
if f, ok := w.(http.Flusher); ok {
f.Flush()
}

152
server/ctrlsubsonic/handlers_raw_test.go
View File

@@ -1,152 +0,0 @@
package ctrlsubsonic
import (
"fmt"
"io/fs"
"net/http"
"net/url"
"os"
"os/exec"
"strconv"
"testing"
"time"
"github.com/matryer/is"
"go.senan.xyz/gonic/db"
"go.senan.xyz/gonic/transcode"
)
func TestServeStreamRaw(t *testing.T) {
t.Parallel()
if _, err := exec.LookPath("ffmpeg"); err != nil {
t.Skipf("no ffmpeg in $PATH")
}
is := is.New(t)
contr := makeControllerAudio(t)
statFlac := stat(t, audioPath10s)
rr, req := makeHTTPMock(url.Values{"id": {"tr-1"}})
serveRaw(t, contr, contr.ServeStream, rr, req)
is.Equal(rr.Code, http.StatusOK)
is.Equal(rr.Header().Get("content-type"), "audio/flac")
is.Equal(atoi(t, rr.Header().Get("content-length")), int(statFlac.Size()))
is.Equal(atoi(t, rr.Header().Get("content-length")), rr.Body.Len())
}
func TestServeStreamOpus(t *testing.T) {
t.Parallel()
if _, err := exec.LookPath("ffmpeg"); err != nil {
t.Skipf("no ffmpeg in $PATH")
}
is := is.New(t)
contr := makeControllerAudio(t)
var user db.User
is.NoErr(contr.DB.Where("name=?", mockUsername).Find(&user).Error)
is.NoErr(contr.DB.Create(&db.TranscodePreference{UserID: user.ID, Client: mockClientName, Profile: "opus"}).Error)
rr, req := makeHTTPMock(url.Values{"id": {"tr-1"}})
serveRaw(t, contr, contr.ServeStream, rr, req)
is.Equal(rr.Code, http.StatusOK)
is.Equal(rr.Header().Get("content-type"), "audio/ogg")
is.Equal(atoi(t, rr.Header().Get("content-length")), transcode.GuessExpectedSize(transcode.Opus, 10*time.Second))
is.Equal(atoi(t, rr.Header().Get("content-length")), rr.Body.Len())
}
func TestServeStreamOpusMaxBitrate(t *testing.T) {
t.Parallel()
if _, err := exec.LookPath("ffmpeg"); err != nil {
t.Skipf("no ffmpeg in $PATH")
}
is := is.New(t)
contr := makeControllerAudio(t)
var user db.User
is.NoErr(contr.DB.Where("name=?", mockUsername).Find(&user).Error)
is.NoErr(contr.DB.Create(&db.TranscodePreference{UserID: user.ID, Client: mockClientName, Profile: "opus"}).Error)
const bitrate = 5
rr, req := makeHTTPMock(url.Values{"id": {"tr-1"}, "maxBitRate": {strconv.Itoa(bitrate)}})
serveRaw(t, contr, contr.ServeStream, rr, req)
profile := transcode.WithBitrate(transcode.Opus, transcode.BitRate(bitrate))
expectedLength := transcode.GuessExpectedSize(profile, 10*time.Second)
is.Equal(rr.Code, http.StatusOK)
is.Equal(rr.Header().Get("content-type"), "audio/ogg")
is.Equal(atoi(t, rr.Header().Get("content-length")), expectedLength)
is.Equal(atoi(t, rr.Header().Get("content-length")), rr.Body.Len())
}
func TestServeStreamMP3Range(t *testing.T) {
t.Parallel()
if _, err := exec.LookPath("ffmpeg"); err != nil {
t.Skipf("no ffmpeg in $PATH")
}
is := is.New(t)
contr := makeControllerAudio(t)
var user db.User
is.NoErr(contr.DB.Where("name=?", mockUsername).Find(&user).Error)
is.NoErr(contr.DB.Create(&db.TranscodePreference{UserID: user.ID, Client: mockClientName, Profile: "mp3"}).Error)
var totalBytes []byte
{
rr, req := makeHTTPMock(url.Values{"id": {"tr-1"}})
serveRaw(t, contr, contr.ServeStream, rr, req)
is.Equal(rr.Code, http.StatusOK)
is.Equal(rr.Header().Get("content-type"), "audio/mpeg")
totalBytes = rr.Body.Bytes()
}
const chunkSize = 2 << 16
var bytes []byte
for i := 0; i < len(totalBytes); i += chunkSize {
rr, req := makeHTTPMock(url.Values{"id": {"tr-1"}})
req.Header.Set("range", fmt.Sprintf("bytes=%d-%d", i, min(i+chunkSize, len(totalBytes))-1))
t.Log(req.Header.Get("range"))
serveRaw(t, contr, contr.ServeStream, rr, req)
is.Equal(rr.Code, http.StatusPartialContent)
is.Equal(rr.Header().Get("content-type"), "audio/mpeg")
is.True(atoi(t, rr.Header().Get("content-length")) == chunkSize || atoi(t, rr.Header().Get("content-length")) == len(totalBytes)%chunkSize)
is.Equal(atoi(t, rr.Header().Get("content-length")), rr.Body.Len())
bytes = append(bytes, rr.Body.Bytes()...)
}
is.Equal(len(totalBytes), len(bytes))
is.Equal(totalBytes, bytes)
}
func stat(t *testing.T, path string) fs.FileInfo {
t.Helper()
info, err := os.Stat(path)
if err != nil {
t.Fatalf("stat %q: %v", path, err)
}
return info
}
func atoi(t *testing.T, in string) int {
t.Helper()
i, err := strconv.Atoi(in)
if err != nil {
t.Fatalf("atoi %q: %v", in, err)
}
return i
}
func min(a, b int) int {
if a < b {
return a
}
return b
}

59
server/ctrlsubsonic/httprange/httprange.go
View File

@@ -1,59 +0,0 @@
package httprange
import (
"fmt"
"regexp"
"strconv"
)
type Unit string
const (
UnitBytes Unit = "bytes"
)
//nolint:gochecknoglobals
var (
reg = regexp.MustCompile(`^(?P<unit>\w+)=(?P<start>(?:\d+)?)\s*-\s*(?P<end>(?:\d+)?)$`)
unit = reg.SubexpIndex("unit")
start = reg.SubexpIndex("start")
end = reg.SubexpIndex("end")
)
var (
ErrInvalidRange = fmt.Errorf("invalid range")
ErrUnknownUnit = fmt.Errorf("unknown range")
)
type Range struct {
Start, End, Length int // bytes
Partial bool
}
func Parse(in string, fullLength int) (Range, error) {
parts := reg.FindStringSubmatch(in)
if len(parts)-1 != reg.NumSubexp() {
return Range{0, fullLength - 1, fullLength, false}, nil
}
switch unit := parts[unit]; Unit(unit) {
case UnitBytes:
default:
return Range{}, fmt.Errorf("%q: %w", unit, ErrUnknownUnit)
}
start, _ := strconv.Atoi(parts[start])
end, _ := strconv.Atoi(parts[end])
length := fullLength
partial := false
switch {
case end > 0 && end < length:
length = end - start + 1
partial = true
case end == 0 && length > 0:
end = length - 1
}
return Range{start, end, length, partial}, nil
}

30
server/ctrlsubsonic/httprange/httprange_test.go
View File

@@ -1,30 +0,0 @@
package httprange_test
import (
"testing"
"github.com/matryer/is"
"go.senan.xyz/gonic/server/ctrlsubsonic/httprange"
)
func TestParse(t *testing.T) {
is := is.New(t)
full := func(start, end, length int) httprange.Range {
return httprange.Range{Start: start, End: end, Length: length}
}
partial := func(start, end, length int) httprange.Range {
return httprange.Range{Start: start, End: end, Length: length, Partial: true}
}
parse := func(in string, length int) httprange.Range {
is.Helper()
rrange, err := httprange.Parse(in, length)
is.NoErr(err)
return rrange
}
is.Equal(parse("bytes=0-0", 0), full(0, 0, 0))
is.Equal(parse("bytes=0-", 10), full(0, 9, 10))
is.Equal(parse("bytes=0-49", 50), partial(0, 49, 50))
is.Equal(parse("bytes=50-99", 100), partial(50, 99, 50))
}

BIN
transcode/testdata/10s.mp3 vendored
View File

Binary file not shown.

BIN
transcode/testdata/5s.mp3 vendored
View File

Binary file not shown.

3
transcode/testdata/fuzz/FuzzGuessExpectedSize/1e10ee326324ee7e1f371585414e7e41d5c802b4db45ebc79abe0397fb915a0c vendored
View File

@@ -1,3 +0,0 @@
go test fuzz v1
byte('Y')
byte('\x05')

3
transcode/testdata/fuzz/FuzzGuessExpectedSize/3eaed9c9c8a57078033cea3337a79e1e6bfebf7e5eb7421878c911df2d78d1ac vendored
View File

@@ -1,3 +0,0 @@
go test fuzz v1
byte('\x15')
byte('}')

3
transcode/testdata/fuzz/FuzzGuessExpectedSize/ce5db7c5c16dcb48567255c77772cacc3ca86d524cbb2d3a769248625bc3edc8 vendored
View File

@@ -1,3 +0,0 @@
go test fuzz v1
byte('\a')
byte('\x02')

23
transcode/transcode.go
View File

@@ -15,7 +15,7 @@ import (
)
type Transcoder interface {
Transcode(ctx context.Context, profile Profile, in string) (io.ReadCloser, error)
Transcode(ctx context.Context, profile Profile, in string, out io.Writer) error
}
var UserProfiles = map[string]Profile{
@@ -106,24 +106,3 @@ func parseProfile(profile Profile, in string) (string, []string, error) {
return name, args, nil
}
// GuessExpectedSize guesses how big the transcoded file will be in bytes.
// Handy if we want to send a Content-Length header to the client before
// the transcode has finished. This way, clients like DSub can render their
// scrub bar and duration as the track is streaming.
//
// The estimate should overshoot a bit (2s in this case) otherwise some HTTP
// clients will shit their trousers given some unexpected bytes.
func GuessExpectedSize(profile Profile, length time.Duration) int {
if length == 0 {
return 0
}
bytesPerSec := int(profile.BitRate() * 1000 / 8)
var guess int
guess += bytesPerSec * int(length.Seconds()-profile.seek.Seconds())
guess += bytesPerSec * 2 // 2s pading
guess += 10000 // 10kb byte padding
return guess
}

47
transcode/transcode_test.go
View File

@@ -1,47 +0,0 @@
//go:build go1.18
// +build go1.18
package transcode_test
import (
"context"
"io"
"testing"
"time"
"github.com/matryer/is"
"go.senan.xyz/gonic/transcode"
)
// FuzzGuessExpectedSize makes sure all of our profile's estimated transcode
// file sizes are slightly bigger than the real thing.
func FuzzGuessExpectedSize(f *testing.F) {
var profiles []transcode.Profile
for _, v := range transcode.UserProfiles {
profiles = append(profiles, v)
}
type track struct {
path string
length time.Duration
}
var tracks []track
tracks = append(tracks, track{"testdata/5s.mp3", 5 * time.Second})
tracks = append(tracks, track{"testdata/10s.mp3", 10 * time.Second})
tr := transcode.NewFFmpegTranscoder()
f.Fuzz(func(t *testing.T, pseed uint8, tseed uint8) {
is := is.New(t)
profile := profiles[int(pseed)%len(profiles)]
track := tracks[int(tseed)%len(tracks)]
sizeGuess := transcode.GuessExpectedSize(profile, track.length)
reader, err := tr.Transcode(context.Background(), profile, track.path)
is.NoErr(err)
actual, err := io.ReadAll(reader)
is.NoErr(err)
is.True(sizeGuess > len(actual))
})
}

23
transcode/transcoder_caching.go
View File

@@ -7,8 +7,6 @@ import (
"io"
"os"
"path/filepath"
"go.senan.xyz/gonic/iout"
)
const perm = 0644
@@ -24,14 +22,14 @@ func NewCachingTranscoder(t Transcoder, cachePath string) *CachingTranscoder {
return &CachingTranscoder{transcoder: t, cachePath: cachePath}
}
func (t *CachingTranscoder) Transcode(ctx context.Context, profile Profile, in string) (io.ReadCloser, error) {
func (t *CachingTranscoder) Transcode(ctx context.Context, profile Profile, in string, out io.Writer) error {
if err := os.MkdirAll(t.cachePath, perm^0111); err != nil {
return nil, fmt.Errorf("make cache path: %w", err)
return fmt.Errorf("make cache path: %w", err)
}
name, args, err := parseProfile(profile, in)
if err != nil {
return nil, fmt.Errorf("split command: %w", err)
return fmt.Errorf("split command: %w", err)
}
key := cacheKey(name, args)
@@ -39,18 +37,21 @@ func (t *CachingTranscoder) Transcode(ctx context.Context, profile Profile, in s
cf, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644)
if err != nil {
return nil, fmt.Errorf("open cache file: %w", err)
return fmt.Errorf("open cache file: %w", err)
}
defer cf.Close()
if i, err := cf.Stat(); err == nil && i.Size() > 0 {
return cf, nil
_, _ = io.Copy(out, cf)
return nil
}
out, err := t.transcoder.Transcode(ctx, profile, in)
if err != nil {
return nil, fmt.Errorf("internal transcode: %w", err)
if err := t.transcoder.Transcode(ctx, profile, in, io.MultiWriter(out, cf)); err != nil {
os.Remove(path)
return fmt.Errorf("internal transcode: %w", err)
}
return iout.NewTeeCloser(out, cf), nil
return nil
}
func cacheKey(cmd string, args []string) string {

25
transcode/transcoder_ffmpeg.go
View File

@@ -2,6 +2,7 @@ package transcode
import (
"context"
"errors"
"fmt"
"io"
"os/exec"
@@ -17,23 +18,25 @@ func NewFFmpegTranscoder() *FFmpegTranscoder {
var ErrFFmpegExit = fmt.Errorf("ffmpeg exited with non 0 status code")
func (*FFmpegTranscoder) Transcode(ctx context.Context, profile Profile, in string) (io.ReadCloser, error) {
func (*FFmpegTranscoder) Transcode(ctx context.Context, profile Profile, in string, out io.Writer) error {
name, args, err := parseProfile(profile, in)
if err != nil {
return nil, fmt.Errorf("split command: %w", err)
return fmt.Errorf("split command: %w", err)
}
preader, pwriter := io.Pipe()
cmd := exec.CommandContext(ctx, name, args...)
cmd.Stdout = pwriter
cmd.Stdout = out
if err := cmd.Start(); err != nil {
return nil, fmt.Errorf("starting cmd: %w", err)
return fmt.Errorf("starting cmd: %w", err)
}
go func() {
_ = pwriter.CloseWithError(cmd.Wait())
}()
return preader, nil
var exitErr *exec.ExitError
if err := cmd.Wait(); err != nil && !errors.As(err, &exitErr) {
return fmt.Errorf("waiting cmd: %w", err)
}
if code := cmd.ProcessState.ExitCode(); code > 1 {
return fmt.Errorf("%w: %d", ErrFFmpegExit, code)
}
return nil
}

13
transcode/transcoder_none.go
View File

@@ -2,6 +2,7 @@ package transcode
import (
"context"
"fmt"
"io"
"os"
)
@@ -14,6 +15,14 @@ func NewNoneTranscoder() *NoneTranscoder {
return &NoneTranscoder{}
}
func (*NoneTranscoder) Transcode(ctx context.Context, _ Profile, in string) (io.ReadCloser, error) {
return os.Open(in)
func (*NoneTranscoder) Transcode(ctx context.Context, _ Profile, in string, out io.Writer) error {
file, err := os.Open(in)
if err != nil {
return fmt.Errorf("open file: %w", err)
}
defer file.Close()
if _, err := io.Copy(out, file); err != nil {
return fmt.Errorf("copy file: %w", err)
}
return nil
}