feat(transcode): add a generic transcoding package for encoding/decoding/caching

Author: sentriz

countrw/null.go

@@ -0,0 +1,15 @@
+package countrw
+
+import "io"
+
+type NullReader struct{}
+
+func NewNullReader() *NullReader {
+	return &NullReader{}
+}
+
+func (*NullReader) Read(p []byte) (n int, err error) {
+	return len(p), nil
+}
+
+var _ io.Reader = (*NullReader)(nil)

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/disintegration/imaging v1.6.2
 	github.com/dustin/go-humanize v1.0.0
 	github.com/faiface/beep v1.1.0
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/securecookie v1.1.1

go.sum

@@ -50,6 +50,8 @@ github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2V
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8=

server/ctrladmin/handlers.go

@@ -10,10 +10,10 @@ import (
 	"github.com/mmcdole/gofeed"
 
 	"go.senan.xyz/gonic/server/db"
-	"go.senan.xyz/gonic/server/encode"
 	"go.senan.xyz/gonic/server/scanner"
 	"go.senan.xyz/gonic/server/scrobble/lastfm"
 	"go.senan.xyz/gonic/server/scrobble/listenbrainz"
+	"go.senan.xyz/gonic/server/transcode"
 )
 
 func doScan(scanner *scanner.Scanner, opts scanner.ScanOptions) {
@@ -67,7 +67,7 @@ func (c *Controller) ServeHome(r *http.Request) *Response {
 	c.DB.
 		Where("user_id=?", user.ID).
 		Find(&data.TranscodePreferences)
-	for profile := range encode.Profiles() {
+	for profile := range transcode.UserProfiles {
 		data.TranscodeProfiles = append(data.TranscodeProfiles, profile)
 	}
 	// podcasts box

server/ctrlsubsonic/ctrl.go

@@ -15,6 +15,7 @@ import (
 	"go.senan.xyz/gonic/server/jukebox"
 	"go.senan.xyz/gonic/server/podcasts"
 	"go.senan.xyz/gonic/server/scrobble"
+	"go.senan.xyz/gonic/server/transcode"
 )
 
 type CtxKey int
@@ -34,6 +35,7 @@ type Controller struct {
 	Jukebox        *jukebox.Jukebox
 	Scrobblers     []scrobble.Scrobbler
 	Podcasts       *podcasts.Podcasts
+	Transcoder     transcode.Transcoder
 }
 
 type metaResponse struct {

server/ctrlsubsonic/handlers_raw.go

@@ -13,12 +13,12 @@ import (
 	"github.com/disintegration/imaging"
 	"github.com/jinzhu/gorm"
 
+	"go.senan.xyz/gonic/countrw"
 	"go.senan.xyz/gonic/server/ctrlsubsonic/params"
 	"go.senan.xyz/gonic/server/ctrlsubsonic/spec"
 	"go.senan.xyz/gonic/server/ctrlsubsonic/specid"
 	"go.senan.xyz/gonic/server/db"
-	"go.senan.xyz/gonic/server/encode"
-	"go.senan.xyz/gonic/server/mime"
+	"go.senan.xyz/gonic/server/transcode"
 )
 
 // "raw" handlers are ones that don't always return a spec response.
@@ -242,111 +242,76 @@ func (c *Controller) ServeStream(w http.ResponseWriter, r *http.Request) *spec.R
 	if err != nil {
 		return spec.NewError(10, "please provide an `id` parameter")
 	}
-	var audioFile db.AudioFile
+	var file db.AudioFile
 	var audioPath string
 	switch id.Type {
 	case specid.Track:
 		track, err := streamGetTrack(c.DB, id.Value)
 		if err != nil {
 			return spec.NewError(70, "track with id `%s` was not found", id)
 		}
-		audioFile = track
+		file = track
 		audioPath = path.Join(track.AbsPath())
 	case specid.PodcastEpisode:
 		podcast, err := streamGetPodcast(c.DB, id.Value)
 		if err != nil {
 			return spec.NewError(70, "podcast with id `%s` was not found", id)
 		}
-		audioFile = podcast
+		file = podcast
 		audioPath = path.Join(c.PodcastsPath, podcast.Path)
 	default:
 		return spec.NewError(70, "media type of `%s` was not found", id.Type)
 	}
 
 	user := r.Context().Value(CtxUser).(*db.User)
-	if track, ok := audioFile.(*db.Track); ok && track.Album != nil {
+	if track, ok := file.(*db.Track); ok && track.Album != nil {
 		defer func() {
 			if err := streamUpdateStats(c.DB, user.ID, track.Album.ID, time.Now()); err != nil {
-				log.Printf("error updating listen stats: %v", err)
+				log.Printf("error updating status: %v", err)
 			}
 		}()
 	}
 
-	pref, err := streamGetTransPref(c.DB, user.ID, params.GetOr("c", ""))
-	if err != nil {
-		return spec.NewError(0, "failed to get transcode stream preference: %v", err)
+	if format, _ := params.Get("format"); format == "raw" {
+		http.ServeFile(w, r, audioPath)
+		return nil
 	}
 
-	onInvalidProfile := func() error {
-		log.Printf("serving raw `%s`\n", audioFile.AudioFilename())
-		w.Header().Set("Content-Type", audioFile.MIME())
+	pref, err := streamGetTransPref(c.DB, user.ID, params.GetOr("c", ""))
+	if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
+		return spec.NewError(0, "couldn't find transcode preference: %v", err)
+	}
+	if pref == nil {
 		http.ServeFile(w, r, audioPath)
 		return nil
 	}
-	onCacheHit := func(profile encode.Profile, path string) error {
-		log.Printf("serving transcode `%s`: cache [%s/%dk] hit!\n",
-			audioFile.AudioFilename(), profile.Format, profile.Bitrate)
-		cacheMime, _ := mime.FromExtension(profile.Format)
-		w.Header().Set("Content-Type", cacheMime)
 
-		cacheFile, err := os.Stat(path)
-		if err != nil {
-			return fmt.Errorf("failed to stat cache file `%s`: %w", path, err)
-		}
-		contentLength := fmt.Sprintf("%d", cacheFile.Size())
-		w.Header().Set("Content-Length", contentLength)
-		http.ServeFile(w, r, path)
-		return nil
+	profile, ok := transcode.UserProfiles[pref.Profile]
+	if !ok {
+		return spec.NewError(0, "unknown transcode user profile %q", pref.Profile)
 	}
-	onCacheMiss := func(profile encode.Profile) (io.Writer, error) {
-		log.Printf("serving transcode `%s`: cache [%s/%dk] miss!\n",
-			audioFile.AudioFilename(), profile.Format, profile.Bitrate)
-		encodeMime, _ := mime.FromExtension(profile.Format)
-		w.Header().Set("Content-Type", encodeMime)
-		return w, nil
-	}
-	encodeOptions := encode.Options{
-		TrackPath:        audioPath,
-		TrackBitrate:     audioFile.AudioBitrate(),
-		CachePath:        c.CachePath,
-		ProfileName:      pref.Profile,
-		PreferredBitrate: params.GetOrInt("maxBitRate", 0),
-		OnInvalidProfile: onInvalidProfile,
-		OnCacheHit:       onCacheHit,
-		OnCacheMiss:      onCacheMiss,
-	}
-	if err := encode.Encode(encodeOptions); err != nil {
-		log.Printf("serving transcode `%s`: error: %v\n", audioFile.AudioFilename(), err)
+	if max, _ := params.GetInt("maxBitRate"); max > 0 && int(profile.BitRate()) > max {
+		profile = transcode.WithBitrate(profile, transcode.BitRate(max))
 	}
-	return nil
-}
 
-func (c *Controller) ServeDownload(w http.ResponseWriter, r *http.Request) *spec.Response {
-	params := r.Context().Value(CtxParams).(params.Params)
-	id, err := params.GetID("id")
-	if err != nil {
-		return spec.NewError(10, "please provide an `id` parameter")
+	log.Printf("trancoding to %q with max bitrate %dk", profile.MIME(), profile.BitRate())
+
+	guessedSize := transcode.GuessExpectedSize(profile, time.Duration(file.AudioLength())*time.Second)
+	w.Header().Set("Content-Type", profile.MIME())
+	w.Header().Set("Content-Length", fmt.Sprintf("%d", guessedSize))
+
+	cw := countrw.NewCountWriter(w)
+	if err := c.Transcoder.Transcode(r.Context(), profile, audioPath, cw); err != nil {
+		return spec.NewError(0, "error transcoding %v", err)
 	}
-	var filePath string
-	var audioFile db.AudioFile
-	switch id.Type {
-	case specid.Track:
-		track, _ := streamGetTrack(c.DB, id.Value)
-		audioFile = track
-		filePath = track.AbsPath()
-		if err != nil {
-			return spec.NewError(70, "track with id `%s` was not found", id)
-		}
-	case specid.PodcastEpisode:
-		podcast, err := streamGetPodcast(c.DB, id.Value)
-		audioFile = podcast
-		filePath = path.Join(c.PodcastsPath, podcast.Path)
-		if err != nil {
-			return spec.NewError(70, "podcast with id `%s` was not found", id)
-		}
+
+	// pad out the response with 0 up until the Content-Length we promised
+	remains := int64(guessedSize) - int64(cw.Count())
+	padding := io.LimitReader(&countrw.NullReader{}, remains)
+	_, _ = io.Copy(cw, padding)
+
+	if f, ok := w.(http.Flusher); ok {
+		f.Flush()
 	}
-	log.Printf("serving raw `%s`\n", audioFile.AudioFilename())
-	w.Header().Set("Content-Type", audioFile.MIME())
-	http.ServeFile(w, r, filePath)
 	return nil
 }

server/db/model.go

@@ -72,10 +72,11 @@ type Genre struct {
 // AudioFile is used to avoid some duplication in handlers_raw.go
 // between Track and Podcast
 type AudioFile interface {
-	AudioFilename() string
 	Ext() string
 	MIME() string
+	AudioFilename() string
 	AudioBitrate() int
+	AudioLength() int
 }
 
 type Track struct {
@@ -100,6 +101,9 @@ type Track struct {
 	TagBrainzID    string   `sql:"default: null"`
 }
 
+func (t *Track) AudioLength() int  { return t.Length }
+func (t *Track) AudioBitrate() int { return t.Bitrate }
+
 func (t *Track) SID() *specid.ID {
 	return &specid.ID{Type: specid.Track, Value: t.ID}
 }
@@ -124,10 +128,6 @@ func (t *Track) AudioFilename() string {
 	return t.Filename
 }
 
-func (t *Track) AudioBitrate() int {
-	return t.Bitrate
-}
-
 func (t *Track) MIME() string {
 	v, _ := mime.FromExtension(t.Ext())
 	return v
@@ -364,6 +364,9 @@ type PodcastEpisode struct {
 	Error       string
 }
 
+func (pe *PodcastEpisode) AudioLength() int  { return pe.Length }
+func (pe *PodcastEpisode) AudioBitrate() int { return pe.Bitrate }
+
 func (pe *PodcastEpisode) SID() *specid.ID {
 	return &specid.ID{Type: specid.PodcastEpisode, Value: pe.ID}
 }
@@ -385,10 +388,6 @@ func (pe *PodcastEpisode) MIME() string {
 	return v
 }
 
-func (pe *PodcastEpisode) AudioBitrate() int {
-	return pe.Bitrate
-}
-
 type Bookmark struct {
 	ID          int `gorm:"primary_key"`
 	User        *User