From a139b2fb669bcee8a19023fc9b8e3b4bdc45735e Mon Sep 17 00:00:00 2001 From: Brad P Date: Thu, 17 Jul 2025 10:35:40 -0500 Subject: [PATCH 01/57] add data channel --- server/ai_http.go | 5 ++ server/ai_live_video.go | 121 ++++++++++++++++++++++++++++- server/ai_mediaserver.go | 163 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 287 insertions(+), 2 deletions(-) diff --git a/server/ai_http.go b/server/ai_http.go index 97a68a6530..d655aff191 100644 --- a/server/ai_http.go +++ b/server/ai_http.go @@ -145,6 +145,7 @@ func (h *lphttp) StartLiveVideoToVideo() http.Handler { subUrl = pubUrl + "-out" controlUrl = pubUrl + "-control" eventsUrl = pubUrl + "-events" + //dataUrl = pubUrl + "-data" ) // Handle initial payment, the rest of the payments are done separately from the stream processing @@ -180,6 +181,8 @@ func (h *lphttp) StartLiveVideoToVideo() http.Handler { controlPubCh.CreateChannel() eventsCh := trickle.NewLocalPublisher(h.trickleSrv, mid+"-events", "application/json") eventsCh.CreateChannel() + dataCh := trickle.NewLocalPublisher(h.trickleSrv, mid+"-data", "application/json") + dataCh.CreateChannel() // Start payment receiver which accounts the payments and stops the stream if the payment is insufficient priceInfo := payment.GetExpectedPrice() @@ -231,6 +234,7 @@ func (h *lphttp) StartLiveVideoToVideo() http.Handler { eventsUrlOverwrite := overwriteHost(h.node.LiveAITrickleHostForRunner, eventsUrl) subscribeUrlOverwrite := overwriteHost(h.node.LiveAITrickleHostForRunner, pubUrl) publishUrlOverwrite := overwriteHost(h.node.LiveAITrickleHostForRunner, subUrl) + //dataUrlOverwrite := overwriteHost(h.node.LiveAITrickleHostForRunner, dataUrl) workerReq := worker.LiveVideoToVideoParams{ ModelId: req.ModelId, @@ -255,6 +259,7 @@ func (h *lphttp) StartLiveVideoToVideo() http.Handler { subCh.Close() controlPubCh.Close() eventsCh.Close() + dataCh.Close() cancel() respondWithError(w, err.Error(), http.StatusInternalServerError) return diff --git a/server/ai_live_video.go b/server/ai_live_video.go index 399358d927..cc0c6b411d 100644 --- a/server/ai_live_video.go +++ b/server/ai_live_video.go @@ -784,6 +784,125 @@ func startEventsSubscribe(ctx context.Context, url *url.URL, params aiRequestPar }() } +func startDataSubscribe(ctx context.Context, url *url.URL, params aiRequestParams, sess *AISession) { + // subscribe to the outputs and send them into LPMS + subscriber := trickle.NewTrickleSubscriber(url.String()) + + // Set up output buffers + rbc := media.RingBufferConfig{BufferLen: 50_000_000} // 50 MB buffer + outWriter, err := media.NewRingBuffer(&rbc) + if err != nil { + stopProcessing(ctx, params, fmt.Errorf("ringbuffer init failed: %w", err)) + return + } + + // Store data segments for SSE endpoint + stream := params.liveParams.stream + dataStore := getDataStore(stream) + + // read segments from trickle subscription + go func() { + defer outWriter.Close() + defer removeDataStore(stream) // Clean up when done + + var err error + firstSegment := true + + retries := 0 + // we're trying to keep (retryPause x maxRetries) duration to fall within one output GOP length + const retryPause = 300 * time.Millisecond + const maxRetries = 5 + for { + select { + case <-ctx.Done(): + clog.Info(ctx, "trickle subscribe done") + return + default: + } + if !params.inputStreamExists() { + clog.Infof(ctx, "trickle subscribe stopping, input stream does not exist.") + break + } + var segment *http.Response + clog.V(8).Infof(ctx, "trickle subscribe read data await") + segment, err = subscriber.Read() + if err != nil { + if errors.Is(err, trickle.EOS) || errors.Is(err, trickle.StreamNotFoundErr) { + stopProcessing(ctx, params, fmt.Errorf("trickle subscribe stopping, stream not found, err=%w", err)) + return + } + var sequenceNonexistent *trickle.SequenceNonexistent + if errors.As(err, &sequenceNonexistent) { + // stream exists but segment doesn't, so skip to leading edge + subscriber.SetSeq(sequenceNonexistent.Latest) + } + // TODO if not EOS then signal a new orchestrator is needed + err = fmt.Errorf("trickle subscribe error reading: %w", err) + clog.Infof(ctx, "%s", err) + if retries > maxRetries { + stopProcessing(ctx, params, errors.New("trickle subscribe stopping, retries exceeded")) + return + } + retries++ + params.liveParams.sendErrorEvent(err) + time.Sleep(retryPause) + continue + } + retries = 0 + seq := trickle.GetSeq(segment) + clog.V(8).Infof(ctx, "trickle subscribe read data received seq=%d", seq) + copyStartTime := time.Now() + + // Read segment data and store it for SSE + body, err := io.ReadAll(segment.Body) + segment.Body.Close() + if err != nil { + clog.InfofErr(ctx, "trickle subscribe error reading segment body seq=%d", seq, err) + subscriber.SetSeq(seq) + retries++ + continue + } + + // Store the raw segment data for SSE endpoint + dataStore.Store(body) + + // Write to output buffer using the body data + n, err := outWriter.Write(body) + if err != nil { + if errors.Is(err, context.Canceled) { + clog.Info(ctx, "trickle subscribe stopping - context canceled") + return + } + + clog.InfofErr(ctx, "trickle subscribe error writing to output buffer seq=%d", seq, err) + subscriber.SetSeq(seq) + retries++ + continue + } + if firstSegment { + firstSegment = false + delayMs := time.Since(params.liveParams.startTime).Milliseconds() + if monitor.Enabled { + monitor.AIFirstSegmentDelay(delayMs, params.liveParams.sess.OrchestratorInfo) + monitor.SendQueueEventAsync("stream_trace", map[string]interface{}{ + "type": "gateway_receive_first_data_segment", + "timestamp": time.Now().UnixMilli(), + "stream_id": params.liveParams.streamID, + "pipeline_id": params.liveParams.pipelineID, + "request_id": params.liveParams.requestID, + "orchestrator_info": map[string]interface{}{ + "address": params.liveParams.sess.Address(), + "url": params.liveParams.sess.Transcoder(), + }, + }) + } + } + + clog.V(8).Info(ctx, "trickle subscribe read data completed", "seq", seq, "bytes", humanize.Bytes(uint64(n)), "took", time.Since(copyStartTime)) + } + }() +} + func (a aiRequestParams) inputStreamExists() bool { if a.node == nil { return false @@ -819,7 +938,7 @@ const maxInflightSegments = 3 // If inflight max is hit, returns true, false otherwise. func (s *SlowOrchChecker) BeginSegment() (int, bool) { // Returns `false` if there are multiple segments in-flight - // this means the orchestrator is slow reading them + // this means the orchestrator is slow reading // If all-OK, returns `true` s.mu.Lock() defer s.mu.Unlock() diff --git a/server/ai_mediaserver.go b/server/ai_mediaserver.go index 27a911f7b2..58b2a54346 100644 --- a/server/ai_mediaserver.go +++ b/server/ai_mediaserver.go @@ -14,6 +14,7 @@ import ( "os" "os/exec" "strings" + "sync" "time" "github.com/livepeer/go-livepeer/monitor" @@ -109,6 +110,9 @@ func startAIMediaServer(ctx context.Context, ls *LivepeerServer) error { ls.HTTPMux.Handle("OPTIONS /live/video-to-video/{streamId}/status", ls.WithCode(http.StatusNoContent)) ls.HTTPMux.Handle("/live/video-to-video/{streamId}/status", ls.GetLiveVideoToVideoStatus()) + // Stream data SSE endpoint + ls.HTTPMux.Handle("/live/video-to-video/{stream}/data", ls.GetLiveVideoToVideoData()) + //API for dynamic capabilities ls.HTTPMux.Handle("/process/request/", ls.SubmitJob()) @@ -787,16 +791,21 @@ func startProcessing(ctx context.Context, params aiRequestParams, res interface{ if err != nil { return fmt.Errorf("invalid events URL: %w", err) } + data, err := common.AppendHostname(strings.Replace(*resp.JSON200.EventsUrl, "-events", "-data", 1), host) + if err != nil { + return fmt.Errorf("invalid data URL: %w", err) + } if resp.JSON200.ManifestId != nil { ctx = clog.AddVal(ctx, "manifest_id", *resp.JSON200.ManifestId) params.liveParams.manifestID = *resp.JSON200.ManifestId } - clog.V(common.VERBOSE).Infof(ctx, "pub %s sub %s control %s events %s", pub, sub, control, events) + clog.V(common.VERBOSE).Infof(ctx, "pub %s sub %s control %s events %s data %s", pub, sub, control, events, data) startControlPublish(ctx, control, params) startTricklePublish(ctx, pub, params, params.liveParams.sess) startTrickleSubscribe(ctx, sub, params, params.liveParams.sess) startEventsSubscribe(ctx, events, params, params.liveParams.sess) + startDataSubscribe(ctx, data, params, params.liveParams.sess) return nil } @@ -1322,6 +1331,158 @@ func (ls *LivepeerServer) SmokeTestLiveVideo() http.Handler { }) } +// DataSegmentStore stores data segments for SSE streaming +type DataSegmentStore struct { + streamID string + segments chan []byte + mu sync.RWMutex + closed bool +} + +func NewDataSegmentStore(streamID string) *DataSegmentStore { + return &DataSegmentStore{ + streamID: streamID, + segments: make(chan []byte, 100), // Buffer up to 100 segments + } +} + +func (d *DataSegmentStore) Store(data []byte) { + d.mu.RLock() + defer d.mu.RUnlock() + if d.closed { + return + } + select { + case d.segments <- data: + default: + // Channel is full, drop oldest segment + select { + case <-d.segments: + default: + } + select { + case d.segments <- data: + default: + } + } +} + +func (d *DataSegmentStore) Subscribe() <-chan []byte { + return d.segments +} + +func (d *DataSegmentStore) Close() { + d.mu.Lock() + defer d.mu.Unlock() + if !d.closed { + d.closed = true + close(d.segments) + } +} + +// Global store for data segments by stream ID +var dataStores = make(map[string]*DataSegmentStore) +var dataStoresMu sync.RWMutex + +func getDataStore(stream string) *DataSegmentStore { + dataStoresMu.RLock() + store, exists := dataStores[stream] + dataStoresMu.RUnlock() + if exists { + return store + } + + dataStoresMu.Lock() + defer dataStoresMu.Unlock() + // Double-check after acquiring write lock + if store, exists := dataStores[stream]; exists { + return store + } + + store = NewDataSegmentStore(stream) + dataStores[stream] = store + return store +} + +func removeDataStore(stream string) { + dataStoresMu.Lock() + defer dataStoresMu.Unlock() + if store, exists := dataStores[stream]; exists { + store.Close() + delete(dataStores, stream) + } +} + +// @Summary Get Live Stream Data +// @Param streamId path string true "Stream ID" +// @Success 200 +// @Router /live/video-to-video/{stream}/data [get] +func (ls *LivepeerServer) GetLiveVideoToVideoData() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + stream := r.PathValue("stream") + if stream == "" { + http.Error(w, "stream name is required", http.StatusBadRequest) + return + } + if r.Method == http.MethodOptions { + corsHeaders(w, r.Method) + w.WriteHeader(http.StatusNoContent) + return + } + + ctx := r.Context() + ctx = clog.AddVal(ctx, "stream", stream) + + // Get the data store for this stream + dataStore := getDataStore(stream) + if dataStore == nil { + http.Error(w, "Stream not found", http.StatusNoContent) + return + } + + // Set up SSE headers + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Connection", "keep-alive") + w.Header().Set("Access-Control-Allow-Origin", "*") + + // Get the subscription channel + dataChan := dataStore.Subscribe() + + flusher, ok := w.(http.Flusher) + if !ok { + http.Error(w, "Streaming not supported", http.StatusInternalServerError) + return + } + + clog.Infof(ctx, "Starting SSE data stream for stream=%s", stream) + + // Send keep-alive ping initially + fmt.Fprintf(w, "event: ping\ndata: {\"type\":\"connected\"}\n\n") + flusher.Flush() + + // Stream data segments as SSE events + for { + select { + case <-ctx.Done(): + clog.Info(ctx, "SSE data stream client disconnected") + return + case data, ok := <-dataChan: + if !ok { + // Channel closed, stream ended + fmt.Fprintf(w, "event: end\ndata: {\"type\":\"stream_ended\"}\n\n") + flusher.Flush() + return + } + + // Send the segment data as a data event + fmt.Fprintf(w, "data: %s\n\n", string(data)) + flusher.Flush() + } + } + }) +} + func startHearbeats(ctx context.Context, node *core.LivepeerNode) { if node.LiveAIHeartbeatURL == "" { return From b2507bfefde7e1a6e6d3d9eaff5c800d558027a9 Mon Sep 17 00:00:00 2001 From: Brad P Date: Tue, 22 Jul 2025 14:08:56 -0500 Subject: [PATCH 02/57] update to new trickle subscriber api --- server/ai_live_video.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/server/ai_live_video.go b/server/ai_live_video.go index cc0c6b411d..e00eeb3b23 100644 --- a/server/ai_live_video.go +++ b/server/ai_live_video.go @@ -786,7 +786,14 @@ func startEventsSubscribe(ctx context.Context, url *url.URL, params aiRequestPar func startDataSubscribe(ctx context.Context, url *url.URL, params aiRequestParams, sess *AISession) { // subscribe to the outputs and send them into LPMS - subscriber := trickle.NewTrickleSubscriber(url.String()) + subscriber, err := trickle.NewTrickleSubscriber(trickle.TrickleSubscriberConfig{ + URL: url.String(), + Ctx: ctx, + }) + if err != nil { + clog.Infof(ctx, "Failed to create trickle subscriber: %s", err) + return + } // Set up output buffers rbc := media.RingBufferConfig{BufferLen: 50_000_000} // 50 MB buffer From 8b7c61b4e5c3644e4b46d7c93e901a8be47ec1c5 Mon Sep 17 00:00:00 2001 From: Brad P Date: Tue, 22 Jul 2025 14:09:11 -0500 Subject: [PATCH 03/57] update ai-runner bindings --- ai/worker/runner.gen.go | 177 ++++++++++++++++++++-------------------- 1 file changed, 90 insertions(+), 87 deletions(-) diff --git a/ai/worker/runner.gen.go b/ai/worker/runner.gen.go index 68197ee283..b4c0826556 100644 --- a/ai/worker/runner.gen.go +++ b/ai/worker/runner.gen.go @@ -317,6 +317,9 @@ type LiveVideoToVideoParams struct { // ControlUrl URL for subscribing via Trickle protocol for updates in the live video-to-video generation params. ControlUrl *string `json:"control_url,omitempty"` + // DataUrl URL for publishing data via Trickle protocol for pipeline status and logs. + DataUrl *string `json:"data_url,omitempty"` + // EventsUrl URL for publishing events via Trickle protocol for pipeline status and logs. EventsUrl *string `json:"events_url,omitempty"` @@ -3160,93 +3163,93 @@ func HandlerWithOptions(si ServerInterface, options ChiServerOptions) http.Handl // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+xdeW/ctrb/KoTeA5IAM97a3D4YuH84SxPj2onhpQtaYy5HOjPDWCJVkrI9zfN3f+Am", - "kRI1I7u229c7f2UscTnr7xySh8rXJGVFyShQKZL9r4lIF1Bg/fPg5PA954yr3xmIlJNSEkaTffUGgXqF", - "OIiSUQGoYBnkW8koKTkrgUsCeoxCzLvdzxdguxcgBJ6D6ieJzCHZT47FXP21LNUfQnJC58nd3Sjh8FtF", - "OGTJ/i961MumS01o3Y9Nv0Aqk7tRclBlhJ1aKruknAb0oxnjCKseaA4UOFatukzpFvpHnn+eJfu/fE3+", - "m8Ms2U/+a7uR5rYV5fYxZARfnB4ld5ejiCTsTJCZmbc63JrpfH4DniJMv2HZcjIHqhues3O4lYrcHi5C", - "ki7KnOHMUYNmJAckGZoCkhxT1XIKmZLJjPECy2Q/mRKK+TJp0ddV4igpQOIMS2xmneEqV/2/3iVtuRxk", - "GVE/cY6+sCki1ExGGLW0lFgIyNQfcgGoJCXkhIZ25OaK0aGUPSFZSEeHio/VfE7oHH2PU2cgh+9QpSZW", - "huLkUTorqac2TbPY1BxkxelEkgKExEUpQhokr6BDx6nug5o+ZvpFoBIk4VZuobOqLBlX1nSN8wrEPnoh", - "gEqgKbwYoRc3jGcvRkiZOTJEoSljOWCKXr5Qk79Q717McC7gxast9M5QhohA9vXLZrxXW64lKgBTgSjz", - "iNyys9l36vd4irXWmjae1CyX541k1sFAxzFidr/CPQ4LPIdzpv/p+se8IhmmKUxEinMI1PTd1uu2jt7T", - "lFUcz0FYS5E1hgAihX6R5kxAvkQ5oVeN8Sq9oZKzopTo5YLMF8Ct7lCBl4hDVqV2CPRbhXMil698uX2w", - "dKIzTWfNL62KKXDFL3EM9ni6GVsyRTmZLdENkYuOX/W7u5FfxNb1uJMVctztyvEdzDloYm4WJDVkNAhp", - "KCUClZVYaBHeYJ4J3YpQIgnOTZutNn1ovZhyxrFYAwkH6IidHqCXR+xmfIrpFTrIcCk1Mr2yisc0Q0QK", - "lDJuomOmvOwGyHwhteMaJrwAg97f4qLMYR99Rb8mOZZA5ThlVBChHG25nafFWFE3Ftlt/muyj3a3dkbo", - "14QCJ1/EdkluIR9jLsfu7d6dL4AjzdiT4WCHn4FQSGGOJbmGiTH+NUScN27yUrzS7lWRDNDNAkv1F9ym", - "eZUBmnFWRER8OKeMKwuaodAg0a/Vzs43Kdr1yf5kSUMnhrQY9VUxMX49KYHHeNhts/BJmxpiMwcIPkaU", - "wC17ASFVgQ5N4xPgHXIIlTA31qvpoTPgoFmT0Aotuzs7/fRkQBkRSse64xY6ZhzMb1SJCucKtQBrzLIQ", - "ZaHIsTKtJBI5uwGOairUMFmVa8+dLlW8ATqXiw5/rj0601THuPPFO8QqVtlkv04FnoFcTtIFpFeB8FTo", - "a0vvBLjCRBVIdTeku2lTFJIUGvdnbexSsFDlmUph2GwGVCgjYxwtMC9mVe6TeWZGfauJqYm10VpTC5B1", - "JXIG1i05phkrkMG3HlGoxlF5O10FUtjZ+p8euGYzk4o0aRouy5w0QY6D07HRzMsd9WY3CGRnbs4ONrfi", - "fukUaAJbJAEIIvv6DCCeIA8OmzXrjxY5HzFBrVUyFJb/EBr3T9nndS3drlPpwJzuB5IB66p01gLFf8QW", - "ZDOOCxAakAWkjGbavIM85FoN73P3fQ9uLXTYD+Z8/V10VtMSEYp0OBcDJv1oBo/NO9h26/iDzfg6fv6p", - "VmvIuH86UTDVejKt0iuQbSp2975rk3HhJlQq1qtNRZQSOS5YRaVSgBmzXm75CYXWmQmF6pWFWfWzULHT", - "9rwhea7AnlD9qqPCY9PsjSY6YMwP7YwImOBqPumB5Z29Tp5as6A7I5xlDRgHDJt0GX0MFh520cFBQDHN", - "ddrc29ckvDTlgIXjOwjxmoCDao76AX59+rL3+v9x9rLJK5wkbkjWst7dnb1vY3ioW94LDn/UY3dnvWeE", - "MaFjRYg5g3kBVB7QpVwQOt/rhpkpu41smqJcGxD6FmHO8RLNyTVQhAXCaMpu3RaA9TONiyPF/08///Qz", - "Mmjsc/uG3fauubuTHzq8F4b4hyI8FlcTQstKRvljN2MOguWVBjXVGOnGLabksiSp9kq9WMOo5HBNWCXU", - "j4ykujeR1q5GTVal/WL39uPtj+jlx3/++M+91//QJnl2cBxkksdq5kNN5l9u1VtUufJicTVhlawFuQIP", - "DlVuXcGokaCJKtzuCi5UAq4GNNuCuJiSeaWEaURvzEqMEJtJoOrPrEr1vh9ICdz2lAtMFeIQOs/BU0PA", - "laMcfTaUx8CDKqPKye8wSRnjmbgfeyUjVCLdk1AsQdQBtB63WVJgOgf0y85o99KaiO5t50VwW0IqTfMp", - "mAYchHqoHhn1ZaRQWMmoCCOWnQu9NTzEGPUn6zrDp9s96+VsZrmyimj5ws0COCDAqSUfEaU49PKn0c+v", - "GvQLEmndrE2Zl79rwnI8hTxC2JF+Xmc0AWmOml1EaEZSLX+smsKcs4pmtrWK9ztBkylOr/wmXXLNtCs2", - "xHM2J/Ie1mK6CVTRsfIAsWC5ynC0eZqxEKFCqqjPZopEjXH6fWTT+cjM3tXz0NjRiQkr4sdFWe+EPnDB", - "+cj7tI8DiJVhK3v4fuCaFPC71/9BG1iDpLnZyVqXcd5758g5Z8R/3y4qehXLe1L1QieoSpnaK3FzyNU9", - "P5Z2u6mb9OoBbKarR/VZDLc+Gl3XM/WM6V53BiYSCkXQnTdHPVY9kQ5jHUlKv6EizJOlEVREgh9OLt6y", - "oqwkHNJZ5Oz5uD6Ez0Biosz/w8kFSk0f/xi4K1QDXzXWxXMv/MWUMzSJ4he/bMDz1wIKxpeTGQcIOujH", - "6Hv1eEU3ySTOI/3O9fNoR0JbpOkH0e0AXAQ0fVJ/r91XUwKhpmVAZMiqk5EjyNNqS3lx9V5IkpPftYrW", - "qVhptmqaIyGxJEKSVDxQuc+ssWFqGCUejxNryX43T2LIyjc6nT+MoblvFEP5gIXwUJOIcRAlKLSWti1E", - "LObj+flJT42RejWwyMiAxfCCnLpeqFuQ887hjpk5QJy2/Oy0HtMNOz28/oBzkunhaq77WHHgvJKT9nge", - "khtOYjDuU9seIEY35tkN5trrrSwG1VEp/16J2POyUvmeqaSqC35OgjarmG8BksfZh7JCfSbnJ7uDjkNs", - "bu23P3HP1qFu2TSs5x01jPumE5HyCmWcSSzFIDVwwPlYRWutkFUgq8gSbtwHKqTt8y2lGLL/ulox7EfU", - "0kv4R8C5XLx1eXYoUTVcJeJp2UJ3RKaJS808yoBWhSL26PPBu8NPH5JR8vlfySh5f3r6+TQZJYfvjt77", - "hJ6ZqdaxbinyOfQYiPCnF433qp+MLVgi69wesbRXGX6eur7K0rc3UzaxLo+1tFy2+q2qrvROIu8lGJ3c", - "r5JL/7qgkYqu8lu7KGgn660EPcZBhNGjo+O3C0ZSiMWp3JRxDou4R0fHx6bUN7m7vBslM0KJWEw4YOGi", - "ibftUB+q6lbo1LSKbXbTDG6DzFA/iKd5ZvoHkty2Gj2PJ9FGVHE5HjfTh4JMGZVApc/EW/sotlnG8gDy", - "Tlm+Hu64aeRmCql2hMXJPoXfKhCR2ogC304kuwLaPqX7h78Lf4vOTZsVGhGDkx1fI36drx2m7egW3Hut", - "S4eX6N6J5ICLoJ+ucg0rVXAR3cqQUJTKVSsOrUPa73ynbRpFzkIlKyfhrs141+vMSvSvqERVv7JdguZ3", - "O1lbWlMrJbQSZwd9VtKgYMu6tU/cS8XWje787QQzSFfBKQcFioHz2EfRooghq0hnM2vNpHIOvYYh7QIX", - "xmxjK7DCTpDWlFfWI5302rpYidjedBGwKcoc9Iqtcd4GdtzLVV5rdtAi3c1e5KquenEZ6amX2P0do1t4", - "bphRhKfWTKH0PPHE5EeuQZ//2mPgE8yxsdouanOWTyqer9k0vzg90sFfVFNdqE/oHF0TjM45Sa/0URuT", - "LGW53ULP9ImKrRbJybUtGRlLNm7XxaBSE+cnA28NWeiCRy0WrpVt3oPosprmRCwUzaZvP+kuc3UJLaYZ", - "ytk8IO+9GaOHujmWcIOXE26wZv15hEqMDt+51PmD6Y5sd/RSUZUzc2RRVrxkAkRYOG97WHDrO6/FlMwG", - "0+NaK8L04bYijfF0AUJyLBkfQNdxPUb2B05rPuEC2ssKJBniFe3al3phfnxh0y30iUmSApK6tHpBBCIC", - "UVxAhtzkrnDKlaibXW8mF8ARZ5UEMdInKkSijIFAlElTA6pmwih6lGRqquAWp9I8eyleoQxKoJlAjIac", - "EOX1BVBpy01phgpdQDfVNUozMq84nuagLVb1/Ldxl38jzOeVK4EYtNSsMaCW9te7zrmkvXygG4ME7h2x", - "Ri4nWVyJIJB1ucZJw90pIQk1HCsftepllZwzc/al8hIlYjtMMKd51Od+pu8wKzf5j7Lx9dbs2kZl61AR", - "4vyesYqn4LNKaMqKkNV6DCSDcqCz+nmU4/bCOKAk1IMfQeIBYkAouddScTXyd1eO949HOtS4gsfV0z1h", - "gPGjomQuyKyKJ4PDyXDc/uzDcwDg2oOJcDFlOEY/rhcjyYb78X3DZ7O1sILd1SHysb34uZ14VUZ9jMWV", - "uJfvmr6u0K7HYf1yl7ZSOL4ZoYp6FU9NPZZAL03XV3V80QVc4XWvsJglLN9bu+/WGU+LIKr3lPG+fTwt", - "jxfCxOJMF0eY5ppuXe8UThlAtxl47TVwS5hwza1UL1u0r9Sv3jaMHOoX6oVTpsJXTEzNs3cXGU9ZJVtl", - "qbpfV+FUzG660/y4AOkqyM2EN1igWY7nc8gQFujT2fc/BuUmapjhJRRKE+qNqdLxy/3rGQeV7Ub9Wg2u", - "nNoUjTUspJiqLAynKQhh7orXx5QDnNi4rjCkaLH5+tTq6tPjxelRTJU62qhk3Fwp7aUy1Nhz89zmUjET", - "YfTxt511PYgYsvFsSkeG78mbQpC7VmVIbCfnabe+R47Hy7D3KmBQ7+0Nsr5dgL/PDfHHvKbUuX+94prS", - "5sr15sr13/fK9ev/6BvX6AxKrOWsrzyUZsdQl8DrzaIX//tCmYaoP1gyXTaF8Zsq1z/tXlUHvwfeq7IG", - "0wqxYQjtjbNnJUC66Au0ARc+ZB2gQuGJKAFfAUcZ5OQauFA6zhX450sEtyUHofWmwgSmWtWZ6gPpwlXr", - "KqPTtqoeZ7plSWSqPaezend/Kdm5qdUSVgLYdEv9ZcaP69Eb5Amvfw+hZFW0aJKy1SHCXDTS+zmrpurN", - "10J7CUwhYjBra+hylgYHjpgu7Xl/m8OvHZu+vPNjeNqqw2pOls1nzVpn0FEZ6gdNU00zOldP16Wuig8z", - "lW3pudaAur0fgAvrMK2z+yettRol183EXfixLx3snFaUAveMxFH9sBouN/XligE9Cd17X3b9Tqz5eMC6", - "pYy7aq/aBqupe5Y6tVdRrnDGELGm9MmS6stq9W6YjmFpxYlcnilSDJ8fz89P3gDmwOvvB+rAZx7Vgyyk", - "LJO7O121EysMP7DfDEnrz7zxiqKDw3on2N/6PSLXUCq0PThsTKi2u2Rna/ebrddKIqwEikuS7CffbO1u", - "7Sh1YbnQdG/rz4eNJRs7nCuZiCU89TfWvE/imWt8dkHKSmsOh5labbW/P2a3U9+wbNmq+DGJEeZyW2Um", - "Y/dpPKPndVYQ+9jZXahjlQbpB0ajmu29nZ0WFZ7Yt7/YeqxhJARraD13K7ep9H7IrMpR02yUfPuIJDQl", - "15H53+DMne6aeXefZ94Liiu5YJz8DpmeePeb55nY7dO/p1KtFM4ZQ0eYm4Kpb3dfPxf3TU6vocqEO0XC", - "3t6jktApf+8S0zRBdYn86+eyv0MqgVOcozPg18AdBR6O6rTER9BfLu8uR4moigLzpfuWJjpnyGVPeC4U", - "eLtYouD7dmyyUCyWY4oLGLNr4JxkGvoDdBgl2wtb0bztYHgOWgQhiPnl6MkTIkis7H0okNz5cnIDmbr/", - "kNO6qH0lq67E+8l5NRP9MS7dGIpNXcDdz555/ZR8eRXkD+PKkKi50atPFZTry9DxqHxQlvnS3YgOPjol", - "TIlJyZnKsrz1bCdMt74S9sRxOpjtmQN1WNO+idT9kXoToe4bocynZc4Zqr8vcM8QRULH8EFgQGau9/QM", - "DqxPzMOPyD2Pw/8ZiXnsgsfG6//i+fkGeh4MPQ9MjkngoT7wXNffj4wiz4fYVxPvlXS4r4w9DwaZ2Z4Z", - "hMLdpA38bJKOJ/D8+mt9D3N95xijZDsn1zAOi2DXLT+iCw+vqt6UN/pfQZYVp5AhoJn+UJaIQkS7PnEl", - "TDxcRz21zM+MEr3FmBvA2ADG4wGGMjMDFn8ENfK2ZxrkyIsBqYI+jq10yQdGOabzSkFYXe3QRYGj46dy", - "/OZC53M7u3d9cePfG/9+RP/W3nJvf84L48K2Wn+M7Qccx3v9Hm2/9Whrw/VtTUxXZPyRb0M+cdbfmfGZ", - "3Tysut84+sbRH8/Rnfc540Z7D/B70XWQUbKtIvSAo4cPraJtc8+3KYuMJ/VeMdwThfVuud3mlGHj9n8T", - "t9eFhn/gkEF67hc4uylZHLTVF3bx/4NM8/8aujvqbhNQNsWRmGZelWrwv0b2IIUpg3xSqAgqLZ8ZK8L/", - "w3SDFRuseHysqF3oYWBhu2u0qLxvtkdhwn43ul4JoOnS/ac4+taoFKj5rzGibt98efqJVwduok12sPH4", - "v4nHe19tv6erV7UzjJJtr3I9WkvV1JI/3aGZneJBhVRBZ6GlKbTsWv8HiKuefpuzKkNvWVFUlMil+4ZS", - "Yi9865ptsb+9nXHAxdh+oGkrt923UtVdX6PoGf9M6hSpb9h6IKHbbeOSbE9B4u1aeXeXd/8XAAD//5FW", - "1i+hfgAA", + "H4sIAAAAAAAC/+xdeW/ctrb/KoTeA+IAM97atO8ZuH84SxPj2onhpQtaw5cjnZlhLJEqSdme5vm7P3CT", + "SImakR3b7e2dvzKWuJz1dw7JQ+VLkrKiZBSoFMnel0Skcyiw/rl/fPCOc8bV7wxEykkpCaPJnnqDQL1C", + "HETJqABUsAzyzWSUlJyVwCUBPUYhZt3uZ3Ow3QsQAs9A9ZNE5pDsJUdipv5alOoPITmhs+TubpRw+L0i", + "HLJk71c96kXTpSa07scmnyGVyd0o2a8ywk4slV1STgL60ZRxhFUPNAMKHKtWXaZ0C/0jzz9Nk71fvyT/", + "zWGa7CX/tdVIc8uKcusIMoLPTw6Tu4tRRBJ2JsjMzJsdbs10Pr8BTxGmX7NscTkDqhuesTO4lYrcHi5C", + "ks7LnOHMUYOmJAckGZoAkhxT1XICmZLJlPECy2QvmRCK+SJp0ddV4igpQOIMS2xmneIqV/2/3CVtuexn", + "GVE/cY4+swki1ExGGLW0lFgIyNQfcg6oJCXkhIZ25OaK0aGUfUmykI4OFR+q2YzQGfoBp85ADt6iSk2s", + "DMXJo3RWUk9tmmaxqTnIitNLSQoQEhelCGmQvIIOHSe6D2r6mOnngUqQhFu5iU6rsmRcWdM1zisQe+iF", + "ACqBpvBihF7cMJ69GCFl5sgQhSaM5YAp2nihJn+h3r2Y4lzAi5eb6K2hDBGB7OuNZryXm64lKgBTgSjz", + "iNy0s9l36vd4grXWmjae1CyXZ41kVsFAxzFidr/EPQ4KPIMzpv/p+sesIhmmKVyKFOcQqOn7zVdtHb2j", + "Kas4noGwliJrDAFECv0izZmAfIFyQq8a41V6QyVnRSnRxpzM5sCt7lCBF4hDVqV2CPR7hXMiFy99ub23", + "dKJTTWfNL62KCXDFL3EM9ni6GVsyRTmZLtANkfOOX/W7u5FfxNb1uJdL5LjTleNbmHHQxNzMSWrIaBDS", + "UEoEKisx1yK8wTwTuhWhRBKcmzabbfrQajHljGOxAhL20SE72Ucbh+xmfILpFdrPcCk1Mr20isc0Q0QK", + "lDJuomOmvOwGyGwuteMaJrwAg97d4qLMYQ99Qb8lOZZA5ThlVBChHG2xlafFWFE3Ftlt/luyh3Y2t0fo", + "t4QCJ5/FVkluIR9jLsfu7e6dL4BDzdiT4WCHn4FQSGGGJbmGS2P8K4g4a9xkQ7zU7lWRDNDNHEv1F9ym", + "eZUBmnJWRER8MKOMKwuaotAg0W/V9vY3Kdrxyf5oSUPHhrQY9VVxafz6sgQe42GnzcJHbWqITR0g+BhR", + "ArfsBYRUBTowjY+Bd8ghVMLMWK+mh06Bg2ZNQiu07Gxv99OTAWVEKB3rjpvoiHEwv1ElKpwr1AKsMctC", + "lIUix8qkkkjk7AY4qqlQw2RVrj13slDxBuhMzjv8ufboVFMd484X7xCrWGaT/ToVeApycZnOIb0KhKdC", + "X1t6x8AVJqpAqrsh3U2bopCk0Lg/bWOXgoUqz1QKw6ZToEIZGeNojnkxrXKfzFMz6htNTE2sjdaaWoCs", + "K5FTsG7JMc1YgQy+9YhCNY7K2+kqkML25v/0wDWbmlSkSdNwWeakCXIcnI6NZja21ZudIJCdujk72NyK", + "+6VToAlskQQgiOyrM4B4gjw4bNasP1rkfMQEtVbJUFj+KjTun7LP61q6XaXSgTndjyQD1lXptAWK38UW", + "ZFOOCxAakAWkjGbavIM85FoN73P3Qw9uzXXYD+Z89X10VtMSEYp0OBcDJv1gBo/NO9h26/iDzfg6fv6p", + "VmvIuH86UTDV+nJSpVcg21Ts7H7fJuPcTahUrFebiiglclywikqlADNmvdzyEwqtMxMK1SsLs+pnoWKn", + "7XlD8lyBPaH6VUeFR6bZa010wJgf2hkRcImr2WUPLG/vdvLUmgXdGeEsa8A4YNiky+hDsPCwiw4OAopJ", + "rtPm3r4m4aUpBywc30GI1wTsVzPUD/Cr05fdV//G2cs6r3CSuCFZy3p3tne/jeGhbnkvOPxJj92d9Z4R", + "xoSOJSHmFGYFULlPF3JO6Gy3G2Ym7DayaYpybUDoW4Q5xws0I9dAERYIowm7dVsA1s80Lo4U/z//8vMv", + "yKCxz+1rdtu75u5OfuDwXhjiH4rwWFxdElpWMsofuxlzECyvNKipxkg3bjElFyVJtVfqxRpGJYdrwiqh", + "fmQk1b2JtHY1arIq7Rc7tx9uf0IbH/7x0z92X32nTfJ0/yjIJI/UzAeazL/cqreocuXF4uqSVbIW5BI8", + "OFC5dQWjRoImqnC7KzhXCbga0GwL4mJCZpUSphG9MSsxQmwqgao/syrV+34gJXDbU84xVYhD6CwHTw0B", + "V45y9MlQHgMPqowqJ3/AZcoYz8T92CsZoRLpnoRiCaIOoPW4zZIC0xmgX7dHOxfWRHRvOy+C2xJSaZpP", + "wDTgINRD9cioLyOFwkpGRRix7FzojeEhxqg/WdcZPt7uWi9nU8uVVUTLF27mwAEBTi35iCjFoY2fR7+8", + "bNAvSKR1szZlXv6uCcvxBPIIYYf6eZ3RBKQ5anYQoRlJtfyxagozziqa2dYq3m8HTSY4vfKbdMk10y7Z", + "EM/ZjMh7WIvpJlBFx8oDxJzlKsPR5mnGQoQKqaI+myoSNcbp95FN50Mze1fPQ2NHJyYsiR/nZb0T+sAF", + "5yPv0z4OIFaGrezh+4ErUsDvX/0HbWANkuZ6J2tVxnnvnSPnnBH/fTOv6FUs70nVC52gKmVqr8TNIVf3", + "/Fja7aZu0qsHsJmuHtVnMdz6aHRdz9QzpnvdGZhIKBRBd94c9Vj1RDqMdSQp/YaKME+WRlARCb4/Pn/D", + "irKScECnkbPno/oQPgOJiTL/98fnKDV9/GPgrlANfNVYF8+98GdTztAkip/9sgHPXwsoGF9cTjlA0EE/", + "Rj+ox0u6SSZxHul3pp9HOxLaIk0/iG4H4CKg6aP6e+W+mhIINS0DIkNWnYwcQZ5WW8qLq/dckpz8oVW0", + "SsVKs1XTHAmJJRGSpOKByn1mjQ1TwyjxeLy0lux38ySGrHyj0/nDGJr7RjGUD1gIDzWJGAdRgkJradtC", + "xGI+nJ0d99QYqVcDi4wMWAwvyKnrhboFOW8d7piZA8Rpy89O6zHdsNPD6484J5kerua6jxUHzks5aY/n", + "IbnhJAbjPrXtAWJ0Y57dYK693spiUB2V8u+liD0rK5XvmUqquuDnOGizjPkWIHmcvS8r1GdyfrI76DjE", + "5tZ++2P3bBXqlk3Det5Rw7hvOhEpL1HGqcRSDFIDB5yPVbTWClkGsoos4cZ9oELaPt9SiiH7r6sVw35E", + "Lb2EfwCcy/kbl2eHElXDVSKels11R2SauNTMowxoVShiDz/tvz34+D4ZJZ/+mYySdycnn06SUXLw9vCd", + "T+ipmWoV65Yin0OPgQh/etF4r/rJ2IIlss7tEUt7leHnqaurLH17M2UTq/JYS8tFq9+y6krvJPJegtHJ", + "/TK59K8LGqnoKr+Vi4J2st5K0GMcRBg9PDx6M2ckhVicyk0Z57CIe3h4dGRKfZO7i7tRMiWUiPklByxc", + "NPG2HepDVd0KnZhWsc1umsFtkBnqB/E0z0z/QJLbVqPn8STaiCoux6Nm+lCQKaMSqPSZeGMfxTbLWB5A", + "3gnLV8MdN43cTCHVjrA42SfwewUiUhtR4NtLya6Atk/pvvN34W/RmWmzRCNicLLja8Sv87XDtB3dgnuv", + "denwEt07kRxwEfTTVa5hpQouolsZEopSuWrFoXVI+73vtE2jyFmoZOVluGsz3vE6sxL9MypR1a9sl6D5", + "3Y5XltbUSgmtxNlBn5U0KNiybu0T91KxdaM7fzvBDNJVcMpBgWLgPPZRtChiyCrS2cxKM6mcQ69gSLvA", + "uTHb2AqssBOkNeWV9UgnvbYuliK2N10EbIoyB71ia5y3gR33cpnXmh20SHezF7msq15cRnrqJXZ/x+gW", + "nhtmFOGpNVMoPU88MfmRa9Dnv/YY+BhzbKy2i9qc5ZcVz1dsmp+fHOrgL6qJLtQndIauCUZnnKRX+qiN", + "SZay3G6hZ/pExVaL5OTaloyMJRu362JQqYnzk4E3hix0zqMWm2GJ70FyWU1yIuaKYtWzn2yXtbpkFtMM", + "5WwWkPZWjdBDF1wrn3kYZabv19H2zozRQ90MS7jBi0tuMHD1OYlK2A7eupT+vemObHe0oajKmTlKKSte", + "MgEiLOi3PSzo9p0jY0qmg+lxrRVh+tBdkcZ4OgchOZaMD6DrqB4j+4pTpI+4gPZyB0mGeEW7dq9emB+f", + "2WQTfWSSpICkLvmeE4GIQBQXkCE3uSvocqXzZjeeyTlwxFklQYz0SQ+RKGMgEGXS1KaqmTCKHnGZWi+4", + "xak0zzbES5RBCTQTiNGQE6LQqAAqbRkszVChC/smunZqSmYVx5MctMWqnv8ybvwvhPmscqUZg5bANTbV", + "0v5y1zkvtZcidGOQwL2j38ilKYt3EWS0Ltc4abhrJiShhmPlo1a9rJIzZs7kVL6kRGyHCeY0j/rcz/Qd", + "ZuUmL1M2vtqaXduobB1aQ5zfU1bxFHxWCU1ZEbJaj4FkUKZ0Wj+PctxesAeUhHrwI1s8cA0Icfdawi6P", + "SN0V7f3jpA6BrhBz+XTDA9+9A4wfrSVzQWZZPBkcTobj9icfngMA1x5MhIspwzH6cb0YSTbcj+8bPpst", + "jyXsLg+Rj+3Fz+3EyzL9IyyuxL181/R1BYA9DuuX4bSVwvHNCFXUq8Rq6sQE2jBdX9bxRReWhdfQwiKb", + "sKxw5X5gZzwtgqjeU8b79he1PF4IE4szXbRhmmu6dR1WOGUA3WbgldfTLWHCNbdSvWjRvlS/ejszUmxQ", + "qBdOmQpfMTG12N4daTxhlWyVy+p+XYVTMb3pTvPTHKSrbDcT3mCBpjmezSBDWKCPpz/8FJTBqGGGl3Yo", + "Tag3pnrIv4ZQzzionDjq12pw5dSmmK1hIcVUZWE4TUEIc4e9Pj4d4MTGdYUhRYvN16dWV58ez08OY6rU", + "0UYl4+aqay+Vocaem+c2l4qZCKOPvx2u61TEkA1xU9Iy/KzAFKjctSpWYjtMT7slP3I8XoS9lwGDem9v", + "tvXtTvx9bq4/5vWpzr3wJden1lfB11fB/75XwV/9R98ER6dQYi1nfRWjNDuGujRfbxa9+L8XyjRE/SGV", + "yaIp2F9X3/5p9706+D3wvpc1mFaIDUNob5w9LQHSeV+gDbjwIWsfFQpPRAn4CjjKICfXwIXSca7AP18g", + "uC05CK03FSYw1arOVB9I566KWBmdtlX1ONMtSyJT7Tmd1bv7S8nOTa2WsBLAplvqLzN+XI/eIE94LX0I", + "JcuiRZOULQ8R5gKU3s9ZNlVvvhbaS2AKEYNZWduXszQ4CMV0YesQ2hx+6dj0xZ0fw9NWfVhz4m0+t9Y6", + "G4/KUD9ommqa0Zl6uip1VXyYqWxLz7UG1BP+CFxYh2nVFDxpDdgouW4m7sKPfelg56SiFLhnJI7qh9WW", + "uakvlgzoSeje+7Krd2LNRw1WLWXcJwBU22A1dc8SrPYqyhX0GCJWlGRZUn1ZLd8N0zEsrTiRi1NFiuHz", + "w9nZ8WvAHHj9XUMd+MyjepC5lGVyd6eriWIF6/v2WyZp/fk5XlG0f1DvBPtbv4fkGkqFtvsHjQnVdpds", + "b+58s/m/SiKsBIpLkuwl32zubG4rdWE513Rv6c+ajSUbO5wrmYglPPW337xP9ZnrhXZBykprDgeZWm21", + "v4tmt1Nfs2zRqkQyiRHmcktlJmP3yT6j51VWEPsI212oY5UG6QdGo5rt3e3tFhWe2Lc+2zqxYSQEa2g9", + "dyu3qfR+yLTKUdNslHz7iCQ0peCR+V/jzJ3umnl3nmfec4orOWec/AGZnnjnm+eZ2O3Tv6NSrRTOGEOH", + "mJtCrm93Xj0X901Or6HKhDtFwu7uo5LQKcvvEtM0QXXp/qvnsr8DKoFTnKNT4NfAHQUejuq0xEfQXy/u", + "LkaJqIoC84X7xic6Y8hlT3gmFHi7WKLg+3ZsslAsFmOKCxiza+CcZBr6A3QYJVtzW2m95WB4BloEIYj5", + "ZfLJEyJIrBx/KJDc+XJyA5n7CCGndbH9UlZd6fmT82om+jou3RiKTV1Y3s+eef2UfHmV7Q/jypCoudGr", + "TxWU60va8ai8X5b5wt3UDj6GJUyJScmZyrK89WwnTLe+XvbEcTqY7ZkDdVhrv47U/ZF6HaHuG6HMJ2/O", + "GKq/e3DPEEVCx/BBYEBmrvf0DA6sTszDj9s9j8P/GYl57OLJ2uv/4vn5GnoeDD0PTI5J4KE+8FzX37WM", + "Is/72Ncc75V0uK+fPQ8GmdmeGYTC3aQ1/KyTjifw/Porgg9zfecYo2QrJ9cwDotgVy0/ogsPr6relDf6", + "X2eWFaeQIaCZ/oCXiEJEuz5xKUw8XEc9tczPjBK9xZhrwFgDxuMBhjIzAxZfgxp52zMNcuTFgFRBH8dW", + "uuQDoxzTWaUgrK526KLA4dFTOX5z0fS5nd27Vrn277V/P6J/a2+5tz/nhXFhW60/xvbDkuPdfo+236C0", + "teH6FimmSzL+yDcrnzjr78z4zG4eVt2vHX3t6I/n6M77nHGj3Qf4veg6yCjZUhF6wNHD+1bRtrnn25RF", + "xpN6rxjuicJ6t9xufcqwdvu/idvrQsOvOGSQnvsFzm5KFgdt9YVd/P+40/x/i+6OutsElE1xJKaZV6Ua", + "/G+WPUhhyiCfFCqCSstnxorw/1ZdY8UaKx4fK2oXehhY2O4aLSrvW/JRmLDfs65XAmiycP9Zj741KgVq", + "/suOqNs3X8R+4tWBm2idHaw9/m/i8d7X5O/p6lXtDKNky6tcj9ZSNbXkT3doZqd4UCFV0FloaQotu9b/", + "TeKqp9/krMrQG1YUFSVy4b6hlNgL37pmW+xtbWUccDG2H2jazG33zVR119coesY/lTpF6hu2Hkjodlu4", + "JFsTkHirVt7dxd3/BwAA//8lGRESOX8AAA==", } // GetSwagger returns the content of the embedded swagger specification file From 00fd90c4c9e67d87eac6768200016f47fe3a67e3 Mon Sep 17 00:00:00 2001 From: Brad P Date: Tue, 22 Jul 2025 14:27:51 -0500 Subject: [PATCH 04/57] remove Orch data url short circuit --- server/ai_http.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/ai_http.go b/server/ai_http.go index d655aff191..5d3b27f637 100644 --- a/server/ai_http.go +++ b/server/ai_http.go @@ -145,7 +145,7 @@ func (h *lphttp) StartLiveVideoToVideo() http.Handler { subUrl = pubUrl + "-out" controlUrl = pubUrl + "-control" eventsUrl = pubUrl + "-events" - //dataUrl = pubUrl + "-data" + dataUrl = pubUrl + "-data" ) // Handle initial payment, the rest of the payments are done separately from the stream processing @@ -234,7 +234,7 @@ func (h *lphttp) StartLiveVideoToVideo() http.Handler { eventsUrlOverwrite := overwriteHost(h.node.LiveAITrickleHostForRunner, eventsUrl) subscribeUrlOverwrite := overwriteHost(h.node.LiveAITrickleHostForRunner, pubUrl) publishUrlOverwrite := overwriteHost(h.node.LiveAITrickleHostForRunner, subUrl) - //dataUrlOverwrite := overwriteHost(h.node.LiveAITrickleHostForRunner, dataUrl) + dataUrlOverwrite := overwriteHost(h.node.LiveAITrickleHostForRunner, dataUrl) workerReq := worker.LiveVideoToVideoParams{ ModelId: req.ModelId, @@ -242,6 +242,7 @@ func (h *lphttp) StartLiveVideoToVideo() http.Handler { SubscribeUrl: subscribeUrlOverwrite, EventsUrl: &eventsUrlOverwrite, ControlUrl: &controlUrlOverwrite, + DataUrl: &dataUrlOverwrite, Params: req.Params, GatewayRequestId: &gatewayRequestID, ManifestId: &mid, From 3a6d24a528598a77fd7a69267315a441e1c2120b Mon Sep 17 00:00:00 2001 From: Brad P Date: Mon, 28 Jul 2025 16:42:32 -0500 Subject: [PATCH 05/57] move datastore items to separte file similar to status --- server/ai_mediaserver.go | 83 ------------------------------------ server/ai_pipeline_data.go | 87 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 83 deletions(-) create mode 100644 server/ai_pipeline_data.go diff --git a/server/ai_mediaserver.go b/server/ai_mediaserver.go index 58b2a54346..358641534e 100644 --- a/server/ai_mediaserver.go +++ b/server/ai_mediaserver.go @@ -14,7 +14,6 @@ import ( "os" "os/exec" "strings" - "sync" "time" "github.com/livepeer/go-livepeer/monitor" @@ -1331,88 +1330,6 @@ func (ls *LivepeerServer) SmokeTestLiveVideo() http.Handler { }) } -// DataSegmentStore stores data segments for SSE streaming -type DataSegmentStore struct { - streamID string - segments chan []byte - mu sync.RWMutex - closed bool -} - -func NewDataSegmentStore(streamID string) *DataSegmentStore { - return &DataSegmentStore{ - streamID: streamID, - segments: make(chan []byte, 100), // Buffer up to 100 segments - } -} - -func (d *DataSegmentStore) Store(data []byte) { - d.mu.RLock() - defer d.mu.RUnlock() - if d.closed { - return - } - select { - case d.segments <- data: - default: - // Channel is full, drop oldest segment - select { - case <-d.segments: - default: - } - select { - case d.segments <- data: - default: - } - } -} - -func (d *DataSegmentStore) Subscribe() <-chan []byte { - return d.segments -} - -func (d *DataSegmentStore) Close() { - d.mu.Lock() - defer d.mu.Unlock() - if !d.closed { - d.closed = true - close(d.segments) - } -} - -// Global store for data segments by stream ID -var dataStores = make(map[string]*DataSegmentStore) -var dataStoresMu sync.RWMutex - -func getDataStore(stream string) *DataSegmentStore { - dataStoresMu.RLock() - store, exists := dataStores[stream] - dataStoresMu.RUnlock() - if exists { - return store - } - - dataStoresMu.Lock() - defer dataStoresMu.Unlock() - // Double-check after acquiring write lock - if store, exists := dataStores[stream]; exists { - return store - } - - store = NewDataSegmentStore(stream) - dataStores[stream] = store - return store -} - -func removeDataStore(stream string) { - dataStoresMu.Lock() - defer dataStoresMu.Unlock() - if store, exists := dataStores[stream]; exists { - store.Close() - delete(dataStores, stream) - } -} - // @Summary Get Live Stream Data // @Param streamId path string true "Stream ID" // @Success 200 diff --git a/server/ai_pipeline_data.go b/server/ai_pipeline_data.go new file mode 100644 index 0000000000..01790e8ccc --- /dev/null +++ b/server/ai_pipeline_data.go @@ -0,0 +1,87 @@ +package server + +import ( + "sync" +) + +// DataSegmentStore stores data segments for SSE streaming +type DataSegmentStore struct { + streamID string + segments chan []byte + mu sync.RWMutex + closed bool +} + +func NewDataSegmentStore(streamID string) *DataSegmentStore { + return &DataSegmentStore{ + streamID: streamID, + segments: make(chan []byte, 100), // Buffer up to 100 segments + } +} + +func (d *DataSegmentStore) Store(data []byte) { + d.mu.RLock() + defer d.mu.RUnlock() + if d.closed { + return + } + select { + case d.segments <- data: + default: + // Channel is full, drop oldest segment + select { + case <-d.segments: + default: + } + select { + case d.segments <- data: + default: + } + } +} + +func (d *DataSegmentStore) Subscribe() <-chan []byte { + return d.segments +} + +func (d *DataSegmentStore) Close() { + d.mu.Lock() + defer d.mu.Unlock() + if !d.closed { + d.closed = true + close(d.segments) + } +} + +// Global store for data segments by stream ID +var dataStores = make(map[string]*DataSegmentStore) +var dataStoresMu sync.RWMutex + +func getDataStore(stream string) *DataSegmentStore { + dataStoresMu.RLock() + store, exists := dataStores[stream] + dataStoresMu.RUnlock() + if exists { + return store + } + + dataStoresMu.Lock() + defer dataStoresMu.Unlock() + // Double-check after acquiring write lock + if store, exists := dataStores[stream]; exists { + return store + } + + store = NewDataSegmentStore(stream) + dataStores[stream] = store + return store +} + +func removeDataStore(stream string) { + dataStoresMu.Lock() + defer dataStoresMu.Unlock() + if store, exists := dataStores[stream]; exists { + store.Close() + delete(dataStores, stream) + } +} From a1323ee83ac913ce7b96eee11240801a1b69f316 Mon Sep 17 00:00:00 2001 From: Brad P Date: Fri, 1 Aug 2025 16:55:44 -0500 Subject: [PATCH 06/57] update to remove datastore and use ringbuffer reader --- core/livepeernode.go | 2 + server/ai_live_video.go | 16 +++---- server/ai_mediaserver.go | 65 +++++++++++++++++++--------- server/ai_pipeline_data.go | 87 -------------------------------------- 4 files changed, 54 insertions(+), 116 deletions(-) delete mode 100644 server/ai_pipeline_data.go diff --git a/core/livepeernode.go b/core/livepeernode.go index 07c5841ff1..439357efb3 100644 --- a/core/livepeernode.go +++ b/core/livepeernode.go @@ -19,6 +19,7 @@ import ( "time" "github.com/golang/glog" + "github.com/livepeer/go-livepeer/media" "github.com/livepeer/go-livepeer/pm" "github.com/livepeer/go-livepeer/trickle" @@ -179,6 +180,7 @@ type LivePipeline struct { Pipeline string ControlPub *trickle.TricklePublisher StopControl func() + DataWriter *media.RingBuffer ReportUpdate func([]byte) } diff --git a/server/ai_live_video.go b/server/ai_live_video.go index e00eeb3b23..da3224835b 100644 --- a/server/ai_live_video.go +++ b/server/ai_live_video.go @@ -798,19 +798,22 @@ func startDataSubscribe(ctx context.Context, url *url.URL, params aiRequestParam // Set up output buffers rbc := media.RingBufferConfig{BufferLen: 50_000_000} // 50 MB buffer outWriter, err := media.NewRingBuffer(&rbc) + //put the data buffer in live pipeline for clients to read from + pipeline := params.node.LivePipelines[params.liveParams.stream] + if pipeline == nil { + clog.Infof(ctx, "No live pipeline found for stream %s", params.liveParams.stream) + return + } + pipeline.DataWriter = outWriter + if err != nil { stopProcessing(ctx, params, fmt.Errorf("ringbuffer init failed: %w", err)) return } - // Store data segments for SSE endpoint - stream := params.liveParams.stream - dataStore := getDataStore(stream) - // read segments from trickle subscription go func() { defer outWriter.Close() - defer removeDataStore(stream) // Clean up when done var err error firstSegment := true @@ -870,9 +873,6 @@ func startDataSubscribe(ctx context.Context, url *url.URL, params aiRequestParam continue } - // Store the raw segment data for SSE endpoint - dataStore.Store(body) - // Write to output buffer using the body data n, err := outWriter.Write(body) if err != nil { diff --git a/server/ai_mediaserver.go b/server/ai_mediaserver.go index 358641534e..94da3a3ec3 100644 --- a/server/ai_mediaserver.go +++ b/server/ai_mediaserver.go @@ -3,6 +3,7 @@ package server import ( "bytes" "context" + "encoding/base64" "encoding/json" "errors" "fmt" @@ -15,6 +16,7 @@ import ( "os/exec" "strings" "time" + "unicode/utf8" "github.com/livepeer/go-livepeer/monitor" @@ -1331,7 +1333,7 @@ func (ls *LivepeerServer) SmokeTestLiveVideo() http.Handler { } // @Summary Get Live Stream Data -// @Param streamId path string true "Stream ID" +// @Param stream path string true "Stream Key" // @Success 200 // @Router /live/video-to-video/{stream}/data [get] func (ls *LivepeerServer) GetLiveVideoToVideoData() http.Handler { @@ -1350,12 +1352,20 @@ func (ls *LivepeerServer) GetLiveVideoToVideoData() http.Handler { ctx := r.Context() ctx = clog.AddVal(ctx, "stream", stream) - // Get the data store for this stream - dataStore := getDataStore(stream) - if dataStore == nil { - http.Error(w, "Stream not found", http.StatusNoContent) + // Get the live pipeline for this stream + livePipeline, ok := ls.LivepeerNode.LivePipelines[stream] + if !ok { + http.Error(w, "Stream not found", http.StatusNotFound) + return + } + + // Get the data readerring buffer + if livePipeline.DataWriter == nil { + clog.Infof(ctx, "No data writer available for stream %s", stream) + http.Error(w, "Stream data not available", http.StatusServiceUnavailable) return } + dataReader := livePipeline.DataWriter.MakeReader() // Set up SSE headers w.Header().Set("Content-Type", "text/event-stream") @@ -1363,9 +1373,6 @@ func (ls *LivepeerServer) GetLiveVideoToVideoData() http.Handler { w.Header().Set("Connection", "keep-alive") w.Header().Set("Access-Control-Allow-Origin", "*") - // Get the subscription channel - dataChan := dataStore.Subscribe() - flusher, ok := w.(http.Flusher) if !ok { http.Error(w, "Streaming not supported", http.StatusInternalServerError) @@ -1374,27 +1381,43 @@ func (ls *LivepeerServer) GetLiveVideoToVideoData() http.Handler { clog.Infof(ctx, "Starting SSE data stream for stream=%s", stream) - // Send keep-alive ping initially - fmt.Fprintf(w, "event: ping\ndata: {\"type\":\"connected\"}\n\n") - flusher.Flush() - - // Stream data segments as SSE events + // Listen for broadcast signals from ring buffer writes + // dataReader.Read() blocks on rb.cond.Wait() until startDataSubscribe broadcasts for { select { case <-ctx.Done(): clog.Info(ctx, "SSE data stream client disconnected") return - case data, ok := <-dataChan: - if !ok { - // Channel closed, stream ended - fmt.Fprintf(w, "event: end\ndata: {\"type\":\"stream_ended\"}\n\n") - flusher.Flush() + default: + // Listen for broadcast from ring buffer writer + buffer := make([]byte, 32*1024) // 32KB read buffer + n, err := dataReader.Read(buffer) + if err != nil { + if err == io.EOF { + // Stream ended + fmt.Fprintf(w, "event: end\ndata: {\"type\":\"stream_ended\"}\n\n") + flusher.Flush() + return + } + clog.Errorf(ctx, "Error reading from ring buffer: %v", err) return } - // Send the segment data as a data event - fmt.Fprintf(w, "data: %s\n\n", string(data)) - flusher.Flush() + if n > 0 { + // Broadcast received - forward segment data as SSE event + data := buffer[:n] + + // Check if data is valid UTF-8 text + if utf8.Valid(data) { + // Send as text string + fmt.Fprintf(w, "data: %s\n\n", string(data)) + } else { + // Send as base64 encoded binary data + encoded := base64.StdEncoding.EncodeToString(data) + fmt.Fprintf(w, "data: %s\n\n", encoded) + } + flusher.Flush() + } } } }) diff --git a/server/ai_pipeline_data.go b/server/ai_pipeline_data.go deleted file mode 100644 index 01790e8ccc..0000000000 --- a/server/ai_pipeline_data.go +++ /dev/null @@ -1,87 +0,0 @@ -package server - -import ( - "sync" -) - -// DataSegmentStore stores data segments for SSE streaming -type DataSegmentStore struct { - streamID string - segments chan []byte - mu sync.RWMutex - closed bool -} - -func NewDataSegmentStore(streamID string) *DataSegmentStore { - return &DataSegmentStore{ - streamID: streamID, - segments: make(chan []byte, 100), // Buffer up to 100 segments - } -} - -func (d *DataSegmentStore) Store(data []byte) { - d.mu.RLock() - defer d.mu.RUnlock() - if d.closed { - return - } - select { - case d.segments <- data: - default: - // Channel is full, drop oldest segment - select { - case <-d.segments: - default: - } - select { - case d.segments <- data: - default: - } - } -} - -func (d *DataSegmentStore) Subscribe() <-chan []byte { - return d.segments -} - -func (d *DataSegmentStore) Close() { - d.mu.Lock() - defer d.mu.Unlock() - if !d.closed { - d.closed = true - close(d.segments) - } -} - -// Global store for data segments by stream ID -var dataStores = make(map[string]*DataSegmentStore) -var dataStoresMu sync.RWMutex - -func getDataStore(stream string) *DataSegmentStore { - dataStoresMu.RLock() - store, exists := dataStores[stream] - dataStoresMu.RUnlock() - if exists { - return store - } - - dataStoresMu.Lock() - defer dataStoresMu.Unlock() - // Double-check after acquiring write lock - if store, exists := dataStores[stream]; exists { - return store - } - - store = NewDataSegmentStore(stream) - dataStores[stream] = store - return store -} - -func removeDataStore(stream string) { - dataStoresMu.Lock() - defer dataStoresMu.Unlock() - if store, exists := dataStores[stream]; exists { - store.Close() - delete(dataStores, stream) - } -} From 642814d933f38eedb29542540a6c6585a3572310 Mon Sep 17 00:00:00 2001 From: Josh Allmann Date: Thu, 14 Aug 2025 05:35:26 +0000 Subject: [PATCH 07/57] ai/live: Read JSONL from a time delimited data channel. --- core/livepeernode.go | 2 +- server/ai_live_video.go | 80 ++++++++++++++++++---------------------- server/ai_mediaserver.go | 38 +++++-------------- server/ai_process.go | 1 + 4 files changed, 47 insertions(+), 74 deletions(-) diff --git a/core/livepeernode.go b/core/livepeernode.go index 439357efb3..eb8e20469c 100644 --- a/core/livepeernode.go +++ b/core/livepeernode.go @@ -180,7 +180,7 @@ type LivePipeline struct { Pipeline string ControlPub *trickle.TricklePublisher StopControl func() - DataWriter *media.RingBuffer + DataWriter *media.SegmentWriter ReportUpdate func([]byte) } diff --git a/server/ai_live_video.go b/server/ai_live_video.go index da3224835b..ce49836b7a 100644 --- a/server/ai_live_video.go +++ b/server/ai_live_video.go @@ -1,6 +1,7 @@ package server import ( + "bufio" "bytes" "context" "encoding/json" @@ -510,9 +511,10 @@ func registerControl(ctx context.Context, params aiRequestParams) { } params.node.LivePipelines[stream] = &core.LivePipeline{ - RequestID: params.liveParams.requestID, - Pipeline: params.liveParams.pipeline, - StreamID: params.liveParams.streamID, + RequestID: params.liveParams.requestID, + Pipeline: params.liveParams.pipeline, + DataWriter: params.liveParams.dataWriter, + StreamID: params.liveParams.streamID, } } @@ -791,29 +793,15 @@ func startDataSubscribe(ctx context.Context, url *url.URL, params aiRequestParam Ctx: ctx, }) if err != nil { - clog.Infof(ctx, "Failed to create trickle subscriber: %s", err) + clog.Infof(ctx, "Failed to create data subscriber: %s", err) return } - // Set up output buffers - rbc := media.RingBufferConfig{BufferLen: 50_000_000} // 50 MB buffer - outWriter, err := media.NewRingBuffer(&rbc) - //put the data buffer in live pipeline for clients to read from - pipeline := params.node.LivePipelines[params.liveParams.stream] - if pipeline == nil { - clog.Infof(ctx, "No live pipeline found for stream %s", params.liveParams.stream) - return - } - pipeline.DataWriter = outWriter - - if err != nil { - stopProcessing(ctx, params, fmt.Errorf("ringbuffer init failed: %w", err)) - return - } + dataWriter := params.liveParams.dataWriter // read segments from trickle subscription go func() { - defer outWriter.Close() + defer dataWriter.Close() var err error firstSegment := true @@ -825,20 +813,21 @@ func startDataSubscribe(ctx context.Context, url *url.URL, params aiRequestParam for { select { case <-ctx.Done(): - clog.Info(ctx, "trickle subscribe done") + clog.Info(ctx, "data subscribe done") return default: } if !params.inputStreamExists() { - clog.Infof(ctx, "trickle subscribe stopping, input stream does not exist.") + clog.Infof(ctx, "data subscribe stopping, input stream does not exist.") break } var segment *http.Response - clog.V(8).Infof(ctx, "trickle subscribe read data await") + readBytes, readMessages := 0, 0 + clog.V(8).Infof(ctx, "data subscribe await") segment, err = subscriber.Read() if err != nil { if errors.Is(err, trickle.EOS) || errors.Is(err, trickle.StreamNotFoundErr) { - stopProcessing(ctx, params, fmt.Errorf("trickle subscribe stopping, stream not found, err=%w", err)) + stopProcessing(ctx, params, fmt.Errorf("data subscribe stopping, stream not found, err=%w", err)) return } var sequenceNonexistent *trickle.SequenceNonexistent @@ -847,10 +836,10 @@ func startDataSubscribe(ctx context.Context, url *url.URL, params aiRequestParam subscriber.SetSeq(sequenceNonexistent.Latest) } // TODO if not EOS then signal a new orchestrator is needed - err = fmt.Errorf("trickle subscribe error reading: %w", err) + err = fmt.Errorf("data subscribe error reading: %w", err) clog.Infof(ctx, "%s", err) if retries > maxRetries { - stopProcessing(ctx, params, errors.New("trickle subscribe stopping, retries exceeded")) + stopProcessing(ctx, params, errors.New("data subscribe stopping, retries exceeded")) return } retries++ @@ -860,32 +849,33 @@ func startDataSubscribe(ctx context.Context, url *url.URL, params aiRequestParam } retries = 0 seq := trickle.GetSeq(segment) - clog.V(8).Infof(ctx, "trickle subscribe read data received seq=%d", seq) + clog.V(8).Infof(ctx, "data subscribe received seq=%d", seq) copyStartTime := time.Now() - // Read segment data and store it for SSE - body, err := io.ReadAll(segment.Body) - segment.Body.Close() - if err != nil { - clog.InfofErr(ctx, "trickle subscribe error reading segment body seq=%d", seq, err) - subscriber.SetSeq(seq) - retries++ - continue - } - - // Write to output buffer using the body data - n, err := outWriter.Write(body) - if err != nil { - if errors.Is(err, context.Canceled) { - clog.Info(ctx, "trickle subscribe stopping - context canceled") + defer segment.Body.Close() + scanner := bufio.NewScanner(segment.Body) + for scanner.Scan() { + writer, err := dataWriter.Next() + if err != nil { + if err != io.EOF { + stopProcessing(ctx, params, fmt.Errorf("data subscribe could not get next: %w", err)) + } return } - - clog.InfofErr(ctx, "trickle subscribe error writing to output buffer seq=%d", seq, err) + n, err := writer.Write(scanner.Bytes()) + if err != nil { + stopProcessing(ctx, params, fmt.Errorf("data subscribe could not write: %w", err)) + } + readBytes += n + readMessages += 1 + } + if err := scanner.Err(); err != nil { + clog.InfofErr(ctx, "data subscribe error reading seq=%d", seq, err) subscriber.SetSeq(seq) retries++ continue } + if firstSegment { firstSegment = false delayMs := time.Since(params.liveParams.startTime).Milliseconds() @@ -905,7 +895,7 @@ func startDataSubscribe(ctx context.Context, url *url.URL, params aiRequestParam } } - clog.V(8).Info(ctx, "trickle subscribe read data completed", "seq", seq, "bytes", humanize.Bytes(uint64(n)), "took", time.Since(copyStartTime)) + clog.V(8).Info(ctx, "data subscribe read completed", "seq", seq, "bytes", humanize.Bytes(uint64(readBytes)), "messages", readMessages, "took", time.Since(copyStartTime)) } }() } diff --git a/server/ai_mediaserver.go b/server/ai_mediaserver.go index 94da3a3ec3..29e9cf8ec5 100644 --- a/server/ai_mediaserver.go +++ b/server/ai_mediaserver.go @@ -3,7 +3,6 @@ package server import ( "bytes" "context" - "encoding/base64" "encoding/json" "errors" "fmt" @@ -16,7 +15,6 @@ import ( "os/exec" "strings" "time" - "unicode/utf8" "github.com/livepeer/go-livepeer/monitor" @@ -112,7 +110,8 @@ func startAIMediaServer(ctx context.Context, ls *LivepeerServer) error { ls.HTTPMux.Handle("/live/video-to-video/{streamId}/status", ls.GetLiveVideoToVideoStatus()) // Stream data SSE endpoint - ls.HTTPMux.Handle("/live/video-to-video/{stream}/data", ls.GetLiveVideoToVideoData()) + ls.HTTPMux.Handle("OPTIONS /live/video-to-video/{stream}/data", ls.WithCode(http.StatusNoContent)) + ls.HTTPMux.Handle("GET /live/video-to-video/{stream}/data", ls.GetLiveVideoToVideoData()) //API for dynamic capabilities ls.HTTPMux.Handle("/process/request/", ls.SubmitJob()) @@ -626,6 +625,7 @@ func (ls *LivepeerServer) StartLiveVideo() http.Handler { liveParams: &liveRequestParams{ segmentReader: ssr, + dataWriter: media.NewSegmentWriter(5), rtmpOutputs: rtmpOutputs, localRTMPPrefix: mediaMTXInputURL, stream: streamName, @@ -1106,6 +1106,7 @@ func (ls *LivepeerServer) CreateWhip(server *media.WHIPServer) http.Handler { liveParams: &liveRequestParams{ segmentReader: ssr, + dataWriter: media.NewSegmentWriter(5), rtmpOutputs: rtmpOutputs, localRTMPPrefix: internalOutputHost, stream: streamName, @@ -1343,11 +1344,6 @@ func (ls *LivepeerServer) GetLiveVideoToVideoData() http.Handler { http.Error(w, "stream name is required", http.StatusBadRequest) return } - if r.Method == http.MethodOptions { - corsHeaders(w, r.Method) - w.WriteHeader(http.StatusNoContent) - return - } ctx := r.Context() ctx = clog.AddVal(ctx, "stream", stream) @@ -1365,7 +1361,7 @@ func (ls *LivepeerServer) GetLiveVideoToVideoData() http.Handler { http.Error(w, "Stream data not available", http.StatusServiceUnavailable) return } - dataReader := livePipeline.DataWriter.MakeReader() + dataReader := livePipeline.DataWriter.MakeReader(media.SegmentReaderConfig{}) // Set up SSE headers w.Header().Set("Content-Type", "text/event-stream") @@ -1389,13 +1385,11 @@ func (ls *LivepeerServer) GetLiveVideoToVideoData() http.Handler { clog.Info(ctx, "SSE data stream client disconnected") return default: - // Listen for broadcast from ring buffer writer - buffer := make([]byte, 32*1024) // 32KB read buffer - n, err := dataReader.Read(buffer) + reader, err := dataReader.Next() if err != nil { if err == io.EOF { // Stream ended - fmt.Fprintf(w, "event: end\ndata: {\"type\":\"stream_ended\"}\n\n") + fmt.Fprintf(w, `event: end\ndata: {"type":"stream_ended"}\n\n`) flusher.Flush() return } @@ -1403,21 +1397,9 @@ func (ls *LivepeerServer) GetLiveVideoToVideoData() http.Handler { return } - if n > 0 { - // Broadcast received - forward segment data as SSE event - data := buffer[:n] - - // Check if data is valid UTF-8 text - if utf8.Valid(data) { - // Send as text string - fmt.Fprintf(w, "data: %s\n\n", string(data)) - } else { - // Send as base64 encoded binary data - encoded := base64.StdEncoding.EncodeToString(data) - fmt.Fprintf(w, "data: %s\n\n", encoded) - } - flusher.Flush() - } + data, err := io.ReadAll(reader) + fmt.Fprintf(w, "data: %s\n\n", data) + flusher.Flush() } } }) diff --git a/server/ai_process.go b/server/ai_process.go index cc50b380dd..a0034e57a1 100644 --- a/server/ai_process.go +++ b/server/ai_process.go @@ -96,6 +96,7 @@ type aiRequestParams struct { // For live video pipelines type liveRequestParams struct { segmentReader *media.SwitchableSegmentReader + dataWriter *media.SegmentWriter stream string requestID string streamID string From 5a0706529f745b90012ff0f49c06b2458be631e9 Mon Sep 17 00:00:00 2001 From: Brad P Date: Tue, 19 Aug 2025 18:03:30 -0500 Subject: [PATCH 08/57] copy dataWriter in newParams --- server/ai_live_video.go | 2 +- server/ai_mediaserver.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/server/ai_live_video.go b/server/ai_live_video.go index ce49836b7a..bb289a55ba 100644 --- a/server/ai_live_video.go +++ b/server/ai_live_video.go @@ -787,7 +787,7 @@ func startEventsSubscribe(ctx context.Context, url *url.URL, params aiRequestPar } func startDataSubscribe(ctx context.Context, url *url.URL, params aiRequestParams, sess *AISession) { - // subscribe to the outputs and send them into LPMS + // subscribe to the outputs subscriber, err := trickle.NewTrickleSubscriber(trickle.TrickleSubscriberConfig{ URL: url.String(), Ctx: ctx, diff --git a/server/ai_mediaserver.go b/server/ai_mediaserver.go index 29e9cf8ec5..fd6cba96dd 100644 --- a/server/ai_mediaserver.go +++ b/server/ai_mediaserver.go @@ -754,6 +754,7 @@ func processStream(ctx context.Context, params aiRequestParams, req worker.GenLi func newParams(params *liveRequestParams, cancelOrch context.CancelCauseFunc) *liveRequestParams { return &liveRequestParams{ segmentReader: params.segmentReader, + dataWriter: params.dataWriter, rtmpOutputs: params.rtmpOutputs, localRTMPPrefix: params.localRTMPPrefix, stream: params.stream, From 35ff4ac06dde6f4ecb934ac922671d5be1a05046 Mon Sep 17 00:00:00 2001 From: Brad P Date: Wed, 20 Aug 2025 09:35:24 -0500 Subject: [PATCH 09/57] update data channel mimetype --- server/ai_http.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/ai_http.go b/server/ai_http.go index 5d3b27f637..bf8f0d9f19 100644 --- a/server/ai_http.go +++ b/server/ai_http.go @@ -181,7 +181,7 @@ func (h *lphttp) StartLiveVideoToVideo() http.Handler { controlPubCh.CreateChannel() eventsCh := trickle.NewLocalPublisher(h.trickleSrv, mid+"-events", "application/json") eventsCh.CreateChannel() - dataCh := trickle.NewLocalPublisher(h.trickleSrv, mid+"-data", "application/json") + dataCh := trickle.NewLocalPublisher(h.trickleSrv, mid+"-data", "application/jsonl") dataCh.CreateChannel() // Start payment receiver which accounts the payments and stops the stream if the payment is insufficient From aa2f0b8e80f268c36ad2078ce3cc0dcaffa70282 Mon Sep 17 00:00:00 2001 From: Brad P Date: Wed, 20 Aug 2025 11:12:13 -0500 Subject: [PATCH 10/57] update code gen for data_url added to LiveVideoToVideoResponse --- ai/worker/runner.gen.go | 63 +++++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 30 deletions(-) diff --git a/ai/worker/runner.gen.go b/ai/worker/runner.gen.go index b4c0826556..a47eb474d0 100644 --- a/ai/worker/runner.gen.go +++ b/ai/worker/runner.gen.go @@ -350,6 +350,9 @@ type LiveVideoToVideoResponse struct { // ControlUrl URL for updating the live video-to-video generation ControlUrl *string `json:"control_url,omitempty"` + // DataUrl URL for publishing data for pipeline + DataUrl *string `json:"data_url,omitempty"` + // EventsUrl URL for subscribing to events for pipeline status and logs EventsUrl *string `json:"events_url,omitempty"` @@ -3220,36 +3223,36 @@ var swaggerSpec = []string{ "xak0zzbES5RBCTQTiNGQE6LQqAAqbRkszVChC/smunZqSmYVx5MctMWqnv8ybvwvhPmscqUZg5bANTbV", "0v5y1zkvtZcidGOQwL2j38ilKYt3EWS0Ltc4abhrJiShhmPlo1a9rJIzZs7kVL6kRGyHCeY0j/rcz/Qd", "ZuUmL1M2vtqaXduobB1aQ5zfU1bxFHxWCU1ZEbJaj4FkUKZ0Wj+PctxesAeUhHrwI1s8cA0Icfdawi6P", - "SN0V7f3jpA6BrhBz+XTDA9+9A4wfrSVzQWZZPBkcTobj9icfngMA1x5MhIspwzH6cb0YSTbcj+8bPpst", - "jyXsLg+Rj+3Fz+3EyzL9IyyuxL181/R1BYA9DuuX4bSVwvHNCFXUq8Rq6sQE2jBdX9bxRReWhdfQwiKb", - "sKxw5X5gZzwtgqjeU8b79he1PF4IE4szXbRhmmu6dR1WOGUA3WbgldfTLWHCNbdSvWjRvlS/ejszUmxQ", - "qBdOmQpfMTG12N4daTxhlWyVy+p+XYVTMb3pTvPTHKSrbDcT3mCBpjmezSBDWKCPpz/8FJTBqGGGl3Yo", - "Tag3pnrIv4ZQzzionDjq12pw5dSmmK1hIcVUZWE4TUEIc4e9Pj4d4MTGdYUhRYvN16dWV58ez08OY6rU", - "0UYl4+aqay+Vocaem+c2l4qZCKOPvx2u61TEkA1xU9Iy/KzAFKjctSpWYjtMT7slP3I8XoS9lwGDem9v", - "tvXtTvx9bq4/5vWpzr3wJden1lfB11fB/75XwV/9R98ER6dQYi1nfRWjNDuGujRfbxa9+L8XyjRE/SGV", - "yaIp2F9X3/5p9706+D3wvpc1mFaIDUNob5w9LQHSeV+gDbjwIWsfFQpPRAn4CjjKICfXwIXSca7AP18g", - "uC05CK03FSYw1arOVB9I566KWBmdtlX1ONMtSyJT7Tmd1bv7S8nOTa2WsBLAplvqLzN+XI/eIE94LX0I", - "JcuiRZOULQ8R5gKU3s9ZNlVvvhbaS2AKEYNZWduXszQ4CMV0YesQ2hx+6dj0xZ0fw9NWfVhz4m0+t9Y6", - "G4/KUD9ommqa0Zl6uip1VXyYqWxLz7UG1BP+CFxYh2nVFDxpDdgouW4m7sKPfelg56SiFLhnJI7qh9WW", - "uakvlgzoSeje+7Krd2LNRw1WLWXcJwBU22A1dc8SrPYqyhX0GCJWlGRZUn1ZLd8N0zEsrTiRi1NFiuHz", - "w9nZ8WvAHHj9XUMd+MyjepC5lGVyd6eriWIF6/v2WyZp/fk5XlG0f1DvBPtbv4fkGkqFtvsHjQnVdpds", - "b+58s/m/SiKsBIpLkuwl32zubG4rdWE513Rv6c+ajSUbO5wrmYglPPW337xP9ZnrhXZBykprDgeZWm21", - "v4tmt1Nfs2zRqkQyiRHmcktlJmP3yT6j51VWEPsI212oY5UG6QdGo5rt3e3tFhWe2Lc+2zqxYSQEa2g9", - "dyu3qfR+yLTKUdNslHz7iCQ0peCR+V/jzJ3umnl3nmfec4orOWec/AGZnnjnm+eZ2O3Tv6NSrRTOGEOH", - "mJtCrm93Xj0X901Or6HKhDtFwu7uo5LQKcvvEtM0QXXp/qvnsr8DKoFTnKNT4NfAHQUejuq0xEfQXy/u", - "LkaJqIoC84X7xic6Y8hlT3gmFHi7WKLg+3ZsslAsFmOKCxiza+CcZBr6A3QYJVtzW2m95WB4BloEIYj5", - "ZfLJEyJIrBx/KJDc+XJyA5n7CCGndbH9UlZd6fmT82om+jou3RiKTV1Y3s+eef2UfHmV7Q/jypCoudGr", - "TxWU60va8ai8X5b5wt3UDj6GJUyJScmZyrK89WwnTLe+XvbEcTqY7ZkDdVhrv47U/ZF6HaHuG6HMJ2/O", - "GKq/e3DPEEVCx/BBYEBmrvf0DA6sTszDj9s9j8P/GYl57OLJ2uv/4vn5GnoeDD0PTI5J4KE+8FzX37WM", - "Is/72Ncc75V0uK+fPQ8GmdmeGYTC3aQ1/KyTjifw/Porgg9zfecYo2QrJ9cwDotgVy0/ogsPr6relDf6", - "X2eWFaeQIaCZ/oCXiEJEuz5xKUw8XEc9tczPjBK9xZhrwFgDxuMBhjIzAxZfgxp52zMNcuTFgFRBH8dW", - "uuQDoxzTWaUgrK526KLA4dFTOX5z0fS5nd27Vrn277V/P6J/a2+5tz/nhXFhW60/xvbDkuPdfo+236C0", - "teH6FimmSzL+yDcrnzjr78z4zG4eVt2vHX3t6I/n6M77nHGj3Qf4veg6yCjZUhF6wNHD+1bRtrnn25RF", - "xpN6rxjuicJ6t9xufcqwdvu/idvrQsOvOGSQnvsFzm5KFgdt9YVd/P+40/x/i+6OutsElE1xJKaZV6Ua", - "/G+WPUhhyiCfFCqCSstnxorw/1ZdY8UaKx4fK2oXehhY2O4aLSrvW/JRmLDfs65XAmiycP9Zj741KgVq", - "/suOqNs3X8R+4tWBm2idHaw9/m/i8d7X5O/p6lXtDKNky6tcj9ZSNbXkT3doZqd4UCFV0FloaQotu9b/", - "TeKqp9/krMrQG1YUFSVy4b6hlNgL37pmW+xtbWUccDG2H2jazG33zVR119coesY/lTpF6hu2Hkjodlu4", - "JFsTkHirVt7dxd3/BwAA//8lGRESOX8AAA==", + "SN0V7f3jpA6BrhBz+XTPFvj8OPJUIc3PDyRzYW1ZBBscwIZHik9+QAhChsYMIlwUGx4VHhc3kGTDkeO+", + "AbvZZFnC7vKg/Ni48dywsWxtcYTFlbgXWpi+ruSwByL8wp+2Uji+GaGKerVfTWWaQBum68s6oulStvDi", + "W1jWExYyrtyB7IynRRDVe8p4346mlscLYaJ/pstETHNNt678CqcMgoUZeOWFeEuYcM2tVC9atC/Vr95A", + "jZQ3FOqFU6ZCdExM9bd3KxtPWCVbBbq6X1fhVExvutP8NAfpaunNhDdYoGmOZzPIEBbo4+kPPwWFN2qY", + "4cUkShPqjalX8i8+1DMOKmCO+rUaXDm1KZ9rWEgxVXkfTlMQwtyarw9sBzixcV1hSNFi8/Wp1dWnx/OT", + "w5gqdbRR6b+5XNtLZaix5+a5zaViJsLo42/A68oYMWQL3hTRDD+dMCUxd60amdie1tMeAowcjxdh72XA", + "oN7bu3R9+yF/n7vyj3lhq3MTfcmFrfXl8/Xl87/v5fNX/9F3z9EplFjLWV/+KM0epb4MoLenXvzfC2Ua", + "ov50y2TRXBFY1/v+aTfMOvg98IaZNZhWiA1DaG+cPS0B0nlfoA248CFrHxUKT0QJ+Ao4yiAn18CF0nGu", + "wD9fILgtOQitNxUmMNWqzlQfSOeublkZnbZV9TjTLUsiU+05ndW7+0vJzk2tlrASwKZb6i8zflyP3iBP", + "eBF+CCXLokWTlC0PEebKld7PWTZVb74W2ktgChGDWVlNmLM0OHrFdGErH9ocfunY9MWdH8PTVkVac8Zu", + "PvDWOo2PylA/aJpqmtGZeroqdVV8mKlsS8+1BlQw/ghcWIdpVTE8adXZKLluJu7Cj33pYOekohS4ZySO", + "6odVs7mpL5YM6Eno3jvBq/d+zWcUVi1l3EcHVNtgNXXPoq/2KsqVEBkiVhSBWVJ9WS3fDdMxLK04kYtT", + "RYrh88PZ2fFrwBx4/SVFHfjMo3qQuZRlcnen65diJfL79uspaf3BO15RtH9Q7wT7W7+H5BpKhbb7B40J", + "1XaXbG/ufLP5v0oirASKS5LsJd9s7mxuK3VhOdd0b+kPqY0lGzucK5mIJTz11+a8jwOaC412QcpKaw4H", + "mVpttb/EZrdTX7Ns0ap9MokR5nJLZSZj95FAo+dVVhD77NtdqGOVBukHRqOa7d3t7RYVnti3PtvKtGEk", + "BGtoPXcrt6n0fsi0ylHTbJR8+4gkNMXnkflf48ydJ5t5d55n3nOKKzlnnPwBmZ5455vnmdjt07+jUq0U", + "zhhDh5ib0rFvd149F/dNTq+hyoQ7RcLu7qOS0LkI0CWmaYLqywKvnsv+DqgETnGOToFfA3cUeDiq0xIf", + "QX+9uLsYJaIqCswX7qui6Iwhlz3hmVDg7WKJgu/bsclCsViMKS5gzK6Bc5Jp6A/QYZRszW1t95aD4Rlo", + "EYQg5hfmJ0+IILELAEOB5M6XkxvI3IAIOa3L+5ey6ordn5xXM9HXcenGUGzqUvZ+9szrp+TLq6V/GFeG", + "RM2NXn2qoFxfC49H5f2yzBfubnjw+S1hilpKzlSW5a1nO2G69b20J47TwWzPHKjD6v51pO6P1OsIdd8I", + "ZT6yc8ZQ/aWFe4YoEjqGDwIDMnO9p2dwYHViHn5O73kc/s9IzGNXXdZe/xfPz9fQ82DoeWByTAIP9YHn", + "uv6SZhR53se+H3mvpMN9b+15MMjM9swgFO4mreFnnXQ8gefX3y18mOs7xxglWzm5hnFYdrtq+RFdeHh1", + "/Ka80f8etKw4hQwBzfQnw0QUItr1iUth4uE66qmefmaU6C3GXAPGGjAeDzCUmRmw+BrUyNueaZAjLwak", + "Cvo4ttIlHxjlmM4qBWF1tUMXBQ6Pnsrxm6utz+3s3kXOtX+v/fsR/Vt7y739OS+MC9tq/TG2n7Ic7/Z7", + "tP3qpa0N1/dWMV2S8Ue+kvnEWX9nxmd287Dqfu3oa0d/PEd33ueMG+0+wO9F10FGyZaK0AOOHt63irbN", + "zeKmLDKe1HvFcE8U1rvldutThrXb/03cXhcafsUhg/TcL3B2U7I4aKsv7OL/V6Hmf3h0t+LdJqBsiiMx", + "zbwq1eD/z+xBClMG+aRQEVRaPjNWhP+b6xor1ljx+FhRu9DDwMJ212hReV+vj8KE/YJ2vRJAk4X774H0", + "rVEpUPOfhETdvvkG9xOvDtxE6+xg7fF/E4/3vl9/T1evamcYJVte5Xq0lqqpJX+6QzM7xYMKqYLOQktT", + "aNm1/jcUVz39JmdVht6woqgokQv31abEXvjWNdtib2sr44CLsf0k1GZuu2+mqru+RtEz/qnUKVLfsPVA", + "QrfbwiXZmoDEW7Xy7i7u/j8AAP//9DvSdKt/AAA=", } // GetSwagger returns the content of the embedded swagger specification file From 3e166c53f275ecf85ebad172142e5bd446df21b6 Mon Sep 17 00:00:00 2001 From: Brad P Date: Wed, 20 Aug 2025 14:27:35 -0500 Subject: [PATCH 11/57] update to use live video response DataUrl field --- server/ai_mediaserver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/ai_mediaserver.go b/server/ai_mediaserver.go index fd6cba96dd..56f6d5b22a 100644 --- a/server/ai_mediaserver.go +++ b/server/ai_mediaserver.go @@ -793,7 +793,7 @@ func startProcessing(ctx context.Context, params aiRequestParams, res interface{ if err != nil { return fmt.Errorf("invalid events URL: %w", err) } - data, err := common.AppendHostname(strings.Replace(*resp.JSON200.EventsUrl, "-events", "-data", 1), host) + data, err := common.AppendHostname(*resp.JSON200.DataUrl, host) if err != nil { return fmt.Errorf("invalid data URL: %w", err) } From 699f7782d96cf6e2d320e9186fb23c944727c57f Mon Sep 17 00:00:00 2001 From: Brad P Date: Wed, 20 Aug 2025 11:32:36 -0500 Subject: [PATCH 12/57] update to make data channel optional --- server/ai_http.go | 45 ++++++++++++++++++++++++++++++++++++---- server/ai_live_video.go | 5 +++++ server/ai_mediaserver.go | 43 +++++++++++++++++++++++++++++++------- 3 files changed, 81 insertions(+), 12 deletions(-) diff --git a/server/ai_http.go b/server/ai_http.go index bf8f0d9f19..b084ba4a16 100644 --- a/server/ai_http.go +++ b/server/ai_http.go @@ -148,6 +148,26 @@ func (h *lphttp) StartLiveVideoToVideo() http.Handler { dataUrl = pubUrl + "-data" ) + //if data is not enabled remove the url and do not start the data channel + if enableData, ok := (*req.Params)["enableData"]; ok { + if val, ok := enableData.(bool); ok { + //turn off data channel if request sets to false + if !val { + dataUrl = "" + } else { + clog.Infof(ctx, "data channel is enabled") + } + } else { + clog.Warningf(ctx, "enableData is not a bool, got type %T", enableData) + } + + //delete the param used for go-livepeer signaling + delete((*req.Params), "enableData") + } else { + //default to no data channel + dataUrl = "" + } + // Handle initial payment, the rest of the payments are done separately from the stream processing // Note that this payment is debit from the balance and acts as a buffer for the AI Realtime Video processing payment, err := getPayment(r.Header.Get(paymentHeader)) @@ -181,8 +201,13 @@ func (h *lphttp) StartLiveVideoToVideo() http.Handler { controlPubCh.CreateChannel() eventsCh := trickle.NewLocalPublisher(h.trickleSrv, mid+"-events", "application/json") eventsCh.CreateChannel() - dataCh := trickle.NewLocalPublisher(h.trickleSrv, mid+"-data", "application/jsonl") - dataCh.CreateChannel() + + //optional channels + var dataCh *trickle.TrickleLocalPublisher + if dataUrl != "" { + dataCh = trickle.NewLocalPublisher(h.trickleSrv, mid+"-data", "application/jsonl") + dataCh.CreateChannel() + } // Start payment receiver which accounts the payments and stops the stream if the payment is insufficient priceInfo := payment.GetExpectedPrice() @@ -203,6 +228,9 @@ func (h *lphttp) StartLiveVideoToVideo() http.Handler { subCh.Close() eventsCh.Close() controlPubCh.Close() + if dataCh != nil { + dataCh.Close() + } cancel() } return err @@ -230,11 +258,17 @@ func (h *lphttp) StartLiveVideoToVideo() http.Handler { }() // Prepare request to worker + // required channels controlUrlOverwrite := overwriteHost(h.node.LiveAITrickleHostForRunner, controlUrl) eventsUrlOverwrite := overwriteHost(h.node.LiveAITrickleHostForRunner, eventsUrl) subscribeUrlOverwrite := overwriteHost(h.node.LiveAITrickleHostForRunner, pubUrl) publishUrlOverwrite := overwriteHost(h.node.LiveAITrickleHostForRunner, subUrl) - dataUrlOverwrite := overwriteHost(h.node.LiveAITrickleHostForRunner, dataUrl) + + // optional channels + var dataUrlOverwrite string + if dataCh != nil { + dataUrlOverwrite = overwriteHost(h.node.LiveAITrickleHostForRunner, dataUrl) + } workerReq := worker.LiveVideoToVideoParams{ ModelId: req.ModelId, @@ -260,7 +294,9 @@ func (h *lphttp) StartLiveVideoToVideo() http.Handler { subCh.Close() controlPubCh.Close() eventsCh.Close() - dataCh.Close() + if dataCh != nil { + dataCh.Close() + } cancel() respondWithError(w, err.Error(), http.StatusInternalServerError) return @@ -272,6 +308,7 @@ func (h *lphttp) StartLiveVideoToVideo() http.Handler { SubscribeUrl: subUrl, ControlUrl: &controlUrl, EventsUrl: &eventsUrl, + DataUrl: &dataUrl, RequestId: &requestID, ManifestId: &mid, }) diff --git a/server/ai_live_video.go b/server/ai_live_video.go index bb289a55ba..9a1c7d1109 100644 --- a/server/ai_live_video.go +++ b/server/ai_live_video.go @@ -787,6 +787,11 @@ func startEventsSubscribe(ctx context.Context, url *url.URL, params aiRequestPar } func startDataSubscribe(ctx context.Context, url *url.URL, params aiRequestParams, sess *AISession) { + //only start DataSubscribe if enabled + if params.liveParams.dataWriter == nil { + return + } + // subscribe to the outputs subscriber, err := trickle.NewTrickleSubscriber(trickle.TrickleSubscriberConfig{ URL: url.String(), diff --git a/server/ai_mediaserver.go b/server/ai_mediaserver.go index 56f6d5b22a..b10098b60f 100644 --- a/server/ai_mediaserver.go +++ b/server/ai_mediaserver.go @@ -625,7 +625,6 @@ func (ls *LivepeerServer) StartLiveVideo() http.Handler { liveParams: &liveRequestParams{ segmentReader: ssr, - dataWriter: media.NewSegmentWriter(5), rtmpOutputs: rtmpOutputs, localRTMPPrefix: mediaMTXInputURL, stream: streamName, @@ -640,6 +639,15 @@ func (ls *LivepeerServer) StartLiveVideo() http.Handler { }, } + //create a dataWriter for data channel if enabled + if enableData, ok := pipelineParams["enableData"]; ok { + if enableData == true || enableData == "true" { + params.liveParams.dataWriter = media.NewSegmentWriter(5) + pipelineParams["enableData"] = true + clog.Infof(ctx, "Data channel enabled for stream %s", streamName) + } + } + registerControl(ctx, params) // Create a special parent context for orchestrator cancellation @@ -777,6 +785,8 @@ func startProcessing(ctx context.Context, params aiRequestParams, res interface{ resp := res.(*worker.GenLiveVideoToVideoResponse) host := params.liveParams.sess.Transcoder() + + //required channels pub, err := common.AppendHostname(resp.JSON200.PublishUrl, host) if err != nil { return fmt.Errorf("invalid publish URL: %w", err) @@ -793,21 +803,30 @@ func startProcessing(ctx context.Context, params aiRequestParams, res interface{ if err != nil { return fmt.Errorf("invalid events URL: %w", err) } - data, err := common.AppendHostname(*resp.JSON200.DataUrl, host) - if err != nil { - return fmt.Errorf("invalid data URL: %w", err) - } + if resp.JSON200.ManifestId != nil { ctx = clog.AddVal(ctx, "manifest_id", *resp.JSON200.ManifestId) params.liveParams.manifestID = *resp.JSON200.ManifestId } - clog.V(common.VERBOSE).Infof(ctx, "pub %s sub %s control %s events %s data %s", pub, sub, control, events, data) + + clog.V(common.VERBOSE).Infof(ctx, "pub %s sub %s control %s events %s", pub, sub, control, events) startControlPublish(ctx, control, params) startTricklePublish(ctx, pub, params, params.liveParams.sess) startTrickleSubscribe(ctx, sub, params, params.liveParams.sess) startEventsSubscribe(ctx, events, params, params.liveParams.sess) - startDataSubscribe(ctx, data, params, params.liveParams.sess) + + //optional channels + var data *url.URL + if *resp.JSON200.DataUrl != "" { + data, err = common.AppendHostname(*resp.JSON200.DataUrl, host) + if err != nil { + return fmt.Errorf("invalid data URL: %w", err) + } + clog.V(common.VERBOSE).Infof(ctx, "data %s", data) + startDataSubscribe(ctx, data, params, params.liveParams.sess) + } + return nil } @@ -1107,7 +1126,6 @@ func (ls *LivepeerServer) CreateWhip(server *media.WHIPServer) http.Handler { liveParams: &liveRequestParams{ segmentReader: ssr, - dataWriter: media.NewSegmentWriter(5), rtmpOutputs: rtmpOutputs, localRTMPPrefix: internalOutputHost, stream: streamName, @@ -1123,6 +1141,15 @@ func (ls *LivepeerServer) CreateWhip(server *media.WHIPServer) http.Handler { }, } + //create a dataWriter for data channel if enabled + if enableData, ok := pipelineParams["enableData"]; ok { + if enableData == true || enableData == "true" { + params.liveParams.dataWriter = media.NewSegmentWriter(5) + pipelineParams["enableData"] = true + clog.Infof(ctx, "Data channel enabled for stream %s", streamName) + } + } + registerControl(ctx, params) req := worker.GenLiveVideoToVideoJSONRequestBody{ From bc624eef79a257baf5ed806ccdce1004835b1dc0 Mon Sep 17 00:00:00 2001 From: Brad P Date: Thu, 21 Aug 2025 14:33:10 -0500 Subject: [PATCH 13/57] add byoc streaming --- server/job_stream.go | 635 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 635 insertions(+) create mode 100644 server/job_stream.go diff --git a/server/job_stream.go b/server/job_stream.go new file mode 100644 index 0000000000..38284afc05 --- /dev/null +++ b/server/job_stream.go @@ -0,0 +1,635 @@ +package server + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + url2 "net/url" + "strings" + "time" + + ethcommon "github.com/ethereum/go-ethereum/common" + "github.com/livepeer/go-livepeer/ai/worker" + "github.com/livepeer/go-livepeer/clog" + "github.com/livepeer/go-livepeer/common" + "github.com/livepeer/go-livepeer/core" + "github.com/livepeer/go-livepeer/media" + "github.com/livepeer/go-livepeer/monitor" + "github.com/livepeer/go-livepeer/trickle" + "github.com/livepeer/go-tools/drivers" +) + +// StartStream handles the POST /stream/start endpoint for the Media Server +func (ls *LivepeerServer) StartStream() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Create fresh context instead of using r.Context() since ctx will outlive the request + ctx := context.Background() + + requestID := string(core.RandomManifestID()) + ctx = clog.AddVal(ctx, "request_id", requestID) + + streamName := r.FormValue("stream") + if streamName == "" { + clog.Errorf(ctx, "Missing stream name") + http.Error(w, "Missing stream name", http.StatusBadRequest) + return + } + + streamRequestTime := time.Now().UnixMilli() + + ctx = clog.AddVal(ctx, "stream", streamName) + sourceID := r.FormValue("source_id") + if sourceID == "" { + clog.Errorf(ctx, "Missing source_id") + http.Error(w, "Missing source_id", http.StatusBadRequest) + return + } + ctx = clog.AddVal(ctx, "source_id", sourceID) + sourceType := r.FormValue("source_type") + if sourceType == "" { + clog.Errorf(ctx, "Missing source_type") + http.Error(w, "Missing source_type", http.StatusBadRequest) + return + } + sourceTypeStr, err := media.MediamtxSourceTypeToString(sourceType) + if err != nil { + clog.Errorf(ctx, "Invalid source type %s", sourceType) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + ctx = clog.AddVal(ctx, "source_type", sourceType) + + remoteHost, err := getRemoteHost(r.RemoteAddr) + if err != nil { + clog.Errorf(ctx, "Could not find callback host: %s", err.Error()) + http.Error(w, "Could not find callback host", http.StatusBadRequest) + return + } + ctx = clog.AddVal(ctx, "remote_addr", remoteHost) + + queryParams := r.FormValue("query") + qp, err := url.ParseQuery(queryParams) + if err != nil { + clog.Errorf(ctx, "invalid query params, err=%w", err) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + // If auth webhook is set and returns an output URL, this will be replaced + outputURL := qp.Get("rtmpOutput") + // Currently for webrtc we need to add a path prefix due to the ingress setup + mediaMTXStreamPrefix := r.PathValue("prefix") + if mediaMTXStreamPrefix != "" { + mediaMTXStreamPrefix = mediaMTXStreamPrefix + "/" + } + mediaMTXInputURL := fmt.Sprintf("rtmp://%s/%s%s", remoteHost, mediaMTXStreamPrefix, streamName) + mediaMTXRtmpURL := r.FormValue("rtmp_url") + if mediaMTXRtmpURL != "" { + mediaMTXInputURL = mediaMTXRtmpURL + } + mediaMTXOutputURL := mediaMTXInputURL + "-out" + mediaMTXOutputAlias := fmt.Sprintf("%s-%s-out", mediaMTXInputURL, requestID) + + // convention to avoid re-subscribing to our own streams + // in case we want to push outputs back into mediamtx - + // use an `-out` suffix for the stream name. + if strings.HasSuffix(streamName, "-out") { + // skip for now; we don't want to re-publish our own outputs + return + } + + // if auth webhook returns pipeline config these will be replaced + pipeline := qp.Get("pipeline") + rawParams := qp.Get("params") + streamID := qp.Get("streamId") + var pipelineID string + var pipelineParams map[string]interface{} + if rawParams != "" { + if err := json.Unmarshal([]byte(rawParams), &pipelineParams); err != nil { + clog.Errorf(ctx, "Invalid pipeline params: %s", err) + http.Error(w, "Invalid model params", http.StatusBadRequest) + return + } + } + + mediaMTXClient := media.NewMediaMTXClient(remoteHost, ls.mediaMTXApiPassword, sourceID, sourceType) + + whepURL := generateWhepUrl(streamName, requestID) + whipURL := fmt.Sprintf("https://%s/stream/%s/%s", ls.LivepeerNode.GatewayHost, streamName, "/whip") + updateURL := fmt.Sprintf("https://%s/stream/%s/%s", ls.LivepeerNode.GatewayHost, streamName, "/update") + statusURL := fmt.Sprintf("https://%s/stream/%s/%s", ls.LivepeerNode.GatewayHost, streamID, "/status") + + if LiveAIAuthWebhookURL != nil { + authResp, err := authenticateAIStream(LiveAIAuthWebhookURL, ls.liveAIAuthApiKey, AIAuthRequest{ + Stream: streamName, + Type: sourceTypeStr, + QueryParams: queryParams, + GatewayHost: ls.LivepeerNode.GatewayHost, + WhepURL: whepURL, + UpdateURL: updateURL, + StatusURL: statusURL, + }) + if err != nil { + kickErr := mediaMTXClient.KickInputConnection(ctx) + if kickErr != nil { + clog.Errorf(ctx, "failed to kick input connection: %s", kickErr.Error()) + } + clog.Errorf(ctx, "Live AI auth failed: %s", err.Error()) + http.Error(w, "Forbidden", http.StatusForbidden) + return + } + + if authResp.RTMPOutputURL != "" { + outputURL = authResp.RTMPOutputURL + } + + if authResp.Pipeline != "" { + pipeline = authResp.Pipeline + } + + if len(authResp.paramsMap) > 0 { + if _, ok := authResp.paramsMap["prompt"]; !ok && pipeline == "comfyui" { + pipelineParams = map[string]interface{}{"prompt": authResp.paramsMap} + } else { + pipelineParams = authResp.paramsMap + } + } + + if authResp.StreamID != "" { + streamID = authResp.StreamID + } + + if authResp.PipelineID != "" { + pipelineID = authResp.PipelineID + } + } + + ctx = clog.AddVal(ctx, "stream_id", streamID) + clog.Infof(ctx, "Received live video AI request for %s. pipelineParams=%v", streamName, pipelineParams) + + // collect all RTMP outputs + var rtmpOutputs []string + if outputURL != "" { + rtmpOutputs = append(rtmpOutputs, outputURL) + } + if mediaMTXOutputURL != "" { + rtmpOutputs = append(rtmpOutputs, mediaMTXOutputURL, mediaMTXOutputAlias) + } + clog.Info(ctx, "RTMP outputs", "destinations", rtmpOutputs) + + // channel that blocks until after orch selection is complete + // avoids a race condition with closing the control channel + orchSelection := make(chan bool) + + // Clear any previous gateway status + GatewayStatus.Clear(streamID) + GatewayStatus.StoreKey(streamID, "whep_url", whepURL) + + monitor.SendQueueEventAsync("stream_trace", map[string]interface{}{ + "type": "gateway_receive_stream_request", + "timestamp": streamRequestTime, + "stream_id": streamID, + "pipeline_id": pipelineID, + "request_id": requestID, + "orchestrator_info": map[string]interface{}{ + "address": "", + "url": "", + }, + }) + + // Count `ai_live_attempts` after successful parameters validation + clog.V(common.VERBOSE).Infof(ctx, "AI Live video attempt") + if monitor.Enabled { + monitor.AILiveVideoAttempt() + } + + sendErrorEvent := LiveErrorEventSender(ctx, streamID, map[string]string{ + "type": "error", + "request_id": requestID, + "stream_id": streamID, + "pipeline_id": pipelineID, + "pipeline": pipeline, + }) + + // this function is called when the pipeline hits a fatal error, we kick the input connection to allow + // the client to reconnect and restart the pipeline + segmenterCtx, cancelSegmenter := context.WithCancel(clog.Clone(context.Background(), ctx)) + kickInput := func(err error) { + defer cancelSegmenter() + if err == nil { + return + } + clog.Errorf(ctx, "Live video pipeline finished with error: %s", err) + + sendErrorEvent(err) + + err = mediaMTXClient.KickInputConnection(ctx) + if err != nil { + clog.Errorf(ctx, "Failed to kick input connection: %s", err) + } + } + + ssr := media.NewSwitchableSegmentReader() + params := aiRequestParams{ + node: ls.LivepeerNode, + os: drivers.NodeStorage.NewSession(requestID), + sessManager: ls.AISessionManager, + + liveParams: &liveRequestParams{ + segmentReader: ssr, + rtmpOutputs: rtmpOutputs, + localRTMPPrefix: mediaMTXInputURL, + stream: streamName, + paymentProcessInterval: ls.livePaymentInterval, + outSegmentTimeout: ls.outSegmentTimeout, + requestID: requestID, + streamID: streamID, + pipelineID: pipelineID, + pipeline: pipeline, + kickInput: kickInput, + sendErrorEvent: sendErrorEvent, + }, + } + + registerControl(ctx, params) + + // Create a special parent context for orchestrator cancellation + orchCtx, orchCancel := context.WithCancel(ctx) + + // Kick off the RTMP pull and segmentation as soon as possible + go func() { + ms := media.MediaSegmenter{Workdir: ls.LivepeerNode.WorkDir, MediaMTXClient: mediaMTXClient} + + // Wait for stream to exist before starting segmentation + // in the case of no input stream but only generating an output stream from instructions + // the segmenter will never start + for { + streamExists, err := mediaMTXClient.StreamExists() + if err != nil { + clog.Errorf(ctx, "Error checking if stream exists: %v", err) + } else if streamExists { + break + } + select { + case <-segmenterCtx.Done(): + return + case <-time.After(200 * time.Millisecond): + // Continue waiting + } + } + + //blocks until error or stream ends + ms.RunSegmentation(segmenterCtx, mediaMTXInputURL, ssr.Read) + sendErrorEvent(errors.New("mediamtx ingest disconnected")) + monitor.SendQueueEventAsync("stream_trace", map[string]interface{}{ + "type": "gateway_ingest_stream_closed", + "timestamp": time.Now().UnixMilli(), + "stream_id": streamID, + "pipeline_id": pipelineID, + "request_id": requestID, + "orchestrator_info": map[string]interface{}{ + "address": "", + "url": "", + }, + }) + ssr.Close() + <-orchSelection // wait for selection to complete + cleanupControl(ctx, params) + orchCancel() + }() + + req := worker.GenLiveVideoToVideoJSONRequestBody{ + ModelId: &pipeline, + Params: &pipelineParams, + GatewayRequestId: &requestID, + StreamId: &streamID, + } + + startStream(orchCtx, params, req) + if err != nil { + clog.Errorf(ctx, "Error starting stream: %s", err) + } + close(orchSelection) + + resp := map[string]string{} + resp["whip_url"] = whipURL + resp["update_url"] = updateURL + resp["status_url"] = statusURL + + respJson, err := json.Marshal(resp) + if err != nil { + http.Error(w, "Failed to marshal response", http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + w.Write(respJson) + }) +} + +func startStream(ctx context.Context, params aiRequestParams, req worker.GenLiveVideoToVideoJSONRequestBody) { + orchSwapper := NewOrchestratorSwapper(params) + isFirst, firstProcessed := true, make(chan interface{}) + go func() { + var err error + for { + perOrchCtx, perOrchCancel := context.WithCancelCause(ctx) + params.liveParams = newParams(params.liveParams, perOrchCancel) + + //need to update this to do a standard BYOC request + var resp interface{} + resp, err = processAIRequest(perOrchCtx, params, req) + + if err != nil { + clog.Errorf(ctx, "Error processing AI Request: %s", err) + perOrchCancel(err) + break + } + + if err = startStreamProcessing(perOrchCtx, params, resp); err != nil { + clog.Errorf(ctx, "Error starting processing: %s", err) + perOrchCancel(err) + break + } + if isFirst { + isFirst = false + firstProcessed <- struct{}{} + } + <-perOrchCtx.Done() + err = context.Cause(perOrchCtx) + if errors.Is(err, context.Canceled) { + // this happens if parent ctx was cancelled without a CancelCause + // or if passing `nil` as a CancelCause + err = nil + } + if !params.inputStreamExists() { + clog.Info(ctx, "No input stream, skipping orchestrator swap") + break + } + if swapErr := orchSwapper.checkSwap(ctx); swapErr != nil { + if err != nil { + err = fmt.Errorf("%w: %w", swapErr, err) + } else { + err = swapErr + } + break + } + clog.Infof(ctx, "Retrying stream with a different orchestrator") + + // will swap, but first notify with the reason for the swap + if err == nil { + err = errors.New("unknown swap reason") + } + params.liveParams.sendErrorEvent(err) + } + if isFirst { + // failed before selecting an orchestrator + firstProcessed <- struct{}{} + } + params.liveParams.kickInput(err) + }() + <-firstProcessed +} + +func startStreamProcessing(ctx context.Context, params aiRequestParams, res interface{}) error { + resp := res.(*worker.GenLiveVideoToVideoResponse) + + host := params.liveParams.sess.Transcoder() + pub, err := common.AppendHostname(resp.JSON200.PublishUrl, host) + if err != nil { + return fmt.Errorf("invalid publish URL: %w", err) + } + sub, err := common.AppendHostname(resp.JSON200.SubscribeUrl, host) + if err != nil { + return fmt.Errorf("invalid subscribe URL: %w", err) + } + control, err := common.AppendHostname(*resp.JSON200.ControlUrl, host) + if err != nil { + return fmt.Errorf("invalid control URL: %w", err) + } + events, err := common.AppendHostname(*resp.JSON200.EventsUrl, host) + if err != nil { + return fmt.Errorf("invalid events URL: %w", err) + } + data, err := common.AppendHostname(strings.Replace(*resp.JSON200.EventsUrl, "-events", "-data", 1), host) + if err != nil { + return fmt.Errorf("invalid data URL: %w", err) + } + if resp.JSON200.ManifestId != nil { + ctx = clog.AddVal(ctx, "manifest_id", *resp.JSON200.ManifestId) + params.liveParams.manifestID = *resp.JSON200.ManifestId + } + clog.V(common.VERBOSE).Infof(ctx, "pub %s sub %s control %s events %s data %s", pub, sub, control, events, data) + + //TODO make this configurable + startControlPublish(ctx, control, params) + startTricklePublish(ctx, pub, params, params.liveParams.sess) + startTrickleSubscribe(ctx, sub, params, params.liveParams.sess) + startEventsSubscribe(ctx, events, params, params.liveParams.sess) + startDataSubscribe(ctx, data, params, params.liveParams.sess) + return nil +} + +// StartStreamOrchestrator handles the POST /stream/start endpoint for the Orchestrator +func (h *lphttp) StartStreamOrchestrator() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + remoteAddr := getRemoteAddr(r) + ctx := clog.AddVal(r.Context(), clog.ClientIP, remoteAddr) + + streamID := r.Header.Get("streamID") + gatewayRequestID := r.Header.Get("requestID") + requestID := string(core.RandomManifestID()) + ctx = clog.AddVal(ctx, "orch_request_id", requestID) + ctx = clog.AddVal(ctx, "gateway_request_id", gatewayRequestID) + ctx = clog.AddVal(ctx, "manifest_id", requestID) + ctx = clog.AddVal(ctx, "stream_id", streamID) + + var req worker.GenLiveVideoToVideoJSONRequestBody + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + respondWithError(w, err.Error(), http.StatusBadRequest) + return + } + + orch := h.orchestrator + pipeline := "byoc-stream" + cap := core.Capability_LiveVideoToVideo + modelID := *req.ModelId + clog.V(common.VERBOSE).Infof(ctx, "Received request id=%v cap=%v modelID=%v", requestID, cap, modelID) + + // Create storage for the request (for AI Workers, must run before CheckAICapacity) + err := orch.CreateStorageForRequest(requestID) + if err != nil { + respondWithError(w, "Could not create storage to receive results", http.StatusInternalServerError) + } + + // Check if there is capacity for the request + hasCapacity, _ := orch.CheckAICapacity(pipeline, modelID) + if !hasCapacity { + clog.Errorf(ctx, "Insufficient capacity for pipeline=%v modelID=%v", pipeline, modelID) + respondWithError(w, "insufficient capacity", http.StatusServiceUnavailable) + return + } + + // Start trickle server for live-video + var ( + mid = requestID // Request ID is used for the manifest ID + pubUrl = orch.ServiceURI().JoinPath(TrickleHTTPPath, mid).String() + subUrl = pubUrl + "-out" + controlUrl = pubUrl + "-control" + eventsUrl = pubUrl + "-events" + dataUrl = pubUrl + "-data" + ) + + // Handle initial payment, the rest of the payments are done separately from the stream processing + // Note that this payment is debit from the balance and acts as a buffer for the AI Realtime Video processing + payment, err := getPayment(r.Header.Get(paymentHeader)) + if err != nil { + respondWithError(w, err.Error(), http.StatusPaymentRequired) + return + } + sender := getPaymentSender(payment) + _, ctx, err = verifySegCreds(ctx, h.orchestrator, r.Header.Get(segmentHeader), sender) + if err != nil { + respondWithError(w, err.Error(), http.StatusForbidden) + return + } + if err := orch.ProcessPayment(ctx, payment, core.ManifestID(mid)); err != nil { + respondWithError(w, err.Error(), http.StatusBadRequest) + return + } + if payment.GetExpectedPrice().GetPricePerUnit() > 0 && !orch.SufficientBalance(sender, core.ManifestID(mid)) { + respondWithError(w, "Insufficient balance", http.StatusBadRequest) + return + } + + // If successful, then create the trickle channels + // Precreate the channels to avoid race conditions + // TODO get the expected mime type from the request + pubCh := trickle.NewLocalPublisher(h.trickleSrv, mid, "video/MP2T") + pubCh.CreateChannel() + subCh := trickle.NewLocalPublisher(h.trickleSrv, mid+"-out", "video/MP2T") + subCh.CreateChannel() + controlPubCh := trickle.NewLocalPublisher(h.trickleSrv, mid+"-control", "application/json") + controlPubCh.CreateChannel() + eventsCh := trickle.NewLocalPublisher(h.trickleSrv, mid+"-events", "application/json") + eventsCh.CreateChannel() + dataCh := trickle.NewLocalPublisher(h.trickleSrv, mid+"-data", "application/json") + dataCh.CreateChannel() + + // Start payment receiver which accounts the payments and stops the stream if the payment is insufficient + priceInfo := payment.GetExpectedPrice() + var paymentProcessor *LivePaymentProcessor + ctx, cancel := context.WithCancel(context.Background()) + if priceInfo != nil && priceInfo.PricePerUnit != 0 { + paymentReceiver := livePaymentReceiver{orchestrator: h.orchestrator} + accountPaymentFunc := func(inPixels int64) error { + err := paymentReceiver.AccountPayment(context.Background(), &SegmentInfoReceiver{ + sender: sender, + inPixels: inPixels, + priceInfo: priceInfo, + sessionID: mid, + }) + if err != nil { + slog.Warn("Error accounting payment, stopping stream processing", "err", err) + pubCh.Close() + subCh.Close() + eventsCh.Close() + controlPubCh.Close() + cancel() + } + return err + } + paymentProcessor = NewLivePaymentProcessor(ctx, h.node.LivePaymentInterval, accountPaymentFunc) + } else { + clog.Warningf(ctx, "No price info found for model %v, Orchestrator will not charge for video processing", modelID) + } + + // Subscribe to the publishUrl for payments monitoring and payment processing + go func() { + sub := trickle.NewLocalSubscriber(h.trickleSrv, mid) + for { + segment, err := sub.Read() + if err != nil { + clog.Infof(ctx, "Error getting local trickle segment err=%v", err) + return + } + reader := segment.Reader + if paymentProcessor != nil { + reader = paymentProcessor.process(ctx, segment.Reader) + } + io.Copy(io.Discard, reader) + } + }() + + // Prepare request to worker + controlUrlOverwrite := overwriteHostInStream(h.node.LiveAITrickleHostForRunner, controlUrl) + eventsUrlOverwrite := overwriteHostInStream(h.node.LiveAITrickleHostForRunner, eventsUrl) + subscribeUrlOverwrite := overwriteHostInStream(h.node.LiveAITrickleHostForRunner, pubUrl) + publishUrlOverwrite := overwriteHostInStream(h.node.LiveAITrickleHostForRunner, subUrl) + dataUrlOverwrite := overwriteHostInStream(h.node.LiveAITrickleHostForRunner, dataUrl) + + workerReq := worker.LiveVideoToVideoParams{ + ModelId: req.ModelId, + PublishUrl: publishUrlOverwrite, + SubscribeUrl: subscribeUrlOverwrite, + EventsUrl: &eventsUrlOverwrite, + ControlUrl: &controlUrlOverwrite, + DataUrl: &dataUrlOverwrite, + Params: req.Params, + GatewayRequestId: &gatewayRequestID, + ManifestId: &mid, + StreamId: &streamID, + } + + // Send request to the worker + _, err = orch.LiveVideoToVideo(ctx, requestID, workerReq) + if err != nil { + if monitor.Enabled { + monitor.AIProcessingError(err.Error(), pipeline, modelID, ethcommon.Address{}.String()) + } + + pubCh.Close() + subCh.Close() + controlPubCh.Close() + eventsCh.Close() + dataCh.Close() + cancel() + respondWithError(w, err.Error(), http.StatusInternalServerError) + return + } + + // Prepare the response + jsonData, err := json.Marshal(&worker.LiveVideoToVideoResponse{ + PublishUrl: pubUrl, + SubscribeUrl: subUrl, + ControlUrl: &controlUrl, + EventsUrl: &eventsUrl, + RequestId: &requestID, + ManifestId: &mid, + }) + if err != nil { + respondWithError(w, err.Error(), http.StatusInternalServerError) + return + } + + clog.Infof(ctx, "Processed request id=%v cap=%v modelID=%v took=%v", requestID, cap, modelID) + respondJsonOk(w, jsonData) + }) +} + +// This function is copied from ai_http.go to avoid import cycle issues +func overwriteHostInStream(hostOverwrite, url string) string { + if hostOverwrite == "" { + return url + } + u, err := url2.ParseRequestURI(url) + if err != nil { + slog.Warn("Couldn't parse url to overwrite for worker, using original url", "url", url, "err", err) + return url + } + u.Host = hostOverwrite + return u.String() +} From 9ac928d31f173ecbc498714646bfa4b0012799d4 Mon Sep 17 00:00:00 2001 From: Brad P Date: Thu, 21 Aug 2025 16:04:53 -0500 Subject: [PATCH 14/57] refactor to move job setup to separate function --- server/job_rpc.go | 137 +++++++++++++++++++++++++++------------------- 1 file changed, 80 insertions(+), 57 deletions(-) diff --git a/server/job_rpc.go b/server/job_rpc.go index 88d27b33e7..803c7a96f1 100644 --- a/server/job_rpc.go +++ b/server/job_rpc.go @@ -73,6 +73,9 @@ type JobRequest struct { orchSearchRespTimeout time.Duration } +type JobRequestDetails struct { + StreamId string `json:"stream_id"` +} type JobParameters struct { Orchestrators JobOrchestratorsFilter `json:"orchestrators,omitempty"` //list of orchestrators to use for the job } @@ -82,6 +85,14 @@ type JobOrchestratorsFilter struct { Include []string `json:"include,omitempty"` } +type orchJob struct { + Req *JobRequest + Details *JobRequestDetails + Params *JobParameters + Orchs []JobToken + JobReqHdr string +} + // worker registers to Orchestrator func (h *lphttp) RegisterCapability(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { @@ -253,6 +264,63 @@ func (h *lphttp) GetJobToken(w http.ResponseWriter, r *http.Request) { json.NewEncoder(w).Encode(jobToken) } +func (ls *LivepeerServer) setupJob(ctx context.Context, r *http.Request) (*orchJob, error) { + clog.Infof(ctx, "processing job request") + var orchs []JobToken + + jobReqHdr := r.Header.Get(jobRequestHdr) + jobReq, err := verifyJobCreds(ctx, nil, jobReqHdr) + if err != nil { + return nil, errors.New(fmt.Sprintf("Unable to parse job request, err=%v", err)) + } + + var jobDetails JobRequestDetails + if err := json.Unmarshal([]byte(jobReq.Request), &jobDetails); err != nil { + return nil, errors.New(fmt.Sprintf("Unable to unmarshal job request err=%v", err)) + } + + var jobParams JobParameters + if err := json.Unmarshal([]byte(jobReq.Parameters), &jobParams); err != nil { + return nil, errors.New(fmt.Sprintf("Unable to unmarshal job parameters err=%v", err)) + } + + searchTimeout, respTimeout := getOrchSearchTimeouts(ctx, r.Header.Get(jobOrchSearchTimeoutHdr), r.Header.Get(jobOrchSearchRespTimeoutHdr)) + jobReq.orchSearchTimeout = searchTimeout + jobReq.orchSearchRespTimeout = respTimeout + + //get pool of Orchestrators that can do the job + orchs, err = getJobOrchestrators(ctx, ls.LivepeerNode, jobReq.Capability, jobParams, jobReq.orchSearchTimeout, jobReq.orchSearchRespTimeout) + if err != nil { + return nil, errors.New(fmt.Sprintf("Unable to find orchestrators for capability %v err=%v", jobReq.Capability, err)) + } + + if len(orchs) == 0 { + return nil, errors.New(fmt.Sprintf("No orchestrators found for capability %v", jobReq.Capability)) + } + + //sign the request + gateway := ls.LivepeerNode.OrchestratorPool.Broadcaster() + sig, err := gateway.Sign([]byte(jobReq.Request + jobReq.Parameters)) + if err != nil { + return nil, errors.New(fmt.Sprintf("Unable to sign request err=%v", err)) + } + jobReq.Sender = gateway.Address().Hex() + jobReq.Sig = "0x" + hex.EncodeToString(sig) + + //create the job request header with the signature + jobReqEncoded, err := json.Marshal(jobReq) + if err != nil { + return nil, errors.New(fmt.Sprintf("Unable to encode job request err=%v", err)) + } + jobReqHdr = base64.StdEncoding.EncodeToString(jobReqEncoded) + + return &orchJob{Req: jobReq, + Details: &jobDetails, + Params: &jobParams, + Orchs: orchs, + JobReqHdr: jobReqHdr}, nil +} + func (h *lphttp) ProcessJob(w http.ResponseWriter, r *http.Request) { ctx := r.Context() @@ -280,42 +348,16 @@ func (ls *LivepeerServer) SubmitJob() http.Handler { } func (ls *LivepeerServer) submitJob(ctx context.Context, w http.ResponseWriter, r *http.Request) { - jobReqHdr := r.Header.Get(jobRequestHdr) - jobReq, err := verifyJobCreds(ctx, nil, jobReqHdr) - if err != nil { - clog.Errorf(ctx, "Unable to verify job creds err=%v", err) - http.Error(w, fmt.Sprintf("Unable to parse job request, err=%v", err), http.StatusBadRequest) - return - } - ctx = clog.AddVal(ctx, "job_id", jobReq.ID) - ctx = clog.AddVal(ctx, "capability", jobReq.Capability) - clog.Infof(ctx, "processing job request") - searchTimeout, respTimeout := getOrchSearchTimeouts(ctx, r.Header.Get(jobOrchSearchTimeoutHdr), r.Header.Get(jobOrchSearchRespTimeoutHdr)) - jobReq.orchSearchTimeout = searchTimeout - jobReq.orchSearchRespTimeout = respTimeout + orchJob, err := ls.setupJob(ctx, r) + clog.Infof(ctx, "Job request setup complete details=%v params=%v", orchJob.Details, orchJob.Params) - var params JobParameters - if err := json.Unmarshal([]byte(jobReq.Parameters), ¶ms); err != nil { - clog.Errorf(ctx, "Unable to unmarshal job parameters err=%v", err) - http.Error(w, fmt.Sprintf("Unable to unmarshal job parameters err=%v", err), http.StatusBadRequest) - return - } - - //get pool of Orchestrators that can do the job - orchs, err := getJobOrchestrators(ctx, ls.LivepeerNode, jobReq.Capability, params, jobReq.orchSearchTimeout, jobReq.orchSearchRespTimeout) if err != nil { - clog.Errorf(ctx, "Unable to find orchestrators for capability %v err=%v", jobReq.Capability, err) - http.Error(w, fmt.Sprintf("Unable to find orchestrators for capability %v err=%v", jobReq.Capability, err), http.StatusBadRequest) - return - } - - if len(orchs) == 0 { - clog.Errorf(ctx, "No orchestrators found for capability %v", jobReq.Capability) - http.Error(w, fmt.Sprintf("No orchestrators found for capability %v", jobReq.Capability), http.StatusServiceUnavailable) + http.Error(w, fmt.Sprintf("Unable to setup job err=%v", err), http.StatusBadRequest) return } - + ctx = clog.AddVal(ctx, "job_id", orchJob.Req.ID) + ctx = clog.AddVal(ctx, "capability", orchJob.Req.Capability) // Read the original request body body, err := io.ReadAll(r.Body) if err != nil { @@ -323,29 +365,10 @@ func (ls *LivepeerServer) submitJob(ctx context.Context, w http.ResponseWriter, return } r.Body.Close() - //sign the request - gateway := ls.LivepeerNode.OrchestratorPool.Broadcaster() - sig, err := gateway.Sign([]byte(jobReq.Request + jobReq.Parameters)) - if err != nil { - clog.Errorf(ctx, "Unable to sign request err=%v", err) - http.Error(w, fmt.Sprintf("Unable to sign request err=%v", err), http.StatusInternalServerError) - return - } - jobReq.Sender = gateway.Address().Hex() - jobReq.Sig = "0x" + hex.EncodeToString(sig) - - //create the job request header with the signature - jobReqEncoded, err := json.Marshal(jobReq) - if err != nil { - clog.Errorf(ctx, "Unable to encode job request err=%v", err) - http.Error(w, fmt.Sprintf("Unable to encode job request err=%v", err), http.StatusInternalServerError) - return - } - jobReqHdr = base64.StdEncoding.EncodeToString(jobReqEncoded) //send the request to the Orchestrator(s) //the loop ends on Gateway error and bad request errors - for _, orchToken := range orchs { + for _, orchToken := range orchJob.Orchs { // Extract the worker resource route from the URL path // The prefix is "/process/request/" @@ -370,9 +393,9 @@ func (ls *LivepeerServer) submitJob(ctx context.Context, w http.ResponseWriter, req.Header.Add("Content-Length", r.Header.Get("Content-Length")) req.Header.Add("Content-Type", r.Header.Get("Content-Type")) - req.Header.Add(jobRequestHdr, jobReqHdr) + req.Header.Add(jobRequestHdr, orchJob.JobReqHdr) if orchToken.Price.PricePerUnit > 0 { - paymentHdr, err := createPayment(ctx, jobReq, orchToken, ls.LivepeerNode) + paymentHdr, err := createPayment(ctx, orchJob.Req, orchToken, ls.LivepeerNode) if err != nil { clog.Errorf(ctx, "Unable to create payment err=%v", err) http.Error(w, fmt.Sprintf("Unable to create payment err=%v", err), http.StatusBadRequest) @@ -382,7 +405,7 @@ func (ls *LivepeerServer) submitJob(ctx context.Context, w http.ResponseWriter, } start := time.Now() - resp, err := sendJobReqWithTimeout(req, time.Duration(jobReq.Timeout+5)*time.Second) //include 5 second buffer + resp, err := sendJobReqWithTimeout(req, time.Duration(orchJob.Req.Timeout+5)*time.Second) //include 5 second buffer if err != nil { clog.Errorf(ctx, "job not able to be processed by Orchestrator %v err=%v ", orchToken.ServiceAddr, err.Error()) continue @@ -427,7 +450,7 @@ func (ls *LivepeerServer) submitJob(ctx context.Context, w http.ResponseWriter, continue } - gatewayBalance := updateGatewayBalance(ls.LivepeerNode, orchToken, jobReq.Capability, time.Since(start)) + gatewayBalance := updateGatewayBalance(ls.LivepeerNode, orchToken, orchJob.Req.Capability, time.Since(start)) clog.V(common.SHORT).Infof(ctx, "Job processed successfully took=%v balance=%v balance_from_orch=%v", time.Since(start), gatewayBalance.FloatString(0), orchBalance) w.Write(data) return @@ -450,7 +473,7 @@ func (ls *LivepeerServer) submitJob(ctx context.Context, w http.ResponseWriter, w.WriteHeader(http.StatusOK) // Read from upstream and forward to client respChan := make(chan string, 100) - respCtx, _ := context.WithTimeout(ctx, time.Duration(jobReq.Timeout+10)*time.Second) //include a small buffer to let Orchestrator close the connection on the timeout + respCtx, _ := context.WithTimeout(ctx, time.Duration(orchJob.Req.Timeout+10)*time.Second) //include a small buffer to let Orchestrator close the connection on the timeout go func() { defer resp.Body.Close() @@ -491,7 +514,7 @@ func (ls *LivepeerServer) submitJob(ctx context.Context, w http.ResponseWriter, } } - gatewayBalance := updateGatewayBalance(ls.LivepeerNode, orchToken, jobReq.Capability, time.Since(start)) + gatewayBalance := updateGatewayBalance(ls.LivepeerNode, orchToken, orchJob.Req.Capability, time.Since(start)) clog.V(common.SHORT).Infof(ctx, "Job processed successfully took=%v balance=%v balance_from_orch=%v", time.Since(start), gatewayBalance.FloatString(0), orchBalance.FloatString(0)) } From ba8525cababc9b4b9041293bf7fe8f2716d47c66 Mon Sep 17 00:00:00 2001 From: Brad P Date: Thu, 21 Aug 2025 16:05:52 -0500 Subject: [PATCH 15/57] fix --- server/job_rpc.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/job_rpc.go b/server/job_rpc.go index 803c7a96f1..2e93f5257a 100644 --- a/server/job_rpc.go +++ b/server/job_rpc.go @@ -350,6 +350,12 @@ func (ls *LivepeerServer) SubmitJob() http.Handler { func (ls *LivepeerServer) submitJob(ctx context.Context, w http.ResponseWriter, r *http.Request) { orchJob, err := ls.setupJob(ctx, r) + if err != nil { + clog.Errorf(ctx, "Error setting up job: %s", err) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + clog.Infof(ctx, "Job request setup complete details=%v params=%v", orchJob.Details, orchJob.Params) if err != nil { From 33c05aef029ed991ed7d90fc5752c836a4d51d0e Mon Sep 17 00:00:00 2001 From: Brad P Date: Fri, 22 Aug 2025 08:06:57 -0500 Subject: [PATCH 16/57] refactor to reuse job request send to orch --- server/job_rpc.go | 41 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/server/job_rpc.go b/server/job_rpc.go index 2e93f5257a..51f778acfd 100644 --- a/server/job_rpc.go +++ b/server/job_rpc.go @@ -411,13 +411,14 @@ func (ls *LivepeerServer) submitJob(ctx context.Context, w http.ResponseWriter, } start := time.Now() - resp, err := sendJobReqWithTimeout(req, time.Duration(orchJob.Req.Timeout+5)*time.Second) //include 5 second buffer + resp, code, err := ls.sendJobToOrch(ctx, r, orchJob.Req, orchJob.JobReqHdr, orchToken, workerResourceRoute, body) if err != nil { clog.Errorf(ctx, "job not able to be processed by Orchestrator %v err=%v ", orchToken.ServiceAddr, err.Error()) continue } + //error response from Orchestrator - if resp.StatusCode > 399 { + if code > 399 { defer resp.Body.Close() data, err := io.ReadAll(resp.Body) if err != nil { @@ -524,8 +525,44 @@ func (ls *LivepeerServer) submitJob(ctx context.Context, w http.ResponseWriter, clog.V(common.SHORT).Infof(ctx, "Job processed successfully took=%v balance=%v balance_from_orch=%v", time.Since(start), gatewayBalance.FloatString(0), orchBalance.FloatString(0)) } + } +} + +func (ls *LivepeerServer) sendJobToOrch(ctx context.Context, r *http.Request, jobReq *JobRequest, signedReqHdr string, orchToken JobToken, route string, body []byte) (*http.Response, int, error) { + orchUrl := orchToken.ServiceAddr + route + req, err := http.NewRequestWithContext(ctx, "POST", orchUrl, bytes.NewBuffer(body)) + if err != nil { + clog.Errorf(ctx, "Unable to create request err=%v", err) + return nil, http.StatusInternalServerError, err + } + + // set the headers + if r != nil { + req.Header.Add("Content-Length", r.Header.Get("Content-Length")) + req.Header.Add("Content-Type", r.Header.Get("Content-Type")) + } else { + //this is for live requests which will be json to start stream + // update requests should include the content type/length + req.Header.Add("Content-Type", "application/json") + } + + req.Header.Add(jobRequestHdr, signedReqHdr) + if orchToken.Price.PricePerUnit > 0 { + paymentHdr, err := createPayment(ctx, jobReq, orchToken, ls.LivepeerNode) + if err != nil { + clog.Errorf(ctx, "Unable to create payment err=%v", err) + return nil, http.StatusInternalServerError, fmt.Errorf("Unable to create payment err=%v", err) + } + req.Header.Add(jobPaymentHeaderHdr, paymentHdr) + } + resp, err := sendJobReqWithTimeout(req, time.Duration(jobReq.Timeout+5)*time.Second) //include 5 second buffer + if err != nil { + clog.Errorf(ctx, "job not able to be processed by Orchestrator %v err=%v ", orchToken.ServiceAddr, err.Error()) + return nil, http.StatusBadRequest, err } + + return resp, resp.StatusCode, nil } func processJob(ctx context.Context, h *lphttp, w http.ResponseWriter, r *http.Request) { From e52582c0676b6d56ca5e45ce658550c3c31f0725 Mon Sep 17 00:00:00 2001 From: Brad P Date: Fri, 22 Aug 2025 08:58:42 -0500 Subject: [PATCH 17/57] fix --- server/job_rpc.go | 25 ++----------------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/server/job_rpc.go b/server/job_rpc.go index 51f778acfd..199ee3f841 100644 --- a/server/job_rpc.go +++ b/server/job_rpc.go @@ -389,27 +389,6 @@ func (ls *LivepeerServer) submitJob(ctx context.Context, w http.ResponseWriter, workerRoute = workerRoute + "/" + workerResourceRoute } - req, err := http.NewRequestWithContext(ctx, "POST", workerRoute, bytes.NewBuffer(body)) - if err != nil { - clog.Errorf(ctx, "Unable to create request err=%v", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - // set the headers - req.Header.Add("Content-Length", r.Header.Get("Content-Length")) - req.Header.Add("Content-Type", r.Header.Get("Content-Type")) - - req.Header.Add(jobRequestHdr, orchJob.JobReqHdr) - if orchToken.Price.PricePerUnit > 0 { - paymentHdr, err := createPayment(ctx, orchJob.Req, orchToken, ls.LivepeerNode) - if err != nil { - clog.Errorf(ctx, "Unable to create payment err=%v", err) - http.Error(w, fmt.Sprintf("Unable to create payment err=%v", err), http.StatusBadRequest) - return - } - req.Header.Add(jobPaymentHeaderHdr, paymentHdr) - } - start := time.Now() resp, code, err := ls.sendJobToOrch(ctx, r, orchJob.Req, orchJob.JobReqHdr, orchToken, workerResourceRoute, body) if err != nil { @@ -428,10 +407,10 @@ func (ls *LivepeerServer) submitJob(ctx context.Context, w http.ResponseWriter, } clog.Errorf(ctx, "error processing request err=%v ", string(data)) //nonretryable error - if resp.StatusCode < 500 { + if code < 500 { //assume non retryable bad request //return error response from the worker - http.Error(w, string(data), resp.StatusCode) + http.Error(w, string(data), code) return } //retryable error, continue to next orchestrator From 8a7ec3fddf0eef05511d0991a7b232f58bdb21da Mon Sep 17 00:00:00 2001 From: Brad P Date: Fri, 22 Aug 2025 12:50:10 -0500 Subject: [PATCH 18/57] capabilities: add live-ai external capabilities to Capabilities to enable live payments --- core/ai_orchestrator.go | 21 +++++++ core/capabilities.go | 2 + core/external_capabilities.go | 105 ++++++++++++++++++++++++++++++---- 3 files changed, 118 insertions(+), 10 deletions(-) diff --git a/core/ai_orchestrator.go b/core/ai_orchestrator.go index ef282163fe..f13bb244f4 100644 --- a/core/ai_orchestrator.go +++ b/core/ai_orchestrator.go @@ -12,6 +12,7 @@ import ( "os" "path" "strconv" + "strings" "sync" "time" @@ -1130,10 +1131,30 @@ func (orch *orchestrator) RegisterExternalCapability(extCapabilitySettings strin //set the price for the capability orch.node.SetPriceForExternalCapability("default", cap.Name, cap.GetPrice()) + //if a live ai capability add it to capabilities to enable live ai payments + var price *AutoConvertedPrice + if strings.Contains(cap.Name, "live-") { + + price, err = NewAutoConvertedPrice(cap.PriceCurrency, big.NewRat(cap.PricePerUnit, cap.PriceScaling), func(price *big.Rat) { + glog.V(6).Infof("Capability %s price set to %s wei per compute unit", cap.Name, price.FloatString(3)) + }) + + if err != nil { + panic(fmt.Errorf("error converting price: %v", err)) + } + orch.node.SetBasePriceForCap("default", Capability_LiveAI, cap.Name, price) + + orch.node.AddAICapabilities(cap.ToCapabilities()) + } return cap, nil } func (orch *orchestrator) RemoveExternalCapability(extCapability string) error { + if strings.Contains(extCapability, "live-") { + cap := orch.node.ExternalCapabilities.Capabilities[extCapability] + orch.node.RemoveAICapabilities(cap.ToCapabilities()) + } + orch.node.ExternalCapabilities.RemoveCapability(extCapability) return nil } diff --git a/core/capabilities.go b/core/capabilities.go index e0cbc436a9..ba9884f560 100644 --- a/core/capabilities.go +++ b/core/capabilities.go @@ -88,6 +88,7 @@ const ( Capability_ImageToText Capability = 34 Capability_LiveVideoToVideo Capability = 35 Capability_TextToSpeech Capability = 36 + Capability_LiveAI Capability = 37 ) var CapabilityNameLookup = map[Capability]string{ @@ -129,6 +130,7 @@ var CapabilityNameLookup = map[Capability]string{ Capability_ImageToText: "Image to text", Capability_LiveVideoToVideo: "Live video to video", Capability_TextToSpeech: "Text to speech", + Capability_LiveAI: "Live AI", } var CapabilityTestLookup = map[Capability]CapabilityTest{ diff --git a/core/external_capabilities.go b/core/external_capabilities.go index f5802db1d8..c099e18bcf 100644 --- a/core/external_capabilities.go +++ b/core/external_capabilities.go @@ -1,37 +1,105 @@ package core import ( + "context" "encoding/json" "fmt" "math/big" + "time" "sync" + ethcommon "github.com/ethereum/go-ethereum/common" "github.com/golang/glog" + "github.com/livepeer/go-livepeer/core" ) type ExternalCapability struct { - Name string `json:"name"` - Description string `json:"description"` - Url string `json:"url"` - Capacity int `json:"capacity"` - PricePerUnit int64 `json:"price_per_unit"` - PriceScaling int64 `json:"price_scaling"` - PriceCurrency string `json:"currency"` - - price *AutoConvertedPrice + Name string `json:"name"` + Description string `json:"description"` + Url string `json:"url"` + Capacity int `json:"capacity"` + PricePerUnit int64 `json:"price_per_unit"` + PriceScaling int64 `json:"price_scaling"` + PriceCurrency string `json:"currency"` + Requirements CapabilityRequirements `json:"requirements"` + price *AutoConvertedPrice mu sync.RWMutex Load int } +type CapabilityRequirements struct { + VideoIngress bool `json:"video_ingress"` + VideoEgress bool `json:"video_egress"` + DataOutput bool `json:"data_output"` +} + +type StreamData struct { + StreamID string + Capability string + //Gateway fields + orchToken interface{} + OrchUrl string + ExcludeOrchs []string + OrchPublishUrl string + OrchSubscribeUrl string + OrchControlUrl string + OrchEventsUrl string + OrchDataUrl string + + //Orchestrator fields + Sender ethcommon.Address + + //Stream fields + Params interface{} + ControlPub interface{} + StreamCtx context.Context + CancelStream context.CancelFunc + StreamStartedTime time.Time +} type ExternalCapabilities struct { capm sync.Mutex Capabilities map[string]*ExternalCapability + Streams map[string]*StreamData } func NewExternalCapabilities() *ExternalCapabilities { - return &ExternalCapabilities{Capabilities: make(map[string]*ExternalCapability)} + return &ExternalCapabilities{Capabilities: make(map[string]*ExternalCapability), + Streams: make(map[string]*StreamData), + } +} + +func (extCaps *ExternalCapabilities) AddStream(streamID string, params interface{}, orchUrl, publishUrl, subscribeUrl, controlUrl, eventsUrl, dataUrl string) error { + extCaps.capm.Lock() + defer extCaps.capm.Unlock() + _, ok := extCaps.Streams[streamID] + if ok { + return fmt.Errorf("stream already exists: %s", streamID) + } + + //add to streams + ctx, cancel := context.WithCancel(context.Background()) + extCaps.Streams[streamID] = &StreamData{ + StreamID: streamID, + Params: params, + StreamCtx: ctx, + CancelStream: cancel, + OrchUrl: orchUrl, + OrchPublishUrl: publishUrl, + OrchSubscribeUrl: subscribeUrl, + OrchEventsUrl: eventsUrl, + OrchDataUrl: dataUrl, + } + + return nil +} + +func (extCaps *ExternalCapabilities) RemoveStream(streamID string) { + extCaps.capm.Lock() + defer extCaps.capm.Unlock() + + delete(extCaps.Streams, streamID) } func (extCaps *ExternalCapabilities) RemoveCapability(extCap string) { @@ -80,3 +148,20 @@ func (extCap *ExternalCapability) GetPrice() *big.Rat { defer extCap.mu.RUnlock() return extCap.price.Value() } + +func (extCap *ExternalCapability) ToCapabilities() *Capabilities { + constraint := ModelConstraint{Capacity: extCap.Capacity} + + capConstraints := core.CapabilityConstraints{ + Models: map[string]ModelConstraint{ + extCap.Name: constraint, + }, + } + + capConstraints[Capability_LiveAI].Models[extCap.Name] = constraint + + caps := core.NewCapabilities([]core.Capability{Capability_LiveAI}) + caps.SetPerCapabilityConstraints(capConstraints) + + return caps +} From fa105de4034fa18267a283719308b151f27db6a6 Mon Sep 17 00:00:00 2001 From: Brad P Date: Fri, 22 Aug 2025 13:05:31 -0500 Subject: [PATCH 19/57] capabilities: fix --- core/ai_orchestrator.go | 2 ++ core/external_capabilities.go | 15 ++++----------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/core/ai_orchestrator.go b/core/ai_orchestrator.go index f13bb244f4..d1a9088f7b 100644 --- a/core/ai_orchestrator.go +++ b/core/ai_orchestrator.go @@ -1146,10 +1146,12 @@ func (orch *orchestrator) RegisterExternalCapability(extCapabilitySettings strin orch.node.AddAICapabilities(cap.ToCapabilities()) } + return cap, nil } func (orch *orchestrator) RemoveExternalCapability(extCapability string) error { + //if a live-ai external capability remove from Capabilities if strings.Contains(extCapability, "live-") { cap := orch.node.ExternalCapabilities.Capabilities[extCapability] orch.node.RemoveAICapabilities(cap.ToCapabilities()) diff --git a/core/external_capabilities.go b/core/external_capabilities.go index c099e18bcf..bd240d2ca0 100644 --- a/core/external_capabilities.go +++ b/core/external_capabilities.go @@ -11,7 +11,6 @@ import ( ethcommon "github.com/ethereum/go-ethereum/common" "github.com/golang/glog" - "github.com/livepeer/go-livepeer/core" ) type ExternalCapability struct { @@ -150,17 +149,11 @@ func (extCap *ExternalCapability) GetPrice() *big.Rat { } func (extCap *ExternalCapability) ToCapabilities() *Capabilities { - constraint := ModelConstraint{Capacity: extCap.Capacity} + capConstraints := make(PerCapabilityConstraints) + capConstraints[Capability_LiveAI].Models = make(ModelConstraints) + capConstraints[Capability_LiveAI].Models[extCap.Name] = &ModelConstraint{Capacity: extCap.Capacity} - capConstraints := core.CapabilityConstraints{ - Models: map[string]ModelConstraint{ - extCap.Name: constraint, - }, - } - - capConstraints[Capability_LiveAI].Models[extCap.Name] = constraint - - caps := core.NewCapabilities([]core.Capability{Capability_LiveAI}) + caps := NewCapabilities([]Capability{Capability_LiveAI}, MandatoryOCapabilities()) caps.SetPerCapabilityConstraints(capConstraints) return caps From 5e8676b436c7bd1f2fe9fdb6ee0c06bc23b04f6a Mon Sep 17 00:00:00 2001 From: Brad P Date: Fri, 22 Aug 2025 16:36:16 -0500 Subject: [PATCH 20/57] updates to add streaming to byoc --- core/external_capabilities.go | 19 +- server/ai_live_video.go | 28 +- server/ai_mediaserver.go | 5 + server/job_rpc.go | 19 + server/job_stream.go | 740 +++++++++++++++++++++------------- server/rpc.go | 2 + 6 files changed, 508 insertions(+), 305 deletions(-) diff --git a/core/external_capabilities.go b/core/external_capabilities.go index bd240d2ca0..66a7c2fb25 100644 --- a/core/external_capabilities.go +++ b/core/external_capabilities.go @@ -38,7 +38,8 @@ type StreamData struct { StreamID string Capability string //Gateway fields - orchToken interface{} + StreamRequest []byte + OrchToken interface{} OrchUrl string ExcludeOrchs []string OrchPublishUrl string @@ -69,7 +70,7 @@ func NewExternalCapabilities() *ExternalCapabilities { } } -func (extCaps *ExternalCapabilities) AddStream(streamID string, params interface{}, orchUrl, publishUrl, subscribeUrl, controlUrl, eventsUrl, dataUrl string) error { +func (extCaps *ExternalCapabilities) AddStream(streamID string, params interface{}, streamReq []byte) error { extCaps.capm.Lock() defer extCaps.capm.Unlock() _, ok := extCaps.Streams[streamID] @@ -80,15 +81,11 @@ func (extCaps *ExternalCapabilities) AddStream(streamID string, params interface //add to streams ctx, cancel := context.WithCancel(context.Background()) extCaps.Streams[streamID] = &StreamData{ - StreamID: streamID, - Params: params, - StreamCtx: ctx, - CancelStream: cancel, - OrchUrl: orchUrl, - OrchPublishUrl: publishUrl, - OrchSubscribeUrl: subscribeUrl, - OrchEventsUrl: eventsUrl, - OrchDataUrl: dataUrl, + StreamID: streamID, + Params: params, + StreamRequest: streamReq, + StreamCtx: ctx, + CancelStream: cancel, } return nil diff --git a/server/ai_live_video.go b/server/ai_live_video.go index 9a1c7d1109..acfc574f40 100644 --- a/server/ai_live_video.go +++ b/server/ai_live_video.go @@ -80,21 +80,23 @@ func startTricklePublish(ctx context.Context, url *url.URL, params aiRequestPara // Start payments which probes a segment every "paymentProcessInterval" and sends a payment ctx, cancel := context.WithCancel(ctx) - priceInfo := sess.OrchestratorInfo.PriceInfo var paymentProcessor *LivePaymentProcessor - if priceInfo != nil && priceInfo.PricePerUnit != 0 { - paymentSender := livePaymentSender{} - sendPaymentFunc := func(inPixels int64) error { - return paymentSender.SendPayment(context.Background(), &SegmentInfoSender{ - sess: sess.BroadcastSession, - inPixels: inPixels, - priceInfo: priceInfo, - mid: params.liveParams.manifestID, - }) + if sess != nil { + priceInfo := sess.OrchestratorInfo.PriceInfo + if priceInfo != nil && priceInfo.PricePerUnit != 0 { + paymentSender := livePaymentSender{} + sendPaymentFunc := func(inPixels int64) error { + return paymentSender.SendPayment(context.Background(), &SegmentInfoSender{ + sess: sess.BroadcastSession, + inPixels: inPixels, + priceInfo: priceInfo, + mid: params.liveParams.manifestID, + }) + } + paymentProcessor = NewLivePaymentProcessor(ctx, params.liveParams.paymentProcessInterval, sendPaymentFunc) + } else { + clog.Warningf(ctx, "No price info found from Orchestrator, Gateway will not send payments for the video processing") } - paymentProcessor = NewLivePaymentProcessor(ctx, params.liveParams.paymentProcessInterval, sendPaymentFunc) - } else { - clog.Warningf(ctx, "No price info found from Orchestrator, Gateway will not send payments for the video processing") } slowOrchChecker := &SlowOrchChecker{} diff --git a/server/ai_mediaserver.go b/server/ai_mediaserver.go index b10098b60f..23e832e4c4 100644 --- a/server/ai_mediaserver.go +++ b/server/ai_mediaserver.go @@ -115,6 +115,11 @@ func startAIMediaServer(ctx context.Context, ls *LivepeerServer) error { //API for dynamic capabilities ls.HTTPMux.Handle("/process/request/", ls.SubmitJob()) + ls.HTTPMux.Handle("/stream/start", ls.StartStream()) + ls.HTTPMux.Handle("/stream/{streamId}/whip", ls.StartStreamWhipIngest()) + ls.HTTPMux.Handle("/stream/{streamId}/rtmp", ls.StartStreamRTMPIngest()) + ls.HTTPMux.Handle("/stream/{streamId}/update", ls.UpdateStream()) + ls.HTTPMux.Handle("/stream/{streamId}/status", ls.GetStreamStatus()) media.StartFileCleanup(ctx, ls.LivepeerNode.WorkDir) diff --git a/server/job_rpc.go b/server/job_rpc.go index 199ee3f841..698a148910 100644 --- a/server/job_rpc.go +++ b/server/job_rpc.go @@ -544,6 +544,25 @@ func (ls *LivepeerServer) sendJobToOrch(ctx context.Context, r *http.Request, jo return resp, resp.StatusCode, nil } +func (ls *LivepeerServer) sendPayment(ctx context.Context, orchUrl string, payment string) error { + req, err := http.NewRequestWithContext(ctx, "POST", orchUrl+"/stream/payment", nil) + if err != nil { + clog.Errorf(ctx, "Unable to create request err=%v", err) + return err + } + + req.Header.Add("Content-Type", "application/json") + req.Header.Add(jobPaymentHeaderHdr, payment) + + _, err = sendJobReqWithTimeout(req, 10) + if err != nil { + clog.Errorf(ctx, "job payment not able to be processed by Orchestrator %v err=%v ", orchUrl, err.Error()) + return err + } + + return nil +} + func processJob(ctx context.Context, h *lphttp, w http.ResponseWriter, r *http.Request) { remoteAddr := getRemoteAddr(r) ctx = clog.AddVal(ctx, "client_ip", remoteAddr) diff --git a/server/job_stream.go b/server/job_stream.go index 38284afc05..9257e6e508 100644 --- a/server/job_stream.go +++ b/server/job_stream.go @@ -9,7 +9,7 @@ import ( "log/slog" "net/http" "net/url" - url2 "net/url" + "os" "strings" "time" @@ -28,192 +28,404 @@ import ( func (ls *LivepeerServer) StartStream() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // Create fresh context instead of using r.Context() since ctx will outlive the request - ctx := context.Background() + ctx := r.Context() - requestID := string(core.RandomManifestID()) - ctx = clog.AddVal(ctx, "request_id", requestID) - - streamName := r.FormValue("stream") - if streamName == "" { - clog.Errorf(ctx, "Missing stream name") - http.Error(w, "Missing stream name", http.StatusBadRequest) + orchJob, err := ls.setupJob(ctx, r) + if err != nil { + clog.Errorf(ctx, "Error setting up job: %s", err) + http.Error(w, err.Error(), http.StatusBadRequest) return } - streamRequestTime := time.Now().UnixMilli() - - ctx = clog.AddVal(ctx, "stream", streamName) - sourceID := r.FormValue("source_id") - if sourceID == "" { - clog.Errorf(ctx, "Missing source_id") - http.Error(w, "Missing source_id", http.StatusBadRequest) + resp, code, err := ls.setupStream(ctx, r, orchJob.Req) + if err != nil { + clog.Errorf(ctx, "Error setting up stream: %s", err) + http.Error(w, err.Error(), code) return } - ctx = clog.AddVal(ctx, "source_id", sourceID) - sourceType := r.FormValue("source_type") - if sourceType == "" { - clog.Errorf(ctx, "Missing source_type") - http.Error(w, "Missing source_type", http.StatusBadRequest) - return + + go ls.runStream(orchJob, resp["stream_id"]) + + if resp != nil { + // Stream started successfully + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(resp) + } else { + //case where we are subscribing to own streams in setupStream + w.WriteHeader(http.StatusNoContent) } - sourceTypeStr, err := media.MediamtxSourceTypeToString(sourceType) + }) +} + +func (ls *LivepeerServer) runStream(job *orchJob, streamID string) { + ctx := context.Background() + stream, exists := ls.LivepeerNode.ExternalCapabilities.Streams[streamID] + params := stream.Params.(aiRequestParams) + if !exists { + clog.Errorf(ctx, "Stream %s not found", streamID) + return + } + + for _, orch := range job.Orchs { + orchResp, _, err := ls.sendJobToOrch(ctx, nil, job.Req, job.JobReqHdr, orch, "/stream/start", stream.StreamRequest) if err != nil { - clog.Errorf(ctx, "Invalid source type %s", sourceType) - http.Error(w, err.Error(), http.StatusBadRequest) + clog.Errorf(ctx, "job not able to be processed by Orchestrator %v err=%v ", orch.ServiceAddr, err.Error()) + continue + } + stream.OrchPublishUrl = orchResp.Header.Get("Publish-Url") + stream.OrchSubscribeUrl = orchResp.Header.Get("Subscribe-Url") + stream.OrchControlUrl = orchResp.Header.Get("Control-Url") + stream.OrchEventsUrl = orchResp.Header.Get("Events-Url") + stream.OrchDataUrl = orchResp.Header.Get("Data-Url") + + perOrchCtx, perOrchCancel := context.WithCancelCause(ctx) + params.liveParams = newParams(params.liveParams, perOrchCancel) + if err = startStreamProcessing(perOrchCtx, stream); err != nil { + clog.Errorf(ctx, "Error starting processing: %s", err) + perOrchCancel(err) + break + } + <-perOrchCtx.Done() + err = context.Cause(perOrchCtx) + if errors.Is(err, context.Canceled) { + // this happens if parent ctx was cancelled without a CancelCause + // or if passing `nil` as a CancelCause + err = nil + } + if !params.inputStreamExists() { + clog.Info(ctx, "No input stream, skipping orchestrator swap") + break + } + //if swapErr := orchSwapper.checkSwap(ctx); swapErr != nil { + // if err != nil { + // err = fmt.Errorf("%w: %w", swapErr, err) + // } else { + // err = swapErr + // } + // break + //} + clog.Infof(ctx, "Retrying stream with a different orchestrator") + + // will swap, but first notify with the reason for the swap + if err == nil { + err = errors.New("unknown swap reason") + } + params.liveParams.sendErrorEvent(err) + + //if isFirst { + // // failed before selecting an orchestrator + // firstProcessed <- struct{}{} + //} + params.liveParams.kickInput(err) + } +} + +func (ls *LivepeerServer) monitorStream(ctx context.Context, streamId string) { + stream, exists := ls.LivepeerNode.ExternalCapabilities.Streams[streamId] + if !exists { + clog.Errorf(ctx, "Stream %s not found", streamId) + return + } + + // Create a ticker that runs every minute for payments + ticker := time.NewTicker(45 * time.Second) + defer ticker.Stop() + + for { + select { + case <-stream.StreamCtx.Done(): + clog.Infof(ctx, "Stream %s stopped, ending monitoring", streamId) return + case <-ticker.C: + // Run payment and fetch new JobToken every minute + req := &JobRequest{Capability: stream.Capability, + Sender: ls.LivepeerNode.OrchestratorPool.Broadcaster().Address().Hex(), + Timeout: 60, + } + pmtHdr, err := createPayment(ctx, req, stream.OrchToken.(JobToken), ls.LivepeerNode) + if err != nil { + clog.Errorf(ctx, "Error processing stream payment for %s: %v", streamId, err) + // Continue monitoring even if payment fails + } + } - ctx = clog.AddVal(ctx, "source_type", sourceType) + } - remoteHost, err := getRemoteHost(r.RemoteAddr) - if err != nil { - clog.Errorf(ctx, "Could not find callback host: %s", err.Error()) - http.Error(w, "Could not find callback host", http.StatusBadRequest) - return +} + +func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, req *JobRequest) (map[string]string, int, error) { + requestID := string(core.RandomManifestID()) + ctx = clog.AddVal(ctx, "request_id", requestID) + + //live-video-to-video uses path value for this + streamName := r.FormValue("stream") + + streamRequestTime := time.Now().UnixMilli() + + ctx = clog.AddVal(ctx, "stream", streamName) + sourceID := r.FormValue("source_id") + if sourceID == "" { + return nil, http.StatusBadRequest, errors.New("missing source_id") + } + ctx = clog.AddVal(ctx, "source_id", sourceID) + sourceType := r.FormValue("source_type") + if sourceType == "" { + return nil, http.StatusBadRequest, errors.New("missing source_type") + } + sourceTypeStr, err := media.MediamtxSourceTypeToString(sourceType) + if err != nil { + return nil, http.StatusBadRequest, errors.New("invalid source type") + } + ctx = clog.AddVal(ctx, "source_type", sourceType) + + //TODO: change the params in query to separate form fields since we + // are setting things up with a POST request + queryParams := r.FormValue("query") + qp, err := url.ParseQuery(queryParams) + if err != nil { + return nil, http.StatusBadRequest, errors.New("invalid query params") + } + // If auth webhook is set and returns an output URL, this will be replaced + outputURL := qp.Get("rtmpOutput") + + // convention to avoid re-subscribing to our own streams + // in case we want to push outputs back into mediamtx - + // use an `-out` suffix for the stream name. + if strings.HasSuffix(streamName, "-out") { + // skip for now; we don't want to re-publish our own outputs + return nil, 0, nil + } + + // if auth webhook returns pipeline config these will be replaced + pipeline := qp.Get("pipeline") + if pipeline == "" { + pipeline = req.Capability + } + rawParams := qp.Get("params") + streamID := qp.Get("streamId") + + var pipelineID string + var pipelineParams map[string]interface{} + if rawParams != "" { + if err := json.Unmarshal([]byte(rawParams), &pipelineParams); err != nil { + return nil, http.StatusBadRequest, errors.New("invalid model params") } - ctx = clog.AddVal(ctx, "remote_addr", remoteHost) + } - queryParams := r.FormValue("query") - qp, err := url.ParseQuery(queryParams) + //ensure a streamid exists and includes the streamName if provided + if streamID == "" { + streamID = string(core.RandomManifestID()) + } + if streamName != "" { + streamID = fmt.Sprintf("%s-%s", streamName, streamID) + } + // Currently for webrtc we need to add a path prefix due to the ingress setup + //mediaMTXStreamPrefix := r.PathValue("prefix") + //if mediaMTXStreamPrefix != "" { + // mediaMTXStreamPrefix = mediaMTXStreamPrefix + "/" + //} + mediaMtxHost := os.Getenv("LIVE_AI_PLAYBACK_HOST") + if mediaMtxHost == "" { + mediaMtxHost = "localhost:1935" + } + mediaMTXInputURL := fmt.Sprintf("rtmp://%s/%s%s", mediaMtxHost, "", streamID) + mediaMTXOutputURL := mediaMTXInputURL + "-out" + mediaMTXOutputAlias := fmt.Sprintf("%s-%s-out", mediaMTXInputURL, requestID) + + whepURL := generateWhepUrl(streamID, requestID) + whipURL := fmt.Sprintf("https://%s/stream/%s/%s", ls.LivepeerNode.GatewayHost, streamID, "/whip") + rtmpURL := mediaMTXInputURL + updateURL := fmt.Sprintf("https://%s/stream/%s/%s", ls.LivepeerNode.GatewayHost, streamID, "/update") + statusURL := fmt.Sprintf("https://%s/stream/%s/%s", ls.LivepeerNode.GatewayHost, streamID, "/status") + + //if set this will overwrite settings above + if LiveAIAuthWebhookURL != nil { + authResp, err := authenticateAIStream(LiveAIAuthWebhookURL, ls.liveAIAuthApiKey, AIAuthRequest{ + Stream: streamName, + Type: sourceTypeStr, + QueryParams: queryParams, + GatewayHost: ls.LivepeerNode.GatewayHost, + WhepURL: whepURL, + UpdateURL: updateURL, + StatusURL: statusURL, + }) if err != nil { - clog.Errorf(ctx, "invalid query params, err=%w", err) - http.Error(w, err.Error(), http.StatusBadRequest) - return + return nil, http.StatusForbidden, fmt.Errorf("live ai auth failed: %w", err) } - // If auth webhook is set and returns an output URL, this will be replaced - outputURL := qp.Get("rtmpOutput") - // Currently for webrtc we need to add a path prefix due to the ingress setup - mediaMTXStreamPrefix := r.PathValue("prefix") - if mediaMTXStreamPrefix != "" { - mediaMTXStreamPrefix = mediaMTXStreamPrefix + "/" - } - mediaMTXInputURL := fmt.Sprintf("rtmp://%s/%s%s", remoteHost, mediaMTXStreamPrefix, streamName) - mediaMTXRtmpURL := r.FormValue("rtmp_url") - if mediaMTXRtmpURL != "" { - mediaMTXInputURL = mediaMTXRtmpURL - } - mediaMTXOutputURL := mediaMTXInputURL + "-out" - mediaMTXOutputAlias := fmt.Sprintf("%s-%s-out", mediaMTXInputURL, requestID) - - // convention to avoid re-subscribing to our own streams - // in case we want to push outputs back into mediamtx - - // use an `-out` suffix for the stream name. - if strings.HasSuffix(streamName, "-out") { - // skip for now; we don't want to re-publish our own outputs - return + + if authResp.RTMPOutputURL != "" { + outputURL = authResp.RTMPOutputURL } - // if auth webhook returns pipeline config these will be replaced - pipeline := qp.Get("pipeline") - rawParams := qp.Get("params") - streamID := qp.Get("streamId") - var pipelineID string - var pipelineParams map[string]interface{} - if rawParams != "" { - if err := json.Unmarshal([]byte(rawParams), &pipelineParams); err != nil { - clog.Errorf(ctx, "Invalid pipeline params: %s", err) - http.Error(w, "Invalid model params", http.StatusBadRequest) - return + if authResp.Pipeline != "" { + pipeline = authResp.Pipeline + } + + if len(authResp.paramsMap) > 0 { + if _, ok := authResp.paramsMap["prompt"]; !ok && pipeline == "comfyui" { + pipelineParams = map[string]interface{}{"prompt": authResp.paramsMap} + } else { + pipelineParams = authResp.paramsMap } } - mediaMTXClient := media.NewMediaMTXClient(remoteHost, ls.mediaMTXApiPassword, sourceID, sourceType) + //don't want to update the streamid or pipeline id with byoc + //if authResp.StreamID != "" { + // streamID = authResp.StreamID + //} - whepURL := generateWhepUrl(streamName, requestID) - whipURL := fmt.Sprintf("https://%s/stream/%s/%s", ls.LivepeerNode.GatewayHost, streamName, "/whip") - updateURL := fmt.Sprintf("https://%s/stream/%s/%s", ls.LivepeerNode.GatewayHost, streamName, "/update") - statusURL := fmt.Sprintf("https://%s/stream/%s/%s", ls.LivepeerNode.GatewayHost, streamID, "/status") + //if authResp.PipelineID != "" { + // pipelineID = authResp.PipelineID + //} + } - if LiveAIAuthWebhookURL != nil { - authResp, err := authenticateAIStream(LiveAIAuthWebhookURL, ls.liveAIAuthApiKey, AIAuthRequest{ - Stream: streamName, - Type: sourceTypeStr, - QueryParams: queryParams, - GatewayHost: ls.LivepeerNode.GatewayHost, - WhepURL: whepURL, - UpdateURL: updateURL, - StatusURL: statusURL, - }) - if err != nil { - kickErr := mediaMTXClient.KickInputConnection(ctx) - if kickErr != nil { - clog.Errorf(ctx, "failed to kick input connection: %s", kickErr.Error()) - } - clog.Errorf(ctx, "Live AI auth failed: %s", err.Error()) - http.Error(w, "Forbidden", http.StatusForbidden) - return - } + ctx = clog.AddVal(ctx, "stream_id", streamID) + clog.Infof(ctx, "Received live video AI request for %s. pipelineParams=%v", streamName, pipelineParams) - if authResp.RTMPOutputURL != "" { - outputURL = authResp.RTMPOutputURL - } + // collect all RTMP outputs + var rtmpOutputs []string + if outputURL != "" { + rtmpOutputs = append(rtmpOutputs, outputURL) + } + if mediaMTXOutputURL != "" { + rtmpOutputs = append(rtmpOutputs, mediaMTXOutputURL, mediaMTXOutputAlias) + } + clog.Info(ctx, "RTMP outputs", "destinations", rtmpOutputs) + + // Clear any previous gateway status + GatewayStatus.Clear(streamID) + GatewayStatus.StoreKey(streamID, "whep_url", whepURL) + + monitor.SendQueueEventAsync("stream_trace", map[string]interface{}{ + "type": "gateway_receive_stream_request", + "timestamp": streamRequestTime, + "stream_id": streamID, + "pipeline_id": pipelineID, + "request_id": requestID, + "orchestrator_info": map[string]interface{}{ + "address": "", + "url": "", + }, + }) - if authResp.Pipeline != "" { - pipeline = authResp.Pipeline - } + // Count `ai_live_attempts` after successful parameters validation + clog.V(common.VERBOSE).Infof(ctx, "AI Live video attempt") + if monitor.Enabled { + monitor.AILiveVideoAttempt(req.Capability) + } - if len(authResp.paramsMap) > 0 { - if _, ok := authResp.paramsMap["prompt"]; !ok && pipeline == "comfyui" { - pipelineParams = map[string]interface{}{"prompt": authResp.paramsMap} - } else { - pipelineParams = authResp.paramsMap - } - } + sendErrorEvent := LiveErrorEventSender(ctx, streamID, map[string]string{ + "type": "error", + "request_id": requestID, + "stream_id": streamID, + "pipeline_id": pipelineID, + "pipeline": pipeline, + }) - if authResp.StreamID != "" { - streamID = authResp.StreamID - } + //params set with ingest types: + // RTMP + // kickInput will kick the input from MediaMTX to force a reconnect + // localRTMPPrefix mediaMTXInputURL matches to get the ingest from MediaMTX + // WHIP + // kickInput will close the whip connection + // localRTMPPrefix set by ENV variable LIVE_AI_PLAYBACK_HOST + //kickInput is set with RTMP ingest + ssr := media.NewSwitchableSegmentReader() //this converts ingest to segments to send to Orchestrator + params := aiRequestParams{ + node: ls.LivepeerNode, + os: drivers.NodeStorage.NewSession(requestID), + sessManager: ls.AISessionManager, + + liveParams: &liveRequestParams{ + segmentReader: ssr, + rtmpOutputs: rtmpOutputs, + stream: streamName, + paymentProcessInterval: ls.livePaymentInterval, + outSegmentTimeout: ls.outSegmentTimeout, + requestID: requestID, + streamID: streamID, + pipelineID: pipelineID, + pipeline: pipeline, + sendErrorEvent: sendErrorEvent, + }, + } - if authResp.PipelineID != "" { - pipelineID = authResp.PipelineID - } + //create a dataWriter for data channel if enabled + if enableData, ok := pipelineParams["enableData"]; ok { + if enableData == true || enableData == "true" { + params.liveParams.dataWriter = media.NewSegmentWriter(5) + pipelineParams["enableData"] = true + clog.Infof(ctx, "Data channel enabled for stream %s", streamName) } + } - ctx = clog.AddVal(ctx, "stream_id", streamID) - clog.Infof(ctx, "Received live video AI request for %s. pipelineParams=%v", streamName, pipelineParams) - - // collect all RTMP outputs - var rtmpOutputs []string - if outputURL != "" { - rtmpOutputs = append(rtmpOutputs, outputURL) - } - if mediaMTXOutputURL != "" { - rtmpOutputs = append(rtmpOutputs, mediaMTXOutputURL, mediaMTXOutputAlias) - } - clog.Info(ctx, "RTMP outputs", "destinations", rtmpOutputs) - - // channel that blocks until after orch selection is complete - // avoids a race condition with closing the control channel - orchSelection := make(chan bool) - - // Clear any previous gateway status - GatewayStatus.Clear(streamID) - GatewayStatus.StoreKey(streamID, "whep_url", whepURL) - - monitor.SendQueueEventAsync("stream_trace", map[string]interface{}{ - "type": "gateway_receive_stream_request", - "timestamp": streamRequestTime, - "stream_id": streamID, - "pipeline_id": pipelineID, - "request_id": requestID, - "orchestrator_info": map[string]interface{}{ - "address": "", - "url": "", - }, - }) + if _, ok := pipelineParams["enableVideoIngress"]; !ok { + pipelineParams["enableVideoIngress"] = true + } - // Count `ai_live_attempts` after successful parameters validation - clog.V(common.VERBOSE).Infof(ctx, "AI Live video attempt") - if monitor.Enabled { - monitor.AILiveVideoAttempt() + if _, ok := pipelineParams["enableVideoEgress"]; !ok { + pipelineParams["enableVideoEgress"] = true + } + + //send start request to Orchestrator + _, exists := ls.LivepeerNode.ExternalCapabilities.Streams[streamID] + if exists { + return nil, http.StatusBadRequest, fmt.Errorf("stream already exists: %s", streamID) + } + + // read entire body to ensure valid and send to orchestrator + body, err := io.ReadAll(r.Body) + if err != nil { + return nil, http.StatusInternalServerError, errors.New("Error reading request body") + } + r.Body.Close() + + //save the stream setup + if err := ls.LivepeerNode.ExternalCapabilities.AddStream(streamID, params, body); err != nil { + return nil, http.StatusBadRequest, err + } + + resp := make(map[string]string) + resp["stream_id"] = streamID + resp["whip_url"] = whipURL + resp["whep_url"] = whepURL + resp["rtmp_url"] = rtmpURL + resp["rtmp_output_url"] = strings.Join(rtmpOutputs, ",") + resp["update_url"] = updateURL + resp["status_url"] = statusURL + + return resp, http.StatusOK, nil +} + +// mediamtx sends this request to go-livepeer when rtmp strem received +func (ls *LivepeerServer) StartStreamRTMPIngest() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + remoteAddr := getRemoteAddr(r) + ctx := clog.AddVal(r.Context(), clog.ClientIP, remoteAddr) + requestID := string(core.RandomManifestID()) + ctx = clog.AddVal(ctx, "request_id", requestID) + + streamId := r.PathValue("streamId") + ctx = clog.AddVal(ctx, "stream_id", streamId) + + stream, ok := ls.LivepeerNode.ExternalCapabilities.Streams[streamId] + if !ok { + respondJsonError(ctx, w, fmt.Errorf("stream not found: %s", streamId), http.StatusNotFound) + return } - sendErrorEvent := LiveErrorEventSender(ctx, streamID, map[string]string{ - "type": "error", - "request_id": requestID, - "stream_id": streamID, - "pipeline_id": pipelineID, - "pipeline": pipeline, - }) + params := stream.Params.(aiRequestParams) + + //note that mediaMtxHost is the ip address of media mtx + // media sends a post request in the runOnReady event setup in mediamtx.yml + // StartLiveVideo calls this remoteHost + mediaMtxHost, err := getRemoteHost(r.RemoteAddr) + if err != nil { + respondJsonError(ctx, w, err, http.StatusBadRequest) + return + } + mediaMTXClient := media.NewMediaMTXClient(mediaMtxHost, ls.mediaMTXApiPassword, "rtmp_ingest", "rtmp") // this function is called when the pipeline hits a fatal error, we kick the input connection to allow // the client to reconnect and restart the pipeline @@ -225,7 +437,7 @@ func (ls *LivepeerServer) StartStream() http.Handler { } clog.Errorf(ctx, "Live video pipeline finished with error: %s", err) - sendErrorEvent(err) + params.liveParams.sendErrorEvent(err) err = mediaMTXClient.KickInputConnection(ctx) if err != nil { @@ -233,104 +445,43 @@ func (ls *LivepeerServer) StartStream() http.Handler { } } - ssr := media.NewSwitchableSegmentReader() - params := aiRequestParams{ - node: ls.LivepeerNode, - os: drivers.NodeStorage.NewSession(requestID), - sessManager: ls.AISessionManager, - - liveParams: &liveRequestParams{ - segmentReader: ssr, - rtmpOutputs: rtmpOutputs, - localRTMPPrefix: mediaMTXInputURL, - stream: streamName, - paymentProcessInterval: ls.livePaymentInterval, - outSegmentTimeout: ls.outSegmentTimeout, - requestID: requestID, - streamID: streamID, - pipelineID: pipelineID, - pipeline: pipeline, - kickInput: kickInput, - sendErrorEvent: sendErrorEvent, - }, - } - - registerControl(ctx, params) + params.liveParams.kickInput = kickInput // Create a special parent context for orchestrator cancellation - orchCtx, orchCancel := context.WithCancel(ctx) + //orchCtx, orchCancel := context.WithCancel(ctx) - // Kick off the RTMP pull and segmentation as soon as possible + // Kick off the RTMP pull and segmentation go func() { ms := media.MediaSegmenter{Workdir: ls.LivepeerNode.WorkDir, MediaMTXClient: mediaMTXClient} + //segmenter blocks until done + ms.RunSegmentation(segmenterCtx, params.liveParams.localRTMPPrefix, params.liveParams.segmentReader.Read) - // Wait for stream to exist before starting segmentation - // in the case of no input stream but only generating an output stream from instructions - // the segmenter will never start - for { - streamExists, err := mediaMTXClient.StreamExists() - if err != nil { - clog.Errorf(ctx, "Error checking if stream exists: %v", err) - } else if streamExists { - break - } - select { - case <-segmenterCtx.Done(): - return - case <-time.After(200 * time.Millisecond): - // Continue waiting - } - } - - //blocks until error or stream ends - ms.RunSegmentation(segmenterCtx, mediaMTXInputURL, ssr.Read) - sendErrorEvent(errors.New("mediamtx ingest disconnected")) + params.liveParams.sendErrorEvent(errors.New("mediamtx ingest disconnected")) monitor.SendQueueEventAsync("stream_trace", map[string]interface{}{ "type": "gateway_ingest_stream_closed", "timestamp": time.Now().UnixMilli(), - "stream_id": streamID, - "pipeline_id": pipelineID, - "request_id": requestID, + "stream_id": params.liveParams.streamID, + "pipeline_id": params.liveParams.pipelineID, + "request_id": params.liveParams.requestID, "orchestrator_info": map[string]interface{}{ "address": "", "url": "", }, }) - ssr.Close() - <-orchSelection // wait for selection to complete + params.liveParams.segmentReader.Close() cleanupControl(ctx, params) - orchCancel() }() - req := worker.GenLiveVideoToVideoJSONRequestBody{ - ModelId: &pipeline, - Params: &pipelineParams, - GatewayRequestId: &requestID, - StreamId: &streamID, - } - - startStream(orchCtx, params, req) - if err != nil { - clog.Errorf(ctx, "Error starting stream: %s", err) - } - close(orchSelection) - - resp := map[string]string{} - resp["whip_url"] = whipURL - resp["update_url"] = updateURL - resp["status_url"] = statusURL - - respJson, err := json.Marshal(resp) - if err != nil { - http.Error(w, "Failed to marshal response", http.StatusInternalServerError) + var req worker.GenLiveVideoToVideoJSONRequestBody + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + respondJsonError(ctx, w, err, http.StatusBadRequest) return } - w.Header().Set("Content-Type", "application/json") - w.Write(respJson) }) } -func startStream(ctx context.Context, params aiRequestParams, req worker.GenLiveVideoToVideoJSONRequestBody) { +func startStream(ctx context.Context, streamData *core.StreamData, req worker.GenLiveVideoToVideoJSONRequestBody) { + params := streamData.Params.(aiRequestParams) orchSwapper := NewOrchestratorSwapper(params) isFirst, firstProcessed := true, make(chan interface{}) go func() { @@ -339,17 +490,13 @@ func startStream(ctx context.Context, params aiRequestParams, req worker.GenLive perOrchCtx, perOrchCancel := context.WithCancelCause(ctx) params.liveParams = newParams(params.liveParams, perOrchCancel) - //need to update this to do a standard BYOC request - var resp interface{} - resp, err = processAIRequest(perOrchCtx, params, req) - if err != nil { clog.Errorf(ctx, "Error processing AI Request: %s", err) perOrchCancel(err) break } - if err = startStreamProcessing(perOrchCtx, params, resp); err != nil { + if err = startStreamProcessing(perOrchCtx, streamData); err != nil { clog.Errorf(ctx, "Error starting processing: %s", err) perOrchCancel(err) break @@ -386,7 +533,7 @@ func startStream(ctx context.Context, params aiRequestParams, req worker.GenLive params.liveParams.sendErrorEvent(err) } if isFirst { - // failed before selecting an orchestrator + // failed before selecting an orchestrator, exit the stream, something is wrong firstProcessed <- struct{}{} } params.liveParams.kickInput(err) @@ -394,42 +541,59 @@ func startStream(ctx context.Context, params aiRequestParams, req worker.GenLive <-firstProcessed } -func startStreamProcessing(ctx context.Context, params aiRequestParams, res interface{}) error { - resp := res.(*worker.GenLiveVideoToVideoResponse) +func startStreamProcessing(ctx context.Context, streamData *core.StreamData) error { + params := streamData.Params.(aiRequestParams) + host := streamData.OrchUrl + var channels []string - host := params.liveParams.sess.Transcoder() - pub, err := common.AppendHostname(resp.JSON200.PublishUrl, host) - if err != nil { - return fmt.Errorf("invalid publish URL: %w", err) - } - sub, err := common.AppendHostname(resp.JSON200.SubscribeUrl, host) - if err != nil { - return fmt.Errorf("invalid subscribe URL: %w", err) - } - control, err := common.AppendHostname(*resp.JSON200.ControlUrl, host) + //this adds the stream to LivePipelines which the Control Publisher and Data Writer + //are accessible for reading data and sending updates + registerControl(ctx, params) + + //required channels + control, err := common.AppendHostname(streamData.OrchControlUrl, host) if err != nil { return fmt.Errorf("invalid control URL: %w", err) } - events, err := common.AppendHostname(*resp.JSON200.EventsUrl, host) + events, err := common.AppendHostname(streamData.OrchEventsUrl, host) if err != nil { return fmt.Errorf("invalid events URL: %w", err) } - data, err := common.AppendHostname(strings.Replace(*resp.JSON200.EventsUrl, "-events", "-data", 1), host) - if err != nil { - return fmt.Errorf("invalid data URL: %w", err) - } - if resp.JSON200.ManifestId != nil { - ctx = clog.AddVal(ctx, "manifest_id", *resp.JSON200.ManifestId) - params.liveParams.manifestID = *resp.JSON200.ManifestId - } - clog.V(common.VERBOSE).Infof(ctx, "pub %s sub %s control %s events %s data %s", pub, sub, control, events, data) - //TODO make this configurable + channels = append(channels, control.String()) + channels = append(channels, events.String()) startControlPublish(ctx, control, params) - startTricklePublish(ctx, pub, params, params.liveParams.sess) - startTrickleSubscribe(ctx, sub, params, params.liveParams.sess) startEventsSubscribe(ctx, events, params, params.liveParams.sess) - startDataSubscribe(ctx, data, params, params.liveParams.sess) + + //Optional channels + if streamData.OrchPublishUrl == "" { + pub, err := common.AppendHostname(streamData.OrchPublishUrl, host) + if err != nil { + return fmt.Errorf("invalid publish URL: %w", err) + } + channels = append(channels, pub.String()) + startTricklePublish(ctx, pub, params, params.liveParams.sess) + } + + if streamData.OrchSubscribeUrl == "" { + sub, err := common.AppendHostname(streamData.OrchSubscribeUrl, host) + if err != nil { + return fmt.Errorf("invalid subscribe URL: %w", err) + } + channels = append(channels, sub.String()) + startTrickleSubscribe(ctx, sub, params, params.liveParams.sess) + } + + if streamData.OrchDataUrl == "" { + data, err := common.AppendHostname(streamData.OrchDataUrl, host) + if err != nil { + return fmt.Errorf("invalid data URL: %w", err) + } + streamData.Params.(aiRequestParams).liveParams.manifestID = streamData.Capability + + startDataSubscribe(ctx, data, params, params.liveParams.sess) + } + return nil } @@ -483,6 +647,26 @@ func (h *lphttp) StartStreamOrchestrator() http.Handler { dataUrl = pubUrl + "-data" ) + //if data is not enabled remove the url and do not start the data channel + if enableData, ok := (*req.Params)["enableData"]; ok { + if val, ok := enableData.(bool); ok { + //turn off data channel if request sets to false + if !val { + dataUrl = "" + } else { + clog.Infof(ctx, "data channel is enabled") + } + } else { + clog.Warningf(ctx, "enableData is not a bool, got type %T", enableData) + } + + //delete the param used for go-livepeer signaling + delete((*req.Params), "enableData") + } else { + //default to no data channel + dataUrl = "" + } + // Handle initial payment, the rest of the payments are done separately from the stream processing // Note that this payment is debit from the balance and acts as a buffer for the AI Realtime Video processing payment, err := getPayment(r.Header.Get(paymentHeader)) @@ -516,8 +700,12 @@ func (h *lphttp) StartStreamOrchestrator() http.Handler { controlPubCh.CreateChannel() eventsCh := trickle.NewLocalPublisher(h.trickleSrv, mid+"-events", "application/json") eventsCh.CreateChannel() - dataCh := trickle.NewLocalPublisher(h.trickleSrv, mid+"-data", "application/json") - dataCh.CreateChannel() + //optional channels + var dataCh *trickle.TrickleLocalPublisher + if dataUrl != "" { + dataCh = trickle.NewLocalPublisher(h.trickleSrv, mid+"-data", "application/jsonl") + dataCh.CreateChannel() + } // Start payment receiver which accounts the payments and stops the stream if the payment is insufficient priceInfo := payment.GetExpectedPrice() @@ -569,7 +757,11 @@ func (h *lphttp) StartStreamOrchestrator() http.Handler { eventsUrlOverwrite := overwriteHostInStream(h.node.LiveAITrickleHostForRunner, eventsUrl) subscribeUrlOverwrite := overwriteHostInStream(h.node.LiveAITrickleHostForRunner, pubUrl) publishUrlOverwrite := overwriteHostInStream(h.node.LiveAITrickleHostForRunner, subUrl) - dataUrlOverwrite := overwriteHostInStream(h.node.LiveAITrickleHostForRunner, dataUrl) + // optional channels + var dataUrlOverwrite string + if dataCh != nil { + dataUrlOverwrite = overwriteHost(h.node.LiveAITrickleHostForRunner, dataUrl) + } workerReq := worker.LiveVideoToVideoParams{ ModelId: req.ModelId, @@ -619,17 +811,3 @@ func (h *lphttp) StartStreamOrchestrator() http.Handler { respondJsonOk(w, jsonData) }) } - -// This function is copied from ai_http.go to avoid import cycle issues -func overwriteHostInStream(hostOverwrite, url string) string { - if hostOverwrite == "" { - return url - } - u, err := url2.ParseRequestURI(url) - if err != nil { - slog.Warn("Couldn't parse url to overwrite for worker, using original url", "url", url, "err", err) - return url - } - u.Host = hostOverwrite - return u.String() -} diff --git a/server/rpc.go b/server/rpc.go index e6f5368a2a..ab3e848e0e 100644 --- a/server/rpc.go +++ b/server/rpc.go @@ -253,6 +253,8 @@ func StartTranscodeServer(orch Orchestrator, bind string, mux *http.ServeMux, wo lp.transRPC.HandleFunc("/process/token", lp.GetJobToken) lp.transRPC.HandleFunc("/capability/register", lp.RegisterCapability) lp.transRPC.HandleFunc("/capability/unregister", lp.UnregisterCapability) + lp.transRPC.HandleFunc("/stream/start", lp.StartStream) + lp.transRPC.HandleFunc("/stream/payment", lp.StreamPayment) cert, key, err := getCert(orch.ServiceURI(), workDir) if err != nil { From b78405dbfe9febff9b4f87e99c897881d4c849aa Mon Sep 17 00:00:00 2001 From: Brad P Date: Mon, 25 Aug 2025 23:07:27 -0500 Subject: [PATCH 21/57] continue build out of gateway and orchestrator buildout --- core/external_capabilities.go | 73 ++- server/ai_live_video.go | 20 +- server/ai_mediaserver.go | 7 +- server/job_rpc.go | 322 ++++++++----- server/job_stream.go | 834 +++++++++++++++++++++------------- server/job_trickle.go | 513 +++++++++++++++++++++ server/rpc.go | 3 +- 7 files changed, 1329 insertions(+), 443 deletions(-) create mode 100644 server/job_trickle.go diff --git a/core/external_capabilities.go b/core/external_capabilities.go index 66a7c2fb25..8c6c6130f8 100644 --- a/core/external_capabilities.go +++ b/core/external_capabilities.go @@ -11,6 +11,8 @@ import ( ethcommon "github.com/ethereum/go-ethereum/common" "github.com/golang/glog" + "github.com/livepeer/go-livepeer/media" + "github.com/livepeer/go-livepeer/trickle" ) type ExternalCapability struct { @@ -34,14 +36,15 @@ type CapabilityRequirements struct { DataOutput bool `json:"data_output"` } -type StreamData struct { +type StreamInfo struct { StreamID string Capability string //Gateway fields - StreamRequest []byte - OrchToken interface{} - OrchUrl string - ExcludeOrchs []string + StreamRequest []byte + ExcludeOrchs []string + OrchToken interface{} + OrchUrl string + OrchPublishUrl string OrchSubscribeUrl string OrchControlUrl string @@ -53,20 +56,42 @@ type StreamData struct { //Stream fields Params interface{} - ControlPub interface{} + DataWriter *media.SegmentWriter + ControlPub *trickle.TricklePublisher + StopControl func() + JobParams string StreamCtx context.Context CancelStream context.CancelFunc StreamStartedTime time.Time + + sdm sync.Mutex } + +func (sd *StreamInfo) IsActive() bool { + return sd.StreamCtx.Err() == nil +} + +func (sd *StreamInfo) ExcludeOrch(orchUrl string) { + sd.sdm.Lock() + defer sd.sdm.Unlock() + sd.ExcludeOrchs = append(sd.ExcludeOrchs, orchUrl) +} + +func (sd *StreamInfo) UpdateParams(params string) { + sd.sdm.Lock() + defer sd.sdm.Unlock() + sd.JobParams = params +} + type ExternalCapabilities struct { capm sync.Mutex Capabilities map[string]*ExternalCapability - Streams map[string]*StreamData + Streams map[string]*StreamInfo } func NewExternalCapabilities() *ExternalCapabilities { return &ExternalCapabilities{Capabilities: make(map[string]*ExternalCapability), - Streams: make(map[string]*StreamData), + Streams: make(map[string]*StreamInfo), } } @@ -80,13 +105,30 @@ func (extCaps *ExternalCapabilities) AddStream(streamID string, params interface //add to streams ctx, cancel := context.WithCancel(context.Background()) - extCaps.Streams[streamID] = &StreamData{ + stream := StreamInfo{ StreamID: streamID, Params: params, StreamRequest: streamReq, StreamCtx: ctx, CancelStream: cancel, } + extCaps.Streams[streamID] = &stream + + go func() { + for { + select { + case <-ctx.Done(): + if stream.DataWriter != nil { + stream.DataWriter.Close() + } + if stream.ControlPub != nil { + stream.StopControl() + stream.ControlPub.Close() + } + return + } + } + }() return nil } @@ -95,9 +137,22 @@ func (extCaps *ExternalCapabilities) RemoveStream(streamID string) { extCaps.capm.Lock() defer extCaps.capm.Unlock() + streamInfo, ok := extCaps.Streams[streamID] + if ok { + streamInfo.CancelStream() + } + delete(extCaps.Streams, streamID) } +func (extCaps *ExternalCapabilities) StreamExists(streamID string) bool { + extCaps.capm.Lock() + defer extCaps.capm.Unlock() + + _, ok := extCaps.Streams[streamID] + return ok +} + func (extCaps *ExternalCapabilities) RemoveCapability(extCap string) { extCaps.capm.Lock() defer extCaps.capm.Unlock() diff --git a/server/ai_live_video.go b/server/ai_live_video.go index acfc574f40..1e9709e4d6 100644 --- a/server/ai_live_video.go +++ b/server/ai_live_video.go @@ -78,10 +78,15 @@ func startTricklePublish(ctx context.Context, url *url.URL, params aiRequestPara return } + var orchAddr string + var orchUrl string // Start payments which probes a segment every "paymentProcessInterval" and sends a payment ctx, cancel := context.WithCancel(ctx) var paymentProcessor *LivePaymentProcessor if sess != nil { + orchAddr = sess.Address() + orchUrl = sess.Transcoder() + priceInfo := sess.OrchestratorInfo.PriceInfo if priceInfo != nil && priceInfo.PricePerUnit != 0 { paymentSender := livePaymentSender{} @@ -97,6 +102,10 @@ func startTricklePublish(ctx context.Context, url *url.URL, params aiRequestPara } else { clog.Warningf(ctx, "No price info found from Orchestrator, Gateway will not send payments for the video processing") } + } else { + //byoc sets as context values + orchAddr = clog.GetVal(ctx, "orch") + orchUrl = clog.GetVal(ctx, "orch_url") } slowOrchChecker := &SlowOrchChecker{} @@ -166,8 +175,8 @@ func startTricklePublish(ctx context.Context, url *url.URL, params aiRequestPara "pipeline_id": params.liveParams.pipelineID, "request_id": params.liveParams.requestID, "orchestrator_info": map[string]interface{}{ - "address": sess.Address(), - "url": sess.Transcoder(), + "address": orchAddr, + "url": orchUrl, }, }) } @@ -722,6 +731,13 @@ func startEventsSubscribe(ctx context.Context, url *url.URL, params aiRequestPar "address": sess.Address(), "url": sess.Transcoder(), } + } else { + address := clog.GetVal(ctx, "orch") + url := clog.GetVal(ctx, "orch_url") + event["orchestrator_info"] = map[string]interface{}{ + "address": address, + "url": url, + } } clog.V(8).Infof(ctx, "Received event for seq=%d event=%+v", trickle.GetSeq(segment), event) diff --git a/server/ai_mediaserver.go b/server/ai_mediaserver.go index 23e832e4c4..a48784a38c 100644 --- a/server/ai_mediaserver.go +++ b/server/ai_mediaserver.go @@ -116,10 +116,15 @@ func startAIMediaServer(ctx context.Context, ls *LivepeerServer) error { //API for dynamic capabilities ls.HTTPMux.Handle("/process/request/", ls.SubmitJob()) ls.HTTPMux.Handle("/stream/start", ls.StartStream()) - ls.HTTPMux.Handle("/stream/{streamId}/whip", ls.StartStreamWhipIngest()) + ls.HTTPMux.Handle("/stream/stop", ls.StopStream()) + if os.Getenv("LIVE_AI_WHIP_ADDR") != "" { + streamWhipServer := media.NewWHIPServer() + ls.HTTPMux.Handle("/stream/{streamId}/whip", ls.StartStreamWhipIngest(streamWhipServer)) + } ls.HTTPMux.Handle("/stream/{streamId}/rtmp", ls.StartStreamRTMPIngest()) ls.HTTPMux.Handle("/stream/{streamId}/update", ls.UpdateStream()) ls.HTTPMux.Handle("/stream/{streamId}/status", ls.GetStreamStatus()) + ls.HTTPMux.Handle("/stream/{streamId}/data", ls.GetStreamData()) media.StartFileCleanup(ctx, ls.LivepeerNode.WorkDir) diff --git a/server/job_rpc.go b/server/job_rpc.go index 698a148910..304ec2cfe4 100644 --- a/server/job_rpc.go +++ b/server/job_rpc.go @@ -44,6 +44,11 @@ const jobOrchSearchTimeoutDefault = 1 * time.Second const jobOrchSearchRespTimeoutDefault = 500 * time.Millisecond var errNoTimeoutSet = errors.New("no timeout_seconds set with request, timeout_seconds is required") +var errNoCapabilityCapacity = errors.New("No capacity available for capability") +var errNoJobCreds = errors.New("Could not verify job creds") +var errPaymentError = errors.New("Could not parse payment") +var errInsufficientBalance = errors.New("Insufficient balance for request") + var sendJobReqWithTimeout = sendReqWithTimeout type JobSender struct { @@ -77,7 +82,13 @@ type JobRequestDetails struct { StreamId string `json:"stream_id"` } type JobParameters struct { + //Gateway Orchestrators JobOrchestratorsFilter `json:"orchestrators,omitempty"` //list of orchestrators to use for the job + + //Orchestrator + EnableVideoIngress bool `json:"enable_video_ingress,omitempty"` + EnableVideoEgress bool `json:"enable_video_egress,omitempty"` + EnableDataOutput bool `json:"enable_data_output,omitempty"` } type JobOrchestratorsFilter struct { @@ -86,9 +97,16 @@ type JobOrchestratorsFilter struct { } type orchJob struct { - Req *JobRequest - Details *JobRequestDetails - Params *JobParameters + Req *JobRequest + Details *JobRequestDetails + Params *JobParameters + + //Orchestrator fields + Sender ethcommon.Address + JobPrice *net.PriceInfo +} +type gatewayJob struct { + Job *orchJob Orchs []JobToken JobReqHdr string } @@ -264,7 +282,7 @@ func (h *lphttp) GetJobToken(w http.ResponseWriter, r *http.Request) { json.NewEncoder(w).Encode(jobToken) } -func (ls *LivepeerServer) setupJob(ctx context.Context, r *http.Request) (*orchJob, error) { +func (ls *LivepeerServer) setupGatewayJob(ctx context.Context, r *http.Request) (*gatewayJob, error) { clog.Infof(ctx, "processing job request") var orchs []JobToken @@ -314,9 +332,12 @@ func (ls *LivepeerServer) setupJob(ctx context.Context, r *http.Request) (*orchJ } jobReqHdr = base64.StdEncoding.EncodeToString(jobReqEncoded) - return &orchJob{Req: jobReq, - Details: &jobDetails, - Params: &jobParams, + job := orchJob{Req: jobReq, + Details: &jobDetails, + Params: &jobParams, + } + + return &gatewayJob{Job: &job, Orchs: orchs, JobReqHdr: jobReqHdr}, nil } @@ -349,21 +370,21 @@ func (ls *LivepeerServer) SubmitJob() http.Handler { func (ls *LivepeerServer) submitJob(ctx context.Context, w http.ResponseWriter, r *http.Request) { - orchJob, err := ls.setupJob(ctx, r) + gatewayJob, err := ls.setupGatewayJob(ctx, r) if err != nil { clog.Errorf(ctx, "Error setting up job: %s", err) http.Error(w, err.Error(), http.StatusBadRequest) return } - clog.Infof(ctx, "Job request setup complete details=%v params=%v", orchJob.Details, orchJob.Params) + clog.Infof(ctx, "Job request setup complete details=%v params=%v", gatewayJob.Job.Details, gatewayJob.Job.Params) if err != nil { http.Error(w, fmt.Sprintf("Unable to setup job err=%v", err), http.StatusBadRequest) return } - ctx = clog.AddVal(ctx, "job_id", orchJob.Req.ID) - ctx = clog.AddVal(ctx, "capability", orchJob.Req.Capability) + ctx = clog.AddVal(ctx, "job_id", gatewayJob.Job.Req.ID) + ctx = clog.AddVal(ctx, "capability", gatewayJob.Job.Req.Capability) // Read the original request body body, err := io.ReadAll(r.Body) if err != nil { @@ -374,7 +395,7 @@ func (ls *LivepeerServer) submitJob(ctx context.Context, w http.ResponseWriter, //send the request to the Orchestrator(s) //the loop ends on Gateway error and bad request errors - for _, orchToken := range orchJob.Orchs { + for _, orchToken := range gatewayJob.Orchs { // Extract the worker resource route from the URL path // The prefix is "/process/request/" @@ -390,7 +411,7 @@ func (ls *LivepeerServer) submitJob(ctx context.Context, w http.ResponseWriter, } start := time.Now() - resp, code, err := ls.sendJobToOrch(ctx, r, orchJob.Req, orchJob.JobReqHdr, orchToken, workerResourceRoute, body) + resp, code, err := ls.sendJobToOrch(ctx, r, gatewayJob.Job.Req, gatewayJob.JobReqHdr, orchToken, workerResourceRoute, body) if err != nil { clog.Errorf(ctx, "job not able to be processed by Orchestrator %v err=%v ", orchToken.ServiceAddr, err.Error()) continue @@ -436,7 +457,7 @@ func (ls *LivepeerServer) submitJob(ctx context.Context, w http.ResponseWriter, continue } - gatewayBalance := updateGatewayBalance(ls.LivepeerNode, orchToken, orchJob.Req.Capability, time.Since(start)) + gatewayBalance := updateGatewayBalance(ls.LivepeerNode, orchToken, gatewayJob.Job.Req.Capability, time.Since(start)) clog.V(common.SHORT).Infof(ctx, "Job processed successfully took=%v balance=%v balance_from_orch=%v", time.Since(start), gatewayBalance.FloatString(0), orchBalance) w.Write(data) return @@ -459,7 +480,7 @@ func (ls *LivepeerServer) submitJob(ctx context.Context, w http.ResponseWriter, w.WriteHeader(http.StatusOK) // Read from upstream and forward to client respChan := make(chan string, 100) - respCtx, _ := context.WithTimeout(ctx, time.Duration(orchJob.Req.Timeout+10)*time.Second) //include a small buffer to let Orchestrator close the connection on the timeout + respCtx, _ := context.WithTimeout(ctx, time.Duration(gatewayJob.Job.Req.Timeout+10)*time.Second) //include a small buffer to let Orchestrator close the connection on the timeout go func() { defer resp.Body.Close() @@ -500,7 +521,7 @@ func (ls *LivepeerServer) submitJob(ctx context.Context, w http.ResponseWriter, } } - gatewayBalance := updateGatewayBalance(ls.LivepeerNode, orchToken, orchJob.Req.Capability, time.Since(start)) + gatewayBalance := updateGatewayBalance(ls.LivepeerNode, orchToken, gatewayJob.Job.Req.Capability, time.Since(start)) clog.V(common.SHORT).Infof(ctx, "Job processed successfully took=%v balance=%v balance_from_orch=%v", time.Since(start), gatewayBalance.FloatString(0), orchBalance.FloatString(0)) } @@ -544,23 +565,31 @@ func (ls *LivepeerServer) sendJobToOrch(ctx context.Context, r *http.Request, jo return resp, resp.StatusCode, nil } -func (ls *LivepeerServer) sendPayment(ctx context.Context, orchUrl string, payment string) error { - req, err := http.NewRequestWithContext(ctx, "POST", orchUrl+"/stream/payment", nil) +func (ls *LivepeerServer) sendPayment(ctx context.Context, orchPmtUrl, capability, jobReq, payment string) (*JobToken, error) { + req, err := http.NewRequestWithContext(ctx, "POST", orchPmtUrl, nil) if err != nil { clog.Errorf(ctx, "Unable to create request err=%v", err) - return err + return nil, err } req.Header.Add("Content-Type", "application/json") + req.Header.Add(jobRequestHdr, jobReq) req.Header.Add(jobPaymentHeaderHdr, payment) - _, err = sendJobReqWithTimeout(req, 10) + resp, err := sendJobReqWithTimeout(req, 10) if err != nil { - clog.Errorf(ctx, "job payment not able to be processed by Orchestrator %v err=%v ", orchUrl, err.Error()) - return err + clog.Errorf(ctx, "job payment not able to be processed by Orchestrator %v err=%v ", orchPmtUrl, err.Error()) + return nil, err } - return nil + var jobToken JobToken + if err := json.NewDecoder(resp.Body).Decode(&jobToken); err != nil { + clog.Errorf(ctx, "Error decoding job token response: %v", err) + return nil, err + } + + //return the job token with new ticketparams, balance and price. Not all fields filled out + return &jobToken, nil } func processJob(ctx context.Context, h *lphttp, w http.ResponseWriter, r *http.Request) { @@ -569,77 +598,20 @@ func processJob(ctx context.Context, h *lphttp, w http.ResponseWriter, r *http.R orch := h.orchestrator // check the prompt sig from the request // confirms capacity available before processing payment info - job := r.Header.Get(jobRequestHdr) - jobReq, err := verifyJobCreds(ctx, orch, job) + orchJob, err := h.setupOrchJob(ctx, r) if err != nil { - if err == errZeroCapacity { - clog.Errorf(ctx, "No capacity available for capability err=%q", err) + if err == errNoCapabilityCapacity { http.Error(w, err.Error(), http.StatusServiceUnavailable) - } else if err == errNoTimeoutSet { - clog.Errorf(ctx, "Timeout not set in request err=%q", err) - http.Error(w, err.Error(), http.StatusBadRequest) } else { - clog.Errorf(ctx, "Could not verify job creds err=%q", err) - http.Error(w, err.Error(), http.StatusForbidden) + http.Error(w, err.Error(), http.StatusBadRequest) } - return } - - sender := ethcommon.HexToAddress(jobReq.Sender) - jobPrice, err := orch.JobPriceInfo(sender, jobReq.Capability) - if err != nil { - clog.Errorf(ctx, "could not get price err=%v", err.Error()) - http.Error(w, fmt.Sprintf("Could not get price err=%v", err.Error()), http.StatusBadRequest) - return - } - clog.V(common.DEBUG).Infof(ctx, "job price=%v units=%v", jobPrice.PricePerUnit, jobPrice.PixelsPerUnit) taskId := core.RandomManifestID() - jobId := jobReq.Capability - ctx = clog.AddVal(ctx, "job_id", jobReq.ID) + ctx = clog.AddVal(ctx, "job_id", orchJob.Req.ID) ctx = clog.AddVal(ctx, "worker_task_id", string(taskId)) - ctx = clog.AddVal(ctx, "capability", jobReq.Capability) - ctx = clog.AddVal(ctx, "sender", jobReq.Sender) - - //no payment included, confirm if balance remains - jobPriceRat := big.NewRat(jobPrice.PricePerUnit, jobPrice.PixelsPerUnit) - var payment net.Payment - // if price is 0, no payment required - if jobPriceRat.Cmp(big.NewRat(0, 1)) > 0 { - // get payment information - payment, err = getPayment(r.Header.Get(jobPaymentHeaderHdr)) - if err != nil { - clog.Errorf(r.Context(), "Could not parse payment: %v", err) - http.Error(w, err.Error(), http.StatusPaymentRequired) - return - } - - if payment.TicketParams == nil { - - //if price is not 0, confirm balance - if jobPriceRat.Cmp(big.NewRat(0, 1)) > 0 { - minBal := jobPriceRat.Mul(jobPriceRat, big.NewRat(60, 1)) //minimum 1 minute balance - orchBal := getPaymentBalance(orch, sender, jobId) - - if orchBal.Cmp(minBal) < 0 { - clog.Errorf(ctx, "Insufficient balance for request") - http.Error(w, "Insufficient balance", http.StatusPaymentRequired) - orch.FreeExternalCapabilityCapacity(jobReq.Capability) - return - } - } - } else { - if err := orch.ProcessPayment(ctx, payment, core.ManifestID(jobId)); err != nil { - clog.Errorf(ctx, "error processing payment err=%q", err) - http.Error(w, err.Error(), http.StatusBadRequest) - orch.FreeExternalCapabilityCapacity(jobReq.Capability) - return - } - } - - clog.Infof(ctx, "balance after payment is %v", getPaymentBalance(orch, sender, jobId).FloatString(0)) - } - + ctx = clog.AddVal(ctx, "capability", orchJob.Req.Capability) + ctx = clog.AddVal(ctx, "sender", orchJob.Req.Sender) clog.V(common.SHORT).Infof(ctx, "Received job, sending for processing") // Read the original body @@ -659,7 +631,7 @@ func processJob(ctx context.Context, h *lphttp, w http.ResponseWriter, r *http.R workerResourceRoute = workerResourceRoute[len(prefix):] } - workerRoute := jobReq.CapabilityUrl + workerRoute := orchJob.Req.CapabilityUrl if workerResourceRoute != "" { workerRoute = workerRoute + "/" + workerResourceRoute } @@ -674,18 +646,18 @@ func processJob(ctx context.Context, h *lphttp, w http.ResponseWriter, r *http.R req.Header.Add("Content-Type", r.Header.Get("Content-Type")) start := time.Now() - resp, err := sendReqWithTimeout(req, time.Duration(jobReq.Timeout)*time.Second) + resp, err := sendReqWithTimeout(req, time.Duration(orchJob.Req.Timeout)*time.Second) if err != nil { clog.Errorf(ctx, "job not able to be processed err=%v ", err.Error()) //if the request failed with connection error, remove the capability //exclude deadline exceeded or context canceled errors does not indicate a fatal error all the time if err != context.DeadlineExceeded && !strings.Contains(err.Error(), "context canceled") { - clog.Errorf(ctx, "removing capability %v due to error %v", jobReq.Capability, err.Error()) - h.orchestrator.RemoveExternalCapability(jobReq.Capability) + clog.Errorf(ctx, "removing capability %v due to error %v", orchJob.Req.Capability, err.Error()) + h.orchestrator.RemoveExternalCapability(orchJob.Req.Capability) } - chargeForCompute(start, jobPrice, orch, sender, jobId) - w.Header().Set(jobPaymentBalanceHdr, getPaymentBalance(orch, sender, jobId).FloatString(0)) + chargeForCompute(start, orchJob.JobPrice, orch, orchJob.Sender, orchJob.Req.Capability) + w.Header().Set(jobPaymentBalanceHdr, getPaymentBalance(orch, orchJob.Sender, orchJob.Req.Capability).FloatString(0)) http.Error(w, fmt.Sprintf("job not able to be processed, removing capability err=%v", err.Error()), http.StatusInternalServerError) return } @@ -695,7 +667,7 @@ func processJob(ctx context.Context, h *lphttp, w http.ResponseWriter, r *http.R //release capacity for another request // if requester closes the connection need to release capacity - defer orch.FreeExternalCapabilityCapacity(jobReq.Capability) + defer orch.FreeExternalCapabilityCapacity(orchJob.Req.Capability) if !strings.Contains(resp.Header.Get("Content-Type"), "text/event-stream") { //non streaming response @@ -705,8 +677,8 @@ func processJob(ctx context.Context, h *lphttp, w http.ResponseWriter, r *http.R if err != nil { clog.Errorf(ctx, "Unable to read response err=%v", err) - chargeForCompute(start, jobPrice, orch, sender, jobId) - w.Header().Set(jobPaymentBalanceHdr, getPaymentBalance(orch, sender, jobId).FloatString(0)) + chargeForCompute(start, orchJob.JobPrice, orch, orchJob.Sender, orchJob.Req.Capability) + w.Header().Set(jobPaymentBalanceHdr, getPaymentBalance(orch, orchJob.Sender, orchJob.Req.Capability).FloatString(0)) http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -715,16 +687,16 @@ func processJob(ctx context.Context, h *lphttp, w http.ResponseWriter, r *http.R if resp.StatusCode > 399 { clog.Errorf(ctx, "error processing request err=%v ", string(data)) - chargeForCompute(start, jobPrice, orch, sender, jobId) - w.Header().Set(jobPaymentBalanceHdr, getPaymentBalance(orch, sender, jobId).FloatString(0)) + chargeForCompute(start, orchJob.JobPrice, orch, orchJob.Sender, orchJob.Req.Capability) + w.Header().Set(jobPaymentBalanceHdr, getPaymentBalance(orch, orchJob.Sender, orchJob.Req.Capability).FloatString(0)) //return error response from the worker http.Error(w, string(data), resp.StatusCode) return } - chargeForCompute(start, jobPrice, orch, sender, jobId) - w.Header().Set(jobPaymentBalanceHdr, getPaymentBalance(orch, sender, jobId).FloatString(0)) - clog.V(common.SHORT).Infof(ctx, "Job processed successfully took=%v balance=%v", time.Since(start), getPaymentBalance(orch, sender, jobId).FloatString(0)) + chargeForCompute(start, orchJob.JobPrice, orch, orchJob.Sender, orchJob.Req.Capability) + w.Header().Set(jobPaymentBalanceHdr, getPaymentBalance(orch, orchJob.Sender, orchJob.Req.Capability).FloatString(0)) + clog.V(common.SHORT).Infof(ctx, "Job processed successfully took=%v balance=%v", time.Since(start), getPaymentBalance(orch, orchJob.Sender, orchJob.Req.Capability).FloatString(0)) w.Write(data) //request completed and returned a response @@ -737,22 +709,22 @@ func processJob(ctx context.Context, h *lphttp, w http.ResponseWriter, r *http.R w.Header().Set("Cache-Control", "no-cache") w.Header().Set("Connection", "keep-alive") //send payment balance back so client can determine if payment is needed - addPaymentBalanceHeader(w, orch, sender, jobId) + addPaymentBalanceHeader(w, orch, orchJob.Sender, orchJob.Req.Capability) // Flush to ensure data is sent immediately flusher, ok := w.(http.Flusher) if !ok { clog.Errorf(ctx, "streaming not supported") - chargeForCompute(start, jobPrice, orch, sender, jobId) - w.Header().Set(jobPaymentBalanceHdr, getPaymentBalance(orch, sender, jobId).FloatString(0)) + chargeForCompute(start, orchJob.JobPrice, orch, orchJob.Sender, orchJob.Req.Capability) + w.Header().Set(jobPaymentBalanceHdr, getPaymentBalance(orch, orchJob.Sender, orchJob.Req.Capability).FloatString(0)) http.Error(w, "Streaming not supported", http.StatusInternalServerError) return } // Read from upstream and forward to client respChan := make(chan string, 100) - respCtx, _ := context.WithTimeout(ctx, time.Duration(jobReq.Timeout)*time.Second) + respCtx, _ := context.WithTimeout(ctx, time.Duration(orchJob.Req.Timeout)*time.Second) go func() { defer resp.Body.Close() @@ -761,7 +733,7 @@ func processJob(ctx context.Context, h *lphttp, w http.ResponseWriter, r *http.R for scanner.Scan() { select { case <-respCtx.Done(): - orchBal := orch.Balance(sender, core.ManifestID(jobId)) + orchBal := orch.Balance(orchJob.Sender, core.ManifestID(orchJob.Req.Capability)) if orchBal == nil { orchBal = big.NewRat(0, 1) } @@ -771,7 +743,7 @@ func processJob(ctx context.Context, h *lphttp, w http.ResponseWriter, r *http.R default: line := scanner.Text() if strings.Contains(line, "[DONE]") { - orchBal := orch.Balance(sender, core.ManifestID(jobId)) + orchBal := orch.Balance(orchJob.Sender, core.ManifestID(orchJob.Req.Capability)) if orchBal == nil { orchBal = big.NewRat(0, 1) } @@ -793,9 +765,10 @@ func processJob(ctx context.Context, h *lphttp, w http.ResponseWriter, r *http.R case <-pmtWatcher.C: //check balance and end response if out of funds //skips if price is 0 + jobPriceRat := big.NewRat(orchJob.JobPrice.PricePerUnit, orchJob.JobPrice.PixelsPerUnit) if jobPriceRat.Cmp(big.NewRat(0, 1)) > 0 { - h.orchestrator.DebitFees(sender, core.ManifestID(jobId), jobPrice, 5) - senderBalance := getPaymentBalance(orch, sender, jobId) + h.orchestrator.DebitFees(orchJob.Sender, core.ManifestID(orchJob.Req.Capability), orchJob.JobPrice, 5) + senderBalance := getPaymentBalance(orch, orchJob.Sender, orchJob.Req.Capability) if senderBalance != nil { if senderBalance.Cmp(big.NewRat(0, 1)) < 0 { w.Write([]byte("event: insufficient balance\n")) @@ -815,10 +788,71 @@ func processJob(ctx context.Context, h *lphttp, w http.ResponseWriter, r *http.R } //capacity released with defer stmt above - clog.V(common.SHORT).Infof(ctx, "Job processed successfully took=%v balance=%v", time.Since(start), getPaymentBalance(orch, sender, jobId).FloatString(0)) + clog.V(common.SHORT).Infof(ctx, "Job processed successfully took=%v balance=%v", time.Since(start), getPaymentBalance(orch, orchJob.Sender, orchJob.Req.Capability).FloatString(0)) } } +// SetupOrchJob prepares the orchestrator job by extracting and validating the job request from the HTTP headers. +// Payment is applied if applicable. +func (h *lphttp) setupOrchJob(ctx context.Context, r *http.Request) (*orchJob, error) { + clog.Infof(ctx, "processing job request") + + job := r.Header.Get(jobRequestHdr) + orch := h.orchestrator + jobReq, err := verifyJobCreds(ctx, orch, job) + if err != nil { + if err == errZeroCapacity { + return nil, errNoCapabilityCapacity + } else if err == errNoTimeoutSet { + return nil, errNoTimeoutSet + } else { + return nil, errNoJobCreds + } + } + + sender := ethcommon.HexToAddress(jobReq.Sender) + jobPrice, err := orch.JobPriceInfo(sender, jobReq.Capability) + if err != nil { + return nil, errors.New("Could not get job price") + } + clog.V(common.DEBUG).Infof(ctx, "job price=%v units=%v", jobPrice.PricePerUnit, jobPrice.PixelsPerUnit) + + //no payment included, confirm if balance remains + jobPriceRat := big.NewRat(jobPrice.PricePerUnit, jobPrice.PixelsPerUnit) + var payment net.Payment + // if price is 0, no payment required + if jobPriceRat.Cmp(big.NewRat(0, 1)) > 0 { + // get payment information + payment, err = getPayment(r.Header.Get(jobPaymentHeaderHdr)) + if err != nil { + return nil, errPaymentError + } + + if payment.TicketParams == nil { + + //if price is not 0, confirm balance + if jobPriceRat.Cmp(big.NewRat(0, 1)) > 0 { + minBal := jobPriceRat.Mul(jobPriceRat, big.NewRat(60, 1)) //minimum 1 minute balance + orchBal := getPaymentBalance(orch, sender, jobReq.Capability) + + if orchBal.Cmp(minBal) < 0 { + orch.FreeExternalCapabilityCapacity(jobReq.Capability) + return nil, errInsufficientBalance + } + } + } else { + if err := orch.ProcessPayment(ctx, payment, core.ManifestID(jobReq.Capability)); err != nil { + orch.FreeExternalCapabilityCapacity(jobReq.Capability) + return nil, errPaymentError + } + } + + clog.Infof(ctx, "balance after payment is %v", getPaymentBalance(orch, sender, jobReq.Capability).FloatString(0)) + } + + return &orchJob{Req: jobReq, Sender: sender, JobPrice: jobPrice}, nil +} + func createPayment(ctx context.Context, jobReq *JobRequest, orchToken JobToken, node *core.LivepeerNode) (string, error) { var payment *net.Payment sender := ethcommon.HexToAddress(jobReq.Sender) @@ -1081,20 +1115,12 @@ func getOrchSearchTimeouts(ctx context.Context, searchTimeoutHdr, respTimeoutHdr func getJobOrchestrators(ctx context.Context, node *core.LivepeerNode, capability string, params JobParameters, timeout time.Duration, respTimeout time.Duration) ([]JobToken, error) { orchs := node.OrchestratorPool.GetInfos() - gateway := node.OrchestratorPool.Broadcaster() - //setup the GET request to get the Orchestrator tokens - //get the address and sig for the sender - gatewayReq, err := genOrchestratorReq(gateway, GetOrchestratorInfoParams{}) + reqSender, err := getJobSender(ctx, node) if err != nil { - clog.Errorf(ctx, "Failed to generate request for Orchestrator to verify to request job token err=%v", err) + clog.Errorf(ctx, "Failed to get job sender err=%v", err) return nil, err } - addr := ethcommon.BytesToAddress(gatewayReq.Address) - reqSender := &JobSender{ - Addr: addr.Hex(), - Sig: "0x" + hex.EncodeToString(gatewayReq.Sig), - } getOrchJobToken := func(ctx context.Context, orchUrl *url.URL, reqSender JobSender, respTimeout time.Duration, tokenCh chan JobToken, errCh chan error) { start := time.Now() @@ -1180,3 +1206,61 @@ func getJobOrchestrators(ctx context.Context, node *core.LivepeerNode, capabilit return jobTokens, nil } + +func getJobSender(ctx context.Context, node *core.LivepeerNode) (*JobSender, error) { + gateway := node.OrchestratorPool.Broadcaster() + orchReq, err := genOrchestratorReq(gateway, GetOrchestratorInfoParams{}) + if err != nil { + clog.Errorf(ctx, "Failed to generate request for Orchestrator to verify to request job token err=%v", err) + return nil, err + } + addr := ethcommon.BytesToAddress(orchReq.Address) + jobSender := &JobSender{ + Addr: addr.Hex(), + Sig: "0x" + hex.EncodeToString(orchReq.Sig), + } + + return jobSender, nil +} +func getToken(ctx context.Context, respTimeout time.Duration, orchUrl, capability, sender, senderSig string) (*JobToken, error) { + start := time.Now() + tokenReq, err := http.NewRequestWithContext(ctx, "GET", orchUrl+"/process/token", nil) + jobSender := JobSender{Addr: sender, Sig: senderSig} + + reqSenderStr, _ := json.Marshal(jobSender) + tokenReq.Header.Set(jobEthAddressHdr, base64.StdEncoding.EncodeToString(reqSenderStr)) + tokenReq.Header.Set(jobCapabilityHdr, capability) + if err != nil { + clog.Errorf(ctx, "Failed to create request for Orchestrator to verify job token request err=%v", err) + return nil, err + } + + resp, err := sendJobReqWithTimeout(tokenReq, respTimeout) + if err != nil { + clog.Errorf(ctx, "failed to get token from Orchestrator err=%v", err) + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + clog.Errorf(ctx, "Failed to get token from Orchestrator %v err=%v", orchUrl, err) + return nil, fmt.Errorf("failed to get token from Orchestrator") + } + + latency := time.Since(start) + clog.V(common.DEBUG).Infof(ctx, "Received job token from uri=%v, latency=%v", orchUrl, latency) + + token, err := io.ReadAll(resp.Body) + if err != nil { + clog.Errorf(ctx, "Failed to read token from Orchestrator %v err=%v", orchUrl, err) + return nil, err + } + var jobToken JobToken + err = json.Unmarshal(token, &jobToken) + if err != nil { + clog.Errorf(ctx, "Failed to unmarshal token from Orchestrator %v err=%v", orchUrl, err) + return nil, err + } + + return &jobToken, nil +} diff --git a/server/job_stream.go b/server/job_stream.go index 9257e6e508..93abb4768f 100644 --- a/server/job_stream.go +++ b/server/job_stream.go @@ -1,20 +1,22 @@ package server import ( + "bytes" "context" + "encoding/base64" + "encoding/hex" "encoding/json" "errors" "fmt" "io" - "log/slog" + "math/big" "net/http" - "net/url" "os" "strings" "time" ethcommon "github.com/ethereum/go-ethereum/common" - "github.com/livepeer/go-livepeer/ai/worker" + "github.com/golang/glog" "github.com/livepeer/go-livepeer/clog" "github.com/livepeer/go-livepeer/common" "github.com/livepeer/go-livepeer/core" @@ -30,21 +32,22 @@ func (ls *LivepeerServer) StartStream() http.Handler { // Create fresh context instead of using r.Context() since ctx will outlive the request ctx := r.Context() - orchJob, err := ls.setupJob(ctx, r) + //verify request, get orchestrators available and sign request + gatewayJob, err := ls.setupGatewayJob(ctx, r) if err != nil { clog.Errorf(ctx, "Error setting up job: %s", err) http.Error(w, err.Error(), http.StatusBadRequest) return } - resp, code, err := ls.setupStream(ctx, r, orchJob.Req) + resp, code, err := ls.setupStream(ctx, r, gatewayJob.Job.Req) if err != nil { clog.Errorf(ctx, "Error setting up stream: %s", err) http.Error(w, err.Error(), code) return } - go ls.runStream(orchJob, resp["stream_id"]) + go ls.runStream(gatewayJob, resp["stream_id"]) if resp != nil { // Stream started successfully @@ -58,7 +61,45 @@ func (ls *LivepeerServer) StartStream() http.Handler { }) } -func (ls *LivepeerServer) runStream(job *orchJob, streamID string) { +func (ls *LivepeerServer) StopStream() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Create fresh context instead of using r.Context() since ctx will outlive the request + ctx := r.Context() + streamId := r.PathValue("streamId") + + if streamInfo, exists := ls.LivepeerNode.ExternalCapabilities.Streams[streamId]; exists { + // Copy streamInfo before deletion + streamInfoCopy := *streamInfo + //remove the stream + ls.LivepeerNode.ExternalCapabilities.RemoveStream(streamId) + + stopJob, err := ls.setupGatewayJob(ctx, r) + if err != nil { + clog.Errorf(ctx, "Error setting up stop job: %s", err) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + resp, code, err := ls.sendJobToOrch(ctx, r, stopJob.Job.Req, stopJob.JobReqHdr, streamInfoCopy.OrchToken.(JobToken), "/stream/stop", streamInfoCopy.StreamRequest) + if err != nil { + clog.Errorf(ctx, "Error sending job to orchestrator: %s", err) + http.Error(w, err.Error(), code) + return + } + + w.WriteHeader(http.StatusOK) + io.Copy(w, resp.Body) + return + } + + //no stream exists + w.WriteHeader(http.StatusNoContent) + return + + }) +} + +func (ls *LivepeerServer) runStream(gatewayJob *gatewayJob, streamID string) { ctx := context.Background() stream, exists := ls.LivepeerNode.ExternalCapabilities.Streams[streamID] params := stream.Params.(aiRequestParams) @@ -67,17 +108,32 @@ func (ls *LivepeerServer) runStream(job *orchJob, streamID string) { return } - for _, orch := range job.Orchs { - orchResp, _, err := ls.sendJobToOrch(ctx, nil, job.Req, job.JobReqHdr, orch, "/stream/start", stream.StreamRequest) + start := time.Now() + for _, orch := range gatewayJob.Orchs { + ctx = clog.AddVal(ctx, "orch", ethcommon.Bytes2Hex(orch.TicketParams.Recipient)) + ctx = clog.AddVal(ctx, "orch_url", orch.ServiceAddr) + //refresh the token if after 5 minutes from start. TicketParams expire in 40 blocks (about 8 minutes). + if time.Since(start) > 5*time.Minute { + orch, err := getToken(ctx, 3*time.Second, orch.ServiceAddr, gatewayJob.Job.Req.Capability, gatewayJob.Job.Req.Sender, gatewayJob.Job.Req.Sig) + if err != nil { + clog.Errorf(ctx, "Error getting token for orch=%v err=%v", orch.ServiceAddr, err) + continue + } + } + + //set request ID to persist from Gateway to Worker + gatewayJob.Job.Req.ID = stream.StreamID + + orchResp, _, err := ls.sendJobToOrch(ctx, nil, gatewayJob.Job.Req, gatewayJob.JobReqHdr, orch, "/stream/start", stream.StreamRequest) if err != nil { clog.Errorf(ctx, "job not able to be processed by Orchestrator %v err=%v ", orch.ServiceAddr, err.Error()) continue } - stream.OrchPublishUrl = orchResp.Header.Get("Publish-Url") - stream.OrchSubscribeUrl = orchResp.Header.Get("Subscribe-Url") - stream.OrchControlUrl = orchResp.Header.Get("Control-Url") - stream.OrchEventsUrl = orchResp.Header.Get("Events-Url") - stream.OrchDataUrl = orchResp.Header.Get("Data-Url") + stream.OrchPublishUrl = orchResp.Header.Get("X-Publish-Url") + stream.OrchSubscribeUrl = orchResp.Header.Get("X-Subscribe-Url") + stream.OrchControlUrl = orchResp.Header.Get("X-Control-Url") + stream.OrchEventsUrl = orchResp.Header.Get("X-Events-Url") + stream.OrchDataUrl = orchResp.Header.Get("X-Data-Url") perOrchCtx, perOrchCancel := context.WithCancelCause(ctx) params.liveParams = newParams(params.liveParams, perOrchCancel) @@ -86,6 +142,7 @@ func (ls *LivepeerServer) runStream(job *orchJob, streamID string) { perOrchCancel(err) break } + //something caused the Orch to stop performing, try to get the error and move to next Orchestrator <-perOrchCtx.Done() err = context.Cause(perOrchCtx) if errors.Is(err, context.Canceled) { @@ -93,7 +150,7 @@ func (ls *LivepeerServer) runStream(job *orchJob, streamID string) { // or if passing `nil` as a CancelCause err = nil } - if !params.inputStreamExists() { + if !ls.LivepeerNode.ExternalCapabilities.StreamExists(streamID) { clog.Info(ctx, "No input stream, skipping orchestrator swap") break } @@ -129,29 +186,54 @@ func (ls *LivepeerServer) monitorStream(ctx context.Context, streamId string) { } // Create a ticker that runs every minute for payments - ticker := time.NewTicker(45 * time.Second) - defer ticker.Stop() + pmtTicker := time.NewTicker(45 * time.Second) + defer pmtTicker.Stop() for { select { case <-stream.StreamCtx.Done(): clog.Infof(ctx, "Stream %s stopped, ending monitoring", streamId) return - case <-ticker.C: - // Run payment and fetch new JobToken every minute + case <-pmtTicker.C: + // Send payment and fetch new JobToken every minute req := &JobRequest{Capability: stream.Capability, Sender: ls.LivepeerNode.OrchestratorPool.Broadcaster().Address().Hex(), Timeout: 60, } + //sign the request + gateway := ls.LivepeerNode.OrchestratorPool.Broadcaster() + sig, err := gateway.Sign([]byte(req.Request + req.Parameters)) + if err != nil { + clog.Errorf(ctx, fmt.Sprintf("Unable to sign request err=%v", err)) + } + req.Sender = gateway.Address().Hex() + req.Sig = "0x" + hex.EncodeToString(sig) + + //create the job request header with the signature + jobReqEncoded, err := json.Marshal(req) + if err != nil { + clog.Errorf(ctx, fmt.Sprintf("Unable to encode job request err=%v", err)) + } + jobReqHdr := base64.StdEncoding.EncodeToString(jobReqEncoded) + pmtHdr, err := createPayment(ctx, req, stream.OrchToken.(JobToken), ls.LivepeerNode) if err != nil { clog.Errorf(ctx, "Error processing stream payment for %s: %v", streamId, err) // Continue monitoring even if payment fails } + //send the payment, update the stream with the refreshed token + token, err := ls.sendPayment(ctx, stream.OrchUrl+"/stream/payment", stream.Capability, jobReqHdr, pmtHdr) + if err != nil { + clog.Errorf(ctx, "Error sending stream payment for %s: %v", streamId, err) + } + streamToken := stream.OrchToken.(JobToken) + streamToken.TicketParams = token.TicketParams + streamToken.Balance = token.Balance + streamToken.Price = token.Price + stream.OrchToken = streamToken } } - } func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, req *JobRequest) (map[string]string, int, error) { @@ -169,25 +251,24 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, req return nil, http.StatusBadRequest, errors.New("missing source_id") } ctx = clog.AddVal(ctx, "source_id", sourceID) - sourceType := r.FormValue("source_type") - if sourceType == "" { - return nil, http.StatusBadRequest, errors.New("missing source_type") - } - sourceTypeStr, err := media.MediamtxSourceTypeToString(sourceType) - if err != nil { - return nil, http.StatusBadRequest, errors.New("invalid source type") - } - ctx = clog.AddVal(ctx, "source_type", sourceType) + //sourceType := r.FormValue("source_type") + //if sourceType == "" { + // return nil, http.StatusBadRequest, errors.New("missing source_type") + //} + //sourceTypeStr, err := media.MediamtxSourceTypeToString(sourceType) + //if err != nil { + // return nil, http.StatusBadRequest, errors.New("invalid source type") + //} + //ctx = clog.AddVal(ctx, "source_type", sourceType) - //TODO: change the params in query to separate form fields since we - // are setting things up with a POST request - queryParams := r.FormValue("query") - qp, err := url.ParseQuery(queryParams) - if err != nil { - return nil, http.StatusBadRequest, errors.New("invalid query params") + streamParams := r.FormValue("params") + var streamParamsJson map[string]interface{} + if err := json.Unmarshal([]byte(streamParams), &streamParamsJson); err != nil { + return nil, http.StatusBadRequest, errors.New("invalid stream params") } + // If auth webhook is set and returns an output URL, this will be replaced - outputURL := qp.Get("rtmpOutput") + outputURL := r.FormValue("rtmpOutput") // convention to avoid re-subscribing to our own streams // in case we want to push outputs back into mediamtx - @@ -198,12 +279,12 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, req } // if auth webhook returns pipeline config these will be replaced - pipeline := qp.Get("pipeline") + pipeline := streamParamsJson["pipeline"].(string) if pipeline == "" { pipeline = req.Capability } - rawParams := qp.Get("params") - streamID := qp.Get("streamId") + rawParams := streamParamsJson["params"].(string) + streamID := streamParamsJson["streamId"].(string) var pipelineID string var pipelineParams map[string]interface{} @@ -243,8 +324,8 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, req if LiveAIAuthWebhookURL != nil { authResp, err := authenticateAIStream(LiveAIAuthWebhookURL, ls.liveAIAuthApiKey, AIAuthRequest{ Stream: streamName, - Type: sourceTypeStr, - QueryParams: queryParams, + Type: "", //sourceTypeStr + QueryParams: streamParams, GatewayHost: ls.LivepeerNode.GatewayHost, WhepURL: whepURL, UpdateURL: updateURL, @@ -330,7 +411,6 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, req // WHIP // kickInput will close the whip connection // localRTMPPrefix set by ENV variable LIVE_AI_PLAYBACK_HOST - //kickInput is set with RTMP ingest ssr := media.NewSwitchableSegmentReader() //this converts ingest to segments to send to Orchestrator params := aiRequestParams{ node: ls.LivepeerNode, @@ -352,29 +432,25 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, req } //create a dataWriter for data channel if enabled - if enableData, ok := pipelineParams["enableData"]; ok { - if enableData == true || enableData == "true" { - params.liveParams.dataWriter = media.NewSegmentWriter(5) - pipelineParams["enableData"] = true - clog.Infof(ctx, "Data channel enabled for stream %s", streamName) - } - } - - if _, ok := pipelineParams["enableVideoIngress"]; !ok { - pipelineParams["enableVideoIngress"] = true + var jobParams JobParameters + err := json.Unmarshal([]byte(req.Parameters), &jobParams) + if err != nil { + return nil, http.StatusBadRequest, errors.New("invalid job parameters") } - - if _, ok := pipelineParams["enableVideoEgress"]; !ok { - pipelineParams["enableVideoEgress"] = true + if jobParams.EnableDataOutput { + params.liveParams.dataWriter = media.NewSegmentWriter(5) } - //send start request to Orchestrator + //check if stream exists _, exists := ls.LivepeerNode.ExternalCapabilities.Streams[streamID] if exists { return nil, http.StatusBadRequest, fmt.Errorf("stream already exists: %s", streamID) } - // read entire body to ensure valid and send to orchestrator + clog.Infof(ctx, "stream setup videoIngress=%v videoEgress=%v dataOutput=%v", jobParams.EnableVideoIngress, jobParams.EnableVideoEgress, jobParams.EnableDataOutput) + + // read entire body to ensure valid and store the request to send to Orchestrator(s) + // there is a round robin mechanism to add resiliency, try body, err := io.ReadAll(r.Body) if err != nil { return nil, http.StatusInternalServerError, errors.New("Error reading request body") @@ -398,7 +474,7 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, req return resp, http.StatusOK, nil } -// mediamtx sends this request to go-livepeer when rtmp strem received +// mediamtx sends this request to go-livepeer when rtmp stream received func (ls *LivepeerServer) StartStreamRTMPIngest() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { remoteAddr := getRemoteAddr(r) @@ -426,10 +502,10 @@ func (ls *LivepeerServer) StartStreamRTMPIngest() http.Handler { return } mediaMTXClient := media.NewMediaMTXClient(mediaMtxHost, ls.mediaMTXApiPassword, "rtmp_ingest", "rtmp") + segmenterCtx, cancelSegmenter := context.WithCancel(clog.Clone(context.Background(), ctx)) // this function is called when the pipeline hits a fatal error, we kick the input connection to allow // the client to reconnect and restart the pipeline - segmenterCtx, cancelSegmenter := context.WithCancel(clog.Clone(context.Background(), ctx)) kickInput := func(err error) { defer cancelSegmenter() if err == nil { @@ -469,93 +545,66 @@ func (ls *LivepeerServer) StartStreamRTMPIngest() http.Handler { }, }) params.liveParams.segmentReader.Close() - cleanupControl(ctx, params) + + stream.CancelStream() //cleanupControl(ctx, params) }() + }) +} - var req worker.GenLiveVideoToVideoJSONRequestBody - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - respondJsonError(ctx, w, err, http.StatusBadRequest) +func (ls *LivepeerServer) StartStreamWhipIngest(whipServer *media.WHIPServer) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + remoteAddr := getRemoteAddr(r) + ctx := clog.AddVal(r.Context(), clog.ClientIP, remoteAddr) + requestID := string(core.RandomManifestID()) + ctx = clog.AddVal(ctx, "request_id", requestID) + + streamId := r.PathValue("streamId") + ctx = clog.AddVal(ctx, "stream_id", streamId) + + stream, ok := ls.LivepeerNode.ExternalCapabilities.Streams[streamId] + if !ok { + respondJsonError(ctx, w, fmt.Errorf("stream not found: %s", streamId), http.StatusNotFound) return } - }) -} -func startStream(ctx context.Context, streamData *core.StreamData, req worker.GenLiveVideoToVideoJSONRequestBody) { - params := streamData.Params.(aiRequestParams) - orchSwapper := NewOrchestratorSwapper(params) - isFirst, firstProcessed := true, make(chan interface{}) - go func() { - var err error - for { - perOrchCtx, perOrchCancel := context.WithCancelCause(ctx) - params.liveParams = newParams(params.liveParams, perOrchCancel) + params := stream.Params.(aiRequestParams) - if err != nil { - clog.Errorf(ctx, "Error processing AI Request: %s", err) - perOrchCancel(err) - break - } + whipConn := media.NewWHIPConnection() + whepURL := generateWhepUrl(streamId, requestID) - if err = startStreamProcessing(perOrchCtx, streamData); err != nil { - clog.Errorf(ctx, "Error starting processing: %s", err) - perOrchCancel(err) - break - } - if isFirst { - isFirst = false - firstProcessed <- struct{}{} - } - <-perOrchCtx.Done() - err = context.Cause(perOrchCtx) - if errors.Is(err, context.Canceled) { - // this happens if parent ctx was cancelled without a CancelCause - // or if passing `nil` as a CancelCause - err = nil - } - if !params.inputStreamExists() { - clog.Info(ctx, "No input stream, skipping orchestrator swap") - break - } - if swapErr := orchSwapper.checkSwap(ctx); swapErr != nil { - if err != nil { - err = fmt.Errorf("%w: %w", swapErr, err) - } else { - err = swapErr - } - break - } - clog.Infof(ctx, "Retrying stream with a different orchestrator") + //wait for the WHIP connection to close and then cleanup + go func() { + statsContext, statsCancel := context.WithCancel(ctx) + defer statsCancel() + go runStats(statsContext, whipConn, streamId, stream.Capability, requestID) + + whipConn.AwaitClose() + stream.CancelStream() //cleanupControl(ctx, params) + params.liveParams.segmentReader.Close() + params.liveParams.kickOrch(errors.New("whip ingest disconnected")) + clog.Info(ctx, "Live cleaned up") + }() + + conn := whipServer.CreateWHIP(ctx, params.liveParams.segmentReader, whepURL, w, r) + whipConn.SetWHIPConnection(conn) // might be nil if theres an error and thats okay + }) - // will swap, but first notify with the reason for the swap - if err == nil { - err = errors.New("unknown swap reason") - } - params.liveParams.sendErrorEvent(err) - } - if isFirst { - // failed before selecting an orchestrator, exit the stream, something is wrong - firstProcessed <- struct{}{} - } - params.liveParams.kickInput(err) - }() - <-firstProcessed } -func startStreamProcessing(ctx context.Context, streamData *core.StreamData) error { - params := streamData.Params.(aiRequestParams) - host := streamData.OrchUrl +func startStreamProcessing(ctx context.Context, streamInfo *core.StreamInfo) error { + params := streamInfo.Params.(aiRequestParams) var channels []string //this adds the stream to LivePipelines which the Control Publisher and Data Writer //are accessible for reading data and sending updates - registerControl(ctx, params) + //registerControl(ctx, params) //required channels - control, err := common.AppendHostname(streamData.OrchControlUrl, host) + control, err := common.AppendHostname(streamInfo.OrchControlUrl, streamInfo.OrchUrl) if err != nil { return fmt.Errorf("invalid control URL: %w", err) } - events, err := common.AppendHostname(streamData.OrchEventsUrl, host) + events, err := common.AppendHostname(streamInfo.OrchEventsUrl, streamInfo.OrchUrl) if err != nil { return fmt.Errorf("invalid events URL: %w", err) } @@ -563,251 +612,414 @@ func startStreamProcessing(ctx context.Context, streamData *core.StreamData) err channels = append(channels, control.String()) channels = append(channels, events.String()) startControlPublish(ctx, control, params) - startEventsSubscribe(ctx, events, params, params.liveParams.sess) + startEventsSubscribe(ctx, events, params, nil) //Optional channels - if streamData.OrchPublishUrl == "" { - pub, err := common.AppendHostname(streamData.OrchPublishUrl, host) + if streamInfo.OrchPublishUrl == "" { + pub, err := common.AppendHostname(streamInfo.OrchPublishUrl, streamInfo.OrchUrl) if err != nil { return fmt.Errorf("invalid publish URL: %w", err) } channels = append(channels, pub.String()) - startTricklePublish(ctx, pub, params, params.liveParams.sess) + startTricklePublish(ctx, pub, params, nil) } - if streamData.OrchSubscribeUrl == "" { - sub, err := common.AppendHostname(streamData.OrchSubscribeUrl, host) + if streamInfo.OrchSubscribeUrl == "" { + sub, err := common.AppendHostname(streamInfo.OrchSubscribeUrl, streamInfo.OrchUrl) if err != nil { return fmt.Errorf("invalid subscribe URL: %w", err) } channels = append(channels, sub.String()) - startTrickleSubscribe(ctx, sub, params, params.liveParams.sess) + startTrickleSubscribe(ctx, sub, params, nil) } - if streamData.OrchDataUrl == "" { - data, err := common.AppendHostname(streamData.OrchDataUrl, host) + if streamInfo.OrchDataUrl == "" { + data, err := common.AppendHostname(streamInfo.OrchDataUrl, streamInfo.OrchUrl) if err != nil { return fmt.Errorf("invalid data URL: %w", err) } - streamData.Params.(aiRequestParams).liveParams.manifestID = streamData.Capability + streamInfo.Params.(aiRequestParams).liveParams.manifestID = streamInfo.Capability - startDataSubscribe(ctx, data, params, params.liveParams.sess) + startDataSubscribe(ctx, data, params, nil) } return nil } -// StartStreamOrchestrator handles the POST /stream/start endpoint for the Orchestrator -func (h *lphttp) StartStreamOrchestrator() http.Handler { +func (ls *LivepeerServer) GetStreamData() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - remoteAddr := getRemoteAddr(r) - ctx := clog.AddVal(r.Context(), clog.ClientIP, remoteAddr) - - streamID := r.Header.Get("streamID") - gatewayRequestID := r.Header.Get("requestID") - requestID := string(core.RandomManifestID()) - ctx = clog.AddVal(ctx, "orch_request_id", requestID) - ctx = clog.AddVal(ctx, "gateway_request_id", gatewayRequestID) - ctx = clog.AddVal(ctx, "manifest_id", requestID) - ctx = clog.AddVal(ctx, "stream_id", streamID) - - var req worker.GenLiveVideoToVideoJSONRequestBody - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - respondWithError(w, err.Error(), http.StatusBadRequest) + streamId := r.PathValue("streamId") + if streamId == "" { + http.Error(w, "stream name is required", http.StatusBadRequest) return } - orch := h.orchestrator - pipeline := "byoc-stream" - cap := core.Capability_LiveVideoToVideo - modelID := *req.ModelId - clog.V(common.VERBOSE).Infof(ctx, "Received request id=%v cap=%v modelID=%v", requestID, cap, modelID) + ctx := r.Context() + ctx = clog.AddVal(ctx, "stream", streamId) - // Create storage for the request (for AI Workers, must run before CheckAICapacity) - err := orch.CreateStorageForRequest(requestID) - if err != nil { - respondWithError(w, "Could not create storage to receive results", http.StatusInternalServerError) + // Get the live pipeline for this stream + stream, ok := ls.LivepeerNode.ExternalCapabilities.Streams[streamId] + if !ok { + http.Error(w, "Stream not found", http.StatusNotFound) + return + } + params := stream.Params.(aiRequestParams) + // Get the data reading buffer + if params.liveParams.dataWriter == nil { + clog.Infof(ctx, "No data writer available for stream %s", stream) + http.Error(w, "Stream data not available", http.StatusServiceUnavailable) + return } + dataReader := params.liveParams.dataWriter.MakeReader(media.SegmentReaderConfig{}) - // Check if there is capacity for the request - hasCapacity, _ := orch.CheckAICapacity(pipeline, modelID) - if !hasCapacity { - clog.Errorf(ctx, "Insufficient capacity for pipeline=%v modelID=%v", pipeline, modelID) - respondWithError(w, "insufficient capacity", http.StatusServiceUnavailable) + // Set up SSE headers + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Connection", "keep-alive") + w.Header().Set("Access-Control-Allow-Origin", "*") + + flusher, ok := w.(http.Flusher) + if !ok { + http.Error(w, "Streaming not supported", http.StatusInternalServerError) return } - // Start trickle server for live-video - var ( - mid = requestID // Request ID is used for the manifest ID - pubUrl = orch.ServiceURI().JoinPath(TrickleHTTPPath, mid).String() - subUrl = pubUrl + "-out" - controlUrl = pubUrl + "-control" - eventsUrl = pubUrl + "-events" - dataUrl = pubUrl + "-data" - ) - - //if data is not enabled remove the url and do not start the data channel - if enableData, ok := (*req.Params)["enableData"]; ok { - if val, ok := enableData.(bool); ok { - //turn off data channel if request sets to false - if !val { - dataUrl = "" - } else { - clog.Infof(ctx, "data channel is enabled") + clog.Infof(ctx, "Starting SSE data stream for stream=%s", stream) + + // Listen for broadcast signals from ring buffer writes + // dataReader.Read() blocks on rb.cond.Wait() until startDataSubscribe broadcasts + for { + select { + case <-ctx.Done(): + clog.Info(ctx, "SSE data stream client disconnected") + return + default: + reader, err := dataReader.Next() + if err != nil { + if err == io.EOF { + // Stream ended + fmt.Fprintf(w, `event: end\ndata: {"type":"stream_ended"}\n\n`) + flusher.Flush() + return + } + clog.Errorf(ctx, "Error reading from ring buffer: %v", err) + return } - } else { - clog.Warningf(ctx, "enableData is not a bool, got type %T", enableData) + + data, err := io.ReadAll(reader) + fmt.Fprintf(w, "data: %s\n\n", data) + flusher.Flush() } + } + }) +} - //delete the param used for go-livepeer signaling - delete((*req.Params), "enableData") - } else { - //default to no data channel - dataUrl = "" +func (ls *LivepeerServer) UpdateStream() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + if r.Method != http.MethodPost { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + // Get stream from path param + streamId := r.PathValue("streamId") + if streamId == "" { + http.Error(w, "Missing stream name", http.StatusBadRequest) + return + } + stream, ok := ls.LivepeerNode.ExternalCapabilities.Streams[streamId] + if !ok { + // Stream not found + http.Error(w, "Stream not found", http.StatusNotFound) + return } - // Handle initial payment, the rest of the payments are done separately from the stream processing - // Note that this payment is debit from the balance and acts as a buffer for the AI Realtime Video processing - payment, err := getPayment(r.Header.Get(paymentHeader)) + reader := http.MaxBytesReader(w, r.Body, 10*1024*1024) // 10 MB + defer reader.Close() + data, err := io.ReadAll(reader) if err != nil { - respondWithError(w, err.Error(), http.StatusPaymentRequired) + http.Error(w, err.Error(), http.StatusBadRequest) return } - sender := getPaymentSender(payment) - _, ctx, err = verifySegCreds(ctx, h.orchestrator, r.Header.Get(segmentHeader), sender) - if err != nil { - respondWithError(w, err.Error(), http.StatusForbidden) + + params := string(data) + stream.JobParams = params + controlPub := stream.ControlPub + + if controlPub == nil { + clog.Info(ctx, "No orchestrator available, caching params", "stream", stream, "params", params) return } - if err := orch.ProcessPayment(ctx, payment, core.ManifestID(mid)); err != nil { - respondWithError(w, err.Error(), http.StatusBadRequest) + + clog.V(6).Infof(ctx, "Sending Live Video Update Control API stream=%s, params=%s", stream, params) + if err := controlPub.Write(strings.NewReader(params)); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) return } - if payment.GetExpectedPrice().GetPricePerUnit() > 0 && !orch.SufficientBalance(sender, core.ManifestID(mid)) { - respondWithError(w, "Insufficient balance", http.StatusBadRequest) + + corsHeaders(w, r.Method) + }) +} + +func (ls *LivepeerServer) GetStreamStatus() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + + corsHeaders(w, r.Method) + + streamId := r.PathValue("streamId") + if streamId == "" { + http.Error(w, "stream id is required", http.StatusBadRequest) return } - // If successful, then create the trickle channels - // Precreate the channels to avoid race conditions - // TODO get the expected mime type from the request + ctx := r.Context() + ctx = clog.AddVal(ctx, "stream", streamId) + + // Get status for specific stream + status, exists := StreamStatusStore.Get(streamId) + gatewayStatus, gatewayExists := GatewayStatus.Get(streamId) + if !exists && !gatewayExists { + http.Error(w, "Stream not found", http.StatusNotFound) + return + } + if gatewayExists { + if status == nil { + status = make(map[string]any) + } + status["gateway_status"] = gatewayStatus + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(status); err != nil { + clog.Errorf(ctx, "Failed to encode stream status err=%v", err) + http.Error(w, "Failed to encode status", http.StatusInternalServerError) + return + } + }) +} + +// StartStream handles the POST /stream/start endpoint for the Orchestrator +func (h *lphttp) StartStream(w http.ResponseWriter, r *http.Request) { + orch := h.orchestrator + remoteAddr := getRemoteAddr(r) + ctx := clog.AddVal(r.Context(), clog.ClientIP, remoteAddr) + + orchJob, err := h.setupOrchJob(ctx, r) + if err != nil { + respondWithError(w, err.Error(), http.StatusBadRequest) + return + } + ctx = clog.AddVal(ctx, "stream_id", orchJob.Req.ID) + + workerRoute := orchJob.Req.CapabilityUrl + "/stream/start" + + // Read the original body + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "Error reading request body", http.StatusBadRequest) + return + } + r.Body.Close() + + req, err := http.NewRequestWithContext(ctx, "POST", workerRoute, bytes.NewBuffer(body)) + var jobParams JobParameters + err = json.Unmarshal([]byte(orchJob.Req.Parameters), &jobParams) + if err != nil { + clog.Errorf(ctx, "unable to parse parameters err=%v", err) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Start trickle server for live-video + var ( + mid = orchJob.Req.ID // Request ID is used for the manifest ID + pubUrl = h.orchestrator.ServiceURI().JoinPath(TrickleHTTPPath, mid).String() + subUrl = pubUrl + "-out" + controlUrl = pubUrl + "-control" + eventsUrl = pubUrl + "-events" + dataUrl = pubUrl + "-data" + ) + + //required channels + controlPubCh := trickle.NewLocalPublisher(h.trickleSrv, mid+"-control", "application/json") + controlPubCh.CreateChannel() + controlUrl = overwriteHost(h.node.LiveAITrickleHostForRunner, controlUrl) + req.Header.Set("X-Control-Url", controlUrl) + + eventsCh := trickle.NewLocalPublisher(h.trickleSrv, mid+"-events", "application/json") + eventsCh.CreateChannel() + eventsUrl = overwriteHost(h.node.LiveAITrickleHostForRunner, eventsUrl) + req.Header.Set("X-Events-Url", eventsUrl) + + //Optional channels + if jobParams.EnableVideoIngress { pubCh := trickle.NewLocalPublisher(h.trickleSrv, mid, "video/MP2T") pubCh.CreateChannel() + pubUrl = overwriteHost(h.node.LiveAITrickleHostForRunner, pubUrl) + req.Header.Set("X-Publish-Url", pubUrl) + } + + if jobParams.EnableVideoEgress { subCh := trickle.NewLocalPublisher(h.trickleSrv, mid+"-out", "video/MP2T") subCh.CreateChannel() - controlPubCh := trickle.NewLocalPublisher(h.trickleSrv, mid+"-control", "application/json") - controlPubCh.CreateChannel() - eventsCh := trickle.NewLocalPublisher(h.trickleSrv, mid+"-events", "application/json") - eventsCh.CreateChannel() - //optional channels - var dataCh *trickle.TrickleLocalPublisher - if dataUrl != "" { - dataCh = trickle.NewLocalPublisher(h.trickleSrv, mid+"-data", "application/jsonl") - dataCh.CreateChannel() - } - - // Start payment receiver which accounts the payments and stops the stream if the payment is insufficient - priceInfo := payment.GetExpectedPrice() - var paymentProcessor *LivePaymentProcessor - ctx, cancel := context.WithCancel(context.Background()) - if priceInfo != nil && priceInfo.PricePerUnit != 0 { - paymentReceiver := livePaymentReceiver{orchestrator: h.orchestrator} - accountPaymentFunc := func(inPixels int64) error { - err := paymentReceiver.AccountPayment(context.Background(), &SegmentInfoReceiver{ - sender: sender, - inPixels: inPixels, - priceInfo: priceInfo, - sessionID: mid, - }) - if err != nil { - slog.Warn("Error accounting payment, stopping stream processing", "err", err) - pubCh.Close() - subCh.Close() - eventsCh.Close() - controlPubCh.Close() - cancel() + subUrl = overwriteHost(h.node.LiveAITrickleHostForRunner, subUrl) + req.Header.Set("X-Subscribe-Url", subUrl) + } + + if jobParams.EnableDataOutput { + dataCh := trickle.NewLocalPublisher(h.trickleSrv, mid+"-data", "application/jsonl") + dataCh.CreateChannel() + dataUrl = overwriteHost(h.node.LiveAITrickleHostForRunner, dataUrl) + req.Header.Set("X-Data-Url", dataUrl) + } + + // set the headers + req.Header.Add("Content-Length", r.Header.Get("Content-Length")) + req.Header.Add("Content-Type", r.Header.Get("Content-Type")) + + start := time.Now() + resp, err := sendReqWithTimeout(req, time.Duration(orchJob.Req.Timeout)*time.Second) + if err != nil { + clog.Errorf(ctx, "Error sending request to worker %v: %v", workerRoute, err) + respondWithError(w, "Error sending request to worker", http.StatusInternalServerError) + return + } + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + clog.Errorf(ctx, "Error reading response body: %v", err) + respondWithError(w, "Error reading response body", http.StatusInternalServerError) + return + } + defer resp.Body.Close() + w.WriteHeader(resp.StatusCode) + //error response from worker but assume can retry and pass along error response and status code + if resp.StatusCode > 399 { + clog.Errorf(ctx, "error processing /stream/start request statusCode=%d", resp.StatusCode) + + chargeForCompute(start, orchJob.JobPrice, orch, orchJob.Sender, orchJob.Req.Capability) + w.Header().Set(jobPaymentBalanceHdr, getPaymentBalance(orch, orchJob.Sender, orchJob.Req.Capability).FloatString(0)) + //return error response from the worker + w.WriteHeader(resp.StatusCode) + w.Write(respBody) + return + } + + chargeForCompute(start, orchJob.JobPrice, orch, orchJob.Sender, orchJob.Req.Capability) + w.Header().Set(jobPaymentBalanceHdr, getPaymentBalance(orch, orchJob.Sender, orchJob.Req.Capability).FloatString(0)) + + clog.V(common.SHORT).Infof(ctx, "Job processed successfully took=%v balance=%v", time.Since(start), getPaymentBalance(orch, orchJob.Sender, orchJob.Req.Capability).FloatString(0)) + + //setup the stream + h.node.ExternalCapabilities.AddStream(orchJob.Req.ID, orchJob.Req, respBody) + + //start payment monitoring + go func() { + pmtTicker := time.NewTicker(10 * time.Second) + defer pmtTicker.Stop() + for range pmtTicker.C { + // Check payment status + jobPriceRat := big.NewRat(orchJob.JobPrice.PricePerUnit, orchJob.JobPrice.PixelsPerUnit) + if jobPriceRat.Cmp(big.NewRat(0, 1)) > 0 { + h.orchestrator.DebitFees(orchJob.Sender, core.ManifestID(orchJob.Req.Capability), orchJob.JobPrice, 5) + senderBalance := getPaymentBalance(orch, orchJob.Sender, orchJob.Req.Capability) + if senderBalance != nil { + if senderBalance.Cmp(big.NewRat(0, 1)) < 0 { + clog.Infof(ctx, "Insufficient balance, stopping stream %s for sender %s", orchJob.Req.ID, orchJob.Sender) + _, exists := h.node.ExternalCapabilities.Streams[orchJob.Req.ID] + if exists { + h.node.ExternalCapabilities.RemoveStream(orchJob.Req.ID) + } + } } - return err } - paymentProcessor = NewLivePaymentProcessor(ctx, h.node.LivePaymentInterval, accountPaymentFunc) - } else { - clog.Warningf(ctx, "No price info found for model %v, Orchestrator will not charge for video processing", modelID) } + }() - // Subscribe to the publishUrl for payments monitoring and payment processing - go func() { - sub := trickle.NewLocalSubscriber(h.trickleSrv, mid) - for { - segment, err := sub.Read() - if err != nil { - clog.Infof(ctx, "Error getting local trickle segment err=%v", err) - return - } - reader := segment.Reader - if paymentProcessor != nil { - reader = paymentProcessor.process(ctx, segment.Reader) - } - io.Copy(io.Discard, reader) - } - }() + //send back trickle urls + w.Header().Set("X-Publish-Url", req.Header.Get("X-Publish-Url")) + w.Header().Set("X-Subscribe-Url", req.Header.Get("X-Subscribe-Url")) + w.Header().Set("X-Control-Url", req.Header.Get("X-Control-Url")) + w.Header().Set("X-Events-Url", req.Header.Get("X-Events-Url")) + w.Header().Set("X-Data-Url", req.Header.Get("X-Data-Url")) - // Prepare request to worker - controlUrlOverwrite := overwriteHostInStream(h.node.LiveAITrickleHostForRunner, controlUrl) - eventsUrlOverwrite := overwriteHostInStream(h.node.LiveAITrickleHostForRunner, eventsUrl) - subscribeUrlOverwrite := overwriteHostInStream(h.node.LiveAITrickleHostForRunner, pubUrl) - publishUrlOverwrite := overwriteHostInStream(h.node.LiveAITrickleHostForRunner, subUrl) - // optional channels - var dataUrlOverwrite string - if dataCh != nil { - dataUrlOverwrite = overwriteHost(h.node.LiveAITrickleHostForRunner, dataUrl) - } - - workerReq := worker.LiveVideoToVideoParams{ - ModelId: req.ModelId, - PublishUrl: publishUrlOverwrite, - SubscribeUrl: subscribeUrlOverwrite, - EventsUrl: &eventsUrlOverwrite, - ControlUrl: &controlUrlOverwrite, - DataUrl: &dataUrlOverwrite, - Params: req.Params, - GatewayRequestId: &gatewayRequestID, - ManifestId: &mid, - StreamId: &streamID, - } - - // Send request to the worker - _, err = orch.LiveVideoToVideo(ctx, requestID, workerReq) - if err != nil { - if monitor.Enabled { - monitor.AIProcessingError(err.Error(), pipeline, modelID, ethcommon.Address{}.String()) - } + w.WriteHeader(http.StatusOK) + return +} - pubCh.Close() - subCh.Close() - controlPubCh.Close() - eventsCh.Close() - dataCh.Close() - cancel() - respondWithError(w, err.Error(), http.StatusInternalServerError) - return +func (h *lphttp) StopStream(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + orchJob, err := h.setupOrchJob(ctx, r) + if err != nil { + respondWithError(w, fmt.Sprintf("Failed to stop stream, request not valid err=%v", err), http.StatusBadRequest) + return + } + + var jobDetails JobRequestDetails + err = json.Unmarshal([]byte(orchJob.Req.Request), &jobDetails) + if err != nil { + respondWithError(w, fmt.Sprintf("Failed to stop stream, request not valid, failed to parse stream id err=%v", err), http.StatusBadRequest) + return + } + + // Stop the stream + h.node.ExternalCapabilities.RemoveStream(jobDetails.StreamId) + + w.Write([]byte("Stream stopped successfully")) +} + +func (h *lphttp) ProcessStreamPayment(w http.ResponseWriter, r *http.Request) { + orch := h.orchestrator + ctx := r.Context() + + //this will process the payment + orchJob, err := h.setupOrchJob(ctx, r) + if err != nil { + respondWithError(w, fmt.Sprintf("Failed to process payment, request not valid err=%v", err), http.StatusBadRequest) + return + } + + senderAddr := ethcommon.HexToAddress(orchJob.Req.Sender) + + jobToken.SenderAddress.Addr = orchJob.Req.Sender + jobPrice, err := orch.JobPriceInfo(senderAddr, orchJob.Req.Capability) + if err != nil { + statusCode := http.StatusBadRequest + if err.Error() == "insufficient sender reserve" { + statusCode = http.StatusServiceUnavailable } + glog.Errorf("could not get price err=%v", err.Error()) + http.Error(w, fmt.Sprintf("Could not get price err=%v", err.Error()), statusCode) + return + } + ticketParams, err := orch.TicketParams(senderAddr, jobPrice) + if err != nil { + glog.Errorf("could not get ticket params err=%v", err.Error()) + http.Error(w, fmt.Sprintf("Could not get ticket params err=%v", err.Error()), http.StatusBadRequest) + return + } - // Prepare the response - jsonData, err := json.Marshal(&worker.LiveVideoToVideoResponse{ - PublishUrl: pubUrl, - SubscribeUrl: subUrl, - ControlUrl: &controlUrl, - EventsUrl: &eventsUrl, - RequestId: &requestID, - ManifestId: &mid, - }) + capBal := orch.Balance(senderAddr, core.ManifestID(orchJob.Req.Capability)) + if capBal != nil { + capBal, err = common.PriceToInt64(capBal) if err != nil { - respondWithError(w, err.Error(), http.StatusInternalServerError) - return + clog.Errorf(context.TODO(), "could not convert balance to int64 sender=%v capability=%v err=%v", senderAddr.Hex(), orchJob.Req.Capability, err.Error()) + capBal = big.NewRat(0, 1) } + } else { + capBal = big.NewRat(0, 1) + } + //convert to int64. Note: returns with 000 more digits to allow for precision of 3 decimal places. + capBalInt, err := common.PriceToFixed(capBal) + if err != nil { + glog.Errorf("could not convert balance to int64 sender=%v capability=%v err=%v", senderAddr.Hex(), orchJob.Req.Capability, err.Error()) + capBalInt = 0 + } else { + // Remove the last three digits from capBalInt + capBalInt = capBalInt / 1000 + } - clog.Infof(ctx, "Processed request id=%v cap=%v modelID=%v took=%v", requestID, cap, modelID) - respondJsonOk(w, jsonData) - }) + jobToken := JobToken{SenderAddress: nil, TicketParams: ticketParams, Balance: capBalInt, Price: jobPrice} + + json.NewEncoder(w).Encode(jobToken) } diff --git a/server/job_trickle.go b/server/job_trickle.go new file mode 100644 index 0000000000..1257fc1411 --- /dev/null +++ b/server/job_trickle.go @@ -0,0 +1,513 @@ +package server + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "sync" + "time" + + "github.com/dustin/go-humanize" + "github.com/livepeer/go-livepeer/clog" + "github.com/livepeer/go-livepeer/common" + "github.com/livepeer/go-livepeer/core" + "github.com/livepeer/go-livepeer/media" + "github.com/livepeer/go-livepeer/monitor" + "github.com/livepeer/go-livepeer/trickle" +) + +func startStreamTricklePublish(ctx context.Context, url *url.URL, streamInfo *core.StreamInfo) { + ctx = clog.AddVal(ctx, "url", url.Redacted()) + params := streamInfo.Params.(aiRequestParams) + + publisher, err := trickle.NewTricklePublisher(url.String()) + if err != nil { + stopProcessing(ctx, params, fmt.Errorf("trickle publish init err: %w", err)) + return + } + + // Start payments which probes a segment every "paymentProcessInterval" and sends a payment + ctx, cancel := context.WithCancel(ctx) + var paymentProcessor *LivePaymentProcessor + //byoc sets as context values + orchAddr := clog.GetVal(ctx, "orch") + orchUrl := clog.GetVal(ctx, "orch_url") + + slowOrchChecker := &SlowOrchChecker{} + + firstSegment := true + + params.liveParams.segmentReader.SwitchReader(func(reader media.CloneableReader) { + // check for end of stream + if _, eos := reader.(*media.EOSReader); eos { + if err := publisher.Close(); err != nil { + clog.Infof(ctx, "Error closing trickle publisher. err=%v", err) + } + cancel() + return + } + thisSeq, atMax := slowOrchChecker.BeginSegment() + if atMax { + clog.Infof(ctx, "Orchestrator is slow - terminating") + streamInfo.ExcludeOrch(orchUrl) //suspendOrchestrator(ctx, params) + cancel() + stopProcessing(ctx, params, errors.New("orchestrator is slow")) + return + } + go func(seq int) { + defer slowOrchChecker.EndSegment() + var r io.Reader = reader + if paymentProcessor != nil { + r = paymentProcessor.process(ctx, reader) + } + + clog.V(8).Infof(ctx, "trickle publish writing data seq=%d", seq) + segment, err := publisher.Next() + if err != nil { + clog.Infof(ctx, "error getting next publish handle; dropping segment err=%v", err) + params.liveParams.sendErrorEvent(fmt.Errorf("Missing next handle %v", err)) + return + } + for { + select { + case <-ctx.Done(): + clog.Info(ctx, "trickle publish done") + return + default: + } + + startTime := time.Now() + currentSeq := slowOrchChecker.GetCount() + if seq != currentSeq { + clog.Infof(ctx, "Next segment has already started; skipping this one seq=%d currentSeq=%d", seq, currentSeq) + params.liveParams.sendErrorEvent(fmt.Errorf("Next segment has started")) + segment.Close() + return + } + params.liveParams.mu.Lock() + params.liveParams.lastSegmentTime = startTime + params.liveParams.mu.Unlock() + logToDisk(ctx, reader, params.node.WorkDir, params.liveParams.requestID, seq) + n, err := segment.Write(r) + if err == nil { + // no error, all done, let's leave + if monitor.Enabled && firstSegment { + firstSegment = false + monitor.SendQueueEventAsync("stream_trace", map[string]interface{}{ + "type": "gateway_send_first_ingest_segment", + "timestamp": time.Now().UnixMilli(), + "stream_id": params.liveParams.streamID, + "pipeline_id": params.liveParams.pipelineID, + "request_id": params.liveParams.requestID, + "orchestrator_info": map[string]interface{}{ + "address": orchAddr, + "url": orchUrl, + }, + }) + } + clog.Info(ctx, "trickle publish complete", "wrote", humanize.Bytes(uint64(n)), "seq", seq, "took", time.Since(startTime)) + return + } + if errors.Is(err, trickle.StreamNotFoundErr) { + stopProcessing(ctx, params, errors.New("stream no longer exists on orchestrator; terminating")) + return + } + // Retry segment only if nothing has been sent yet + // and the next segment has not yet started + // otherwise drop + if n > 0 { + clog.Infof(ctx, "Error publishing segment; dropping remainder wrote=%d err=%v", n, err) + params.liveParams.sendErrorEvent(fmt.Errorf("Error publishing, wrote %d dropping %v", n, err)) + segment.Close() + return + } + clog.Infof(ctx, "Error publishing segment before writing; retrying err=%v", err) + // Clone in case read head was incremented somewhere, which cloning resets + r = reader.Clone() + time.Sleep(250 * time.Millisecond) + } + }(thisSeq) + }) + clog.Infof(ctx, "trickle pub") +} + +func startStreamTrickleSubscribe(ctx context.Context, url *url.URL, streamInfo *core.StreamInfo) { + // subscribe to inference outputs and send them into the world + params := streamInfo.Params.(aiRequestParams) + orchAddr := clog.GetVal(ctx, "orch") + orchUrl := clog.GetVal(ctx, "orch_url") + subscriber, err := trickle.NewTrickleSubscriber(trickle.TrickleSubscriberConfig{ + URL: url.String(), + Ctx: ctx, + }) + if err != nil { + stopProcessing(ctx, params, fmt.Errorf("trickle subscription init failed: %w", err)) + return + } + + ctx = clog.AddVal(ctx, "url", url.Redacted()) + + // Set up output buffers and ffmpeg processes + rbc := media.RingBufferConfig{BufferLen: 5_000_000} // 5 MB, 20-30 seconds at current rates + outWriter, err := media.NewRingBuffer(&rbc) + if err != nil { + stopProcessing(ctx, params, fmt.Errorf("ringbuffer init failed: %w", err)) + return + } + // Launch ffmpeg for each configured RTMP output + for _, outURL := range params.liveParams.rtmpOutputs { + go ffmpegOutput(ctx, outURL, outWriter, params) + } + + // watchdog that gets reset on every segment to catch output stalls + segmentTimeout := params.liveParams.outSegmentTimeout + if segmentTimeout <= 0 { + segmentTimeout = 30 * time.Second + } + segmentTicker := time.NewTicker(segmentTimeout) + + // read segments from trickle subscription + go func() { + defer outWriter.Close() + defer segmentTicker.Stop() + + var err error + firstSegment := true + var segmentsReceived int64 + + retries := 0 + // we're trying to keep (retryPause x maxRetries) duration to fall within one output GOP length + const retryPause = 300 * time.Millisecond + const maxRetries = 5 + for { + select { + case <-ctx.Done(): + clog.Info(ctx, "trickle subscribe done") + return + default: + } + if !params.inputStreamExists() { + clog.Infof(ctx, "trickle subscribe stopping, input stream does not exist.") + break + } + segmentTicker.Reset(segmentTimeout) // reset ticker on each iteration. + var segment *http.Response + clog.V(8).Infof(ctx, "trickle subscribe read data await") + segment, err = subscriber.Read() + if err != nil { + if errors.Is(err, trickle.EOS) || errors.Is(err, trickle.StreamNotFoundErr) { + stopProcessing(ctx, params, fmt.Errorf("trickle subscribe stopping, stream not found, err=%w", err)) + return + } + var sequenceNonexistent *trickle.SequenceNonexistent + if errors.As(err, &sequenceNonexistent) { + // stream exists but segment doesn't, so skip to leading edge + subscriber.SetSeq(sequenceNonexistent.Latest) + } + // TODO if not EOS then signal a new orchestrator is needed + err = fmt.Errorf("trickle subscribe error reading: %w", err) + clog.Infof(ctx, "%s", err) + if retries > maxRetries { + stopProcessing(ctx, params, errors.New("trickle subscribe stopping, retries exceeded")) + return + } + retries++ + params.liveParams.sendErrorEvent(err) + time.Sleep(retryPause) + continue + } + retries = 0 + seq := trickle.GetSeq(segment) + clog.V(8).Infof(ctx, "trickle subscribe read data received seq=%d", seq) + copyStartTime := time.Now() + + n, err := copySegment(ctx, segment, outWriter, seq, params) + if err != nil { + if errors.Is(err, context.Canceled) { + clog.Info(ctx, "trickle subscribe stopping - context canceled") + return + } + // Check whether the client has sent data recently. + // TODO ensure the threshold is some multiple of LIVE_AI_MIN_SEG_DUR + params.liveParams.mu.Lock() + lastSegmentTime := params.liveParams.lastSegmentTime + params.liveParams.mu.Unlock() + segmentAge := time.Since(lastSegmentTime) + maxSegmentDelay := params.liveParams.outSegmentTimeout / 2 + if segmentAge < maxSegmentDelay && params.inputStreamExists() { + // we have some recent input but no output from orch, so kick + suspendOrchestrator(ctx, params) + stopProcessing(ctx, params, fmt.Errorf("trickle subscribe error, swapping: %w", err)) + return + } + clog.InfofErr(ctx, "trickle subscribe error copying segment seq=%d", seq, err) + subscriber.SetSeq(seq) + retries++ + continue + } + if firstSegment { + firstSegment = false + delayMs := time.Since(params.liveParams.startTime).Milliseconds() + if monitor.Enabled { + monitor.AIFirstSegmentDelay(delayMs, params.liveParams.sess.OrchestratorInfo) + monitor.SendQueueEventAsync("stream_trace", map[string]interface{}{ + "type": "gateway_receive_first_processed_segment", + "timestamp": time.Now().UnixMilli(), + "stream_id": params.liveParams.streamID, + "pipeline_id": params.liveParams.pipelineID, + "request_id": params.liveParams.requestID, + "orchestrator_info": map[string]interface{}{ + "address": orchAddr, + "url": orchUrl, + }, + }) + } + clog.V(common.VERBOSE).Infof(ctx, "First Segment delay=%dms streamID=%s", delayMs, params.liveParams.streamID) + } + segmentsReceived += 1 + if segmentsReceived == 3 && monitor.Enabled { + // We assume that after receiving 3 segments, the runner started successfully + // and we should be able to start the playback + monitor.SendQueueEventAsync("stream_trace", map[string]interface{}{ + "type": "gateway_receive_few_processed_segments", + "timestamp": time.Now().UnixMilli(), + "stream_id": params.liveParams.streamID, + "pipeline_id": params.liveParams.pipelineID, + "request_id": params.liveParams.requestID, + "orchestrator_info": map[string]interface{}{ + "address": orchAddr, + "url": orchUrl, + }, + }) + + } + clog.V(8).Info(ctx, "trickle subscribe read data completed", "seq", seq, "bytes", humanize.Bytes(uint64(n)), "took", time.Since(copyStartTime)) + } + }() + + // watchdog: fires if orch does not produce segments for too long + go func() { + for { + select { + case <-segmentTicker.C: + // check whether this timeout is due to missing input + // only suspend orchestrator if there is recent input + // ( no input == no output, so don't suspend for that ) + params.liveParams.mu.Lock() + lastInputSegmentTime := params.liveParams.lastSegmentTime + params.liveParams.mu.Unlock() + lastInputSegmentAge := time.Since(lastInputSegmentTime) + hasRecentInput := lastInputSegmentAge < segmentTimeout/2 + if hasRecentInput && streamInfo.IsActive() { + // abandon the orchestrator + streamInfo.ExcludeOrch(orchUrl) + stopProcessing(ctx, params, fmt.Errorf("timeout waiting for segments")) + segmentTicker.Stop() + return + } + } + } + }() + +} + +func startStreamControlPublish(ctx context.Context, control *url.URL, streamInfo *core.StreamInfo) { + params := streamInfo.Params.(aiRequestParams) + controlPub, err := trickle.NewTricklePublisher(control.String()) + if err != nil { + stopProcessing(ctx, params, fmt.Errorf("error starting control publisher, err=%w", err)) + return + } + params.node.LiveMu.Lock() + defer params.node.LiveMu.Unlock() + + ticker := time.NewTicker(10 * time.Second) + done := make(chan bool, 1) + once := sync.Once{} + stop := func() { + once.Do(func() { + ticker.Stop() + done <- true + }) + } + + //sess, exists := params.node.LivePipelines[stream] + //if !exists || sess.RequestID != params.liveParams.requestID { + // stopProcessing(ctx, params, fmt.Errorf("control session did not exist")) + // return + //} + //if sess.ControlPub != nil { + // // clean up from existing orchestrator + // go sess.ControlPub.Close() + //} + streamInfo.ControlPub = controlPub + streamInfo.StopControl = stop + + if monitor.Enabled { + monitorCurrentLiveSessions(params.node.LivePipelines) + } + + // Send any cached control params in a goroutine outside the lock. + msg := streamInfo.JobParams + go func() { + if msg == "" { + return + } + var err error + for i := 0; i < 3; i++ { + err = controlPub.Write(strings.NewReader(msg)) + if err == nil { + return + } + time.Sleep(100 * time.Millisecond) + } + stopProcessing(ctx, params, fmt.Errorf("control write failed: %w", err)) + }() + + // send a keepalive periodically to keep both ends of the connection alive + go func() { + for { + select { + case <-ticker.C: + const msg = `{"keep":"alive"}` + err := controlPub.Write(strings.NewReader(msg)) + if err == trickle.StreamNotFoundErr { + // the channel doesn't exist anymore, so stop + stop() + stopProcessing(ctx, params, errors.New("control channel does not exist")) + continue // loop back to consume the `done` chan + } + // if there was another type of error, we'll just retry anyway + case <-done: + return + case <-ctx.Done(): + stop() + } + } + }() +} + +func startStreamDataSubscribe(ctx context.Context, url *url.URL, streamInfo *core.StreamInfo) { + //only start DataSubscribe if enabled + params := streamInfo.Params.(aiRequestParams) + if params.liveParams.dataWriter == nil { + return + } + + // subscribe to the outputs + subscriber, err := trickle.NewTrickleSubscriber(trickle.TrickleSubscriberConfig{ + URL: url.String(), + Ctx: ctx, + }) + if err != nil { + clog.Infof(ctx, "Failed to create data subscriber: %s", err) + return + } + + dataWriter := params.liveParams.dataWriter + + // read segments from trickle subscription + go func() { + defer dataWriter.Close() + + var err error + firstSegment := true + + retries := 0 + // we're trying to keep (retryPause x maxRetries) duration to fall within one output GOP length + const retryPause = 300 * time.Millisecond + const maxRetries = 5 + for { + select { + case <-ctx.Done(): + clog.Info(ctx, "data subscribe done") + return + default: + } + if !params.inputStreamExists() { + clog.Infof(ctx, "data subscribe stopping, input stream does not exist.") + break + } + var segment *http.Response + readBytes, readMessages := 0, 0 + clog.V(8).Infof(ctx, "data subscribe await") + segment, err = subscriber.Read() + if err != nil { + if errors.Is(err, trickle.EOS) || errors.Is(err, trickle.StreamNotFoundErr) { + stopProcessing(ctx, params, fmt.Errorf("data subscribe stopping, stream not found, err=%w", err)) + return + } + var sequenceNonexistent *trickle.SequenceNonexistent + if errors.As(err, &sequenceNonexistent) { + // stream exists but segment doesn't, so skip to leading edge + subscriber.SetSeq(sequenceNonexistent.Latest) + } + // TODO if not EOS then signal a new orchestrator is needed + err = fmt.Errorf("data subscribe error reading: %w", err) + clog.Infof(ctx, "%s", err) + if retries > maxRetries { + stopProcessing(ctx, params, errors.New("data subscribe stopping, retries exceeded")) + return + } + retries++ + params.liveParams.sendErrorEvent(err) + time.Sleep(retryPause) + continue + } + retries = 0 + seq := trickle.GetSeq(segment) + clog.V(8).Infof(ctx, "data subscribe received seq=%d", seq) + copyStartTime := time.Now() + + defer segment.Body.Close() + scanner := bufio.NewScanner(segment.Body) + for scanner.Scan() { + writer, err := dataWriter.Next() + if err != nil { + if err != io.EOF { + stopProcessing(ctx, params, fmt.Errorf("data subscribe could not get next: %w", err)) + } + return + } + n, err := writer.Write(scanner.Bytes()) + if err != nil { + stopProcessing(ctx, params, fmt.Errorf("data subscribe could not write: %w", err)) + } + readBytes += n + readMessages += 1 + } + if err := scanner.Err(); err != nil { + clog.InfofErr(ctx, "data subscribe error reading seq=%d", seq, err) + subscriber.SetSeq(seq) + retries++ + continue + } + + if firstSegment { + firstSegment = false + delayMs := time.Since(params.liveParams.startTime).Milliseconds() + if monitor.Enabled { + monitor.AIFirstSegmentDelay(delayMs, params.liveParams.sess.OrchestratorInfo) + monitor.SendQueueEventAsync("stream_trace", map[string]interface{}{ + "type": "gateway_receive_first_data_segment", + "timestamp": time.Now().UnixMilli(), + "stream_id": params.liveParams.streamID, + "pipeline_id": params.liveParams.pipelineID, + "request_id": params.liveParams.requestID, + "orchestrator_info": map[string]interface{}{ + "address": params.liveParams.sess.Address(), + "url": params.liveParams.sess.Transcoder(), + }, + }) + } + } + + clog.V(8).Info(ctx, "data subscribe read completed", "seq", seq, "bytes", humanize.Bytes(uint64(readBytes)), "messages", readMessages, "took", time.Since(copyStartTime)) + } + }() +} diff --git a/server/rpc.go b/server/rpc.go index ab3e848e0e..0cf3554acb 100644 --- a/server/rpc.go +++ b/server/rpc.go @@ -254,7 +254,8 @@ func StartTranscodeServer(orch Orchestrator, bind string, mux *http.ServeMux, wo lp.transRPC.HandleFunc("/capability/register", lp.RegisterCapability) lp.transRPC.HandleFunc("/capability/unregister", lp.UnregisterCapability) lp.transRPC.HandleFunc("/stream/start", lp.StartStream) - lp.transRPC.HandleFunc("/stream/payment", lp.StreamPayment) + lp.transRPC.HandleFunc("/stream/stop", lp.StopStream) + lp.transRPC.HandleFunc("/stream/payment", lp.ProcessStreamPayment) cert, key, err := getCert(orch.ServiceURI(), workDir) if err != nil { From 80b56ba1758f3c7a5806c72185e118aee3e830e0 Mon Sep 17 00:00:00 2001 From: Brad P Date: Wed, 27 Aug 2025 16:25:50 -0500 Subject: [PATCH 22/57] building out byoc stream --- core/external_capabilities.go | 2 +- server/ai_mediaserver.go | 20 +-- server/job_rpc.go | 57 +++++--- server/job_stream.go | 268 +++++++++++++++++++++++----------- server/job_trickle.go | 240 +++++++++++++++++++++++++++++- 5 files changed, 467 insertions(+), 120 deletions(-) diff --git a/core/external_capabilities.go b/core/external_capabilities.go index 8c6c6130f8..7af5118ff5 100644 --- a/core/external_capabilities.go +++ b/core/external_capabilities.go @@ -107,7 +107,7 @@ func (extCaps *ExternalCapabilities) AddStream(streamID string, params interface ctx, cancel := context.WithCancel(context.Background()) stream := StreamInfo{ StreamID: streamID, - Params: params, + Params: params, // Store the interface value directly, not a pointer to it StreamRequest: streamReq, StreamCtx: ctx, CancelStream: cancel, diff --git a/server/ai_mediaserver.go b/server/ai_mediaserver.go index a48784a38c..a9507a9b74 100644 --- a/server/ai_mediaserver.go +++ b/server/ai_mediaserver.go @@ -98,8 +98,9 @@ func startAIMediaServer(ctx context.Context, ls *LivepeerServer) error { // Configure WHIP ingest only if an addr is specified. // TODO use a proper cli flag + var whipServer *media.WHIPServer if os.Getenv("LIVE_AI_WHIP_ADDR") != "" { - whipServer := media.NewWHIPServer() + whipServer = media.NewWHIPServer() ls.HTTPMux.Handle("POST /live/video-to-video/{stream}/whip", ls.CreateWhip(whipServer)) ls.HTTPMux.Handle("HEAD /live/video-to-video/{stream}/whip", ls.WithCode(http.StatusMethodNotAllowed)) ls.HTTPMux.Handle("OPTIONS /live/video-to-video/{stream}/whip", ls.WithCode(http.StatusNoContent)) @@ -115,16 +116,17 @@ func startAIMediaServer(ctx context.Context, ls *LivepeerServer) error { //API for dynamic capabilities ls.HTTPMux.Handle("/process/request/", ls.SubmitJob()) - ls.HTTPMux.Handle("/stream/start", ls.StartStream()) - ls.HTTPMux.Handle("/stream/stop", ls.StopStream()) + + ls.HTTPMux.Handle("OPTIONS /ai/stream/", ls.WithCode(http.StatusNoContent)) + ls.HTTPMux.Handle("POST /ai/stream/start", ls.StartStream()) + ls.HTTPMux.Handle("POST /ai/stream/stop", ls.StopStream()) if os.Getenv("LIVE_AI_WHIP_ADDR") != "" { - streamWhipServer := media.NewWHIPServer() - ls.HTTPMux.Handle("/stream/{streamId}/whip", ls.StartStreamWhipIngest(streamWhipServer)) + ls.HTTPMux.Handle("POST /ai/stream/{streamId}/whip", ls.StartStreamWhipIngest(whipServer)) } - ls.HTTPMux.Handle("/stream/{streamId}/rtmp", ls.StartStreamRTMPIngest()) - ls.HTTPMux.Handle("/stream/{streamId}/update", ls.UpdateStream()) - ls.HTTPMux.Handle("/stream/{streamId}/status", ls.GetStreamStatus()) - ls.HTTPMux.Handle("/stream/{streamId}/data", ls.GetStreamData()) + ls.HTTPMux.Handle("POST /ai/stream/{streamId}/rtmp", ls.StartStreamRTMPIngest()) + ls.HTTPMux.Handle("POST /ai/stream/{streamId}/update", ls.UpdateStream()) + ls.HTTPMux.Handle("GET /ai/stream/{streamId}/status", ls.GetStreamStatus()) + ls.HTTPMux.Handle("GET /ai/stream/{streamId}/data", ls.GetStreamData()) media.StartFileCleanup(ctx, ls.LivepeerNode.WorkDir) diff --git a/server/job_rpc.go b/server/job_rpc.go index 304ec2cfe4..dbfa473325 100644 --- a/server/job_rpc.go +++ b/server/job_rpc.go @@ -106,9 +106,29 @@ type orchJob struct { JobPrice *net.PriceInfo } type gatewayJob struct { - Job *orchJob - Orchs []JobToken - JobReqHdr string + Job *orchJob + Orchs []JobToken + SignedJobReq string +} + +func (g *gatewayJob) sign(node *core.LivepeerNode) error { + //sign the request + gateway := node.OrchestratorPool.Broadcaster() + sig, err := gateway.Sign([]byte(g.Job.Req.Request + g.Job.Req.Parameters)) + if err != nil { + return errors.New(fmt.Sprintf("Unable to sign request err=%v", err)) + } + g.Job.Req.Sender = gateway.Address().Hex() + g.Job.Req.Sig = "0x" + hex.EncodeToString(sig) + + //create the job request header with the signature + jobReqEncoded, err := json.Marshal(g.Job.Req) + if err != nil { + return errors.New(fmt.Sprintf("Unable to encode job request err=%v", err)) + } + g.SignedJobReq = base64.StdEncoding.EncodeToString(jobReqEncoded) + + return nil } // worker registers to Orchestrator @@ -283,10 +303,11 @@ func (h *lphttp) GetJobToken(w http.ResponseWriter, r *http.Request) { } func (ls *LivepeerServer) setupGatewayJob(ctx context.Context, r *http.Request) (*gatewayJob, error) { - clog.Infof(ctx, "processing job request") + var orchs []JobToken jobReqHdr := r.Header.Get(jobRequestHdr) + clog.Infof(ctx, "processing job request req=%v", jobReqHdr) jobReq, err := verifyJobCreds(ctx, nil, jobReqHdr) if err != nil { return nil, errors.New(fmt.Sprintf("Unable to parse job request, err=%v", err)) @@ -316,30 +337,12 @@ func (ls *LivepeerServer) setupGatewayJob(ctx context.Context, r *http.Request) return nil, errors.New(fmt.Sprintf("No orchestrators found for capability %v", jobReq.Capability)) } - //sign the request - gateway := ls.LivepeerNode.OrchestratorPool.Broadcaster() - sig, err := gateway.Sign([]byte(jobReq.Request + jobReq.Parameters)) - if err != nil { - return nil, errors.New(fmt.Sprintf("Unable to sign request err=%v", err)) - } - jobReq.Sender = gateway.Address().Hex() - jobReq.Sig = "0x" + hex.EncodeToString(sig) - - //create the job request header with the signature - jobReqEncoded, err := json.Marshal(jobReq) - if err != nil { - return nil, errors.New(fmt.Sprintf("Unable to encode job request err=%v", err)) - } - jobReqHdr = base64.StdEncoding.EncodeToString(jobReqEncoded) - job := orchJob{Req: jobReq, Details: &jobDetails, Params: &jobParams, } - return &gatewayJob{Job: &job, - Orchs: orchs, - JobReqHdr: jobReqHdr}, nil + return &gatewayJob{Job: &job, Orchs: orchs}, nil } func (h *lphttp) ProcessJob(w http.ResponseWriter, r *http.Request) { @@ -410,8 +413,14 @@ func (ls *LivepeerServer) submitJob(ctx context.Context, w http.ResponseWriter, workerRoute = workerRoute + "/" + workerResourceRoute } + err := gatewayJob.sign(ls.LivepeerNode) + if err != nil { + clog.Errorf(ctx, "Error signing job, exiting stream processing request: %v", err) + return + } + start := time.Now() - resp, code, err := ls.sendJobToOrch(ctx, r, gatewayJob.Job.Req, gatewayJob.JobReqHdr, orchToken, workerResourceRoute, body) + resp, code, err := ls.sendJobToOrch(ctx, r, gatewayJob.Job.Req, gatewayJob.SignedJobReq, orchToken, workerResourceRoute, body) if err != nil { clog.Errorf(ctx, "job not able to be processed by Orchestrator %v err=%v ", orchToken.ServiceAddr, err.Error()) continue diff --git a/server/job_stream.go b/server/job_stream.go index 93abb4768f..55c0dc44ea 100644 --- a/server/job_stream.go +++ b/server/job_stream.go @@ -26,12 +26,18 @@ import ( "github.com/livepeer/go-tools/drivers" ) -// StartStream handles the POST /stream/start endpoint for the Media Server func (ls *LivepeerServer) StartStream() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodOptions { + corsHeaders(w, r.Method) + w.WriteHeader(http.StatusNoContent) + return + } + // Create fresh context instead of using r.Context() since ctx will outlive the request ctx := r.Context() + corsHeaders(w, r.Method) //verify request, get orchestrators available and sign request gatewayJob, err := ls.setupGatewayJob(ctx, r) if err != nil { @@ -47,7 +53,7 @@ func (ls *LivepeerServer) StartStream() http.Handler { return } - go ls.runStream(gatewayJob, resp["stream_id"]) + go ls.runStream(gatewayJob) if resp != nil { // Stream started successfully @@ -80,7 +86,8 @@ func (ls *LivepeerServer) StopStream() http.Handler { return } - resp, code, err := ls.sendJobToOrch(ctx, r, stopJob.Job.Req, stopJob.JobReqHdr, streamInfoCopy.OrchToken.(JobToken), "/stream/stop", streamInfoCopy.StreamRequest) + stopJob.sign(ls.LivepeerNode) + resp, code, err := ls.sendJobToOrch(ctx, r, stopJob.Job.Req, stopJob.SignedJobReq, streamInfoCopy.OrchToken.(JobToken), "/stream/stop", streamInfoCopy.StreamRequest) if err != nil { clog.Errorf(ctx, "Error sending job to orchestrator: %s", err) http.Error(w, err.Error(), code) @@ -99,19 +106,22 @@ func (ls *LivepeerServer) StopStream() http.Handler { }) } -func (ls *LivepeerServer) runStream(gatewayJob *gatewayJob, streamID string) { +func (ls *LivepeerServer) runStream(gatewayJob *gatewayJob) { + streamID := gatewayJob.Job.Req.ID ctx := context.Background() + ctx = clog.AddVal(ctx, "stream_id", streamID) stream, exists := ls.LivepeerNode.ExternalCapabilities.Streams[streamID] - params := stream.Params.(aiRequestParams) if !exists { clog.Errorf(ctx, "Stream %s not found", streamID) return } + params := stream.Params.(aiRequestParams) start := time.Now() for _, orch := range gatewayJob.Orchs { ctx = clog.AddVal(ctx, "orch", ethcommon.Bytes2Hex(orch.TicketParams.Recipient)) ctx = clog.AddVal(ctx, "orch_url", orch.ServiceAddr) + clog.Infof(ctx, "Starting stream processing") //refresh the token if after 5 minutes from start. TicketParams expire in 40 blocks (about 8 minutes). if time.Since(start) > 5*time.Minute { orch, err := getToken(ctx, 3*time.Second, orch.ServiceAddr, gatewayJob.Job.Req.Capability, gatewayJob.Job.Req.Sender, gatewayJob.Job.Req.Sig) @@ -123,12 +133,18 @@ func (ls *LivepeerServer) runStream(gatewayJob *gatewayJob, streamID string) { //set request ID to persist from Gateway to Worker gatewayJob.Job.Req.ID = stream.StreamID - - orchResp, _, err := ls.sendJobToOrch(ctx, nil, gatewayJob.Job.Req, gatewayJob.JobReqHdr, orch, "/stream/start", stream.StreamRequest) + err := gatewayJob.sign(ls.LivepeerNode) + if err != nil { + clog.Errorf(ctx, "Error signing job, exiting stream processing request: %v", err) + stream.CancelStream() + return + } + orchResp, _, err := ls.sendJobToOrch(ctx, nil, gatewayJob.Job.Req, gatewayJob.SignedJobReq, orch, "/stream/start", stream.StreamRequest) if err != nil { clog.Errorf(ctx, "job not able to be processed by Orchestrator %v err=%v ", orch.ServiceAddr, err.Error()) continue } + stream.OrchPublishUrl = orchResp.Header.Get("X-Publish-Url") stream.OrchSubscribeUrl = orchResp.Header.Get("X-Subscribe-Url") stream.OrchControlUrl = orchResp.Header.Get("X-Control-Url") @@ -136,7 +152,8 @@ func (ls *LivepeerServer) runStream(gatewayJob *gatewayJob, streamID string) { stream.OrchDataUrl = orchResp.Header.Get("X-Data-Url") perOrchCtx, perOrchCancel := context.WithCancelCause(ctx) - params.liveParams = newParams(params.liveParams, perOrchCancel) + params.liveParams.kickOrch = perOrchCancel + stream.Params = params //update params used to kickOrch (perOrchCancel) if err = startStreamProcessing(perOrchCtx, stream); err != nil { clog.Errorf(ctx, "Error starting processing: %s", err) perOrchCancel(err) @@ -240,18 +257,37 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, req requestID := string(core.RandomManifestID()) ctx = clog.AddVal(ctx, "request_id", requestID) + // Setup request body to be able to preserve for retries + // Read the entire body first + bodyBytes, err := io.ReadAll(r.Body) + if err != nil { + return nil, http.StatusBadRequest, err + } + + // Create a clone of the request with a new body for parsing form values + bodyForForm := bytes.NewBuffer(bodyBytes) + r.Body = io.NopCloser(bytes.NewBuffer(bodyBytes)) // Reset the original body + + // Create a temporary request for parsing form data + formReq := r.Clone(ctx) + formReq.Body = io.NopCloser(bodyForForm) + + // Parse the form (10MB max) + if err := formReq.ParseMultipartForm(10 << 20); err != nil { + return nil, http.StatusBadRequest, err + } //live-video-to-video uses path value for this - streamName := r.FormValue("stream") + streamName := formReq.FormValue("stream") streamRequestTime := time.Now().UnixMilli() ctx = clog.AddVal(ctx, "stream", streamName) - sourceID := r.FormValue("source_id") - if sourceID == "" { - return nil, http.StatusBadRequest, errors.New("missing source_id") - } - ctx = clog.AddVal(ctx, "source_id", sourceID) - //sourceType := r.FormValue("source_type") + //sourceID := formReq.FormValue("source_id") + //if sourceID == "" { + // return nil, http.StatusBadRequest, errors.New("missing source_id") + //} + //ctx = clog.AddVal(ctx, "source_id", sourceID) + //sourceType := formReq.FormValue("source_type") //if sourceType == "" { // return nil, http.StatusBadRequest, errors.New("missing source_type") //} @@ -261,14 +297,8 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, req //} //ctx = clog.AddVal(ctx, "source_type", sourceType) - streamParams := r.FormValue("params") - var streamParamsJson map[string]interface{} - if err := json.Unmarshal([]byte(streamParams), &streamParamsJson); err != nil { - return nil, http.StatusBadRequest, errors.New("invalid stream params") - } - // If auth webhook is set and returns an output URL, this will be replaced - outputURL := r.FormValue("rtmpOutput") + outputURL := formReq.FormValue("rtmpOutput") // convention to avoid re-subscribing to our own streams // in case we want to push outputs back into mediamtx - @@ -279,12 +309,9 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, req } // if auth webhook returns pipeline config these will be replaced - pipeline := streamParamsJson["pipeline"].(string) - if pipeline == "" { - pipeline = req.Capability - } - rawParams := streamParamsJson["params"].(string) - streamID := streamParamsJson["streamId"].(string) + pipeline := req.Capability //streamParamsJson["pipeline"].(string) + rawParams := formReq.FormValue("params") + streamID := formReq.FormValue("streamId") var pipelineID string var pipelineParams map[string]interface{} @@ -308,24 +335,25 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, req //} mediaMtxHost := os.Getenv("LIVE_AI_PLAYBACK_HOST") if mediaMtxHost == "" { - mediaMtxHost = "localhost:1935" + mediaMtxHost = "rtmp://localhost:1935" } - mediaMTXInputURL := fmt.Sprintf("rtmp://%s/%s%s", mediaMtxHost, "", streamID) + mediaMTXInputURL := fmt.Sprintf("%s/%s%s", mediaMtxHost, "", streamID) mediaMTXOutputURL := mediaMTXInputURL + "-out" mediaMTXOutputAlias := fmt.Sprintf("%s-%s-out", mediaMTXInputURL, requestID) whepURL := generateWhepUrl(streamID, requestID) - whipURL := fmt.Sprintf("https://%s/stream/%s/%s", ls.LivepeerNode.GatewayHost, streamID, "/whip") + whipURL := fmt.Sprintf("https://%s/ai/stream/%s/%s", ls.LivepeerNode.GatewayHost, streamID, "whip") rtmpURL := mediaMTXInputURL - updateURL := fmt.Sprintf("https://%s/stream/%s/%s", ls.LivepeerNode.GatewayHost, streamID, "/update") - statusURL := fmt.Sprintf("https://%s/stream/%s/%s", ls.LivepeerNode.GatewayHost, streamID, "/status") + updateURL := fmt.Sprintf("https://%s/ai/stream/%s/%s", ls.LivepeerNode.GatewayHost, streamID, "update") + statusURL := fmt.Sprintf("https://%s/ai/stream/%s/%s", ls.LivepeerNode.GatewayHost, streamID, "status") + dataURL := fmt.Sprintf("https://%s/ai/stream/%s/%s", ls.LivepeerNode.GatewayHost, streamID, "data") //if set this will overwrite settings above if LiveAIAuthWebhookURL != nil { authResp, err := authenticateAIStream(LiveAIAuthWebhookURL, ls.liveAIAuthApiKey, AIAuthRequest{ Stream: streamName, Type: "", //sourceTypeStr - QueryParams: streamParams, + QueryParams: rawParams, GatewayHost: ls.LivepeerNode.GatewayHost, WhepURL: whepURL, UpdateURL: updateURL, @@ -433,8 +461,7 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, req //create a dataWriter for data channel if enabled var jobParams JobParameters - err := json.Unmarshal([]byte(req.Parameters), &jobParams) - if err != nil { + if err = json.Unmarshal([]byte(req.Parameters), &jobParams); err != nil { return nil, http.StatusBadRequest, errors.New("invalid job parameters") } if jobParams.EnableDataOutput { @@ -449,12 +476,10 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, req clog.Infof(ctx, "stream setup videoIngress=%v videoEgress=%v dataOutput=%v", jobParams.EnableVideoIngress, jobParams.EnableVideoEgress, jobParams.EnableDataOutput) - // read entire body to ensure valid and store the request to send to Orchestrator(s) - // there is a round robin mechanism to add resiliency, try - body, err := io.ReadAll(r.Body) - if err != nil { - return nil, http.StatusInternalServerError, errors.New("Error reading request body") - } + // No need to read the body again since we already have it + // We're using the original request body that was already read and stored + // We already have the bodyBytes variable which contains the full request body + body := bodyBytes r.Body.Close() //save the stream setup @@ -462,6 +487,7 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, req return nil, http.StatusBadRequest, err } + req.ID = streamID resp := make(map[string]string) resp["stream_id"] = streamID resp["whip_url"] = whipURL @@ -470,6 +496,7 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, req resp["rtmp_output_url"] = strings.Join(rtmpOutputs, ",") resp["update_url"] = updateURL resp["status_url"] = statusURL + resp["data_url"] = dataURL return resp, http.StatusOK, nil } @@ -522,6 +549,7 @@ func (ls *LivepeerServer) StartStreamRTMPIngest() http.Handler { } params.liveParams.kickInput = kickInput + stream.Params = params //update params used to kickInput // Create a special parent context for orchestrator cancellation //orchCtx, orchCancel := context.WithCancel(ctx) @@ -572,6 +600,19 @@ func (ls *LivepeerServer) StartStreamWhipIngest(whipServer *media.WHIPServer) ht whipConn := media.NewWHIPConnection() whepURL := generateWhepUrl(streamId, requestID) + // this function is called when the pipeline hits a fatal error, we kick the input connection to allow + // the client to reconnect and restart the pipeline + kickInput := func(err error) { + if err == nil { + return + } + clog.Errorf(ctx, "Live video pipeline finished with error: %s", err) + params.liveParams.sendErrorEvent(err) + whipConn.Close() + } + params.liveParams.kickInput = kickInput + stream.Params = params //update params used to kickInput + //wait for the WHIP connection to close and then cleanup go func() { statsContext, statsCancel := context.WithCancel(ctx) @@ -592,7 +633,6 @@ func (ls *LivepeerServer) StartStreamWhipIngest(whipServer *media.WHIPServer) ht } func startStreamProcessing(ctx context.Context, streamInfo *core.StreamInfo) error { - params := streamInfo.Params.(aiRequestParams) var channels []string //this adds the stream to LivePipelines which the Control Publisher and Data Writer @@ -611,36 +651,39 @@ func startStreamProcessing(ctx context.Context, streamInfo *core.StreamInfo) err channels = append(channels, control.String()) channels = append(channels, events.String()) - startControlPublish(ctx, control, params) - startEventsSubscribe(ctx, events, params, nil) + startStreamControlPublish(ctx, control, streamInfo) + startStreamEventsSubscribe(ctx, events, streamInfo) //Optional channels - if streamInfo.OrchPublishUrl == "" { + if streamInfo.OrchPublishUrl != "" { + clog.Infof(ctx, "Starting video ingress publisher") pub, err := common.AppendHostname(streamInfo.OrchPublishUrl, streamInfo.OrchUrl) if err != nil { return fmt.Errorf("invalid publish URL: %w", err) } channels = append(channels, pub.String()) - startTricklePublish(ctx, pub, params, nil) + startStreamTricklePublish(ctx, pub, streamInfo) } - if streamInfo.OrchSubscribeUrl == "" { + if streamInfo.OrchSubscribeUrl != "" { + clog.Infof(ctx, "Starting video egress subscriber") sub, err := common.AppendHostname(streamInfo.OrchSubscribeUrl, streamInfo.OrchUrl) if err != nil { return fmt.Errorf("invalid subscribe URL: %w", err) } channels = append(channels, sub.String()) - startTrickleSubscribe(ctx, sub, params, nil) + startStreamTrickleSubscribe(ctx, sub, streamInfo) } - if streamInfo.OrchDataUrl == "" { + if streamInfo.OrchDataUrl != "" { + clog.Infof(ctx, "Starting data channel subscriber") data, err := common.AppendHostname(streamInfo.OrchDataUrl, streamInfo.OrchUrl) if err != nil { return fmt.Errorf("invalid data URL: %w", err) } streamInfo.Params.(aiRequestParams).liveParams.manifestID = streamInfo.Capability - startDataSubscribe(ctx, data, params, nil) + startStreamDataSubscribe(ctx, data, streamInfo) } return nil @@ -821,7 +864,6 @@ func (h *lphttp) StartStream(w http.ResponseWriter, r *http.Request) { } r.Body.Close() - req, err := http.NewRequestWithContext(ctx, "POST", workerRoute, bytes.NewBuffer(body)) var jobParams JobParameters err = json.Unmarshal([]byte(orchJob.Req.Parameters), &jobParams) if err != nil { @@ -830,6 +872,7 @@ func (h *lphttp) StartStream(w http.ResponseWriter, r *http.Request) { return } + clog.Infof(ctx, "Processing stream start request videoIngress=%v videoEgress=%v dataOutput=%v", jobParams.EnableVideoIngress, jobParams.EnableVideoEgress, jobParams.EnableDataOutput) // Start trickle server for live-video var ( mid = orchJob.Req.ID // Request ID is used for the manifest ID @@ -840,39 +883,50 @@ func (h *lphttp) StartStream(w http.ResponseWriter, r *http.Request) { dataUrl = pubUrl + "-data" ) + reqBody := make(map[string]interface{}) + reqBody["gateway_request_id"] = mid //required channels controlPubCh := trickle.NewLocalPublisher(h.trickleSrv, mid+"-control", "application/json") controlPubCh.CreateChannel() controlUrl = overwriteHost(h.node.LiveAITrickleHostForRunner, controlUrl) - req.Header.Set("X-Control-Url", controlUrl) + reqBody["control_url"] = controlUrl eventsCh := trickle.NewLocalPublisher(h.trickleSrv, mid+"-events", "application/json") eventsCh.CreateChannel() eventsUrl = overwriteHost(h.node.LiveAITrickleHostForRunner, eventsUrl) - req.Header.Set("X-Events-Url", eventsUrl) + reqBody["events_url"] = eventsUrl //Optional channels if jobParams.EnableVideoIngress { pubCh := trickle.NewLocalPublisher(h.trickleSrv, mid, "video/MP2T") pubCh.CreateChannel() pubUrl = overwriteHost(h.node.LiveAITrickleHostForRunner, pubUrl) - req.Header.Set("X-Publish-Url", pubUrl) + reqBody["publish_url"] = pubUrl } if jobParams.EnableVideoEgress { subCh := trickle.NewLocalPublisher(h.trickleSrv, mid+"-out", "video/MP2T") subCh.CreateChannel() subUrl = overwriteHost(h.node.LiveAITrickleHostForRunner, subUrl) - req.Header.Set("X-Subscribe-Url", subUrl) + reqBody["subscribe_url"] = subUrl } if jobParams.EnableDataOutput { dataCh := trickle.NewLocalPublisher(h.trickleSrv, mid+"-data", "application/jsonl") dataCh.CreateChannel() dataUrl = overwriteHost(h.node.LiveAITrickleHostForRunner, dataUrl) - req.Header.Set("X-Data-Url", dataUrl) + reqBody["data_url"] = dataUrl } + reqBody["request"] = base64.StdEncoding.EncodeToString(body) + reqBodyBytes, err := json.Marshal(reqBody) + if err != nil { + clog.Errorf(ctx, "Failed to marshal request body err=%v", err) + http.Error(w, "Failed to marshal request body", http.StatusInternalServerError) + return + } + + req, err := http.NewRequestWithContext(ctx, "POST", workerRoute, bytes.NewBuffer(reqBodyBytes)) // set the headers req.Header.Add("Content-Length", r.Header.Get("Content-Length")) req.Header.Add("Content-Type", r.Header.Get("Content-Type")) @@ -892,10 +946,10 @@ func (h *lphttp) StartStream(w http.ResponseWriter, r *http.Request) { return } defer resp.Body.Close() - w.WriteHeader(resp.StatusCode) + //error response from worker but assume can retry and pass along error response and status code if resp.StatusCode > 399 { - clog.Errorf(ctx, "error processing /stream/start request statusCode=%d", resp.StatusCode) + clog.Errorf(ctx, "error processing stream start request statusCode=%d", resp.StatusCode) chargeForCompute(start, orchJob.JobPrice, orch, orchJob.Sender, orchJob.Req.Capability) w.Header().Set(jobPaymentBalanceHdr, getPaymentBalance(orch, orchJob.Sender, orchJob.Req.Capability).FloatString(0)) @@ -908,40 +962,54 @@ func (h *lphttp) StartStream(w http.ResponseWriter, r *http.Request) { chargeForCompute(start, orchJob.JobPrice, orch, orchJob.Sender, orchJob.Req.Capability) w.Header().Set(jobPaymentBalanceHdr, getPaymentBalance(orch, orchJob.Sender, orchJob.Req.Capability).FloatString(0)) - clog.V(common.SHORT).Infof(ctx, "Job processed successfully took=%v balance=%v", time.Since(start), getPaymentBalance(orch, orchJob.Sender, orchJob.Req.Capability).FloatString(0)) + clog.V(common.SHORT).Infof(ctx, "stream start processed successfully took=%v balance=%v", time.Since(start), getPaymentBalance(orch, orchJob.Sender, orchJob.Req.Capability).FloatString(0)) //setup the stream - h.node.ExternalCapabilities.AddStream(orchJob.Req.ID, orchJob.Req, respBody) + err = h.node.ExternalCapabilities.AddStream(orchJob.Req.ID, orchJob.Req, respBody) + if err != nil { + clog.Errorf(ctx, "Error adding stream to external capabilities: %v", err) + respondWithError(w, "Error adding stream to external capabilities", http.StatusInternalServerError) + return + } //start payment monitoring go func() { - pmtTicker := time.NewTicker(10 * time.Second) + stream, _ := h.node.ExternalCapabilities.Streams[orchJob.Req.ID] + + pmtTicker := time.NewTicker(30 * time.Second) defer pmtTicker.Stop() - for range pmtTicker.C { - // Check payment status - jobPriceRat := big.NewRat(orchJob.JobPrice.PricePerUnit, orchJob.JobPrice.PixelsPerUnit) - if jobPriceRat.Cmp(big.NewRat(0, 1)) > 0 { - h.orchestrator.DebitFees(orchJob.Sender, core.ManifestID(orchJob.Req.Capability), orchJob.JobPrice, 5) - senderBalance := getPaymentBalance(orch, orchJob.Sender, orchJob.Req.Capability) - if senderBalance != nil { - if senderBalance.Cmp(big.NewRat(0, 1)) < 0 { - clog.Infof(ctx, "Insufficient balance, stopping stream %s for sender %s", orchJob.Req.ID, orchJob.Sender) - _, exists := h.node.ExternalCapabilities.Streams[orchJob.Req.ID] - if exists { - h.node.ExternalCapabilities.RemoveStream(orchJob.Req.ID) + for { + select { + case <-stream.StreamCtx.Done(): + return + case <-pmtTicker.C: + // Check payment status + clog.V(8).Infof(ctx, "Checking payment balance for stream") + jobPriceRat := big.NewRat(orchJob.JobPrice.PricePerUnit, orchJob.JobPrice.PixelsPerUnit) + if jobPriceRat.Cmp(big.NewRat(0, 1)) > 0 { + h.orchestrator.DebitFees(orchJob.Sender, core.ManifestID(orchJob.Req.Capability), orchJob.JobPrice, 5) + senderBalance := getPaymentBalance(orch, orchJob.Sender, orchJob.Req.Capability) + if senderBalance != nil { + if senderBalance.Cmp(big.NewRat(0, 1)) < 0 { + clog.Infof(ctx, "Insufficient balance, stopping stream %s for sender %s", orchJob.Req.ID, orchJob.Sender) + _, exists := h.node.ExternalCapabilities.Streams[orchJob.Req.ID] + if exists { + h.node.ExternalCapabilities.RemoveStream(orchJob.Req.ID) + } } } } + } } }() //send back trickle urls - w.Header().Set("X-Publish-Url", req.Header.Get("X-Publish-Url")) - w.Header().Set("X-Subscribe-Url", req.Header.Get("X-Subscribe-Url")) - w.Header().Set("X-Control-Url", req.Header.Get("X-Control-Url")) - w.Header().Set("X-Events-Url", req.Header.Get("X-Events-Url")) - w.Header().Set("X-Data-Url", req.Header.Get("X-Data-Url")) + w.Header().Set("X-Publish-Url", pubUrl) + w.Header().Set("X-Subscribe-Url", subUrl) + w.Header().Set("X-Control-Url", controlUrl) + w.Header().Set("X-Events-Url", eventsUrl) + w.Header().Set("X-Data-Url", dataUrl) w.WriteHeader(http.StatusOK) return @@ -961,11 +1029,48 @@ func (h *lphttp) StopStream(w http.ResponseWriter, r *http.Request) { respondWithError(w, fmt.Sprintf("Failed to stop stream, request not valid, failed to parse stream id err=%v", err), http.StatusBadRequest) return } + clog.Infof(ctx, "Stopping stream %s", jobDetails.StreamId) + + // Read the original body + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "Error reading request body", http.StatusBadRequest) + return + } + r.Body.Close() + + workerRoute := orchJob.Req.CapabilityUrl + "/stream/stop" + req, err := http.NewRequestWithContext(ctx, "POST", workerRoute, bytes.NewBuffer(body)) + if err != nil { + clog.Errorf(ctx, "failed to create /stop/stream request to worker err=%v", err) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + resp, err := sendReqWithTimeout(req, time.Duration(orchJob.Req.Timeout)*time.Second) + if err != nil { + clog.Errorf(ctx, "Error sending request to worker %v: %v", workerRoute, err) + respondWithError(w, "Error sending request to worker", http.StatusInternalServerError) + return + } + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + clog.Errorf(ctx, "Error reading response body: %v", err) + respondWithError(w, "Error reading response body", http.StatusInternalServerError) + return + } + defer resp.Body.Close() + + if resp.StatusCode > 399 { + clog.Errorf(ctx, "error processing stream stop request statusCode=%d", resp.StatusCode) + } // Stop the stream h.node.ExternalCapabilities.RemoveStream(jobDetails.StreamId) - w.Write([]byte("Stream stopped successfully")) + w.WriteHeader(resp.StatusCode) + w.Write(respBody) } func (h *lphttp) ProcessStreamPayment(w http.ResponseWriter, r *http.Request) { @@ -981,7 +1086,6 @@ func (h *lphttp) ProcessStreamPayment(w http.ResponseWriter, r *http.Request) { senderAddr := ethcommon.HexToAddress(orchJob.Req.Sender) - jobToken.SenderAddress.Addr = orchJob.Req.Sender jobPrice, err := orch.JobPriceInfo(senderAddr, orchJob.Req.Capability) if err != nil { statusCode := http.StatusBadRequest @@ -1019,7 +1123,7 @@ func (h *lphttp) ProcessStreamPayment(w http.ResponseWriter, r *http.Request) { capBalInt = capBalInt / 1000 } - jobToken := JobToken{SenderAddress: nil, TicketParams: ticketParams, Balance: capBalInt, Price: jobPrice} + jobToken := JobToken{TicketParams: ticketParams, Balance: capBalInt, Price: jobPrice} json.NewEncoder(w).Encode(jobToken) } diff --git a/server/job_trickle.go b/server/job_trickle.go index 1257fc1411..479cfb9f8b 100644 --- a/server/job_trickle.go +++ b/server/job_trickle.go @@ -3,13 +3,16 @@ package server import ( "bufio" "context" + "encoding/json" "errors" "fmt" "io" "net/http" "net/url" + "os/exec" "strings" "sync" + "syscall" "time" "github.com/dustin/go-humanize" @@ -161,7 +164,7 @@ func startStreamTrickleSubscribe(ctx context.Context, url *url.URL, streamInfo * } // Launch ffmpeg for each configured RTMP output for _, outURL := range params.liveParams.rtmpOutputs { - go ffmpegOutput(ctx, outURL, outWriter, params) + go ffmpegStreamOutput(ctx, outURL, outWriter, streamInfo) } // watchdog that gets reset on every segment to catch output stalls @@ -191,7 +194,7 @@ func startStreamTrickleSubscribe(ctx context.Context, url *url.URL, streamInfo * return default: } - if !params.inputStreamExists() { + if !streamInfo.IsActive() { clog.Infof(ctx, "trickle subscribe stopping, input stream does not exist.") break } @@ -239,7 +242,7 @@ func startStreamTrickleSubscribe(ctx context.Context, url *url.URL, streamInfo * params.liveParams.mu.Unlock() segmentAge := time.Since(lastSegmentTime) maxSegmentDelay := params.liveParams.outSegmentTimeout / 2 - if segmentAge < maxSegmentDelay && params.inputStreamExists() { + if segmentAge < maxSegmentDelay && streamInfo.IsActive() { // we have some recent input but no output from orch, so kick suspendOrchestrator(ctx, params) stopProcessing(ctx, params, fmt.Errorf("trickle subscribe error, swapping: %w", err)) @@ -429,7 +432,7 @@ func startStreamDataSubscribe(ctx context.Context, url *url.URL, streamInfo *cor return default: } - if !params.inputStreamExists() { + if !streamInfo.IsActive() { clog.Infof(ctx, "data subscribe stopping, input stream does not exist.") break } @@ -511,3 +514,232 @@ func startStreamDataSubscribe(ctx context.Context, url *url.URL, streamInfo *cor } }() } + +func startStreamEventsSubscribe(ctx context.Context, url *url.URL, streamInfo *core.StreamInfo) { + params := streamInfo.Params.(aiRequestParams) + subscriber, err := trickle.NewTrickleSubscriber(trickle.TrickleSubscriberConfig{ + URL: url.String(), + Ctx: ctx, + }) + if err != nil { + stopProcessing(ctx, params, fmt.Errorf("event sub init failed: %w", err)) + return + } + stream := params.liveParams.stream + streamId := params.liveParams.streamID + + // vars to check events periodically to ensure liveness + var ( + eventCheckInterval = 10 * time.Second + maxEventGap = 30 * time.Second + eventTicker = time.NewTicker(eventCheckInterval) + eventsDone = make(chan bool) + // remaining vars in this block must be protected by mutex + lastEventMu = &sync.Mutex{} + lastEvent = time.Now() + ) + + clog.Infof(ctx, "Starting event subscription for URL: %s", url.String()) + + go func() { + defer time.AfterFunc(clearStreamDelay, func() { + StreamStatusStore.Clear(streamId) + GatewayStatus.Clear(streamId) + }) + defer func() { + eventTicker.Stop() + eventsDone <- true + }() + const maxRetries = 5 + const retryPause = 300 * time.Millisecond + retries := 0 + for { + select { + case <-ctx.Done(): + clog.Info(ctx, "event subscription done") + return + default: + } + clog.Infof(ctx, "Reading from event subscription for URL: %s", url.String()) + segment, err := subscriber.Read() + if err == nil { + retries = 0 + } else { + // handle errors from event read + if errors.Is(err, trickle.EOS) || errors.Is(err, trickle.StreamNotFoundErr) { + clog.Infof(ctx, "Stopping subscription due to %s", err) + return + } + var seqErr *trickle.SequenceNonexistent + if errors.As(err, &seqErr) { + // stream exists but segment doesn't, so skip to leading edge + subscriber.SetSeq(seqErr.Latest) + } + if retries > maxRetries { + stopProcessing(ctx, params, fmt.Errorf("too many errors reading events; stopping subscription, err=%w", err)) + return + } + clog.Infof(ctx, "Error reading events subscription: err=%v retry=%d", err, retries) + retries++ + time.Sleep(retryPause) + continue + } + + body, err := io.ReadAll(segment.Body) + segment.Body.Close() + + if err != nil { + clog.Infof(ctx, "Error reading events subscription body: %s", err) + continue + } + + var eventWrapper struct { + QueueEventType string `json:"queue_event_type"` + Event map[string]interface{} `json:"event"` + } + if err := json.Unmarshal(body, &eventWrapper); err != nil { + clog.Infof(ctx, "Failed to parse JSON from events subscription: %s", err) + continue + } + + event := eventWrapper.Event + queueEventType := eventWrapper.QueueEventType + if event == nil { + // revert this once push to prod -- If no "event" field found, treat the entire body as the event + event = make(map[string]interface{}) + if err := json.Unmarshal(body, &event); err != nil { + clog.Infof(ctx, "Failed to parse JSON as direct event: %s", err) + continue + } + queueEventType = "ai_stream_events" + } + + event["stream_id"] = streamId + event["request_id"] = params.liveParams.requestID + event["pipeline_id"] = params.liveParams.pipelineID + event["orchestrator_info"] = map[string]interface{}{ + "address": clog.GetVal(ctx, "orch"), + "url": clog.GetVal(ctx, "orch_url"), + } + + clog.V(8).Infof(ctx, "Received event for seq=%d event=%+v", trickle.GetSeq(segment), event) + + // record the event time + lastEventMu.Lock() + lastEvent = time.Now() + lastEventMu.Unlock() + + eventType, ok := event["type"].(string) + if !ok { + eventType = "unknown" + clog.Warningf(ctx, "Received event without a type stream=%s event=%+v", stream, event) + } + + if eventType == "status" { + queueEventType = "ai_stream_status" + // The large logs and params fields are only sent once and then cleared to save bandwidth. So coalesce the + // incoming status with the last non-null value that we received on such fields for the status API. + lastStreamStatus, _ := StreamStatusStore.Get(streamId) + + // Check if inference_status exists in both current and last status + inferenceStatus, hasInference := event["inference_status"].(map[string]interface{}) + lastInferenceStatus, hasLastInference := lastStreamStatus["inference_status"].(map[string]interface{}) + + if hasInference { + if logs, ok := inferenceStatus["last_restart_logs"]; !ok || logs == nil { + if hasLastInference { + inferenceStatus["last_restart_logs"] = lastInferenceStatus["last_restart_logs"] + } + } + if params, ok := inferenceStatus["last_params"]; !ok || params == nil { + if hasLastInference { + inferenceStatus["last_params"] = lastInferenceStatus["last_params"] + } + } + } + + StreamStatusStore.Store(streamId, event) + } + + monitor.SendQueueEventAsync(queueEventType, event) + } + }() + + // Use events as a heartbeat of sorts: + // if no events arrive for too long, abort the job + go func() { + for { + select { + case <-eventTicker.C: + lastEventMu.Lock() + eventTime := lastEvent + lastEventMu.Unlock() + if time.Now().Sub(eventTime) > maxEventGap { + stopProcessing(ctx, params, fmt.Errorf("timeout waiting for events")) + eventTicker.Stop() + return + } + case <-eventsDone: + return + } + } + }() +} + +func ffmpegStreamOutput(ctx context.Context, outputUrl string, outWriter *media.RingBuffer, streamInfo *core.StreamInfo) { + // Clone the context since we can call this function multiple times + // Adding rtmpOut val multiple times to the same context will just stomp over old ones + ctx = clog.Clone(ctx, ctx) + ctx = clog.AddVal(ctx, "rtmpOut", outputUrl) + params := streamInfo.Params.(aiRequestParams) + defer func() { + if rec := recover(); rec != nil { + // panicked, so shut down the stream and handle it + err, ok := rec.(error) + if !ok { + err = errors.New("unknown error") + } + stopProcessing(ctx, params, fmt.Errorf("ffmpeg panic: %w", err)) + } + }() + for { + clog.V(6).Infof(ctx, "Starting output rtmp") + if !streamInfo.IsActive() { + clog.Errorf(ctx, "Stopping output rtmp stream, input stream does not exist.") + break + } + + // we receive opus by default, but re-encode to AAC for non-local outputs + acodec := "copy" + if !strings.Contains(outputUrl, params.liveParams.localRTMPPrefix) { + acodec = "libfdk_aac" + } + + cmd := exec.CommandContext(ctx, "ffmpeg", + "-analyzeduration", "2500000", // 2.5 seconds + "-i", "pipe:0", + "-c:a", acodec, + "-c:v", "copy", + "-f", "flv", + outputUrl, + ) + // Change Cancel function to send a SIGTERM instead of SIGKILL. Still send a SIGKILL after 5s (WaitDelay) if it's stuck. + cmd.Cancel = func() error { + return cmd.Process.Signal(syscall.SIGTERM) + } + cmd.WaitDelay = 5 * time.Second + cmd.Stdin = outWriter.MakeReader() // start at leading edge of output for each retry + output, err := cmd.CombinedOutput() + clog.Infof(ctx, "Process err=%v output: %s", err, output) + + select { + case <-ctx.Done(): + clog.Info(ctx, "Context done, stopping rtmp output") + return // Returns context.Canceled or context.DeadlineExceeded + default: + // Context is still active, continue with normal processing + } + + time.Sleep(5 * time.Second) + } +} From 56def522e15eb2b209411546f10264dbc0b2d749 Mon Sep 17 00:00:00 2001 From: Brad P Date: Wed, 27 Aug 2025 20:16:53 -0500 Subject: [PATCH 23/57] updates --- server/job_stream.go | 24 ++++++++++++++++++------ server/job_trickle.go | 18 +++++++++--------- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/server/job_stream.go b/server/job_stream.go index 55c0dc44ea..0f43be4c96 100644 --- a/server/job_stream.go +++ b/server/job_stream.go @@ -583,8 +583,6 @@ func (ls *LivepeerServer) StartStreamWhipIngest(whipServer *media.WHIPServer) ht return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { remoteAddr := getRemoteAddr(r) ctx := clog.AddVal(r.Context(), clog.ClientIP, remoteAddr) - requestID := string(core.RandomManifestID()) - ctx = clog.AddVal(ctx, "request_id", requestID) streamId := r.PathValue("streamId") ctx = clog.AddVal(ctx, "stream_id", streamId) @@ -598,7 +596,7 @@ func (ls *LivepeerServer) StartStreamWhipIngest(whipServer *media.WHIPServer) ht params := stream.Params.(aiRequestParams) whipConn := media.NewWHIPConnection() - whepURL := generateWhepUrl(streamId, requestID) + whepURL := generateWhepUrl(streamId, params.liveParams.requestID) // this function is called when the pipeline hits a fatal error, we kick the input connection to allow // the client to reconnect and restart the pipeline @@ -617,7 +615,7 @@ func (ls *LivepeerServer) StartStreamWhipIngest(whipServer *media.WHIPServer) ht go func() { statsContext, statsCancel := context.WithCancel(ctx) defer statsCancel() - go runStats(statsContext, whipConn, streamId, stream.Capability, requestID) + go runStats(statsContext, whipConn, streamId, stream.Capability, params.liveParams.requestID) whipConn.AwaitClose() stream.CancelStream() //cleanupControl(ctx, params) @@ -901,14 +899,14 @@ func (h *lphttp) StartStream(w http.ResponseWriter, r *http.Request) { pubCh := trickle.NewLocalPublisher(h.trickleSrv, mid, "video/MP2T") pubCh.CreateChannel() pubUrl = overwriteHost(h.node.LiveAITrickleHostForRunner, pubUrl) - reqBody["publish_url"] = pubUrl + reqBody["subscribe_url"] = pubUrl //runner needs to subscribe to input } if jobParams.EnableVideoEgress { subCh := trickle.NewLocalPublisher(h.trickleSrv, mid+"-out", "video/MP2T") subCh.CreateChannel() subUrl = overwriteHost(h.node.LiveAITrickleHostForRunner, subUrl) - reqBody["subscribe_url"] = subUrl + reqBody["publish_url"] = subUrl //runner needs to send results -out } if jobParams.EnableDataOutput { @@ -1000,6 +998,20 @@ func (h *lphttp) StartStream(w http.ResponseWriter, r *http.Request) { } } + //check if stream still exists + // if not, send stop to worker and exit monitoring + _, exists := h.node.ExternalCapabilities.Streams[orchJob.Req.ID] + if !exists { + req, err := http.NewRequestWithContext(ctx, "POST", orchJob.Req.CapabilityUrl+"/stream/stop", nil) + // set the headers + _, err = sendReqWithTimeout(req, time.Duration(orchJob.Req.Timeout)*time.Second) + if err != nil { + clog.Errorf(ctx, "Error sending request to worker %v: %v", workerRoute, err) + respondWithError(w, "Error sending request to worker", http.StatusInternalServerError) + return + } + return + } } } }() diff --git a/server/job_trickle.go b/server/job_trickle.go index 479cfb9f8b..fd6600e71a 100644 --- a/server/job_trickle.go +++ b/server/job_trickle.go @@ -36,7 +36,6 @@ func startStreamTricklePublish(ctx context.Context, url *url.URL, streamInfo *co // Start payments which probes a segment every "paymentProcessInterval" and sends a payment ctx, cancel := context.WithCancel(ctx) - var paymentProcessor *LivePaymentProcessor //byoc sets as context values orchAddr := clog.GetVal(ctx, "orch") orchUrl := clog.GetVal(ctx, "orch_url") @@ -65,9 +64,6 @@ func startStreamTricklePublish(ctx context.Context, url *url.URL, streamInfo *co go func(seq int) { defer slowOrchChecker.EndSegment() var r io.Reader = reader - if paymentProcessor != nil { - r = paymentProcessor.process(ctx, reader) - } clog.V(8).Infof(ctx, "trickle publish writing data seq=%d", seq) segment, err := publisher.Next() @@ -244,7 +240,7 @@ func startStreamTrickleSubscribe(ctx context.Context, url *url.URL, streamInfo * maxSegmentDelay := params.liveParams.outSegmentTimeout / 2 if segmentAge < maxSegmentDelay && streamInfo.IsActive() { // we have some recent input but no output from orch, so kick - suspendOrchestrator(ctx, params) + streamInfo.ExcludeOrch(orchUrl) //suspendOrchestrator(ctx, params) stopProcessing(ctx, params, fmt.Errorf("trickle subscribe error, swapping: %w", err)) return } @@ -257,7 +253,7 @@ func startStreamTrickleSubscribe(ctx context.Context, url *url.URL, streamInfo * firstSegment = false delayMs := time.Since(params.liveParams.startTime).Milliseconds() if monitor.Enabled { - monitor.AIFirstSegmentDelay(delayMs, params.liveParams.sess.OrchestratorInfo) + //monitor.AIFirstSegmentDelay(delayMs, streamInfo) //update this to take the address and url as strings monitor.SendQueueEventAsync("stream_trace", map[string]interface{}{ "type": "gateway_receive_first_processed_segment", "timestamp": time.Now().UnixMilli(), @@ -398,6 +394,8 @@ func startStreamControlPublish(ctx context.Context, control *url.URL, streamInfo func startStreamDataSubscribe(ctx context.Context, url *url.URL, streamInfo *core.StreamInfo) { //only start DataSubscribe if enabled params := streamInfo.Params.(aiRequestParams) + orchAddr := clog.GetVal(ctx, "orch") + orchUrl := clog.GetVal(ctx, "orch_url") if params.liveParams.dataWriter == nil { return } @@ -495,7 +493,7 @@ func startStreamDataSubscribe(ctx context.Context, url *url.URL, streamInfo *cor firstSegment = false delayMs := time.Since(params.liveParams.startTime).Milliseconds() if monitor.Enabled { - monitor.AIFirstSegmentDelay(delayMs, params.liveParams.sess.OrchestratorInfo) + //monitor.AIFirstSegmentDelay(delayMs, params.liveParams.sess.OrchestratorInfo) monitor.SendQueueEventAsync("stream_trace", map[string]interface{}{ "type": "gateway_receive_first_data_segment", "timestamp": time.Now().UnixMilli(), @@ -503,11 +501,13 @@ func startStreamDataSubscribe(ctx context.Context, url *url.URL, streamInfo *cor "pipeline_id": params.liveParams.pipelineID, "request_id": params.liveParams.requestID, "orchestrator_info": map[string]interface{}{ - "address": params.liveParams.sess.Address(), - "url": params.liveParams.sess.Transcoder(), + "address": orchAddr, + "url": orchUrl, }, }) } + + clog.V(common.VERBOSE).Infof(ctx, "First Data Segment delay=%dms streamID=%s", delayMs, params.liveParams.streamID) } clog.V(8).Info(ctx, "data subscribe read completed", "seq", seq, "bytes", humanize.Bytes(uint64(readBytes)), "messages", readMessages, "took", time.Since(copyStartTime)) From b480f78bf85d82fd32a44d3eeda6ea5f6c40224f Mon Sep 17 00:00:00 2001 From: Brad P Date: Thu, 28 Aug 2025 08:07:41 -0500 Subject: [PATCH 24/57] only set trickle urls for gateway if enabled --- server/job_stream.go | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/server/job_stream.go b/server/job_stream.go index 0f43be4c96..6200fbd452 100644 --- a/server/job_stream.go +++ b/server/job_stream.go @@ -881,43 +881,48 @@ func (h *lphttp) StartStream(w http.ResponseWriter, r *http.Request) { dataUrl = pubUrl + "-data" ) - reqBody := make(map[string]interface{}) - reqBody["gateway_request_id"] = mid + reqBodyForRunner := make(map[string]interface{}) + reqBodyForRunner["gateway_request_id"] = mid //required channels controlPubCh := trickle.NewLocalPublisher(h.trickleSrv, mid+"-control", "application/json") controlPubCh.CreateChannel() controlUrl = overwriteHost(h.node.LiveAITrickleHostForRunner, controlUrl) - reqBody["control_url"] = controlUrl + reqBodyForRunner["control_url"] = controlUrl + w.Header().Set("X-Control-Url", controlUrl) eventsCh := trickle.NewLocalPublisher(h.trickleSrv, mid+"-events", "application/json") eventsCh.CreateChannel() eventsUrl = overwriteHost(h.node.LiveAITrickleHostForRunner, eventsUrl) - reqBody["events_url"] = eventsUrl + reqBodyForRunner["events_url"] = eventsUrl + w.Header().Set("X-Events-Url", eventsUrl) //Optional channels if jobParams.EnableVideoIngress { pubCh := trickle.NewLocalPublisher(h.trickleSrv, mid, "video/MP2T") pubCh.CreateChannel() pubUrl = overwriteHost(h.node.LiveAITrickleHostForRunner, pubUrl) - reqBody["subscribe_url"] = pubUrl //runner needs to subscribe to input + reqBodyForRunner["subscribe_url"] = pubUrl //runner needs to subscribe to input + w.Header().Set("X-Publish-Url", pubUrl) //gateway will connect to pubUrl to send ingress video } if jobParams.EnableVideoEgress { subCh := trickle.NewLocalPublisher(h.trickleSrv, mid+"-out", "video/MP2T") subCh.CreateChannel() subUrl = overwriteHost(h.node.LiveAITrickleHostForRunner, subUrl) - reqBody["publish_url"] = subUrl //runner needs to send results -out + reqBodyForRunner["publish_url"] = subUrl //runner needs to send results -out + w.Header().Set("X-Subscribe-Url", subUrl) //gateway will connect to subUrl to receive results } if jobParams.EnableDataOutput { dataCh := trickle.NewLocalPublisher(h.trickleSrv, mid+"-data", "application/jsonl") dataCh.CreateChannel() dataUrl = overwriteHost(h.node.LiveAITrickleHostForRunner, dataUrl) - reqBody["data_url"] = dataUrl + reqBodyForRunner["data_url"] = dataUrl + w.Header().Set("X-Data-Url", dataUrl) } - reqBody["request"] = base64.StdEncoding.EncodeToString(body) - reqBodyBytes, err := json.Marshal(reqBody) + reqBodyForRunner["request"] = base64.StdEncoding.EncodeToString(body) + reqBodyBytes, err := json.Marshal(reqBodyForRunner) if err != nil { clog.Errorf(ctx, "Failed to marshal request body err=%v", err) http.Error(w, "Failed to marshal request body", http.StatusInternalServerError) @@ -1016,13 +1021,7 @@ func (h *lphttp) StartStream(w http.ResponseWriter, r *http.Request) { } }() - //send back trickle urls - w.Header().Set("X-Publish-Url", pubUrl) - w.Header().Set("X-Subscribe-Url", subUrl) - w.Header().Set("X-Control-Url", controlUrl) - w.Header().Set("X-Events-Url", eventsUrl) - w.Header().Set("X-Data-Url", dataUrl) - + //send back the trickle urls set in header w.WriteHeader(http.StatusOK) return } From 469da959098c2b588f1d4c425a0f81b4e9ee4786 Mon Sep 17 00:00:00 2001 From: Brad P Date: Fri, 29 Aug 2025 07:19:15 -0500 Subject: [PATCH 25/57] various updates --- core/external_capabilities.go | 6 +++- server/ai_mediaserver.go | 2 +- server/job_stream.go | 56 ++++++++++++++++++++++++----------- server/job_trickle.go | 10 +++---- 4 files changed, 49 insertions(+), 25 deletions(-) diff --git a/core/external_capabilities.go b/core/external_capabilities.go index 7af5118ff5..9f3cd3a846 100644 --- a/core/external_capabilities.go +++ b/core/external_capabilities.go @@ -125,6 +125,7 @@ func (extCaps *ExternalCapabilities) AddStream(streamID string, params interface stream.StopControl() stream.ControlPub.Close() } + return } } @@ -139,7 +140,10 @@ func (extCaps *ExternalCapabilities) RemoveStream(streamID string) { streamInfo, ok := extCaps.Streams[streamID] if ok { - streamInfo.CancelStream() + //confirm stream context is canceled before deleting + if streamInfo.StreamCtx.Err() == nil { + streamInfo.CancelStream() + } } delete(extCaps.Streams, streamID) diff --git a/server/ai_mediaserver.go b/server/ai_mediaserver.go index a9507a9b74..0b5835d860 100644 --- a/server/ai_mediaserver.go +++ b/server/ai_mediaserver.go @@ -119,7 +119,7 @@ func startAIMediaServer(ctx context.Context, ls *LivepeerServer) error { ls.HTTPMux.Handle("OPTIONS /ai/stream/", ls.WithCode(http.StatusNoContent)) ls.HTTPMux.Handle("POST /ai/stream/start", ls.StartStream()) - ls.HTTPMux.Handle("POST /ai/stream/stop", ls.StopStream()) + ls.HTTPMux.Handle("POST /ai/stream/{streamId}/stop", ls.StopStream()) if os.Getenv("LIVE_AI_WHIP_ADDR") != "" { ls.HTTPMux.Handle("POST /ai/stream/{streamId}/whip", ls.StartStreamWhipIngest(whipServer)) } diff --git a/server/job_stream.go b/server/job_stream.go index 6200fbd452..621fa541b9 100644 --- a/server/job_stream.go +++ b/server/job_stream.go @@ -55,6 +55,8 @@ func (ls *LivepeerServer) StartStream() http.Handler { go ls.runStream(gatewayJob) + go ls.monitorStream(gatewayJob.Job.Req.ID) + if resp != nil { // Stream started successfully w.Header().Set("Content-Type", "application/json") @@ -108,14 +110,15 @@ func (ls *LivepeerServer) StopStream() http.Handler { func (ls *LivepeerServer) runStream(gatewayJob *gatewayJob) { streamID := gatewayJob.Job.Req.ID - ctx := context.Background() - ctx = clog.AddVal(ctx, "stream_id", streamID) stream, exists := ls.LivepeerNode.ExternalCapabilities.Streams[streamID] if !exists { - clog.Errorf(ctx, "Stream %s not found", streamID) + glog.Errorf("Stream %s not found", streamID) return } params := stream.Params.(aiRequestParams) + //this context passes to all channels that will close when stream is canceled + ctx := stream.StreamCtx + ctx = clog.AddVal(ctx, "stream_id", streamID) start := time.Now() for _, orch := range gatewayJob.Orchs { @@ -191,16 +194,29 @@ func (ls *LivepeerServer) runStream(gatewayJob *gatewayJob) { // // failed before selecting an orchestrator // firstProcessed <- struct{}{} //} - params.liveParams.kickInput(err) + + //if there is ingress input then force off + if params.liveParams.kickInput != nil { + params.liveParams.kickInput(err) + } } + + //exhausted all Orchestrators, end stream + ls.LivepeerNode.ExternalCapabilities.RemoveStream(streamID) } -func (ls *LivepeerServer) monitorStream(ctx context.Context, streamId string) { +func (ls *LivepeerServer) monitorStream(streamId string) { + ctx := context.Background() + ctx = clog.AddVal(ctx, "stream_id", streamId) + stream, exists := ls.LivepeerNode.ExternalCapabilities.Streams[streamId] if !exists { clog.Errorf(ctx, "Stream %s not found", streamId) return } + params := stream.Params.(aiRequestParams) + + ctx = clog.AddVal(ctx, "request_id", params.liveParams.requestID) // Create a ticker that runs every minute for payments pmtTicker := time.NewTicker(45 * time.Second) @@ -210,6 +226,7 @@ func (ls *LivepeerServer) monitorStream(ctx context.Context, streamId string) { select { case <-stream.StreamCtx.Done(): clog.Infof(ctx, "Stream %s stopped, ending monitoring", streamId) + ls.LivepeerNode.ExternalCapabilities.RemoveStream(streamId) return case <-pmtTicker.C: // Send payment and fetch new JobToken every minute @@ -271,6 +288,8 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, req // Create a temporary request for parsing form data formReq := r.Clone(ctx) formReq.Body = io.NopCloser(bodyForForm) + defer r.Body.Close() + defer formReq.Body.Close() // Parse the form (10MB max) if err := formReq.ParseMultipartForm(10 << 20); err != nil { @@ -328,6 +347,7 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, req if streamName != "" { streamID = fmt.Sprintf("%s-%s", streamName, streamID) } + // BYOC uses Livepeer native WHIP // Currently for webrtc we need to add a path prefix due to the ingress setup //mediaMTXStreamPrefix := r.PathValue("prefix") //if mediaMTXStreamPrefix != "" { @@ -390,7 +410,7 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, req } ctx = clog.AddVal(ctx, "stream_id", streamID) - clog.Infof(ctx, "Received live video AI request for %s. pipelineParams=%v", streamName, pipelineParams) + clog.Infof(ctx, "Received live video AI request pipelineParams=%v", streamID, pipelineParams) // collect all RTMP outputs var rtmpOutputs []string @@ -476,14 +496,11 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, req clog.Infof(ctx, "stream setup videoIngress=%v videoEgress=%v dataOutput=%v", jobParams.EnableVideoIngress, jobParams.EnableVideoEgress, jobParams.EnableDataOutput) - // No need to read the body again since we already have it - // We're using the original request body that was already read and stored - // We already have the bodyBytes variable which contains the full request body - body := bodyBytes + // Close the body, done with all form variables r.Body.Close() //save the stream setup - if err := ls.LivepeerNode.ExternalCapabilities.AddStream(streamID, params, body); err != nil { + if err := ls.LivepeerNode.ExternalCapabilities.AddStream(streamID, params, bodyBytes); err != nil { return nil, http.StatusBadRequest, err } @@ -633,10 +650,6 @@ func (ls *LivepeerServer) StartStreamWhipIngest(whipServer *media.WHIPServer) ht func startStreamProcessing(ctx context.Context, streamInfo *core.StreamInfo) error { var channels []string - //this adds the stream to LivePipelines which the Control Publisher and Data Writer - //are accessible for reading data and sending updates - //registerControl(ctx, params) - //required channels control, err := common.AppendHostname(streamInfo.OrchControlUrl, streamInfo.OrchUrl) if err != nil { @@ -978,8 +991,11 @@ func (h *lphttp) StartStream(w http.ResponseWriter, r *http.Request) { //start payment monitoring go func() { stream, _ := h.node.ExternalCapabilities.Streams[orchJob.Req.ID] + ctx := context.Background() + ctx = clog.AddVal(ctx, "stream_id", orchJob.Req.ID) - pmtTicker := time.NewTicker(30 * time.Second) + pmtCheckDur := 30 * time.Second + pmtTicker := time.NewTicker(pmtCheckDur) defer pmtTicker.Stop() for { select { @@ -987,10 +1003,10 @@ func (h *lphttp) StartStream(w http.ResponseWriter, r *http.Request) { return case <-pmtTicker.C: // Check payment status - clog.V(8).Infof(ctx, "Checking payment balance for stream") + jobPriceRat := big.NewRat(orchJob.JobPrice.PricePerUnit, orchJob.JobPrice.PixelsPerUnit) if jobPriceRat.Cmp(big.NewRat(0, 1)) > 0 { - h.orchestrator.DebitFees(orchJob.Sender, core.ManifestID(orchJob.Req.Capability), orchJob.JobPrice, 5) + h.orchestrator.DebitFees(orchJob.Sender, core.ManifestID(orchJob.Req.Capability), orchJob.JobPrice, int64(pmtCheckDur.Seconds())) senderBalance := getPaymentBalance(orch, orchJob.Sender, orchJob.Req.Capability) if senderBalance != nil { if senderBalance.Cmp(big.NewRat(0, 1)) < 0 { @@ -1000,6 +1016,8 @@ func (h *lphttp) StartStream(w http.ResponseWriter, r *http.Request) { h.node.ExternalCapabilities.RemoveStream(orchJob.Req.ID) } } + + clog.V(8).Infof(ctx, "Payment balance for stream capability is good balance=%v", senderBalance.FloatString(0)) } } @@ -1015,6 +1033,7 @@ func (h *lphttp) StartStream(w http.ResponseWriter, r *http.Request) { respondWithError(w, "Error sending request to worker", http.StatusInternalServerError) return } + //end monitoring of stream return } } @@ -1135,6 +1154,7 @@ func (h *lphttp) ProcessStreamPayment(w http.ResponseWriter, r *http.Request) { } jobToken := JobToken{TicketParams: ticketParams, Balance: capBalInt, Price: jobPrice} + clog.Infof(ctx, "Processed payment request id=%s capability=%s balance=%v pricePerUnit=%v pixelsPerUnit=%v", orchJob.Req.ID, orchJob.Req.Capability, jobToken.Balance, jobPrice.PricePerUnit, jobPrice.PixelsPerUnit) json.NewEncoder(w).Encode(jobToken) } diff --git a/server/job_trickle.go b/server/job_trickle.go index fd6600e71a..4b683624ce 100644 --- a/server/job_trickle.go +++ b/server/job_trickle.go @@ -190,7 +190,7 @@ func startStreamTrickleSubscribe(ctx context.Context, url *url.URL, streamInfo * return default: } - if !streamInfo.IsActive() { + if streamInfo != nil && !streamInfo.IsActive() { clog.Infof(ctx, "trickle subscribe stopping, input stream does not exist.") break } @@ -238,7 +238,7 @@ func startStreamTrickleSubscribe(ctx context.Context, url *url.URL, streamInfo * params.liveParams.mu.Unlock() segmentAge := time.Since(lastSegmentTime) maxSegmentDelay := params.liveParams.outSegmentTimeout / 2 - if segmentAge < maxSegmentDelay && streamInfo.IsActive() { + if segmentAge < maxSegmentDelay && streamInfo != nil && streamInfo.IsActive() { // we have some recent input but no output from orch, so kick streamInfo.ExcludeOrch(orchUrl) //suspendOrchestrator(ctx, params) stopProcessing(ctx, params, fmt.Errorf("trickle subscribe error, swapping: %w", err)) @@ -302,7 +302,7 @@ func startStreamTrickleSubscribe(ctx context.Context, url *url.URL, streamInfo * params.liveParams.mu.Unlock() lastInputSegmentAge := time.Since(lastInputSegmentTime) hasRecentInput := lastInputSegmentAge < segmentTimeout/2 - if hasRecentInput && streamInfo.IsActive() { + if hasRecentInput && streamInfo != nil && streamInfo.IsActive() { // abandon the orchestrator streamInfo.ExcludeOrch(orchUrl) stopProcessing(ctx, params, fmt.Errorf("timeout waiting for segments")) @@ -430,7 +430,7 @@ func startStreamDataSubscribe(ctx context.Context, url *url.URL, streamInfo *cor return default: } - if !streamInfo.IsActive() { + if streamInfo != nil && !streamInfo.IsActive() { clog.Infof(ctx, "data subscribe stopping, input stream does not exist.") break } @@ -704,7 +704,7 @@ func ffmpegStreamOutput(ctx context.Context, outputUrl string, outWriter *media. }() for { clog.V(6).Infof(ctx, "Starting output rtmp") - if !streamInfo.IsActive() { + if streamInfo != nil && !streamInfo.IsActive() { clog.Errorf(ctx, "Stopping output rtmp stream, input stream does not exist.") break } From c6cfec13033a40d72fa423596895b1859e0ae699 Mon Sep 17 00:00:00 2001 From: Brad P Date: Fri, 29 Aug 2025 08:25:03 -0500 Subject: [PATCH 26/57] add OrchToken to stream for each current orch --- server/job_stream.go | 1 + 1 file changed, 1 insertion(+) diff --git a/server/job_stream.go b/server/job_stream.go index 621fa541b9..7408f97222 100644 --- a/server/job_stream.go +++ b/server/job_stream.go @@ -122,6 +122,7 @@ func (ls *LivepeerServer) runStream(gatewayJob *gatewayJob) { start := time.Now() for _, orch := range gatewayJob.Orchs { + stream.OrchToken = orch ctx = clog.AddVal(ctx, "orch", ethcommon.Bytes2Hex(orch.TicketParams.Recipient)) ctx = clog.AddVal(ctx, "orch_url", orch.ServiceAddr) clog.Infof(ctx, "Starting stream processing") From a1b9a57ffc40b819026728da540f1fc36e4226eb Mon Sep 17 00:00:00 2001 From: Brad P Date: Fri, 29 Aug 2025 14:07:46 -0500 Subject: [PATCH 27/57] various updates --- core/external_capabilities.go | 3 +- server/job_rpc.go | 36 +++++++++++------- server/job_stream.go | 72 ++++++++++++++++++++--------------- server/rpc.go | 6 +-- 4 files changed, 69 insertions(+), 48 deletions(-) diff --git a/core/external_capabilities.go b/core/external_capabilities.go index 9f3cd3a846..1331e05177 100644 --- a/core/external_capabilities.go +++ b/core/external_capabilities.go @@ -95,7 +95,7 @@ func NewExternalCapabilities() *ExternalCapabilities { } } -func (extCaps *ExternalCapabilities) AddStream(streamID string, params interface{}, streamReq []byte) error { +func (extCaps *ExternalCapabilities) AddStream(streamID string, pipeline string, params interface{}, streamReq []byte) error { extCaps.capm.Lock() defer extCaps.capm.Unlock() _, ok := extCaps.Streams[streamID] @@ -107,6 +107,7 @@ func (extCaps *ExternalCapabilities) AddStream(streamID string, params interface ctx, cancel := context.WithCancel(context.Background()) stream := StreamInfo{ StreamID: streamID, + Capability: pipeline, Params: params, // Store the interface value directly, not a pointer to it StreamRequest: streamReq, StreamCtx: ctx, diff --git a/server/job_rpc.go b/server/job_rpc.go index dbfa473325..cc56668839 100644 --- a/server/job_rpc.go +++ b/server/job_rpc.go @@ -109,11 +109,13 @@ type gatewayJob struct { Job *orchJob Orchs []JobToken SignedJobReq string + + node *core.LivepeerNode } -func (g *gatewayJob) sign(node *core.LivepeerNode) error { +func (g *gatewayJob) sign() error { //sign the request - gateway := node.OrchestratorPool.Broadcaster() + gateway := g.node.OrchestratorPool.Broadcaster() sig, err := gateway.Sign([]byte(g.Job.Req.Request + g.Job.Req.Parameters)) if err != nil { return errors.New(fmt.Sprintf("Unable to sign request err=%v", err)) @@ -308,7 +310,7 @@ func (ls *LivepeerServer) setupGatewayJob(ctx context.Context, r *http.Request) jobReqHdr := r.Header.Get(jobRequestHdr) clog.Infof(ctx, "processing job request req=%v", jobReqHdr) - jobReq, err := verifyJobCreds(ctx, nil, jobReqHdr) + jobReq, err := verifyJobCreds(ctx, nil, jobReqHdr, true) if err != nil { return nil, errors.New(fmt.Sprintf("Unable to parse job request, err=%v", err)) } @@ -342,7 +344,7 @@ func (ls *LivepeerServer) setupGatewayJob(ctx context.Context, r *http.Request) Params: &jobParams, } - return &gatewayJob{Job: &job, Orchs: orchs}, nil + return &gatewayJob{Job: &job, Orchs: orchs, node: ls.LivepeerNode}, nil } func (h *lphttp) ProcessJob(w http.ResponseWriter, r *http.Request) { @@ -413,7 +415,7 @@ func (ls *LivepeerServer) submitJob(ctx context.Context, w http.ResponseWriter, workerRoute = workerRoute + "/" + workerResourceRoute } - err := gatewayJob.sign(ls.LivepeerNode) + err := gatewayJob.sign() if err != nil { clog.Errorf(ctx, "Error signing job, exiting stream processing request: %v", err) return @@ -585,7 +587,7 @@ func (ls *LivepeerServer) sendPayment(ctx context.Context, orchPmtUrl, capabilit req.Header.Add(jobRequestHdr, jobReq) req.Header.Add(jobPaymentHeaderHdr, payment) - resp, err := sendJobReqWithTimeout(req, 10) + resp, err := sendJobReqWithTimeout(req, 10*time.Second) if err != nil { clog.Errorf(ctx, "job payment not able to be processed by Orchestrator %v err=%v ", orchPmtUrl, err.Error()) return nil, err @@ -607,7 +609,7 @@ func processJob(ctx context.Context, h *lphttp, w http.ResponseWriter, r *http.R orch := h.orchestrator // check the prompt sig from the request // confirms capacity available before processing payment info - orchJob, err := h.setupOrchJob(ctx, r) + orchJob, err := h.setupOrchJob(ctx, r, true) if err != nil { if err == errNoCapabilityCapacity { http.Error(w, err.Error(), http.StatusServiceUnavailable) @@ -803,18 +805,19 @@ func processJob(ctx context.Context, h *lphttp, w http.ResponseWriter, r *http.R // SetupOrchJob prepares the orchestrator job by extracting and validating the job request from the HTTP headers. // Payment is applied if applicable. -func (h *lphttp) setupOrchJob(ctx context.Context, r *http.Request) (*orchJob, error) { +func (h *lphttp) setupOrchJob(ctx context.Context, r *http.Request, reserveCapacity bool) (*orchJob, error) { clog.Infof(ctx, "processing job request") job := r.Header.Get(jobRequestHdr) orch := h.orchestrator - jobReq, err := verifyJobCreds(ctx, orch, job) + jobReq, err := verifyJobCreds(ctx, orch, job, reserveCapacity) if err != nil { - if err == errZeroCapacity { + if err == errZeroCapacity && reserveCapacity { return nil, errNoCapabilityCapacity } else if err == errNoTimeoutSet { return nil, errNoTimeoutSet } else { + clog.Errorf(ctx, "job failed verification: %v", err) return nil, errNoJobCreds } } @@ -834,6 +837,7 @@ func (h *lphttp) setupOrchJob(ctx context.Context, r *http.Request) (*orchJob, e // get payment information payment, err = getPayment(r.Header.Get(jobPaymentHeaderHdr)) if err != nil { + clog.Errorf(ctx, "job payment invalid: %v", err) return nil, errPaymentError } @@ -859,7 +863,13 @@ func (h *lphttp) setupOrchJob(ctx context.Context, r *http.Request) (*orchJob, e clog.Infof(ctx, "balance after payment is %v", getPaymentBalance(orch, sender, jobReq.Capability).FloatString(0)) } - return &orchJob{Req: jobReq, Sender: sender, JobPrice: jobPrice}, nil + var jobDetails JobRequestDetails + err = json.Unmarshal([]byte(jobReq.Request), &jobDetails) + if err != nil { + return nil, fmt.Errorf("Unable to unmarshal job request details err=%v", err) + } + + return &orchJob{Req: jobReq, Sender: sender, JobPrice: jobPrice, Details: &jobDetails}, nil } func createPayment(ctx context.Context, jobReq *JobRequest, orchToken JobToken, node *core.LivepeerNode) (string, error) { @@ -1062,7 +1072,7 @@ func parseJobRequest(jobReq string) (*JobRequest, error) { return &jobData, nil } -func verifyJobCreds(ctx context.Context, orch Orchestrator, jobCreds string) (*JobRequest, error) { +func verifyJobCreds(ctx context.Context, orch Orchestrator, jobCreds string, reserveCapacity bool) (*JobRequest, error) { //Gateway needs JobRequest parsed and verification of required fields jobData, err := parseJobRequest(jobCreds) if err != nil { @@ -1092,7 +1102,7 @@ func verifyJobCreds(ctx context.Context, orch Orchestrator, jobCreds string) (*J return nil, errSegSig } - if orch.ReserveExternalCapabilityCapacity(jobData.Capability) != nil { + if reserveCapacity && orch.ReserveExternalCapabilityCapacity(jobData.Capability) != nil { return nil, errZeroCapacity } diff --git a/server/job_stream.go b/server/job_stream.go index 7408f97222..8a3e63b98c 100644 --- a/server/job_stream.go +++ b/server/job_stream.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "encoding/base64" - "encoding/hex" "encoding/json" "errors" "fmt" @@ -88,8 +87,8 @@ func (ls *LivepeerServer) StopStream() http.Handler { return } - stopJob.sign(ls.LivepeerNode) - resp, code, err := ls.sendJobToOrch(ctx, r, stopJob.Job.Req, stopJob.SignedJobReq, streamInfoCopy.OrchToken.(JobToken), "/stream/stop", streamInfoCopy.StreamRequest) + stopJob.sign() + resp, code, err := ls.sendJobToOrch(ctx, r, stopJob.Job.Req, stopJob.SignedJobReq, streamInfoCopy.OrchToken.(JobToken), "/ai/stream/stop", streamInfoCopy.StreamRequest) if err != nil { clog.Errorf(ctx, "Error sending job to orchestrator: %s", err) http.Error(w, err.Error(), code) @@ -123,6 +122,7 @@ func (ls *LivepeerServer) runStream(gatewayJob *gatewayJob) { start := time.Now() for _, orch := range gatewayJob.Orchs { stream.OrchToken = orch + stream.OrchUrl = orch.ServiceAddr ctx = clog.AddVal(ctx, "orch", ethcommon.Bytes2Hex(orch.TicketParams.Recipient)) ctx = clog.AddVal(ctx, "orch_url", orch.ServiceAddr) clog.Infof(ctx, "Starting stream processing") @@ -137,13 +137,13 @@ func (ls *LivepeerServer) runStream(gatewayJob *gatewayJob) { //set request ID to persist from Gateway to Worker gatewayJob.Job.Req.ID = stream.StreamID - err := gatewayJob.sign(ls.LivepeerNode) + err := gatewayJob.sign() if err != nil { clog.Errorf(ctx, "Error signing job, exiting stream processing request: %v", err) stream.CancelStream() return } - orchResp, _, err := ls.sendJobToOrch(ctx, nil, gatewayJob.Job.Req, gatewayJob.SignedJobReq, orch, "/stream/start", stream.StreamRequest) + orchResp, _, err := ls.sendJobToOrch(ctx, nil, gatewayJob.Job.Req, gatewayJob.SignedJobReq, orch, "/ai/stream/start", stream.StreamRequest) if err != nil { clog.Errorf(ctx, "job not able to be processed by Orchestrator %v err=%v ", orch.ServiceAddr, err.Error()) continue @@ -231,25 +231,25 @@ func (ls *LivepeerServer) monitorStream(streamId string) { return case <-pmtTicker.C: // Send payment and fetch new JobToken every minute - req := &JobRequest{Capability: stream.Capability, + // TicketParams expire in about 8 minutes so have multiple chances to get new token + // Each payment is for 60 seconds of compute + jobDetails := JobRequestDetails{StreamId: streamId} + jobDetailsStr, err := json.Marshal(jobDetails) + if err != nil { + clog.Errorf(ctx, "Error marshalling job details: %v", err) + continue + } + req := &JobRequest{Request: string(jobDetailsStr), Parameters: "{}", Capability: stream.Capability, Sender: ls.LivepeerNode.OrchestratorPool.Broadcaster().Address().Hex(), Timeout: 60, } //sign the request - gateway := ls.LivepeerNode.OrchestratorPool.Broadcaster() - sig, err := gateway.Sign([]byte(req.Request + req.Parameters)) - if err != nil { - clog.Errorf(ctx, fmt.Sprintf("Unable to sign request err=%v", err)) - } - req.Sender = gateway.Address().Hex() - req.Sig = "0x" + hex.EncodeToString(sig) - - //create the job request header with the signature - jobReqEncoded, err := json.Marshal(req) + job := gatewayJob{Job: &orchJob{Req: req}, node: ls.LivepeerNode} + err = job.sign() if err != nil { - clog.Errorf(ctx, fmt.Sprintf("Unable to encode job request err=%v", err)) + clog.Errorf(ctx, "Error signing job, continuing monitoring: %v", err) + continue } - jobReqHdr := base64.StdEncoding.EncodeToString(jobReqEncoded) pmtHdr, err := createPayment(ctx, req, stream.OrchToken.(JobToken), ls.LivepeerNode) if err != nil { @@ -258,14 +258,20 @@ func (ls *LivepeerServer) monitorStream(streamId string) { } //send the payment, update the stream with the refreshed token - token, err := ls.sendPayment(ctx, stream.OrchUrl+"/stream/payment", stream.Capability, jobReqHdr, pmtHdr) + clog.Infof(ctx, "Sending stream payment for %s", streamId) + newToken, err := ls.sendPayment(ctx, stream.OrchUrl+"/ai/stream/payment", stream.Capability, job.SignedJobReq, pmtHdr) if err != nil { clog.Errorf(ctx, "Error sending stream payment for %s: %v", streamId, err) + continue + } + if newToken == nil { + clog.Errorf(ctx, "Updated token not received for %s", streamId) + continue } streamToken := stream.OrchToken.(JobToken) - streamToken.TicketParams = token.TicketParams - streamToken.Balance = token.Balance - streamToken.Price = token.Price + streamToken.TicketParams = newToken.TicketParams + streamToken.Balance = newToken.Balance + streamToken.Price = newToken.Price stream.OrchToken = streamToken } } @@ -501,7 +507,7 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, req r.Body.Close() //save the stream setup - if err := ls.LivepeerNode.ExternalCapabilities.AddStream(streamID, params, bodyBytes); err != nil { + if err := ls.LivepeerNode.ExternalCapabilities.AddStream(streamID, pipeline, params, bodyBytes); err != nil { return nil, http.StatusBadRequest, err } @@ -721,7 +727,6 @@ func (ls *LivepeerServer) GetStreamData() http.Handler { params := stream.Params.(aiRequestParams) // Get the data reading buffer if params.liveParams.dataWriter == nil { - clog.Infof(ctx, "No data writer available for stream %s", stream) http.Error(w, "Stream data not available", http.StatusServiceUnavailable) return } @@ -859,7 +864,7 @@ func (h *lphttp) StartStream(w http.ResponseWriter, r *http.Request) { remoteAddr := getRemoteAddr(r) ctx := clog.AddVal(r.Context(), clog.ClientIP, remoteAddr) - orchJob, err := h.setupOrchJob(ctx, r) + orchJob, err := h.setupOrchJob(ctx, r, false) if err != nil { respondWithError(w, err.Error(), http.StatusBadRequest) return @@ -982,7 +987,7 @@ func (h *lphttp) StartStream(w http.ResponseWriter, r *http.Request) { clog.V(common.SHORT).Infof(ctx, "stream start processed successfully took=%v balance=%v", time.Since(start), getPaymentBalance(orch, orchJob.Sender, orchJob.Req.Capability).FloatString(0)) //setup the stream - err = h.node.ExternalCapabilities.AddStream(orchJob.Req.ID, orchJob.Req, respBody) + err = h.node.ExternalCapabilities.AddStream(orchJob.Req.ID, orchJob.Req.Capability, orchJob.Req, respBody) if err != nil { clog.Errorf(ctx, "Error adding stream to external capabilities: %v", err) respondWithError(w, "Error adding stream to external capabilities", http.StatusInternalServerError) @@ -1001,6 +1006,7 @@ func (h *lphttp) StartStream(w http.ResponseWriter, r *http.Request) { for { select { case <-stream.StreamCtx.Done(): + h.orchestrator.FreeExternalCapabilityCapacity(orchJob.Req.Capability) return case <-pmtTicker.C: // Check payment status @@ -1048,7 +1054,7 @@ func (h *lphttp) StartStream(w http.ResponseWriter, r *http.Request) { func (h *lphttp) StopStream(w http.ResponseWriter, r *http.Request) { ctx := r.Context() - orchJob, err := h.setupOrchJob(ctx, r) + orchJob, err := h.setupOrchJob(ctx, r, false) if err != nil { respondWithError(w, fmt.Sprintf("Failed to stop stream, request not valid err=%v", err), http.StatusBadRequest) return @@ -1097,7 +1103,7 @@ func (h *lphttp) StopStream(w http.ResponseWriter, r *http.Request) { clog.Errorf(ctx, "error processing stream stop request statusCode=%d", resp.StatusCode) } - // Stop the stream + // Stop the stream and free capacity h.node.ExternalCapabilities.RemoveStream(jobDetails.StreamId) w.WriteHeader(resp.StatusCode) @@ -1108,14 +1114,18 @@ func (h *lphttp) ProcessStreamPayment(w http.ResponseWriter, r *http.Request) { orch := h.orchestrator ctx := r.Context() - //this will process the payment - orchJob, err := h.setupOrchJob(ctx, r) + //this will validate the request and process the payment + orchJob, err := h.setupOrchJob(ctx, r, false) if err != nil { respondWithError(w, fmt.Sprintf("Failed to process payment, request not valid err=%v", err), http.StatusBadRequest) return } + ctx = clog.AddVal(ctx, "stream_id", orchJob.Details.StreamId) + ctx = clog.AddVal(ctx, "capability", orchJob.Req.Capability) + ctx = clog.AddVal(ctx, "sender", orchJob.Req.Sender) senderAddr := ethcommon.HexToAddress(orchJob.Req.Sender) + clog.Infof(ctx, "Processing payment request") jobPrice, err := orch.JobPriceInfo(senderAddr, orchJob.Req.Capability) if err != nil { @@ -1155,7 +1165,7 @@ func (h *lphttp) ProcessStreamPayment(w http.ResponseWriter, r *http.Request) { } jobToken := JobToken{TicketParams: ticketParams, Balance: capBalInt, Price: jobPrice} - clog.Infof(ctx, "Processed payment request id=%s capability=%s balance=%v pricePerUnit=%v pixelsPerUnit=%v", orchJob.Req.ID, orchJob.Req.Capability, jobToken.Balance, jobPrice.PricePerUnit, jobPrice.PixelsPerUnit) + clog.Infof(ctx, "Processed payment request stream_id=%s capability=%s balance=%v pricePerUnit=%v pixelsPerUnit=%v", orchJob.Details.StreamId, orchJob.Req.Capability, jobToken.Balance, jobPrice.PricePerUnit, jobPrice.PixelsPerUnit) json.NewEncoder(w).Encode(jobToken) } diff --git a/server/rpc.go b/server/rpc.go index 0cf3554acb..4f1a0c1720 100644 --- a/server/rpc.go +++ b/server/rpc.go @@ -253,9 +253,9 @@ func StartTranscodeServer(orch Orchestrator, bind string, mux *http.ServeMux, wo lp.transRPC.HandleFunc("/process/token", lp.GetJobToken) lp.transRPC.HandleFunc("/capability/register", lp.RegisterCapability) lp.transRPC.HandleFunc("/capability/unregister", lp.UnregisterCapability) - lp.transRPC.HandleFunc("/stream/start", lp.StartStream) - lp.transRPC.HandleFunc("/stream/stop", lp.StopStream) - lp.transRPC.HandleFunc("/stream/payment", lp.ProcessStreamPayment) + lp.transRPC.HandleFunc("/ai/stream/start", lp.StartStream) + lp.transRPC.HandleFunc("/ai/stream/stop", lp.StopStream) + lp.transRPC.HandleFunc("/ai/stream/payment", lp.ProcessStreamPayment) cert, key, err := getCert(orch.ServiceURI(), workDir) if err != nil { From 9c51a2f06344d20c6411c7b6f93e5bce1a211972 Mon Sep 17 00:00:00 2001 From: Brad P Date: Fri, 29 Aug 2025 14:34:24 -0500 Subject: [PATCH 28/57] small update to add vals to ctx --- server/job_stream.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/job_stream.go b/server/job_stream.go index 8a3e63b98c..8ddd14b016 100644 --- a/server/job_stream.go +++ b/server/job_stream.go @@ -999,6 +999,7 @@ func (h *lphttp) StartStream(w http.ResponseWriter, r *http.Request) { stream, _ := h.node.ExternalCapabilities.Streams[orchJob.Req.ID] ctx := context.Background() ctx = clog.AddVal(ctx, "stream_id", orchJob.Req.ID) + ctx = clog.AddVal(ctx, "capability", orchJob.Req.Capability) pmtCheckDur := 30 * time.Second pmtTicker := time.NewTicker(pmtCheckDur) @@ -1007,6 +1008,7 @@ func (h *lphttp) StartStream(w http.ResponseWriter, r *http.Request) { select { case <-stream.StreamCtx.Done(): h.orchestrator.FreeExternalCapabilityCapacity(orchJob.Req.Capability) + clog.Infof(ctx, "Stream ended, stopping payment monitoring and released capacity") return case <-pmtTicker.C: // Check payment status From 015c433de441eaf58f17a837963763999e0ee3d7 Mon Sep 17 00:00:00 2001 From: Brad P Date: Sat, 30 Aug 2025 06:59:40 -0500 Subject: [PATCH 29/57] various updates --- core/ai_orchestrator.go | 6 ++ core/external_capabilities.go | 42 +++++++++++-- server/job_rpc.go | 25 ++++---- server/job_stream.go | 108 +++++++++++++++------------------- 4 files changed, 105 insertions(+), 76 deletions(-) diff --git a/core/ai_orchestrator.go b/core/ai_orchestrator.go index d1a9088f7b..90f52bb1c4 100644 --- a/core/ai_orchestrator.go +++ b/core/ai_orchestrator.go @@ -1223,6 +1223,12 @@ func (orch *orchestrator) JobPriceInfo(sender ethcommon.Address, jobCapability s return nil, err } + //ensure price numerator and denominator can be int64 + jobPrice, err = common.PriceToInt64(jobPrice) + if err != nil { + return nil, err + } + return &net.PriceInfo{ PricePerUnit: jobPrice.Num().Int64(), PixelsPerUnit: jobPrice.Denom().Int64(), diff --git a/core/external_capabilities.go b/core/external_capabilities.go index 1331e05177..1b1363f604 100644 --- a/core/external_capabilities.go +++ b/core/external_capabilities.go @@ -52,8 +52,12 @@ type StreamInfo struct { OrchDataUrl string //Orchestrator fields - Sender ethcommon.Address - + Sender ethcommon.Address + pubChannel *trickle.TrickleLocalPublisher + subChannel *trickle.TrickleLocalPublisher + controlChannel *trickle.TrickleLocalPublisher + eventsChannel *trickle.TrickleLocalPublisher + dataChannel *trickle.TrickleLocalPublisher //Stream fields Params interface{} DataWriter *media.SegmentWriter @@ -83,6 +87,16 @@ func (sd *StreamInfo) UpdateParams(params string) { sd.JobParams = params } +func (sd *StreamInfo) SetChannels(pub, sub, control, events, data *trickle.TrickleLocalPublisher) { + sd.sdm.Lock() + defer sd.sdm.Unlock() + sd.pubChannel = pub + sd.subChannel = sub + sd.controlChannel = control + sd.eventsChannel = events + sd.dataChannel = data +} + type ExternalCapabilities struct { capm sync.Mutex Capabilities map[string]*ExternalCapability @@ -95,12 +109,12 @@ func NewExternalCapabilities() *ExternalCapabilities { } } -func (extCaps *ExternalCapabilities) AddStream(streamID string, pipeline string, params interface{}, streamReq []byte) error { +func (extCaps *ExternalCapabilities) AddStream(streamID string, pipeline string, params interface{}, streamReq []byte) (*StreamInfo, error) { extCaps.capm.Lock() defer extCaps.capm.Unlock() _, ok := extCaps.Streams[streamID] if ok { - return fmt.Errorf("stream already exists: %s", streamID) + return nil, fmt.Errorf("stream already exists: %s", streamID) } //add to streams @@ -115,10 +129,12 @@ func (extCaps *ExternalCapabilities) AddStream(streamID string, pipeline string, } extCaps.Streams[streamID] = &stream + //clean up when stream ends go func() { for { select { case <-ctx.Done(): + //gateway channels shutdown if stream.DataWriter != nil { stream.DataWriter.Close() } @@ -127,12 +143,28 @@ func (extCaps *ExternalCapabilities) AddStream(streamID string, pipeline string, stream.ControlPub.Close() } + //orchestrator channels shutdown + if stream.pubChannel != nil { + stream.pubChannel.Close() + } + if stream.subChannel != nil { + stream.subChannel.Close() + } + if stream.controlChannel != nil { + stream.controlChannel.Close() + } + if stream.eventsChannel != nil { + stream.eventsChannel.Close() + } + if stream.dataChannel != nil { + stream.dataChannel.Close() + } return } } }() - return nil + return &stream, nil } func (extCaps *ExternalCapabilities) RemoveStream(streamID string) { diff --git a/server/job_rpc.go b/server/job_rpc.go index cc56668839..a4d4c65caf 100644 --- a/server/job_rpc.go +++ b/server/job_rpc.go @@ -576,11 +576,11 @@ func (ls *LivepeerServer) sendJobToOrch(ctx context.Context, r *http.Request, jo return resp, resp.StatusCode, nil } -func (ls *LivepeerServer) sendPayment(ctx context.Context, orchPmtUrl, capability, jobReq, payment string) (*JobToken, error) { +func (ls *LivepeerServer) sendPayment(ctx context.Context, orchPmtUrl, capability, jobReq, payment string) (int, error) { req, err := http.NewRequestWithContext(ctx, "POST", orchPmtUrl, nil) if err != nil { clog.Errorf(ctx, "Unable to create request err=%v", err) - return nil, err + return http.StatusBadRequest, err } req.Header.Add("Content-Type", "application/json") @@ -590,17 +590,10 @@ func (ls *LivepeerServer) sendPayment(ctx context.Context, orchPmtUrl, capabilit resp, err := sendJobReqWithTimeout(req, 10*time.Second) if err != nil { clog.Errorf(ctx, "job payment not able to be processed by Orchestrator %v err=%v ", orchPmtUrl, err.Error()) - return nil, err + return http.StatusBadRequest, err } - var jobToken JobToken - if err := json.NewDecoder(resp.Body).Decode(&jobToken); err != nil { - clog.Errorf(ctx, "Error decoding job token response: %v", err) - return nil, err - } - - //return the job token with new ticketparams, balance and price. Not all fields filled out - return &jobToken, nil + return resp.StatusCode, nil } func processJob(ctx context.Context, h *lphttp, w http.ResponseWriter, r *http.Request) { @@ -856,6 +849,7 @@ func (h *lphttp) setupOrchJob(ctx context.Context, r *http.Request, reserveCapac } else { if err := orch.ProcessPayment(ctx, payment, core.ManifestID(jobReq.Capability)); err != nil { orch.FreeExternalCapabilityCapacity(jobReq.Capability) + clog.Errorf(ctx, "Error processing payment: %v", err) return nil, errPaymentError } } @@ -874,6 +868,7 @@ func (h *lphttp) setupOrchJob(ctx context.Context, r *http.Request, reserveCapac func createPayment(ctx context.Context, jobReq *JobRequest, orchToken JobToken, node *core.LivepeerNode) (string, error) { var payment *net.Payment + clog.Infof(ctx, "creating payment for job request %s", jobReq.Capability) sender := ethcommon.HexToAddress(jobReq.Sender) orchAddr := ethcommon.BytesToAddress(orchToken.TicketParams.Recipient) balance := node.Balances.Balance(orchAddr, core.ManifestID(jobReq.Capability)) @@ -885,7 +880,12 @@ func createPayment(ctx context.Context, jobReq *JobRequest, orchToken JobToken, balance = node.Balances.Balance(orchAddr, core.ManifestID(jobReq.Capability)) } else { price := big.NewRat(orchToken.Price.PricePerUnit, orchToken.Price.PixelsPerUnit) - cost := price.Mul(price, big.NewRat(int64(jobReq.Timeout), 1)) + cost := new(big.Rat).Mul(price, big.NewRat(int64(jobReq.Timeout), 1)) + minBal := new(big.Rat).Mul(price, big.NewRat(60, 1)) //minimum 1 minute balance + if cost.Cmp(minBal) < 0 { + cost = minBal + } + if balance.Cmp(cost) > 0 { createTickets = false payment = &net.Payment{ @@ -893,6 +893,7 @@ func createPayment(ctx context.Context, jobReq *JobRequest, orchToken JobToken, ExpectedPrice: orchToken.Price, } } + clog.Infof(ctx, "current balance for sender=%v capability=%v is %v, cost=%v price=%v", sender.Hex(), jobReq.Capability, balance.FloatString(3), cost.FloatString(3), price.FloatString(3)) } if !createTickets { diff --git a/server/job_stream.go b/server/job_stream.go index 8ddd14b016..619eaaf8ec 100644 --- a/server/job_stream.go +++ b/server/job_stream.go @@ -172,7 +172,7 @@ func (ls *LivepeerServer) runStream(gatewayJob *gatewayJob) { err = nil } if !ls.LivepeerNode.ExternalCapabilities.StreamExists(streamID) { - clog.Info(ctx, "No input stream, skipping orchestrator swap") + clog.Info(ctx, "No stream exists, skipping orchestrator swap") break } //if swapErr := orchSwapper.checkSwap(ctx); swapErr != nil { @@ -220,9 +220,14 @@ func (ls *LivepeerServer) monitorStream(streamId string) { ctx = clog.AddVal(ctx, "request_id", params.liveParams.requestID) // Create a ticker that runs every minute for payments - pmtTicker := time.NewTicker(45 * time.Second) + dur := 50 * time.Second + pmtTicker := time.NewTicker(dur) defer pmtTicker.Stop() - + jobSender, err := getJobSender(ctx, ls.LivepeerNode) + if err != nil { + clog.Errorf(ctx, "Error getting job sender: %v", err) + return + } for { select { case <-stream.StreamCtx.Done(): @@ -230,9 +235,18 @@ func (ls *LivepeerServer) monitorStream(streamId string) { ls.LivepeerNode.ExternalCapabilities.RemoveStream(streamId) return case <-pmtTicker.C: - // Send payment and fetch new JobToken every minute - // TicketParams expire in about 8 minutes so have multiple chances to get new token - // Each payment is for 60 seconds of compute + // fetch new JobToken with each payment + token := stream.OrchToken.(JobToken) + updateGatewayBalance(ls.LivepeerNode, token, stream.Capability, dur) + + newToken, err := getToken(ctx, 3*time.Second, stream.OrchUrl, stream.Capability, jobSender.Addr, jobSender.Sig) + if err != nil { + clog.Errorf(ctx, "Error getting new token for %s: %v", stream.OrchUrl, err) + continue + } + stream.OrchToken = *newToken + + // send the payment jobDetails := JobRequestDetails{StreamId: streamId} jobDetailsStr, err := json.Marshal(jobDetails) if err != nil { @@ -259,20 +273,15 @@ func (ls *LivepeerServer) monitorStream(streamId string) { //send the payment, update the stream with the refreshed token clog.Infof(ctx, "Sending stream payment for %s", streamId) - newToken, err := ls.sendPayment(ctx, stream.OrchUrl+"/ai/stream/payment", stream.Capability, job.SignedJobReq, pmtHdr) + statusCode, err := ls.sendPayment(ctx, stream.OrchUrl+"/ai/stream/payment", stream.Capability, job.SignedJobReq, pmtHdr) if err != nil { clog.Errorf(ctx, "Error sending stream payment for %s: %v", streamId, err) continue } - if newToken == nil { - clog.Errorf(ctx, "Updated token not received for %s", streamId) + if statusCode != http.StatusOK { + clog.Errorf(ctx, "Unexpected status code %d received for %s", statusCode, streamId) continue } - streamToken := stream.OrchToken.(JobToken) - streamToken.TicketParams = newToken.TicketParams - streamToken.Balance = newToken.Balance - streamToken.Price = newToken.Price - stream.OrchToken = streamToken } } } @@ -507,7 +516,8 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, req r.Body.Close() //save the stream setup - if err := ls.LivepeerNode.ExternalCapabilities.AddStream(streamID, pipeline, params, bodyBytes); err != nil { + _, err = ls.LivepeerNode.ExternalCapabilities.AddStream(streamID, pipeline, params, bodyBytes) + if err != nil { return nil, http.StatusBadRequest, err } @@ -892,24 +902,29 @@ func (h *lphttp) StartStream(w http.ResponseWriter, r *http.Request) { clog.Infof(ctx, "Processing stream start request videoIngress=%v videoEgress=%v dataOutput=%v", jobParams.EnableVideoIngress, jobParams.EnableVideoEgress, jobParams.EnableDataOutput) // Start trickle server for live-video var ( - mid = orchJob.Req.ID // Request ID is used for the manifest ID - pubUrl = h.orchestrator.ServiceURI().JoinPath(TrickleHTTPPath, mid).String() - subUrl = pubUrl + "-out" - controlUrl = pubUrl + "-control" - eventsUrl = pubUrl + "-events" - dataUrl = pubUrl + "-data" + mid = orchJob.Req.ID // Request ID is used for the manifest ID + pubUrl = h.orchestrator.ServiceURI().JoinPath(TrickleHTTPPath, mid).String() + subUrl = pubUrl + "-out" + controlUrl = pubUrl + "-control" + eventsUrl = pubUrl + "-events" + dataUrl = pubUrl + "-data" + pubCh *trickle.TrickleLocalPublisher + subCh *trickle.TrickleLocalPublisher + controlPubCh *trickle.TrickleLocalPublisher + eventsCh *trickle.TrickleLocalPublisher + dataCh *trickle.TrickleLocalPublisher ) reqBodyForRunner := make(map[string]interface{}) reqBodyForRunner["gateway_request_id"] = mid //required channels - controlPubCh := trickle.NewLocalPublisher(h.trickleSrv, mid+"-control", "application/json") + controlPubCh = trickle.NewLocalPublisher(h.trickleSrv, mid+"-control", "application/json") controlPubCh.CreateChannel() controlUrl = overwriteHost(h.node.LiveAITrickleHostForRunner, controlUrl) reqBodyForRunner["control_url"] = controlUrl w.Header().Set("X-Control-Url", controlUrl) - eventsCh := trickle.NewLocalPublisher(h.trickleSrv, mid+"-events", "application/json") + eventsCh = trickle.NewLocalPublisher(h.trickleSrv, mid+"-events", "application/json") eventsCh.CreateChannel() eventsUrl = overwriteHost(h.node.LiveAITrickleHostForRunner, eventsUrl) reqBodyForRunner["events_url"] = eventsUrl @@ -917,7 +932,7 @@ func (h *lphttp) StartStream(w http.ResponseWriter, r *http.Request) { //Optional channels if jobParams.EnableVideoIngress { - pubCh := trickle.NewLocalPublisher(h.trickleSrv, mid, "video/MP2T") + pubCh = trickle.NewLocalPublisher(h.trickleSrv, mid, "video/MP2T") pubCh.CreateChannel() pubUrl = overwriteHost(h.node.LiveAITrickleHostForRunner, pubUrl) reqBodyForRunner["subscribe_url"] = pubUrl //runner needs to subscribe to input @@ -925,7 +940,7 @@ func (h *lphttp) StartStream(w http.ResponseWriter, r *http.Request) { } if jobParams.EnableVideoEgress { - subCh := trickle.NewLocalPublisher(h.trickleSrv, mid+"-out", "video/MP2T") + subCh = trickle.NewLocalPublisher(h.trickleSrv, mid+"-out", "video/MP2T") subCh.CreateChannel() subUrl = overwriteHost(h.node.LiveAITrickleHostForRunner, subUrl) reqBodyForRunner["publish_url"] = subUrl //runner needs to send results -out @@ -933,7 +948,7 @@ func (h *lphttp) StartStream(w http.ResponseWriter, r *http.Request) { } if jobParams.EnableDataOutput { - dataCh := trickle.NewLocalPublisher(h.trickleSrv, mid+"-data", "application/jsonl") + dataCh = trickle.NewLocalPublisher(h.trickleSrv, mid+"-data", "application/jsonl") dataCh.CreateChannel() dataUrl = overwriteHost(h.node.LiveAITrickleHostForRunner, dataUrl) reqBodyForRunner["data_url"] = dataUrl @@ -987,13 +1002,15 @@ func (h *lphttp) StartStream(w http.ResponseWriter, r *http.Request) { clog.V(common.SHORT).Infof(ctx, "stream start processed successfully took=%v balance=%v", time.Since(start), getPaymentBalance(orch, orchJob.Sender, orchJob.Req.Capability).FloatString(0)) //setup the stream - err = h.node.ExternalCapabilities.AddStream(orchJob.Req.ID, orchJob.Req.Capability, orchJob.Req, respBody) + stream, err := h.node.ExternalCapabilities.AddStream(orchJob.Req.ID, orchJob.Req.Capability, orchJob.Req, respBody) if err != nil { clog.Errorf(ctx, "Error adding stream to external capabilities: %v", err) respondWithError(w, "Error adding stream to external capabilities", http.StatusInternalServerError) return } + stream.SetChannels(pubCh, subCh, controlPubCh, eventsCh, dataCh) + //start payment monitoring go func() { stream, _ := h.node.ExternalCapabilities.Streams[orchJob.Req.ID] @@ -1024,6 +1041,8 @@ func (h *lphttp) StartStream(w http.ResponseWriter, r *http.Request) { if exists { h.node.ExternalCapabilities.RemoveStream(orchJob.Req.ID) } + + return } clog.V(8).Infof(ctx, "Payment balance for stream capability is good balance=%v", senderBalance.FloatString(0)) @@ -1127,47 +1146,18 @@ func (h *lphttp) ProcessStreamPayment(w http.ResponseWriter, r *http.Request) { ctx = clog.AddVal(ctx, "sender", orchJob.Req.Sender) senderAddr := ethcommon.HexToAddress(orchJob.Req.Sender) - clog.Infof(ctx, "Processing payment request") - - jobPrice, err := orch.JobPriceInfo(senderAddr, orchJob.Req.Capability) - if err != nil { - statusCode := http.StatusBadRequest - if err.Error() == "insufficient sender reserve" { - statusCode = http.StatusServiceUnavailable - } - glog.Errorf("could not get price err=%v", err.Error()) - http.Error(w, fmt.Sprintf("Could not get price err=%v", err.Error()), statusCode) - return - } - ticketParams, err := orch.TicketParams(senderAddr, jobPrice) - if err != nil { - glog.Errorf("could not get ticket params err=%v", err.Error()) - http.Error(w, fmt.Sprintf("Could not get ticket params err=%v", err.Error()), http.StatusBadRequest) - return - } capBal := orch.Balance(senderAddr, core.ManifestID(orchJob.Req.Capability)) if capBal != nil { capBal, err = common.PriceToInt64(capBal) if err != nil { - clog.Errorf(context.TODO(), "could not convert balance to int64 sender=%v capability=%v err=%v", senderAddr.Hex(), orchJob.Req.Capability, err.Error()) + clog.Errorf(ctx, "could not convert balance to int64 sender=%v capability=%v err=%v", senderAddr.Hex(), orchJob.Req.Capability, err.Error()) capBal = big.NewRat(0, 1) } } else { capBal = big.NewRat(0, 1) } - //convert to int64. Note: returns with 000 more digits to allow for precision of 3 decimal places. - capBalInt, err := common.PriceToFixed(capBal) - if err != nil { - glog.Errorf("could not convert balance to int64 sender=%v capability=%v err=%v", senderAddr.Hex(), orchJob.Req.Capability, err.Error()) - capBalInt = 0 - } else { - // Remove the last three digits from capBalInt - capBalInt = capBalInt / 1000 - } - - jobToken := JobToken{TicketParams: ticketParams, Balance: capBalInt, Price: jobPrice} - clog.Infof(ctx, "Processed payment request stream_id=%s capability=%s balance=%v pricePerUnit=%v pixelsPerUnit=%v", orchJob.Details.StreamId, orchJob.Req.Capability, jobToken.Balance, jobPrice.PricePerUnit, jobPrice.PixelsPerUnit) - json.NewEncoder(w).Encode(jobToken) + w.Header().Set(jobPaymentBalanceHdr, capBal.FloatString(0)) + w.WriteHeader(http.StatusOK) } From 3f3b1567e18a565c59cfd6a0a15892da95a42985 Mon Sep 17 00:00:00 2001 From: Brad P Date: Sat, 30 Aug 2025 07:15:49 -0500 Subject: [PATCH 30/57] refactor create payment and job payment processing --- server/job_rpc.go | 45 ++++++++++++++++++++++---------------------- server/job_stream.go | 3 +++ 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/server/job_rpc.go b/server/job_rpc.go index a4d4c65caf..ae8d26f664 100644 --- a/server/job_rpc.go +++ b/server/job_rpc.go @@ -564,7 +564,9 @@ func (ls *LivepeerServer) sendJobToOrch(ctx context.Context, r *http.Request, jo clog.Errorf(ctx, "Unable to create payment err=%v", err) return nil, http.StatusInternalServerError, fmt.Errorf("Unable to create payment err=%v", err) } - req.Header.Add(jobPaymentHeaderHdr, paymentHdr) + if paymentHdr != "" { + req.Header.Add(jobPaymentHeaderHdr, paymentHdr) + } } resp, err := sendJobReqWithTimeout(req, time.Duration(jobReq.Timeout+5)*time.Second) //include 5 second buffer @@ -799,8 +801,6 @@ func processJob(ctx context.Context, h *lphttp, w http.ResponseWriter, r *http.R // SetupOrchJob prepares the orchestrator job by extracting and validating the job request from the HTTP headers. // Payment is applied if applicable. func (h *lphttp) setupOrchJob(ctx context.Context, r *http.Request, reserveCapacity bool) (*orchJob, error) { - clog.Infof(ctx, "processing job request") - job := r.Header.Get(jobRequestHdr) orch := h.orchestrator jobReq, err := verifyJobCreds(ctx, orch, job, reserveCapacity) @@ -825,36 +825,34 @@ func (h *lphttp) setupOrchJob(ctx context.Context, r *http.Request, reserveCapac //no payment included, confirm if balance remains jobPriceRat := big.NewRat(jobPrice.PricePerUnit, jobPrice.PixelsPerUnit) var payment net.Payment + var orchBal *big.Rat // if price is 0, no payment required if jobPriceRat.Cmp(big.NewRat(0, 1)) > 0 { // get payment information - payment, err = getPayment(r.Header.Get(jobPaymentHeaderHdr)) - if err != nil { - clog.Errorf(ctx, "job payment invalid: %v", err) - return nil, errPaymentError - } - - if payment.TicketParams == nil { - - //if price is not 0, confirm balance - if jobPriceRat.Cmp(big.NewRat(0, 1)) > 0 { - minBal := jobPriceRat.Mul(jobPriceRat, big.NewRat(60, 1)) //minimum 1 minute balance - orchBal := getPaymentBalance(orch, sender, jobReq.Capability) - - if orchBal.Cmp(minBal) < 0 { - orch.FreeExternalCapabilityCapacity(jobReq.Capability) - return nil, errInsufficientBalance - } + paymentHdr := r.Header.Get(jobPaymentHeaderHdr) + minBal := jobPriceRat.Mul(jobPriceRat, big.NewRat(60, 1)) //minimum 1 minute balance + if paymentHdr != "" { + payment, err = getPayment(paymentHdr) + if err != nil { + clog.Errorf(ctx, "job payment invalid: %v", err) + return nil, errPaymentError } - } else { + if err := orch.ProcessPayment(ctx, payment, core.ManifestID(jobReq.Capability)); err != nil { orch.FreeExternalCapabilityCapacity(jobReq.Capability) clog.Errorf(ctx, "Error processing payment: %v", err) return nil, errPaymentError } + //update balance for payment + orchBal = getPaymentBalance(orch, sender, jobReq.Capability) + } else { + orchBal = getPaymentBalance(orch, sender, jobReq.Capability) } - clog.Infof(ctx, "balance after payment is %v", getPaymentBalance(orch, sender, jobReq.Capability).FloatString(0)) + if orchBal.Cmp(minBal) < 0 { + orch.FreeExternalCapabilityCapacity(jobReq.Capability) + return nil, errInsufficientBalance + } } var jobDetails JobRequestDetails @@ -863,6 +861,8 @@ func (h *lphttp) setupOrchJob(ctx context.Context, r *http.Request, reserveCapac return nil, fmt.Errorf("Unable to unmarshal job request details err=%v", err) } + clog.Infof(ctx, "job request verified id=%v sender=%v capability=%v timeout=%v price=%v balance=%v", jobReq.ID, jobReq.Sender, jobReq.Capability, jobReq.Timeout, jobPriceRat.FloatString(0), orchBal.FloatString(0)) + return &orchJob{Req: jobReq, Sender: sender, JobPrice: jobPrice, Details: &jobDetails}, nil } @@ -898,6 +898,7 @@ func createPayment(ctx context.Context, jobReq *JobRequest, orchToken JobToken, if !createTickets { clog.V(common.DEBUG).Infof(ctx, "No payment required, using balance=%v", balance.FloatString(3)) + return "", nil } else { //calc ticket count ticketCnt := math.Ceil(float64(jobReq.Timeout)) diff --git a/server/job_stream.go b/server/job_stream.go index 619eaaf8ec..e812e77e32 100644 --- a/server/job_stream.go +++ b/server/job_stream.go @@ -270,6 +270,9 @@ func (ls *LivepeerServer) monitorStream(streamId string) { clog.Errorf(ctx, "Error processing stream payment for %s: %v", streamId, err) // Continue monitoring even if payment fails } + if pmtHdr == "" { + continue + } //send the payment, update the stream with the refreshed token clog.Infof(ctx, "Sending stream payment for %s", streamId) From 8bd0849574443c451b0841585150899b776ab40a Mon Sep 17 00:00:00 2001 From: Brad P Date: Sat, 30 Aug 2025 16:34:47 -0500 Subject: [PATCH 31/57] report not active if lost control pub --- core/external_capabilities.go | 10 ++- server/job_stream.go | 146 +++++++++++++++++++--------------- 2 files changed, 92 insertions(+), 64 deletions(-) diff --git a/core/external_capabilities.go b/core/external_capabilities.go index 1b1363f604..3e54782bd9 100644 --- a/core/external_capabilities.go +++ b/core/external_capabilities.go @@ -72,7 +72,15 @@ type StreamInfo struct { } func (sd *StreamInfo) IsActive() bool { - return sd.StreamCtx.Err() == nil + if sd.StreamCtx.Err() != nil { + return false + } + + if sd.controlChannel == nil && sd.ControlPub == nil { + return false + } + + return true } func (sd *StreamInfo) ExcludeOrch(orchUrl string) { diff --git a/server/job_stream.go b/server/job_stream.go index e812e77e32..c49e4b6b13 100644 --- a/server/job_stream.go +++ b/server/job_stream.go @@ -45,7 +45,7 @@ func (ls *LivepeerServer) StartStream() http.Handler { return } - resp, code, err := ls.setupStream(ctx, r, gatewayJob.Job.Req) + streamUrls, code, err := ls.setupStream(ctx, r, gatewayJob.Job.Req) if err != nil { clog.Errorf(ctx, "Error setting up stream: %s", err) http.Error(w, err.Error(), code) @@ -56,11 +56,11 @@ func (ls *LivepeerServer) StartStream() http.Handler { go ls.monitorStream(gatewayJob.Job.Req.ID) - if resp != nil { + if streamUrls != nil { // Stream started successfully w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(resp) + json.NewEncoder(w).Encode(streamUrls) } else { //case where we are subscribing to own streams in setupStream w.WriteHeader(http.StatusNoContent) @@ -119,15 +119,18 @@ func (ls *LivepeerServer) runStream(gatewayJob *gatewayJob) { ctx := stream.StreamCtx ctx = clog.AddVal(ctx, "stream_id", streamID) - start := time.Now() + //monitor for lots of fast swaps, likely something wrong with request + orchSwapper := NewOrchestratorSwapper(params) + + firstProcessed := false for _, orch := range gatewayJob.Orchs { stream.OrchToken = orch stream.OrchUrl = orch.ServiceAddr ctx = clog.AddVal(ctx, "orch", ethcommon.Bytes2Hex(orch.TicketParams.Recipient)) ctx = clog.AddVal(ctx, "orch_url", orch.ServiceAddr) clog.Infof(ctx, "Starting stream processing") - //refresh the token if after 5 minutes from start. TicketParams expire in 40 blocks (about 8 minutes). - if time.Since(start) > 5*time.Minute { + //refresh the token if not first Orch to confirm capacity and new ticket params + if firstProcessed { orch, err := getToken(ctx, 3*time.Second, orch.ServiceAddr, gatewayJob.Job.Req.Capability, gatewayJob.Job.Req.Sender, gatewayJob.Job.Req.Sig) if err != nil { clog.Errorf(ctx, "Error getting token for orch=%v err=%v", orch.ServiceAddr, err) @@ -175,14 +178,17 @@ func (ls *LivepeerServer) runStream(gatewayJob *gatewayJob) { clog.Info(ctx, "No stream exists, skipping orchestrator swap") break } - //if swapErr := orchSwapper.checkSwap(ctx); swapErr != nil { - // if err != nil { - // err = fmt.Errorf("%w: %w", swapErr, err) - // } else { - // err = swapErr - // } - // break - //} + + //if swapping too fast, stop trying since likely a bad request + if swapErr := orchSwapper.checkSwap(ctx); swapErr != nil { + if err != nil { + err = fmt.Errorf("%w: %w", swapErr, err) + } else { + err = swapErr + } + break + } + firstProcessed = true clog.Infof(ctx, "Retrying stream with a different orchestrator") // will swap, but first notify with the reason for the swap @@ -191,11 +197,6 @@ func (ls *LivepeerServer) runStream(gatewayJob *gatewayJob) { } params.liveParams.sendErrorEvent(err) - //if isFirst { - // // failed before selecting an orchestrator - // firstProcessed <- struct{}{} - //} - //if there is ingress input then force off if params.liveParams.kickInput != nil { params.liveParams.kickInput(err) @@ -219,7 +220,7 @@ func (ls *LivepeerServer) monitorStream(streamId string) { ctx = clog.AddVal(ctx, "request_id", params.liveParams.requestID) - // Create a ticker that runs every minute for payments + // Create a ticker that runs every minute for payments with buffer to ensure payment is completed dur := 50 * time.Second pmtTicker := time.NewTicker(dur) defer pmtTicker.Stop() @@ -228,6 +229,7 @@ func (ls *LivepeerServer) monitorStream(streamId string) { clog.Errorf(ctx, "Error getting job sender: %v", err) return } + for { select { case <-stream.StreamCtx.Done(): @@ -289,37 +291,51 @@ func (ls *LivepeerServer) monitorStream(streamId string) { } } -func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, req *JobRequest) (map[string]string, int, error) { +type StartRequest struct { + Stream string `json:"stream_name"` + RtmpOutput string `json:"rtmp_output"` + StreamId string `json:"stream_id"` + Params string `json:"params"` +} + +type StreamUrls struct { + StreamId string `json:"stream_id"` + WhipUrl string `json:"whip_url"` + WhepUrl string `json:"whep_url"` + RtmpUrl string `json:"rtmp_url"` + RtmpOutputUrl string `json:"rtmp_output_url"` + UpdateUrl string `json:"update_url"` + StatusUrl string `json:"status_url"` + DataUrl string `json:"data_url"` +} + +func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, req *JobRequest) (*StreamUrls, int, error) { requestID := string(core.RandomManifestID()) ctx = clog.AddVal(ctx, "request_id", requestID) // Setup request body to be able to preserve for retries - // Read the entire body first - bodyBytes, err := io.ReadAll(r.Body) + // Read the entire body first with 10MB limit + bodyBytes, err := io.ReadAll(io.LimitReader(r.Body, 10<<20)) if err != nil { return nil, http.StatusBadRequest, err } + r.Body.Close() - // Create a clone of the request with a new body for parsing form values - bodyForForm := bytes.NewBuffer(bodyBytes) - r.Body = io.NopCloser(bytes.NewBuffer(bodyBytes)) // Reset the original body - - // Create a temporary request for parsing form data - formReq := r.Clone(ctx) - formReq.Body = io.NopCloser(bodyForForm) - defer r.Body.Close() - defer formReq.Body.Close() - - // Parse the form (10MB max) - if err := formReq.ParseMultipartForm(10 << 20); err != nil { - return nil, http.StatusBadRequest, err + // Decode the StartRequest from JSON body + var startReq StartRequest + if err := json.NewDecoder(bytes.NewBuffer(bodyBytes)).Decode(&startReq); err != nil { + return nil, http.StatusBadRequest, fmt.Errorf("invalid JSON request body: %w", err) } + //live-video-to-video uses path value for this - streamName := formReq.FormValue("stream") + streamName := startReq.Stream streamRequestTime := time.Now().UnixMilli() ctx = clog.AddVal(ctx, "stream", streamName) + + //I think these are for mediamtx ingest + //sourceID := formReq.FormValue("source_id") //if sourceID == "" { // return nil, http.StatusBadRequest, errors.New("missing source_id") @@ -336,7 +352,7 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, req //ctx = clog.AddVal(ctx, "source_type", sourceType) // If auth webhook is set and returns an output URL, this will be replaced - outputURL := formReq.FormValue("rtmpOutput") + outputURL := startReq.RtmpOutput // convention to avoid re-subscribing to our own streams // in case we want to push outputs back into mediamtx - @@ -348,8 +364,8 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, req // if auth webhook returns pipeline config these will be replaced pipeline := req.Capability //streamParamsJson["pipeline"].(string) - rawParams := formReq.FormValue("params") - streamID := formReq.FormValue("streamId") + rawParams := startReq.Params + streamID := startReq.StreamId var pipelineID string var pipelineParams map[string]interface{} @@ -418,14 +434,13 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, req } } - //don't want to update the streamid or pipeline id with byoc - //if authResp.StreamID != "" { - // streamID = authResp.StreamID - //} + if authResp.StreamID != "" { + streamID = authResp.StreamID + } - //if authResp.PipelineID != "" { - // pipelineID = authResp.PipelineID - //} + if authResp.PipelineID != "" { + pipelineID = authResp.PipelineID + } } ctx = clog.AddVal(ctx, "stream_id", streamID) @@ -515,9 +530,6 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, req clog.Infof(ctx, "stream setup videoIngress=%v videoEgress=%v dataOutput=%v", jobParams.EnableVideoIngress, jobParams.EnableVideoEgress, jobParams.EnableDataOutput) - // Close the body, done with all form variables - r.Body.Close() - //save the stream setup _, err = ls.LivepeerNode.ExternalCapabilities.AddStream(streamID, pipeline, params, bodyBytes) if err != nil { @@ -525,17 +537,18 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, req } req.ID = streamID - resp := make(map[string]string) - resp["stream_id"] = streamID - resp["whip_url"] = whipURL - resp["whep_url"] = whepURL - resp["rtmp_url"] = rtmpURL - resp["rtmp_output_url"] = strings.Join(rtmpOutputs, ",") - resp["update_url"] = updateURL - resp["status_url"] = statusURL - resp["data_url"] = dataURL - - return resp, http.StatusOK, nil + streamUrls := StreamUrls{ + StreamId: streamID, + WhipUrl: whipURL, + WhepUrl: whepURL, + RtmpUrl: rtmpURL, + RtmpOutputUrl: strings.Join(rtmpOutputs, ","), + UpdateUrl: updateURL, + StatusUrl: statusURL, + DataUrl: dataURL, + } + + return &streamUrls, http.StatusOK, nil } // mediamtx sends this request to go-livepeer when rtmp stream received @@ -1021,7 +1034,7 @@ func (h *lphttp) StartStream(w http.ResponseWriter, r *http.Request) { ctx = clog.AddVal(ctx, "stream_id", orchJob.Req.ID) ctx = clog.AddVal(ctx, "capability", orchJob.Req.Capability) - pmtCheckDur := 30 * time.Second + pmtCheckDur := 25 * time.Second pmtTicker := time.NewTicker(pmtCheckDur) defer pmtTicker.Stop() for { @@ -1054,7 +1067,7 @@ func (h *lphttp) StartStream(w http.ResponseWriter, r *http.Request) { //check if stream still exists // if not, send stop to worker and exit monitoring - _, exists := h.node.ExternalCapabilities.Streams[orchJob.Req.ID] + stream, exists := h.node.ExternalCapabilities.Streams[orchJob.Req.ID] if !exists { req, err := http.NewRequestWithContext(ctx, "POST", orchJob.Req.CapabilityUrl+"/stream/stop", nil) // set the headers @@ -1067,6 +1080,13 @@ func (h *lphttp) StartStream(w http.ResponseWriter, r *http.Request) { //end monitoring of stream return } + + //check if control channel is still open, end if not + if !stream.IsActive() { + // Stop the stream and free capacity + h.node.ExternalCapabilities.RemoveStream(orchJob.Req.ID) + return + } } } }() From cbaba08bf35238e9ac1334e3ef339aba72caf11d Mon Sep 17 00:00:00 2001 From: Brad P Date: Tue, 2 Sep 2025 08:06:55 -0500 Subject: [PATCH 32/57] fix init of orchBal when 0 price set --- server/job_rpc.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/job_rpc.go b/server/job_rpc.go index ae8d26f664..4ae0441913 100644 --- a/server/job_rpc.go +++ b/server/job_rpc.go @@ -825,7 +825,7 @@ func (h *lphttp) setupOrchJob(ctx context.Context, r *http.Request, reserveCapac //no payment included, confirm if balance remains jobPriceRat := big.NewRat(jobPrice.PricePerUnit, jobPrice.PixelsPerUnit) var payment net.Payment - var orchBal *big.Rat + orchBal := big.NewRat(0, 1) // if price is 0, no payment required if jobPriceRat.Cmp(big.NewRat(0, 1)) > 0 { // get payment information From 3bb300083efc17270b50630d81d12adb90b69a23 Mon Sep 17 00:00:00 2001 From: Brad P Date: Tue, 2 Sep 2025 08:18:40 -0500 Subject: [PATCH 33/57] remove changes in separate pr --- ai/worker/runner.gen.go | 180 ++++++++++++++++------------------ core/ai_orchestrator.go | 23 ----- core/capabilities.go | 2 - core/external_capabilities.go | 60 ++++-------- server/ai_http.go | 43 -------- server/ai_live_video.go | 168 +++---------------------------- server/ai_mediaserver.go | 111 --------------------- 7 files changed, 122 insertions(+), 465 deletions(-) diff --git a/ai/worker/runner.gen.go b/ai/worker/runner.gen.go index a47eb474d0..68197ee283 100644 --- a/ai/worker/runner.gen.go +++ b/ai/worker/runner.gen.go @@ -317,9 +317,6 @@ type LiveVideoToVideoParams struct { // ControlUrl URL for subscribing via Trickle protocol for updates in the live video-to-video generation params. ControlUrl *string `json:"control_url,omitempty"` - // DataUrl URL for publishing data via Trickle protocol for pipeline status and logs. - DataUrl *string `json:"data_url,omitempty"` - // EventsUrl URL for publishing events via Trickle protocol for pipeline status and logs. EventsUrl *string `json:"events_url,omitempty"` @@ -350,9 +347,6 @@ type LiveVideoToVideoResponse struct { // ControlUrl URL for updating the live video-to-video generation ControlUrl *string `json:"control_url,omitempty"` - // DataUrl URL for publishing data for pipeline - DataUrl *string `json:"data_url,omitempty"` - // EventsUrl URL for subscribing to events for pipeline status and logs EventsUrl *string `json:"events_url,omitempty"` @@ -3166,93 +3160,93 @@ func HandlerWithOptions(si ServerInterface, options ChiServerOptions) http.Handl // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+xdeW/ctrb/KoTeA+IAM97atO8ZuH84SxPj2onhpQtaw5cjnZlhLJEqSdme5vm7P3CT", - "SImakR3b7e2dvzKWuJz1dw7JQ+VLkrKiZBSoFMnel0Skcyiw/rl/fPCOc8bV7wxEykkpCaPJnnqDQL1C", - "HETJqABUsAzyzWSUlJyVwCUBPUYhZt3uZ3Ow3QsQAs9A9ZNE5pDsJUdipv5alOoPITmhs+TubpRw+L0i", - "HLJk71c96kXTpSa07scmnyGVyd0o2a8ywk4slV1STgL60ZRxhFUPNAMKHKtWXaZ0C/0jzz9Nk71fvyT/", - "zWGa7CX/tdVIc8uKcusIMoLPTw6Tu4tRRBJ2JsjMzJsdbs10Pr8BTxGmX7NscTkDqhuesTO4lYrcHi5C", - "ks7LnOHMUYOmJAckGZoAkhxT1XICmZLJlPECy2QvmRCK+SJp0ddV4igpQOIMS2xmneIqV/2/3CVtuexn", - "GVE/cY4+swki1ExGGLW0lFgIyNQfcg6oJCXkhIZ25OaK0aGUfUmykI4OFR+q2YzQGfoBp85ADt6iSk2s", - "DMXJo3RWUk9tmmaxqTnIitNLSQoQEhelCGmQvIIOHSe6D2r6mOnngUqQhFu5iU6rsmRcWdM1zisQe+iF", - "ACqBpvBihF7cMJ69GCFl5sgQhSaM5YAp2nihJn+h3r2Y4lzAi5eb6K2hDBGB7OuNZryXm64lKgBTgSjz", - "iNy0s9l36vd4grXWmjae1CyXZ41kVsFAxzFidr/EPQ4KPIMzpv/p+sesIhmmKVyKFOcQqOn7zVdtHb2j", - "Kas4noGwliJrDAFECv0izZmAfIFyQq8a41V6QyVnRSnRxpzM5sCt7lCBF4hDVqV2CPR7hXMiFy99ub23", - "dKJTTWfNL62KCXDFL3EM9ni6GVsyRTmZLtANkfOOX/W7u5FfxNb1uJdL5LjTleNbmHHQxNzMSWrIaBDS", - "UEoEKisx1yK8wTwTuhWhRBKcmzabbfrQajHljGOxAhL20SE72Ucbh+xmfILpFdrPcCk1Mr20isc0Q0QK", - "lDJuomOmvOwGyGwuteMaJrwAg97d4qLMYQ99Qb8lOZZA5ThlVBChHG2xlafFWFE3Ftlt/luyh3Y2t0fo", - "t4QCJ5/FVkluIR9jLsfu7e6dL4BDzdiT4WCHn4FQSGGGJbmGS2P8K4g4a9xkQ7zU7lWRDNDNHEv1F9ym", - "eZUBmnJWRER8MKOMKwuaotAg0W/V9vY3Kdrxyf5oSUPHhrQY9VVxafz6sgQe42GnzcJHbWqITR0g+BhR", - "ArfsBYRUBTowjY+Bd8ghVMLMWK+mh06Bg2ZNQiu07Gxv99OTAWVEKB3rjpvoiHEwv1ElKpwr1AKsMctC", - "lIUix8qkkkjk7AY4qqlQw2RVrj13slDxBuhMzjv8ufboVFMd484X7xCrWGaT/ToVeApycZnOIb0KhKdC", - "X1t6x8AVJqpAqrsh3U2bopCk0Lg/bWOXgoUqz1QKw6ZToEIZGeNojnkxrXKfzFMz6htNTE2sjdaaWoCs", - "K5FTsG7JMc1YgQy+9YhCNY7K2+kqkML25v/0wDWbmlSkSdNwWeakCXIcnI6NZja21ZudIJCdujk72NyK", - "+6VToAlskQQgiOyrM4B4gjw4bNasP1rkfMQEtVbJUFj+KjTun7LP61q6XaXSgTndjyQD1lXptAWK38UW", - "ZFOOCxAakAWkjGbavIM85FoN73P3Qw9uzXXYD+Z89X10VtMSEYp0OBcDJv1gBo/NO9h26/iDzfg6fv6p", - "VmvIuH86UTDV+nJSpVcg21Ts7H7fJuPcTahUrFebiiglclywikqlADNmvdzyEwqtMxMK1SsLs+pnoWKn", - "7XlD8lyBPaH6VUeFR6bZa010wJgf2hkRcImr2WUPLG/vdvLUmgXdGeEsa8A4YNiky+hDsPCwiw4OAopJ", - "rtPm3r4m4aUpBywc30GI1wTsVzPUD/Cr05fdV//G2cs6r3CSuCFZy3p3tne/jeGhbnkvOPxJj92d9Z4R", - "xoSOJSHmFGYFULlPF3JO6Gy3G2Ym7DayaYpybUDoW4Q5xws0I9dAERYIowm7dVsA1s80Lo4U/z//8vMv", - "yKCxz+1rdtu75u5OfuDwXhjiH4rwWFxdElpWMsofuxlzECyvNKipxkg3bjElFyVJtVfqxRpGJYdrwiqh", - "fmQk1b2JtHY1arIq7Rc7tx9uf0IbH/7x0z92X32nTfJ0/yjIJI/UzAeazL/cqreocuXF4uqSVbIW5BI8", - "OFC5dQWjRoImqnC7KzhXCbga0GwL4mJCZpUSphG9MSsxQmwqgao/syrV+34gJXDbU84xVYhD6CwHTw0B", - "V45y9MlQHgMPqowqJ3/AZcoYz8T92CsZoRLpnoRiCaIOoPW4zZIC0xmgX7dHOxfWRHRvOy+C2xJSaZpP", - "wDTgINRD9cioLyOFwkpGRRix7FzojeEhxqg/WdcZPt7uWi9nU8uVVUTLF27mwAEBTi35iCjFoY2fR7+8", - "bNAvSKR1szZlXv6uCcvxBPIIYYf6eZ3RBKQ5anYQoRlJtfyxagozziqa2dYq3m8HTSY4vfKbdMk10y7Z", - "EM/ZjMh7WIvpJlBFx8oDxJzlKsPR5mnGQoQKqaI+myoSNcbp95FN50Mze1fPQ2NHJyYsiR/nZb0T+sAF", - "5yPv0z4OIFaGrezh+4ErUsDvX/0HbWANkuZ6J2tVxnnvnSPnnBH/fTOv6FUs70nVC52gKmVqr8TNIVf3", - "/Fja7aZu0qsHsJmuHtVnMdz6aHRdz9QzpnvdGZhIKBRBd94c9Vj1RDqMdSQp/YaKME+WRlARCb4/Pn/D", - "irKScECnkbPno/oQPgOJiTL/98fnKDV9/GPgrlANfNVYF8+98GdTztAkip/9sgHPXwsoGF9cTjlA0EE/", - "Rj+ox0u6SSZxHul3pp9HOxLaIk0/iG4H4CKg6aP6e+W+mhIINS0DIkNWnYwcQZ5WW8qLq/dckpz8oVW0", - "SsVKs1XTHAmJJRGSpOKByn1mjQ1TwyjxeLy0lux38ySGrHyj0/nDGJr7RjGUD1gIDzWJGAdRgkJradtC", - "xGI+nJ0d99QYqVcDi4wMWAwvyKnrhboFOW8d7piZA8Rpy89O6zHdsNPD6484J5kerua6jxUHzks5aY/n", - "IbnhJAbjPrXtAWJ0Y57dYK693spiUB2V8u+liD0rK5XvmUqquuDnOGizjPkWIHmcvS8r1GdyfrI76DjE", - "5tZ++2P3bBXqlk3Det5Rw7hvOhEpL1HGqcRSDFIDB5yPVbTWClkGsoos4cZ9oELaPt9SiiH7r6sVw35E", - "Lb2EfwCcy/kbl2eHElXDVSKels11R2SauNTMowxoVShiDz/tvz34+D4ZJZ/+mYySdycnn06SUXLw9vCd", - "T+ipmWoV65Yin0OPgQh/etF4r/rJ2IIlss7tEUt7leHnqaurLH17M2UTq/JYS8tFq9+y6krvJPJegtHJ", - "/TK59K8LGqnoKr+Vi4J2st5K0GMcRBg9PDx6M2ckhVicyk0Z57CIe3h4dGRKfZO7i7tRMiWUiPklByxc", - "NPG2HepDVd0KnZhWsc1umsFtkBnqB/E0z0z/QJLbVqPn8STaiCoux6Nm+lCQKaMSqPSZeGMfxTbLWB5A", - "3gnLV8MdN43cTCHVjrA42SfwewUiUhtR4NtLya6Atk/pvvN34W/RmWmzRCNicLLja8Sv87XDtB3dgnuv", - "denwEt07kRxwEfTTVa5hpQouolsZEopSuWrFoXVI+73vtE2jyFmoZOVluGsz3vE6sxL9MypR1a9sl6D5", - "3Y5XltbUSgmtxNlBn5U0KNiybu0T91KxdaM7fzvBDNJVcMpBgWLgPPZRtChiyCrS2cxKM6mcQ69gSLvA", - "uTHb2AqssBOkNeWV9UgnvbYuliK2N10EbIoyB71ia5y3gR33cpnXmh20SHezF7msq15cRnrqJXZ/x+gW", - "nhtmFOGpNVMoPU88MfmRa9Dnv/YY+BhzbKy2i9qc5ZcVz1dsmp+fHOrgL6qJLtQndIauCUZnnKRX+qiN", - "SZay3G6hZ/pExVaL5OTaloyMJRu362JQqYnzk4E3hix0zqMWm2GJ70FyWU1yIuaKYtWzn2yXtbpkFtMM", - "5WwWkPZWjdBDF1wrn3kYZabv19H2zozRQ90MS7jBi0tuMHD1OYlK2A7eupT+vemObHe0oajKmTlKKSte", - "MgEiLOi3PSzo9p0jY0qmg+lxrRVh+tBdkcZ4OgchOZaMD6DrqB4j+4pTpI+4gPZyB0mGeEW7dq9emB+f", - "2WQTfWSSpICkLvmeE4GIQBQXkCE3uSvocqXzZjeeyTlwxFklQYz0SQ+RKGMgEGXS1KaqmTCKHnGZWi+4", - "xak0zzbES5RBCTQTiNGQE6LQqAAqbRkszVChC/smunZqSmYVx5MctMWqnv8ybvwvhPmscqUZg5bANTbV", - "0v5y1zkvtZcidGOQwL2j38ilKYt3EWS0Ltc4abhrJiShhmPlo1a9rJIzZs7kVL6kRGyHCeY0j/rcz/Qd", - "ZuUmL1M2vtqaXduobB1aQ5zfU1bxFHxWCU1ZEbJaj4FkUKZ0Wj+PctxesAeUhHrwI1s8cA0Icfdawi6P", - "SN0V7f3jpA6BrhBz+XTPFvj8OPJUIc3PDyRzYW1ZBBscwIZHik9+QAhChsYMIlwUGx4VHhc3kGTDkeO+", - "AbvZZFnC7vKg/Ni48dywsWxtcYTFlbgXWpi+ruSwByL8wp+2Uji+GaGKerVfTWWaQBum68s6oulStvDi", - "W1jWExYyrtyB7IynRRDVe8p4346mlscLYaJ/pstETHNNt678CqcMgoUZeOWFeEuYcM2tVC9atC/Vr95A", - "jZQ3FOqFU6ZCdExM9bd3KxtPWCVbBbq6X1fhVExvutP8NAfpaunNhDdYoGmOZzPIEBbo4+kPPwWFN2qY", - "4cUkShPqjalX8i8+1DMOKmCO+rUaXDm1KZ9rWEgxVXkfTlMQwtyarw9sBzixcV1hSNFi8/Wp1dWnx/OT", - "w5gqdbRR6b+5XNtLZaix5+a5zaViJsLo42/A68oYMWQL3hTRDD+dMCUxd60amdie1tMeAowcjxdh72XA", - "oN7bu3R9+yF/n7vyj3lhq3MTfcmFrfXl8/Xl87/v5fNX/9F3z9EplFjLWV/+KM0epb4MoLenXvzfC2Ua", - "ov50y2TRXBFY1/v+aTfMOvg98IaZNZhWiA1DaG+cPS0B0nlfoA248CFrHxUKT0QJ+Ao4yiAn18CF0nGu", - "wD9fILgtOQitNxUmMNWqzlQfSOeublkZnbZV9TjTLUsiU+05ndW7+0vJzk2tlrASwKZb6i8zflyP3iBP", - "eBF+CCXLokWTlC0PEebKld7PWTZVb74W2ktgChGDWVlNmLM0OHrFdGErH9ocfunY9MWdH8PTVkVac8Zu", - "PvDWOo2PylA/aJpqmtGZeroqdVV8mKlsS8+1BlQw/ghcWIdpVTE8adXZKLluJu7Cj33pYOekohS4ZySO", - "6odVs7mpL5YM6Eno3jvBq/d+zWcUVi1l3EcHVNtgNXXPoq/2KsqVEBkiVhSBWVJ9WS3fDdMxLK04kYtT", - "RYrh88PZ2fFrwBx4/SVFHfjMo3qQuZRlcnen65diJfL79uspaf3BO15RtH9Q7wT7W7+H5BpKhbb7B40J", - "1XaXbG/ufLP5v0oirASKS5LsJd9s7mxuK3VhOdd0b+kPqY0lGzucK5mIJTz11+a8jwOaC412QcpKaw4H", - "mVpttb/EZrdTX7Ns0ap9MokR5nJLZSZj95FAo+dVVhD77NtdqGOVBukHRqOa7d3t7RYVnti3PtvKtGEk", - "BGtoPXcrt6n0fsi0ylHTbJR8+4gkNMXnkflf48ydJ5t5d55n3nOKKzlnnPwBmZ5455vnmdjt07+jUq0U", - "zhhDh5ib0rFvd149F/dNTq+hyoQ7RcLu7qOS0LkI0CWmaYLqywKvnsv+DqgETnGOToFfA3cUeDiq0xIf", - "QX+9uLsYJaIqCswX7qui6Iwhlz3hmVDg7WKJgu/bsclCsViMKS5gzK6Bc5Jp6A/QYZRszW1t95aD4Rlo", - "EYQg5hfmJ0+IILELAEOB5M6XkxvI3IAIOa3L+5ey6ordn5xXM9HXcenGUGzqUvZ+9szrp+TLq6V/GFeG", - "RM2NXn2qoFxfC49H5f2yzBfubnjw+S1hilpKzlSW5a1nO2G69b20J47TwWzPHKjD6v51pO6P1OsIdd8I", - "ZT6yc8ZQ/aWFe4YoEjqGDwIDMnO9p2dwYHViHn5O73kc/s9IzGNXXdZe/xfPz9fQ82DoeWByTAIP9YHn", - "uv6SZhR53se+H3mvpMN9b+15MMjM9swgFO4mreFnnXQ8gefX3y18mOs7xxglWzm5hnFYdrtq+RFdeHh1", - "/Ka80f8etKw4hQwBzfQnw0QUItr1iUth4uE66qmefmaU6C3GXAPGGjAeDzCUmRmw+BrUyNueaZAjLwak", - "Cvo4ttIlHxjlmM4qBWF1tUMXBQ6Pnsrxm6utz+3s3kXOtX+v/fsR/Vt7y739OS+MC9tq/TG2n7Ic7/Z7", - "tP3qpa0N1/dWMV2S8Ue+kvnEWX9nxmd287Dqfu3oa0d/PEd33ueMG+0+wO9F10FGyZaK0AOOHt63irbN", - "zeKmLDKe1HvFcE8U1rvldutThrXb/03cXhcafsUhg/TcL3B2U7I4aKsv7OL/V6Hmf3h0t+LdJqBsiiMx", - "zbwq1eD/z+xBClMG+aRQEVRaPjNWhP+b6xor1ljx+FhRu9DDwMJ212hReV+vj8KE/YJ2vRJAk4X774H0", - "rVEpUPOfhETdvvkG9xOvDtxE6+xg7fF/E4/3vl9/T1evamcYJVte5Xq0lqqpJX+6QzM7xYMKqYLOQktT", - "aNm1/jcUVz39JmdVht6woqgokQv31abEXvjWNdtib2sr44CLsf0k1GZuu2+mqru+RtEz/qnUKVLfsPVA", - "QrfbwiXZmoDEW7Xy7i7u/j8AAP//9DvSdKt/AAA=", + "H4sIAAAAAAAC/+xdeW/ctrb/KoTeA5IAM97a3D4YuH84SxPj2onhpQtaYy5HOjPDWCJVkrI9zfN3f+Am", + "kRI1I7u229c7f2UscTnr7xySh8rXJGVFyShQKZL9r4lIF1Bg/fPg5PA954yr3xmIlJNSEkaTffUGgXqF", + "OIiSUQGoYBnkW8koKTkrgUsCeoxCzLvdzxdguxcgBJ6D6ieJzCHZT47FXP21LNUfQnJC58nd3Sjh8FtF", + "OGTJ/i961MumS01o3Y9Nv0Aqk7tRclBlhJ1aKruknAb0oxnjCKseaA4UOFatukzpFvpHnn+eJfu/fE3+", + "m8Ms2U/+a7uR5rYV5fYxZARfnB4ld5ejiCTsTJCZmbc63JrpfH4DniJMv2HZcjIHqhues3O4lYrcHi5C", + "ki7KnOHMUYNmJAckGZoCkhxT1XIKmZLJjPECy2Q/mRKK+TJp0ddV4igpQOIMS2xmneEqV/2/3iVtuRxk", + "GVE/cY6+sCki1ExGGLW0lFgIyNQfcgGoJCXkhIZ25OaK0aGUPSFZSEeHio/VfE7oHH2PU2cgh+9QpSZW", + "huLkUTorqac2TbPY1BxkxelEkgKExEUpQhokr6BDx6nug5o+ZvpFoBIk4VZuobOqLBlX1nSN8wrEPnoh", + "gEqgKbwYoRc3jGcvRkiZOTJEoSljOWCKXr5Qk79Q717McC7gxast9M5QhohA9vXLZrxXW64lKgBTgSjz", + "iNyys9l36vd4irXWmjae1CyX541k1sFAxzFidr/CPQ4LPIdzpv/p+se8IhmmKUxEinMI1PTd1uu2jt7T", + "lFUcz0FYS5E1hgAihX6R5kxAvkQ5oVeN8Sq9oZKzopTo5YLMF8Ct7lCBl4hDVqV2CPRbhXMil698uX2w", + "dKIzTWfNL62KKXDFL3EM9ni6GVsyRTmZLdENkYuOX/W7u5FfxNb1uJMVctztyvEdzDloYm4WJDVkNAhp", + "KCUClZVYaBHeYJ4J3YpQIgnOTZutNn1ovZhyxrFYAwkH6IidHqCXR+xmfIrpFTrIcCk1Mr2yisc0Q0QK", + "lDJuomOmvOwGyHwhteMaJrwAg97f4qLMYR99Rb8mOZZA5ThlVBChHG25nafFWFE3Ftlt/muyj3a3dkbo", + "14QCJ1/EdkluIR9jLsfu7d6dL4AjzdiT4WCHn4FQSGGOJbmGiTH+NUScN27yUrzS7lWRDNDNAkv1F9ym", + "eZUBmnFWRER8OKeMKwuaodAg0a/Vzs43Kdr1yf5kSUMnhrQY9VUxMX49KYHHeNhts/BJmxpiMwcIPkaU", + "wC17ASFVgQ5N4xPgHXIIlTA31qvpoTPgoFmT0Aotuzs7/fRkQBkRSse64xY6ZhzMb1SJCucKtQBrzLIQ", + "ZaHIsTKtJBI5uwGOairUMFmVa8+dLlW8ATqXiw5/rj0601THuPPFO8QqVtlkv04FnoFcTtIFpFeB8FTo", + "a0vvBLjCRBVIdTeku2lTFJIUGvdnbexSsFDlmUph2GwGVCgjYxwtMC9mVe6TeWZGfauJqYm10VpTC5B1", + "JXIG1i05phkrkMG3HlGoxlF5O10FUtjZ+p8euGYzk4o0aRouy5w0QY6D07HRzMsd9WY3CGRnbs4ONrfi", + "fukUaAJbJAEIIvv6DCCeIA8OmzXrjxY5HzFBrVUyFJb/EBr3T9nndS3drlPpwJzuB5IB66p01gLFf8QW", + "ZDOOCxAakAWkjGbavIM85FoN73P3fQ9uLXTYD+Z8/V10VtMSEYp0OBcDJv1oBo/NO9h26/iDzfg6fv6p", + "VmvIuH86UTDVejKt0iuQbSp2975rk3HhJlQq1qtNRZQSOS5YRaVSgBmzXm75CYXWmQmF6pWFWfWzULHT", + "9rwhea7AnlD9qqPCY9PsjSY6YMwP7YwImOBqPumB5Z29Tp5as6A7I5xlDRgHDJt0GX0MFh520cFBQDHN", + "ddrc29ckvDTlgIXjOwjxmoCDao76AX59+rL3+v9x9rLJK5wkbkjWst7dnb1vY3ioW94LDn/UY3dnvWeE", + "MaFjRYg5g3kBVB7QpVwQOt/rhpkpu41smqJcGxD6FmHO8RLNyTVQhAXCaMpu3RaA9TONiyPF/08///Qz", + "Mmjsc/uG3fauubuTHzq8F4b4hyI8FlcTQstKRvljN2MOguWVBjXVGOnGLabksiSp9kq9WMOo5HBNWCXU", + "j4ykujeR1q5GTVal/WL39uPtj+jlx3/++M+91//QJnl2cBxkksdq5kNN5l9u1VtUufJicTVhlawFuQIP", + "DlVuXcGokaCJKtzuCi5UAq4GNNuCuJiSeaWEaURvzEqMEJtJoOrPrEr1vh9ICdz2lAtMFeIQOs/BU0PA", + "laMcfTaUx8CDKqPKye8wSRnjmbgfeyUjVCLdk1AsQdQBtB63WVJgOgf0y85o99KaiO5t50VwW0IqTfMp", + "mAYchHqoHhn1ZaRQWMmoCCOWnQu9NTzEGPUn6zrDp9s96+VsZrmyimj5ws0COCDAqSUfEaU49PKn0c+v", + "GvQLEmndrE2Zl79rwnI8hTxC2JF+Xmc0AWmOml1EaEZSLX+smsKcs4pmtrWK9ztBkylOr/wmXXLNtCs2", + "xHM2J/Ie1mK6CVTRsfIAsWC5ynC0eZqxEKFCqqjPZopEjXH6fWTT+cjM3tXz0NjRiQkr4sdFWe+EPnDB", + "+cj7tI8DiJVhK3v4fuCaFPC71/9BG1iDpLnZyVqXcd5758g5Z8R/3y4qehXLe1L1QieoSpnaK3FzyNU9", + "P5Z2u6mb9OoBbKarR/VZDLc+Gl3XM/WM6V53BiYSCkXQnTdHPVY9kQ5jHUlKv6EizJOlEVREgh9OLt6y", + "oqwkHNJZ5Oz5uD6Ez0Biosz/w8kFSk0f/xi4K1QDXzXWxXMv/MWUMzSJ4he/bMDz1wIKxpeTGQcIOujH", + "6Hv1eEU3ySTOI/3O9fNoR0JbpOkH0e0AXAQ0fVJ/r91XUwKhpmVAZMiqk5EjyNNqS3lx9V5IkpPftYrW", + "qVhptmqaIyGxJEKSVDxQuc+ssWFqGCUejxNryX43T2LIyjc6nT+MoblvFEP5gIXwUJOIcRAlKLSWti1E", + "LObj+flJT42RejWwyMiAxfCCnLpeqFuQ887hjpk5QJy2/Oy0HtMNOz28/oBzkunhaq77WHHgvJKT9nge", + "khtOYjDuU9seIEY35tkN5trrrSwG1VEp/16J2POyUvmeqaSqC35OgjarmG8BksfZh7JCfSbnJ7uDjkNs", + "bu23P3HP1qFu2TSs5x01jPumE5HyCmWcSSzFIDVwwPlYRWutkFUgq8gSbtwHKqTt8y2lGLL/ulox7EfU", + "0kv4R8C5XLx1eXYoUTVcJeJp2UJ3RKaJS808yoBWhSL26PPBu8NPH5JR8vlfySh5f3r6+TQZJYfvjt77", + "hJ6ZqdaxbinyOfQYiPCnF433qp+MLVgi69wesbRXGX6eur7K0rc3UzaxLo+1tFy2+q2qrvROIu8lGJ3c", + "r5JL/7qgkYqu8lu7KGgn660EPcZBhNGjo+O3C0ZSiMWp3JRxDou4R0fHx6bUN7m7vBslM0KJWEw4YOGi", + "ibftUB+q6lbo1LSKbXbTDG6DzFA/iKd5ZvoHkty2Gj2PJ9FGVHE5HjfTh4JMGZVApc/EW/sotlnG8gDy", + "Tlm+Hu64aeRmCql2hMXJPoXfKhCR2ogC304kuwLaPqX7h78Lf4vOTZsVGhGDkx1fI36drx2m7egW3Hut", + "S4eX6N6J5ICLoJ+ucg0rVXAR3cqQUJTKVSsOrUPa73ynbRpFzkIlKyfhrs141+vMSvSvqERVv7JdguZ3", + "O1lbWlMrJbQSZwd9VtKgYMu6tU/cS8XWje787QQzSFfBKQcFioHz2EfRooghq0hnM2vNpHIOvYYh7QIX", + "xmxjK7DCTpDWlFfWI5302rpYidjedBGwKcoc9Iqtcd4GdtzLVV5rdtAi3c1e5KquenEZ6amX2P0do1t4", + "bphRhKfWTKH0PPHE5EeuQZ//2mPgE8yxsdouanOWTyqer9k0vzg90sFfVFNdqE/oHF0TjM45Sa/0URuT", + "LGW53ULP9ImKrRbJybUtGRlLNm7XxaBSE+cnA28NWeiCRy0WrpVt3oPosprmRCwUzaZvP+kuc3UJLaYZ", + "ytk8IO+9GaOHujmWcIOXE26wZv15hEqMDt+51PmD6Y5sd/RSUZUzc2RRVrxkAkRYOG97WHDrO6/FlMwG", + "0+NaK8L04bYijfF0AUJyLBkfQNdxPUb2B05rPuEC2ssKJBniFe3al3phfnxh0y30iUmSApK6tHpBBCIC", + "UVxAhtzkrnDKlaibXW8mF8ARZ5UEMdInKkSijIFAlElTA6pmwih6lGRqquAWp9I8eyleoQxKoJlAjIac", + "EOX1BVBpy01phgpdQDfVNUozMq84nuagLVb1/Ldxl38jzOeVK4EYtNSsMaCW9te7zrmkvXygG4ME7h2x", + "Ri4nWVyJIJB1ucZJw90pIQk1HCsftepllZwzc/al8hIlYjtMMKd51Od+pu8wKzf5j7Lx9dbs2kZl61AR", + "4vyesYqn4LNKaMqKkNV6DCSDcqCz+nmU4/bCOKAk1IMfQeIBYkAouddScTXyd1eO949HOtS4gsfV0z1h", + "gPGjomQuyKyKJ4PDyXDc/uzDcwDg2oOJcDFlOEY/rhcjyYb78X3DZ7O1sILd1SHysb34uZ14VUZ9jMWV", + "uJfvmr6u0K7HYf1yl7ZSOL4ZoYp6FU9NPZZAL03XV3V80QVc4XWvsJglLN9bu+/WGU+LIKr3lPG+fTwt", + "jxfCxOJMF0eY5ppuXe8UThlAtxl47TVwS5hwza1UL1u0r9Sv3jaMHOoX6oVTpsJXTEzNs3cXGU9ZJVtl", + "qbpfV+FUzG660/y4AOkqyM2EN1igWY7nc8gQFujT2fc/BuUmapjhJRRKE+qNqdLxy/3rGQeV7Ub9Wg2u", + "nNoUjTUspJiqLAynKQhh7orXx5QDnNi4rjCkaLH5+tTq6tPjxelRTJU62qhk3Fwp7aUy1Nhz89zmUjET", + "YfTxt511PYgYsvFsSkeG78mbQpC7VmVIbCfnabe+R47Hy7D3KmBQ7+0Nsr5dgL/PDfHHvKbUuX+94prS", + "5sr15sr13/fK9ev/6BvX6AxKrOWsrzyUZsdQl8DrzaIX//tCmYaoP1gyXTaF8Zsq1z/tXlUHvwfeq7IG", + "0wqxYQjtjbNnJUC66Au0ARc+ZB2gQuGJKAFfAUcZ5OQauFA6zhX450sEtyUHofWmwgSmWtWZ6gPpwlXr", + "KqPTtqoeZ7plSWSqPaezend/Kdm5qdUSVgLYdEv9ZcaP69Eb5Amvfw+hZFW0aJKy1SHCXDTS+zmrpurN", + "10J7CUwhYjBra+hylgYHjpgu7Xl/m8OvHZu+vPNjeNqqw2pOls1nzVpn0FEZ6gdNU00zOldP16Wuig8z", + "lW3pudaAur0fgAvrMK2z+yettRol183EXfixLx3snFaUAveMxFH9sBouN/XligE9Cd17X3b9Tqz5eMC6", + "pYy7aq/aBqupe5Y6tVdRrnDGELGm9MmS6stq9W6YjmFpxYlcnilSDJ8fz89P3gDmwOvvB+rAZx7Vgyyk", + "LJO7O121EysMP7DfDEnrz7zxiqKDw3on2N/6PSLXUCq0PThsTKi2u2Rna/ebrddKIqwEikuS7CffbO1u", + "7Sh1YbnQdG/rz4eNJRs7nCuZiCU89TfWvE/imWt8dkHKSmsOh5labbW/P2a3U9+wbNmq+DGJEeZyW2Um", + "Y/dpPKPndVYQ+9jZXahjlQbpB0ajmu29nZ0WFZ7Yt7/YeqxhJARraD13K7ep9H7IrMpR02yUfPuIJDQl", + "15H53+DMne6aeXefZ94Liiu5YJz8DpmeePeb55nY7dO/p1KtFM4ZQ0eYm4Kpb3dfPxf3TU6vocqEO0XC", + "3t6jktApf+8S0zRBdYn86+eyv0MqgVOcozPg18AdBR6O6rTER9BfLu8uR4moigLzpfuWJjpnyGVPeC4U", + "eLtYouD7dmyyUCyWY4oLGLNr4JxkGvoDdBgl2wtb0bztYHgOWgQhiPnl6MkTIkis7H0okNz5cnIDmbr/", + "kNO6qH0lq67E+8l5NRP9MS7dGIpNXcDdz555/ZR8eRXkD+PKkKi50atPFZTry9DxqHxQlvnS3YgOPjol", + "TIlJyZnKsrz1bCdMt74S9sRxOpjtmQN1WNO+idT9kXoToe4bocynZc4Zqr8vcM8QRULH8EFgQGau9/QM", + "DqxPzMOPyD2Pw/8ZiXnsgsfG6//i+fkGeh4MPQ9MjkngoT7wXNffj4wiz4fYVxPvlXS4r4w9DwaZ2Z4Z", + "hMLdpA38bJKOJ/D8+mt9D3N95xijZDsn1zAOi2DXLT+iCw+vqt6UN/pfQZYVp5AhoJn+UJaIQkS7PnEl", + "TDxcRz21zM+MEr3FmBvA2ADG4wGGMjMDFn8ENfK2ZxrkyIsBqYI+jq10yQdGOabzSkFYXe3QRYGj46dy", + "/OZC53M7u3d9cePfG/9+RP/W3nJvf84L48K2Wn+M7Qccx3v9Hm2/9Whrw/VtTUxXZPyRb0M+cdbfmfGZ", + "3Tysut84+sbRH8/Rnfc540Z7D/B70XWQUbKtIvSAo4cPraJtc8+3KYuMJ/VeMdwThfVuud3mlGHj9n8T", + "t9eFhn/gkEF67hc4uylZHLTVF3bx/4NM8/8aujvqbhNQNsWRmGZelWrwv0b2IIUpg3xSqAgqLZ8ZK8L/", + "w3SDFRuseHysqF3oYWBhu2u0qLxvtkdhwn43ul4JoOnS/ac4+taoFKj5rzGibt98efqJVwduok12sPH4", + "v4nHe19tv6erV7UzjJJtr3I9WkvV1JI/3aGZneJBhVRBZ6GlKbTsWv8HiKuefpuzKkNvWVFUlMil+4ZS", + "Yi9865ptsb+9nXHAxdh+oGkrt923UtVdX6PoGf9M6hSpb9h6IKHbbeOSbE9B4u1aeXeXd/8XAAD//5FW", + "1i+hfgAA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/core/ai_orchestrator.go b/core/ai_orchestrator.go index 90f52bb1c4..ddd81e90e1 100644 --- a/core/ai_orchestrator.go +++ b/core/ai_orchestrator.go @@ -12,7 +12,6 @@ import ( "os" "path" "strconv" - "strings" "sync" "time" @@ -1131,32 +1130,10 @@ func (orch *orchestrator) RegisterExternalCapability(extCapabilitySettings strin //set the price for the capability orch.node.SetPriceForExternalCapability("default", cap.Name, cap.GetPrice()) - //if a live ai capability add it to capabilities to enable live ai payments - var price *AutoConvertedPrice - if strings.Contains(cap.Name, "live-") { - - price, err = NewAutoConvertedPrice(cap.PriceCurrency, big.NewRat(cap.PricePerUnit, cap.PriceScaling), func(price *big.Rat) { - glog.V(6).Infof("Capability %s price set to %s wei per compute unit", cap.Name, price.FloatString(3)) - }) - - if err != nil { - panic(fmt.Errorf("error converting price: %v", err)) - } - orch.node.SetBasePriceForCap("default", Capability_LiveAI, cap.Name, price) - - orch.node.AddAICapabilities(cap.ToCapabilities()) - } - return cap, nil } func (orch *orchestrator) RemoveExternalCapability(extCapability string) error { - //if a live-ai external capability remove from Capabilities - if strings.Contains(extCapability, "live-") { - cap := orch.node.ExternalCapabilities.Capabilities[extCapability] - orch.node.RemoveAICapabilities(cap.ToCapabilities()) - } - orch.node.ExternalCapabilities.RemoveCapability(extCapability) return nil } diff --git a/core/capabilities.go b/core/capabilities.go index ba9884f560..e0cbc436a9 100644 --- a/core/capabilities.go +++ b/core/capabilities.go @@ -88,7 +88,6 @@ const ( Capability_ImageToText Capability = 34 Capability_LiveVideoToVideo Capability = 35 Capability_TextToSpeech Capability = 36 - Capability_LiveAI Capability = 37 ) var CapabilityNameLookup = map[Capability]string{ @@ -130,7 +129,6 @@ var CapabilityNameLookup = map[Capability]string{ Capability_ImageToText: "Image to text", Capability_LiveVideoToVideo: "Live video to video", Capability_TextToSpeech: "Text to speech", - Capability_LiveAI: "Live AI", } var CapabilityTestLookup = map[Capability]CapabilityTest{ diff --git a/core/external_capabilities.go b/core/external_capabilities.go index 3e54782bd9..82682190b0 100644 --- a/core/external_capabilities.go +++ b/core/external_capabilities.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "math/big" - "time" "sync" @@ -16,40 +15,35 @@ import ( ) type ExternalCapability struct { - Name string `json:"name"` - Description string `json:"description"` - Url string `json:"url"` - Capacity int `json:"capacity"` - PricePerUnit int64 `json:"price_per_unit"` - PriceScaling int64 `json:"price_scaling"` - PriceCurrency string `json:"currency"` - Requirements CapabilityRequirements `json:"requirements"` - price *AutoConvertedPrice + Name string `json:"name"` + Description string `json:"description"` + Url string `json:"url"` + Capacity int `json:"capacity"` + PricePerUnit int64 `json:"price_per_unit"` + PriceScaling int64 `json:"price_scaling"` + PriceCurrency string `json:"currency"` + + price *AutoConvertedPrice mu sync.RWMutex Load int } -type CapabilityRequirements struct { - VideoIngress bool `json:"video_ingress"` - VideoEgress bool `json:"video_egress"` - DataOutput bool `json:"data_output"` -} - type StreamInfo struct { StreamID string Capability string //Gateway fields - StreamRequest []byte - ExcludeOrchs []string - OrchToken interface{} - OrchUrl string - + StreamRequest []byte + ExcludeOrchs []string + OrchToken interface{} + OrchUrl string OrchPublishUrl string OrchSubscribeUrl string OrchControlUrl string OrchEventsUrl string OrchDataUrl string + ControlPub *trickle.TricklePublisher + StopControl func() //Orchestrator fields Sender ethcommon.Address @@ -59,14 +53,11 @@ type StreamInfo struct { eventsChannel *trickle.TrickleLocalPublisher dataChannel *trickle.TrickleLocalPublisher //Stream fields - Params interface{} - DataWriter *media.SegmentWriter - ControlPub *trickle.TricklePublisher - StopControl func() - JobParams string - StreamCtx context.Context - CancelStream context.CancelFunc - StreamStartedTime time.Time + Params interface{} + DataWriter *media.SegmentWriter + JobParams string + StreamCtx context.Context + CancelStream context.CancelFunc sdm sync.Mutex } @@ -244,14 +235,3 @@ func (extCap *ExternalCapability) GetPrice() *big.Rat { defer extCap.mu.RUnlock() return extCap.price.Value() } - -func (extCap *ExternalCapability) ToCapabilities() *Capabilities { - capConstraints := make(PerCapabilityConstraints) - capConstraints[Capability_LiveAI].Models = make(ModelConstraints) - capConstraints[Capability_LiveAI].Models[extCap.Name] = &ModelConstraint{Capacity: extCap.Capacity} - - caps := NewCapabilities([]Capability{Capability_LiveAI}, MandatoryOCapabilities()) - caps.SetPerCapabilityConstraints(capConstraints) - - return caps -} diff --git a/server/ai_http.go b/server/ai_http.go index b084ba4a16..97a68a6530 100644 --- a/server/ai_http.go +++ b/server/ai_http.go @@ -145,29 +145,8 @@ func (h *lphttp) StartLiveVideoToVideo() http.Handler { subUrl = pubUrl + "-out" controlUrl = pubUrl + "-control" eventsUrl = pubUrl + "-events" - dataUrl = pubUrl + "-data" ) - //if data is not enabled remove the url and do not start the data channel - if enableData, ok := (*req.Params)["enableData"]; ok { - if val, ok := enableData.(bool); ok { - //turn off data channel if request sets to false - if !val { - dataUrl = "" - } else { - clog.Infof(ctx, "data channel is enabled") - } - } else { - clog.Warningf(ctx, "enableData is not a bool, got type %T", enableData) - } - - //delete the param used for go-livepeer signaling - delete((*req.Params), "enableData") - } else { - //default to no data channel - dataUrl = "" - } - // Handle initial payment, the rest of the payments are done separately from the stream processing // Note that this payment is debit from the balance and acts as a buffer for the AI Realtime Video processing payment, err := getPayment(r.Header.Get(paymentHeader)) @@ -202,13 +181,6 @@ func (h *lphttp) StartLiveVideoToVideo() http.Handler { eventsCh := trickle.NewLocalPublisher(h.trickleSrv, mid+"-events", "application/json") eventsCh.CreateChannel() - //optional channels - var dataCh *trickle.TrickleLocalPublisher - if dataUrl != "" { - dataCh = trickle.NewLocalPublisher(h.trickleSrv, mid+"-data", "application/jsonl") - dataCh.CreateChannel() - } - // Start payment receiver which accounts the payments and stops the stream if the payment is insufficient priceInfo := payment.GetExpectedPrice() var paymentProcessor *LivePaymentProcessor @@ -228,9 +200,6 @@ func (h *lphttp) StartLiveVideoToVideo() http.Handler { subCh.Close() eventsCh.Close() controlPubCh.Close() - if dataCh != nil { - dataCh.Close() - } cancel() } return err @@ -258,25 +227,17 @@ func (h *lphttp) StartLiveVideoToVideo() http.Handler { }() // Prepare request to worker - // required channels controlUrlOverwrite := overwriteHost(h.node.LiveAITrickleHostForRunner, controlUrl) eventsUrlOverwrite := overwriteHost(h.node.LiveAITrickleHostForRunner, eventsUrl) subscribeUrlOverwrite := overwriteHost(h.node.LiveAITrickleHostForRunner, pubUrl) publishUrlOverwrite := overwriteHost(h.node.LiveAITrickleHostForRunner, subUrl) - // optional channels - var dataUrlOverwrite string - if dataCh != nil { - dataUrlOverwrite = overwriteHost(h.node.LiveAITrickleHostForRunner, dataUrl) - } - workerReq := worker.LiveVideoToVideoParams{ ModelId: req.ModelId, PublishUrl: publishUrlOverwrite, SubscribeUrl: subscribeUrlOverwrite, EventsUrl: &eventsUrlOverwrite, ControlUrl: &controlUrlOverwrite, - DataUrl: &dataUrlOverwrite, Params: req.Params, GatewayRequestId: &gatewayRequestID, ManifestId: &mid, @@ -294,9 +255,6 @@ func (h *lphttp) StartLiveVideoToVideo() http.Handler { subCh.Close() controlPubCh.Close() eventsCh.Close() - if dataCh != nil { - dataCh.Close() - } cancel() respondWithError(w, err.Error(), http.StatusInternalServerError) return @@ -308,7 +266,6 @@ func (h *lphttp) StartLiveVideoToVideo() http.Handler { SubscribeUrl: subUrl, ControlUrl: &controlUrl, EventsUrl: &eventsUrl, - DataUrl: &dataUrl, RequestId: &requestID, ManifestId: &mid, }) diff --git a/server/ai_live_video.go b/server/ai_live_video.go index 1e9709e4d6..aa7f3264d7 100644 --- a/server/ai_live_video.go +++ b/server/ai_live_video.go @@ -1,7 +1,6 @@ package server import ( - "bufio" "bytes" "context" "encoding/json" @@ -78,34 +77,23 @@ func startTricklePublish(ctx context.Context, url *url.URL, params aiRequestPara return } - var orchAddr string - var orchUrl string // Start payments which probes a segment every "paymentProcessInterval" and sends a payment ctx, cancel := context.WithCancel(ctx) + priceInfo := sess.OrchestratorInfo.PriceInfo var paymentProcessor *LivePaymentProcessor - if sess != nil { - orchAddr = sess.Address() - orchUrl = sess.Transcoder() - - priceInfo := sess.OrchestratorInfo.PriceInfo - if priceInfo != nil && priceInfo.PricePerUnit != 0 { - paymentSender := livePaymentSender{} - sendPaymentFunc := func(inPixels int64) error { - return paymentSender.SendPayment(context.Background(), &SegmentInfoSender{ - sess: sess.BroadcastSession, - inPixels: inPixels, - priceInfo: priceInfo, - mid: params.liveParams.manifestID, - }) - } - paymentProcessor = NewLivePaymentProcessor(ctx, params.liveParams.paymentProcessInterval, sendPaymentFunc) - } else { - clog.Warningf(ctx, "No price info found from Orchestrator, Gateway will not send payments for the video processing") + if priceInfo != nil && priceInfo.PricePerUnit != 0 { + paymentSender := livePaymentSender{} + sendPaymentFunc := func(inPixels int64) error { + return paymentSender.SendPayment(context.Background(), &SegmentInfoSender{ + sess: sess.BroadcastSession, + inPixels: inPixels, + priceInfo: priceInfo, + mid: params.liveParams.manifestID, + }) } + paymentProcessor = NewLivePaymentProcessor(ctx, params.liveParams.paymentProcessInterval, sendPaymentFunc) } else { - //byoc sets as context values - orchAddr = clog.GetVal(ctx, "orch") - orchUrl = clog.GetVal(ctx, "orch_url") + clog.Warningf(ctx, "No price info found from Orchestrator, Gateway will not send payments for the video processing") } slowOrchChecker := &SlowOrchChecker{} @@ -175,8 +163,8 @@ func startTricklePublish(ctx context.Context, url *url.URL, params aiRequestPara "pipeline_id": params.liveParams.pipelineID, "request_id": params.liveParams.requestID, "orchestrator_info": map[string]interface{}{ - "address": orchAddr, - "url": orchUrl, + "address": sess.Address(), + "url": sess.Transcoder(), }, }) } @@ -731,13 +719,6 @@ func startEventsSubscribe(ctx context.Context, url *url.URL, params aiRequestPar "address": sess.Address(), "url": sess.Transcoder(), } - } else { - address := clog.GetVal(ctx, "orch") - url := clog.GetVal(ctx, "orch_url") - event["orchestrator_info"] = map[string]interface{}{ - "address": address, - "url": url, - } } clog.V(8).Infof(ctx, "Received event for seq=%d event=%+v", trickle.GetSeq(segment), event) @@ -804,125 +785,6 @@ func startEventsSubscribe(ctx context.Context, url *url.URL, params aiRequestPar }() } -func startDataSubscribe(ctx context.Context, url *url.URL, params aiRequestParams, sess *AISession) { - //only start DataSubscribe if enabled - if params.liveParams.dataWriter == nil { - return - } - - // subscribe to the outputs - subscriber, err := trickle.NewTrickleSubscriber(trickle.TrickleSubscriberConfig{ - URL: url.String(), - Ctx: ctx, - }) - if err != nil { - clog.Infof(ctx, "Failed to create data subscriber: %s", err) - return - } - - dataWriter := params.liveParams.dataWriter - - // read segments from trickle subscription - go func() { - defer dataWriter.Close() - - var err error - firstSegment := true - - retries := 0 - // we're trying to keep (retryPause x maxRetries) duration to fall within one output GOP length - const retryPause = 300 * time.Millisecond - const maxRetries = 5 - for { - select { - case <-ctx.Done(): - clog.Info(ctx, "data subscribe done") - return - default: - } - if !params.inputStreamExists() { - clog.Infof(ctx, "data subscribe stopping, input stream does not exist.") - break - } - var segment *http.Response - readBytes, readMessages := 0, 0 - clog.V(8).Infof(ctx, "data subscribe await") - segment, err = subscriber.Read() - if err != nil { - if errors.Is(err, trickle.EOS) || errors.Is(err, trickle.StreamNotFoundErr) { - stopProcessing(ctx, params, fmt.Errorf("data subscribe stopping, stream not found, err=%w", err)) - return - } - var sequenceNonexistent *trickle.SequenceNonexistent - if errors.As(err, &sequenceNonexistent) { - // stream exists but segment doesn't, so skip to leading edge - subscriber.SetSeq(sequenceNonexistent.Latest) - } - // TODO if not EOS then signal a new orchestrator is needed - err = fmt.Errorf("data subscribe error reading: %w", err) - clog.Infof(ctx, "%s", err) - if retries > maxRetries { - stopProcessing(ctx, params, errors.New("data subscribe stopping, retries exceeded")) - return - } - retries++ - params.liveParams.sendErrorEvent(err) - time.Sleep(retryPause) - continue - } - retries = 0 - seq := trickle.GetSeq(segment) - clog.V(8).Infof(ctx, "data subscribe received seq=%d", seq) - copyStartTime := time.Now() - - defer segment.Body.Close() - scanner := bufio.NewScanner(segment.Body) - for scanner.Scan() { - writer, err := dataWriter.Next() - if err != nil { - if err != io.EOF { - stopProcessing(ctx, params, fmt.Errorf("data subscribe could not get next: %w", err)) - } - return - } - n, err := writer.Write(scanner.Bytes()) - if err != nil { - stopProcessing(ctx, params, fmt.Errorf("data subscribe could not write: %w", err)) - } - readBytes += n - readMessages += 1 - } - if err := scanner.Err(); err != nil { - clog.InfofErr(ctx, "data subscribe error reading seq=%d", seq, err) - subscriber.SetSeq(seq) - retries++ - continue - } - - if firstSegment { - firstSegment = false - delayMs := time.Since(params.liveParams.startTime).Milliseconds() - if monitor.Enabled { - monitor.AIFirstSegmentDelay(delayMs, params.liveParams.sess.OrchestratorInfo) - monitor.SendQueueEventAsync("stream_trace", map[string]interface{}{ - "type": "gateway_receive_first_data_segment", - "timestamp": time.Now().UnixMilli(), - "stream_id": params.liveParams.streamID, - "pipeline_id": params.liveParams.pipelineID, - "request_id": params.liveParams.requestID, - "orchestrator_info": map[string]interface{}{ - "address": params.liveParams.sess.Address(), - "url": params.liveParams.sess.Transcoder(), - }, - }) - } - } - - clog.V(8).Info(ctx, "data subscribe read completed", "seq", seq, "bytes", humanize.Bytes(uint64(readBytes)), "messages", readMessages, "took", time.Since(copyStartTime)) - } - }() -} - func (a aiRequestParams) inputStreamExists() bool { if a.node == nil { return false @@ -958,7 +820,7 @@ const maxInflightSegments = 3 // If inflight max is hit, returns true, false otherwise. func (s *SlowOrchChecker) BeginSegment() (int, bool) { // Returns `false` if there are multiple segments in-flight - // this means the orchestrator is slow reading + // this means the orchestrator is slow reading them // If all-OK, returns `true` s.mu.Lock() defer s.mu.Unlock() diff --git a/server/ai_mediaserver.go b/server/ai_mediaserver.go index 0b5835d860..41bb955fbd 100644 --- a/server/ai_mediaserver.go +++ b/server/ai_mediaserver.go @@ -110,10 +110,6 @@ func startAIMediaServer(ctx context.Context, ls *LivepeerServer) error { ls.HTTPMux.Handle("OPTIONS /live/video-to-video/{streamId}/status", ls.WithCode(http.StatusNoContent)) ls.HTTPMux.Handle("/live/video-to-video/{streamId}/status", ls.GetLiveVideoToVideoStatus()) - // Stream data SSE endpoint - ls.HTTPMux.Handle("OPTIONS /live/video-to-video/{stream}/data", ls.WithCode(http.StatusNoContent)) - ls.HTTPMux.Handle("GET /live/video-to-video/{stream}/data", ls.GetLiveVideoToVideoData()) - //API for dynamic capabilities ls.HTTPMux.Handle("/process/request/", ls.SubmitJob()) @@ -651,15 +647,6 @@ func (ls *LivepeerServer) StartLiveVideo() http.Handler { }, } - //create a dataWriter for data channel if enabled - if enableData, ok := pipelineParams["enableData"]; ok { - if enableData == true || enableData == "true" { - params.liveParams.dataWriter = media.NewSegmentWriter(5) - pipelineParams["enableData"] = true - clog.Infof(ctx, "Data channel enabled for stream %s", streamName) - } - } - registerControl(ctx, params) // Create a special parent context for orchestrator cancellation @@ -774,7 +761,6 @@ func processStream(ctx context.Context, params aiRequestParams, req worker.GenLi func newParams(params *liveRequestParams, cancelOrch context.CancelCauseFunc) *liveRequestParams { return &liveRequestParams{ segmentReader: params.segmentReader, - dataWriter: params.dataWriter, rtmpOutputs: params.rtmpOutputs, localRTMPPrefix: params.localRTMPPrefix, stream: params.stream, @@ -797,8 +783,6 @@ func startProcessing(ctx context.Context, params aiRequestParams, res interface{ resp := res.(*worker.GenLiveVideoToVideoResponse) host := params.liveParams.sess.Transcoder() - - //required channels pub, err := common.AppendHostname(resp.JSON200.PublishUrl, host) if err != nil { return fmt.Errorf("invalid publish URL: %w", err) @@ -815,30 +799,16 @@ func startProcessing(ctx context.Context, params aiRequestParams, res interface{ if err != nil { return fmt.Errorf("invalid events URL: %w", err) } - if resp.JSON200.ManifestId != nil { ctx = clog.AddVal(ctx, "manifest_id", *resp.JSON200.ManifestId) params.liveParams.manifestID = *resp.JSON200.ManifestId } - clog.V(common.VERBOSE).Infof(ctx, "pub %s sub %s control %s events %s", pub, sub, control, events) startControlPublish(ctx, control, params) startTricklePublish(ctx, pub, params, params.liveParams.sess) startTrickleSubscribe(ctx, sub, params, params.liveParams.sess) startEventsSubscribe(ctx, events, params, params.liveParams.sess) - - //optional channels - var data *url.URL - if *resp.JSON200.DataUrl != "" { - data, err = common.AppendHostname(*resp.JSON200.DataUrl, host) - if err != nil { - return fmt.Errorf("invalid data URL: %w", err) - } - clog.V(common.VERBOSE).Infof(ctx, "data %s", data) - startDataSubscribe(ctx, data, params, params.liveParams.sess) - } - return nil } @@ -1153,15 +1123,6 @@ func (ls *LivepeerServer) CreateWhip(server *media.WHIPServer) http.Handler { }, } - //create a dataWriter for data channel if enabled - if enableData, ok := pipelineParams["enableData"]; ok { - if enableData == true || enableData == "true" { - params.liveParams.dataWriter = media.NewSegmentWriter(5) - pipelineParams["enableData"] = true - clog.Infof(ctx, "Data channel enabled for stream %s", streamName) - } - } - registerControl(ctx, params) req := worker.GenLiveVideoToVideoJSONRequestBody{ @@ -1373,78 +1334,6 @@ func (ls *LivepeerServer) SmokeTestLiveVideo() http.Handler { }) } -// @Summary Get Live Stream Data -// @Param stream path string true "Stream Key" -// @Success 200 -// @Router /live/video-to-video/{stream}/data [get] -func (ls *LivepeerServer) GetLiveVideoToVideoData() http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - stream := r.PathValue("stream") - if stream == "" { - http.Error(w, "stream name is required", http.StatusBadRequest) - return - } - - ctx := r.Context() - ctx = clog.AddVal(ctx, "stream", stream) - - // Get the live pipeline for this stream - livePipeline, ok := ls.LivepeerNode.LivePipelines[stream] - if !ok { - http.Error(w, "Stream not found", http.StatusNotFound) - return - } - - // Get the data readerring buffer - if livePipeline.DataWriter == nil { - clog.Infof(ctx, "No data writer available for stream %s", stream) - http.Error(w, "Stream data not available", http.StatusServiceUnavailable) - return - } - dataReader := livePipeline.DataWriter.MakeReader(media.SegmentReaderConfig{}) - - // Set up SSE headers - w.Header().Set("Content-Type", "text/event-stream") - w.Header().Set("Cache-Control", "no-cache") - w.Header().Set("Connection", "keep-alive") - w.Header().Set("Access-Control-Allow-Origin", "*") - - flusher, ok := w.(http.Flusher) - if !ok { - http.Error(w, "Streaming not supported", http.StatusInternalServerError) - return - } - - clog.Infof(ctx, "Starting SSE data stream for stream=%s", stream) - - // Listen for broadcast signals from ring buffer writes - // dataReader.Read() blocks on rb.cond.Wait() until startDataSubscribe broadcasts - for { - select { - case <-ctx.Done(): - clog.Info(ctx, "SSE data stream client disconnected") - return - default: - reader, err := dataReader.Next() - if err != nil { - if err == io.EOF { - // Stream ended - fmt.Fprintf(w, `event: end\ndata: {"type":"stream_ended"}\n\n`) - flusher.Flush() - return - } - clog.Errorf(ctx, "Error reading from ring buffer: %v", err) - return - } - - data, err := io.ReadAll(reader) - fmt.Fprintf(w, "data: %s\n\n", data) - flusher.Flush() - } - } - }) -} - func startHearbeats(ctx context.Context, node *core.LivepeerNode) { if node.LiveAIHeartbeatURL == "" { return From c4c9bd627349fe0acd5c29932731606d4003907c Mon Sep 17 00:00:00 2001 From: Brad P Date: Tue, 2 Sep 2025 16:26:13 -0500 Subject: [PATCH 34/57] updates for payment stability --- server/job_rpc.go | 64 ++++++++++++++++++++++++++++-------------- server/job_rpc_test.go | 6 ++-- server/job_stream.go | 19 ++++++++----- 3 files changed, 58 insertions(+), 31 deletions(-) diff --git a/server/job_rpc.go b/server/job_rpc.go index 4ae0441913..386822d616 100644 --- a/server/job_rpc.go +++ b/server/job_rpc.go @@ -62,6 +62,8 @@ type JobToken struct { Balance int64 `json:"balance,omitempty"` Price *net.PriceInfo `json:"price,omitempty"` ServiceAddr string `json:"service_addr,omitempty"` + + lastNonce uint32 } type JobRequest struct { @@ -559,7 +561,7 @@ func (ls *LivepeerServer) sendJobToOrch(ctx context.Context, r *http.Request, jo req.Header.Add(jobRequestHdr, signedReqHdr) if orchToken.Price.PricePerUnit > 0 { - paymentHdr, err := createPayment(ctx, jobReq, orchToken, ls.LivepeerNode) + paymentHdr, err := createPayment(ctx, jobReq, &orchToken, ls.LivepeerNode) if err != nil { clog.Errorf(ctx, "Unable to create payment err=%v", err) return nil, http.StatusInternalServerError, fmt.Errorf("Unable to create payment err=%v", err) @@ -830,7 +832,7 @@ func (h *lphttp) setupOrchJob(ctx context.Context, r *http.Request, reserveCapac if jobPriceRat.Cmp(big.NewRat(0, 1)) > 0 { // get payment information paymentHdr := r.Header.Get(jobPaymentHeaderHdr) - minBal := jobPriceRat.Mul(jobPriceRat, big.NewRat(60, 1)) //minimum 1 minute balance + minBal := new(big.Rat).Mul(jobPriceRat, big.NewRat(60, 1)) //minimum 1 minute balance if paymentHdr != "" { payment, err = getPayment(paymentHdr) if err != nil { @@ -866,35 +868,55 @@ func (h *lphttp) setupOrchJob(ctx context.Context, r *http.Request, reserveCapac return &orchJob{Req: jobReq, Sender: sender, JobPrice: jobPrice, Details: &jobDetails}, nil } -func createPayment(ctx context.Context, jobReq *JobRequest, orchToken JobToken, node *core.LivepeerNode) (string, error) { +func createPayment(ctx context.Context, jobReq *JobRequest, orchToken *JobToken, node *core.LivepeerNode) (string, error) { var payment *net.Payment + createTickets := true clog.Infof(ctx, "creating payment for job request %s", jobReq.Capability) sender := ethcommon.HexToAddress(jobReq.Sender) orchAddr := ethcommon.BytesToAddress(orchToken.TicketParams.Recipient) - balance := node.Balances.Balance(orchAddr, core.ManifestID(jobReq.Capability)) sessionID := node.Sender.StartSession(*pmTicketParams(orchToken.TicketParams)) - createTickets := true + + //setup balances and update Gateway balance to Orchestrator balance, log differences + orchBal := big.NewRat(orchToken.Balance, 1) + balance := node.Balances.Balance(orchAddr, core.ManifestID(jobReq.Capability)) if balance == nil { //create a balance of 0 node.Balances.Debit(orchAddr, core.ManifestID(jobReq.Capability), big.NewRat(0, 1)) balance = node.Balances.Balance(orchAddr, core.ManifestID(jobReq.Capability)) + } + + diff := new(big.Rat).Sub(orchBal, balance) + if balance.Cmp(orchBal) != 0 { + clog.Infof(ctx, "Adjusting gateway balance to Orchestrator provided balance for sender=%v capability=%v balance=%v orchBal=%v diff=%v", sender.Hex(), jobReq.Capability, balance.FloatString(0), orchBal.FloatString(0), diff.FloatString(0)) + } + + if diff.Sign() > 0 { + node.Balances.Credit(orchAddr, core.ManifestID(jobReq.Capability), diff) } else { - price := big.NewRat(orchToken.Price.PricePerUnit, orchToken.Price.PixelsPerUnit) - cost := new(big.Rat).Mul(price, big.NewRat(int64(jobReq.Timeout), 1)) - minBal := new(big.Rat).Mul(price, big.NewRat(60, 1)) //minimum 1 minute balance - if cost.Cmp(minBal) < 0 { - cost = minBal - } + node.Balances.Debit(orchAddr, core.ManifestID(jobReq.Capability), new(big.Rat).Abs(diff)) + } - if balance.Cmp(cost) > 0 { - createTickets = false - payment = &net.Payment{ - Sender: sender.Bytes(), - ExpectedPrice: orchToken.Price, - } + price := big.NewRat(orchToken.Price.PricePerUnit, orchToken.Price.PixelsPerUnit) + cost := new(big.Rat).Mul(price, big.NewRat(int64(jobReq.Timeout), 1)) + minBal := new(big.Rat).Mul(price, big.NewRat(60, 1)) //minimum 1 minute balance + if cost.Cmp(minBal) < 0 { + cost = minBal + } + + if balance.Sign() > 0 && orchToken.Balance == 0 { + clog.Infof(ctx, "Updating balance to 0 because orchestrator balance reset for sender=%v capability=%v balance=%v", sender.Hex(), jobReq.Capability, balance.FloatString(0)) + node.Balances.Debit(orchAddr, core.ManifestID(jobReq.Capability), balance) + balance = node.Balances.Balance(orchAddr, core.ManifestID(jobReq.Capability)) + } + + if balance.Cmp(cost) > 0 { + createTickets = false + payment = &net.Payment{ + Sender: sender.Bytes(), + ExpectedPrice: orchToken.Price, } - clog.Infof(ctx, "current balance for sender=%v capability=%v is %v, cost=%v price=%v", sender.Hex(), jobReq.Capability, balance.FloatString(3), cost.FloatString(3), price.FloatString(3)) } + clog.Infof(ctx, "current balance for sender=%v capability=%v is %v, cost=%v price=%v", sender.Hex(), jobReq.Capability, balance.FloatString(3), cost.FloatString(3), price.FloatString(3)) if !createTickets { clog.V(common.DEBUG).Infof(ctx, "No payment required, using balance=%v", balance.FloatString(3)) @@ -929,12 +951,12 @@ func createPayment(ctx context.Context, jobReq *JobRequest, orchToken JobToken, senderParams := make([]*net.TicketSenderParams, len(tickets.SenderParams)) for i := 0; i < len(tickets.SenderParams); i++ { senderParams[i] = &net.TicketSenderParams{ - SenderNonce: tickets.SenderParams[i].SenderNonce, + SenderNonce: orchToken.lastNonce + tickets.SenderParams[i].SenderNonce, Sig: tickets.SenderParams[i].Sig, } totalEV = totalEV.Add(totalEV, tickets.WinProbRat()) } - + orchToken.lastNonce = tickets.SenderParams[len(tickets.SenderParams)-1].SenderNonce + 1 payment.TicketSenderParams = senderParams ratPrice, _ := common.RatPriceInfo(payment.ExpectedPrice) @@ -967,7 +989,7 @@ func updateGatewayBalance(node *core.LivepeerNode, orchToken JobToken, capabilit orchAddr := ethcommon.BytesToAddress(orchToken.TicketParams.Recipient) // update for usage of compute orchPrice := big.NewRat(orchToken.Price.PricePerUnit, orchToken.Price.PixelsPerUnit) - cost := orchPrice.Mul(orchPrice, big.NewRat(int64(math.Ceil(took.Seconds())), 1)) + cost := new(big.Rat).Mul(orchPrice, big.NewRat(int64(math.Ceil(took.Seconds())), 1)) node.Balances.Debit(orchAddr, core.ManifestID(capability), cost) //get the updated balance diff --git a/server/job_rpc_test.go b/server/job_rpc_test.go index 29c05b5cc6..84d3976887 100644 --- a/server/job_rpc_test.go +++ b/server/job_rpc_test.go @@ -945,7 +945,7 @@ func TestCreatePayment(t *testing.T) { //payment with one ticket jobReq.Timeout = 1 mockSender.On("CreateTicketBatch", "foo", jobReq.Timeout).Return(mockTicketBatch(jobReq.Timeout), nil).Once() - payment, err := createPayment(ctx, &jobReq, orchTocken, node) + payment, err := createPayment(ctx, &jobReq, &orchTocken, node) assert.Nil(t, err) pmPayment, err := base64.StdEncoding.DecodeString(payment) assert.Nil(t, err) @@ -956,7 +956,7 @@ func TestCreatePayment(t *testing.T) { //test 2 tickets jobReq.Timeout = 2 mockSender.On("CreateTicketBatch", "foo", jobReq.Timeout).Return(mockTicketBatch(jobReq.Timeout), nil).Once() - payment, err = createPayment(ctx, &jobReq, orchTocken, node) + payment, err = createPayment(ctx, &jobReq, &orchTocken, node) assert.Nil(t, err) pmPayment, err = base64.StdEncoding.DecodeString(payment) assert.Nil(t, err) @@ -967,7 +967,7 @@ func TestCreatePayment(t *testing.T) { //test 600 tickets jobReq.Timeout = 600 mockSender.On("CreateTicketBatch", "foo", jobReq.Timeout).Return(mockTicketBatch(jobReq.Timeout), nil).Once() - payment, err = createPayment(ctx, &jobReq, orchTocken, node) + payment, err = createPayment(ctx, &jobReq, &orchTocken, node) assert.Nil(t, err) pmPayment, err = base64.StdEncoding.DecodeString(payment) assert.Nil(t, err) diff --git a/server/job_stream.go b/server/job_stream.go index c49e4b6b13..65061119f0 100644 --- a/server/job_stream.go +++ b/server/job_stream.go @@ -126,6 +126,7 @@ func (ls *LivepeerServer) runStream(gatewayJob *gatewayJob) { for _, orch := range gatewayJob.Orchs { stream.OrchToken = orch stream.OrchUrl = orch.ServiceAddr + ctx = clog.AddVal(ctx, "orch", ethcommon.Bytes2Hex(orch.TicketParams.Recipient)) ctx = clog.AddVal(ctx, "orch_url", orch.ServiceAddr) clog.Infof(ctx, "Starting stream processing") @@ -266,8 +267,8 @@ func (ls *LivepeerServer) monitorStream(streamId string) { clog.Errorf(ctx, "Error signing job, continuing monitoring: %v", err) continue } - - pmtHdr, err := createPayment(ctx, req, stream.OrchToken.(JobToken), ls.LivepeerNode) + orchToken := stream.OrchToken.(JobToken) + pmtHdr, err := createPayment(ctx, req, &orchToken, ls.LivepeerNode) if err != nil { clog.Errorf(ctx, "Error processing stream payment for %s: %v", streamId, err) // Continue monitoring even if payment fails @@ -745,8 +746,8 @@ func (ls *LivepeerServer) GetStreamData() http.Handler { ctx = clog.AddVal(ctx, "stream", streamId) // Get the live pipeline for this stream - stream, ok := ls.LivepeerNode.ExternalCapabilities.Streams[streamId] - if !ok { + stream, exists := ls.LivepeerNode.ExternalCapabilities.Streams[streamId] + if !exists { http.Error(w, "Stream not found", http.StatusNotFound) return } @@ -770,7 +771,7 @@ func (ls *LivepeerServer) GetStreamData() http.Handler { return } - clog.Infof(ctx, "Starting SSE data stream for stream=%s", stream) + clog.Infof(ctx, "Starting SSE data stream for stream=%s", streamId) // Listen for broadcast signals from ring buffer writes // dataReader.Read() blocks on rb.cond.Wait() until startDataSubscribe broadcasts @@ -892,7 +893,11 @@ func (h *lphttp) StartStream(w http.ResponseWriter, r *http.Request) { orchJob, err := h.setupOrchJob(ctx, r, false) if err != nil { - respondWithError(w, err.Error(), http.StatusBadRequest) + code := http.StatusBadRequest + if err == errInsufficientBalance { + code = http.StatusPaymentRequired + } + respondWithError(w, err.Error(), code) return } ctx = clog.AddVal(ctx, "stream_id", orchJob.Req.ID) @@ -1034,7 +1039,7 @@ func (h *lphttp) StartStream(w http.ResponseWriter, r *http.Request) { ctx = clog.AddVal(ctx, "stream_id", orchJob.Req.ID) ctx = clog.AddVal(ctx, "capability", orchJob.Req.Capability) - pmtCheckDur := 25 * time.Second + pmtCheckDur := 23 * time.Second //run slightly faster than gateway so can return updated balance pmtTicker := time.NewTicker(pmtCheckDur) defer pmtTicker.Stop() for { From 24551610ee3bfe300e13c629ac0500e6f10c39f2 Mon Sep 17 00:00:00 2001 From: Brad P Date: Tue, 2 Sep 2025 16:27:44 -0500 Subject: [PATCH 35/57] simplify stream cleanup --- core/external_capabilities.go | 57 +++++++++++++++++------------------ 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/core/external_capabilities.go b/core/external_capabilities.go index 82682190b0..cf7bfb8610 100644 --- a/core/external_capabilities.go +++ b/core/external_capabilities.go @@ -130,37 +130,34 @@ func (extCaps *ExternalCapabilities) AddStream(streamID string, pipeline string, //clean up when stream ends go func() { - for { - select { - case <-ctx.Done(): - //gateway channels shutdown - if stream.DataWriter != nil { - stream.DataWriter.Close() - } - if stream.ControlPub != nil { - stream.StopControl() - stream.ControlPub.Close() - } - - //orchestrator channels shutdown - if stream.pubChannel != nil { - stream.pubChannel.Close() - } - if stream.subChannel != nil { - stream.subChannel.Close() - } - if stream.controlChannel != nil { - stream.controlChannel.Close() - } - if stream.eventsChannel != nil { - stream.eventsChannel.Close() - } - if stream.dataChannel != nil { - stream.dataChannel.Close() - } - return - } + <-ctx.Done() + + //gateway channels shutdown + if stream.DataWriter != nil { + stream.DataWriter.Close() + } + if stream.ControlPub != nil { + stream.StopControl() + stream.ControlPub.Close() + } + + //orchestrator channels shutdown + if stream.pubChannel != nil { + stream.pubChannel.Close() + } + if stream.subChannel != nil { + stream.subChannel.Close() + } + if stream.controlChannel != nil { + stream.controlChannel.Close() + } + if stream.eventsChannel != nil { + stream.eventsChannel.Close() + } + if stream.dataChannel != nil { + stream.dataChannel.Close() } + return }() return &stream, nil From e2e23ed355a684849be837d177e845dcd31448f7 Mon Sep 17 00:00:00 2001 From: Brad P Date: Wed, 3 Sep 2025 10:08:51 -0500 Subject: [PATCH 36/57] fix workerRoute and stream_ingest_metrics --- server/job_stream.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/job_stream.go b/server/job_stream.go index 65061119f0..0fc2303a1d 100644 --- a/server/job_stream.go +++ b/server/job_stream.go @@ -556,7 +556,7 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, req func (ls *LivepeerServer) StartStreamRTMPIngest() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { remoteAddr := getRemoteAddr(r) - ctx := clog.AddVal(r.Context(), clog.ClientIP, remoteAddr) + ctx := clog.AddVal(context.Background(), clog.ClientIP, remoteAddr) requestID := string(core.RandomManifestID()) ctx = clog.AddVal(ctx, "request_id", requestID) @@ -633,7 +633,7 @@ func (ls *LivepeerServer) StartStreamRTMPIngest() http.Handler { func (ls *LivepeerServer) StartStreamWhipIngest(whipServer *media.WHIPServer) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { remoteAddr := getRemoteAddr(r) - ctx := clog.AddVal(r.Context(), clog.ClientIP, remoteAddr) + ctx := clog.AddVal(context.Background(), clog.ClientIP, remoteAddr) streamId := r.PathValue("streamId") ctx = clog.AddVal(ctx, "stream_id", streamId) @@ -1078,7 +1078,7 @@ func (h *lphttp) StartStream(w http.ResponseWriter, r *http.Request) { // set the headers _, err = sendReqWithTimeout(req, time.Duration(orchJob.Req.Timeout)*time.Second) if err != nil { - clog.Errorf(ctx, "Error sending request to worker %v: %v", workerRoute, err) + clog.Errorf(ctx, "Error sending request to worker %v: %v", orchJob.Req.CapabilityUrl, err) respondWithError(w, "Error sending request to worker", http.StatusInternalServerError) return } From f8198f8d69c839e558fb7a6901f8c6bb58f683fd Mon Sep 17 00:00:00 2001 From: Brad P Date: Wed, 3 Sep 2025 16:50:10 -0500 Subject: [PATCH 37/57] move JobToken and JobSender to core package and fix runStream --- core/external_capabilities.go | 17 ++++++++- server/job_rpc.go | 71 ++++++++++++++--------------------- server/job_rpc_test.go | 24 ++++++------ server/job_stream.go | 63 ++++++++++++++++--------------- 4 files changed, 89 insertions(+), 86 deletions(-) diff --git a/core/external_capabilities.go b/core/external_capabilities.go index cf7bfb8610..602af447df 100644 --- a/core/external_capabilities.go +++ b/core/external_capabilities.go @@ -11,9 +11,24 @@ import ( ethcommon "github.com/ethereum/go-ethereum/common" "github.com/golang/glog" "github.com/livepeer/go-livepeer/media" + "github.com/livepeer/go-livepeer/net" "github.com/livepeer/go-livepeer/trickle" ) +type JobToken struct { + SenderAddress *JobSender `json:"sender_address,omitempty"` + TicketParams *net.TicketParams `json:"ticket_params,omitempty"` + Balance int64 `json:"balance,omitempty"` + Price *net.PriceInfo `json:"price,omitempty"` + ServiceAddr string `json:"service_addr,omitempty"` + + LastNonce uint32 +} +type JobSender struct { + Addr string `json:"addr"` + Sig string `json:"sig"` +} + type ExternalCapability struct { Name string `json:"name"` Description string `json:"description"` @@ -35,7 +50,7 @@ type StreamInfo struct { //Gateway fields StreamRequest []byte ExcludeOrchs []string - OrchToken interface{} + OrchToken *JobToken OrchUrl string OrchPublishUrl string OrchSubscribeUrl string diff --git a/server/job_rpc.go b/server/job_rpc.go index 386822d616..3ad8db720a 100644 --- a/server/job_rpc.go +++ b/server/job_rpc.go @@ -51,21 +51,6 @@ var errInsufficientBalance = errors.New("Insufficient balance for request") var sendJobReqWithTimeout = sendReqWithTimeout -type JobSender struct { - Addr string `json:"addr"` - Sig string `json:"sig"` -} - -type JobToken struct { - SenderAddress *JobSender `json:"sender_address,omitempty"` - TicketParams *net.TicketParams `json:"ticket_params,omitempty"` - Balance int64 `json:"balance,omitempty"` - Price *net.PriceInfo `json:"price,omitempty"` - ServiceAddr string `json:"service_addr,omitempty"` - - lastNonce uint32 -} - type JobRequest struct { ID string `json:"id"` Request string `json:"request"` @@ -76,10 +61,9 @@ type JobRequest struct { Sig string `json:"sig"` Timeout int `json:"timeout_seconds"` - orchSearchTimeout time.Duration - orchSearchRespTimeout time.Duration + OrchSearchTimeout time.Duration + OrchSearchRespTimeout time.Duration } - type JobRequestDetails struct { StreamId string `json:"stream_id"` } @@ -92,7 +76,6 @@ type JobParameters struct { EnableVideoEgress bool `json:"enable_video_egress,omitempty"` EnableDataOutput bool `json:"enable_data_output,omitempty"` } - type JobOrchestratorsFilter struct { Exclude []string `json:"exclude,omitempty"` Include []string `json:"include,omitempty"` @@ -109,7 +92,7 @@ type orchJob struct { } type gatewayJob struct { Job *orchJob - Orchs []JobToken + Orchs []core.JobToken SignedJobReq string node *core.LivepeerNode @@ -246,7 +229,7 @@ func (h *lphttp) GetJobToken(w http.ResponseWriter, r *http.Request) { } w.Header().Set("Content-Type", "application/json") - jobToken := JobToken{SenderAddress: nil, TicketParams: nil, Balance: 0, Price: nil} + jobToken := core.JobToken{SenderAddress: nil, TicketParams: nil, Balance: 0, Price: nil} if !orch.CheckExternalCapabilityCapacity(jobCapsHdr) { //send response indicating no capacity available @@ -291,7 +274,7 @@ func (h *lphttp) GetJobToken(w http.ResponseWriter, r *http.Request) { capBalInt = capBalInt / 1000 } - jobToken = JobToken{ + jobToken = core.JobToken{ SenderAddress: jobSenderAddr, TicketParams: ticketParams, Balance: capBalInt, @@ -308,7 +291,7 @@ func (h *lphttp) GetJobToken(w http.ResponseWriter, r *http.Request) { func (ls *LivepeerServer) setupGatewayJob(ctx context.Context, r *http.Request) (*gatewayJob, error) { - var orchs []JobToken + var orchs []core.JobToken jobReqHdr := r.Header.Get(jobRequestHdr) clog.Infof(ctx, "processing job request req=%v", jobReqHdr) @@ -328,11 +311,11 @@ func (ls *LivepeerServer) setupGatewayJob(ctx context.Context, r *http.Request) } searchTimeout, respTimeout := getOrchSearchTimeouts(ctx, r.Header.Get(jobOrchSearchTimeoutHdr), r.Header.Get(jobOrchSearchRespTimeoutHdr)) - jobReq.orchSearchTimeout = searchTimeout - jobReq.orchSearchRespTimeout = respTimeout + jobReq.OrchSearchTimeout = searchTimeout + jobReq.OrchSearchRespTimeout = respTimeout //get pool of Orchestrators that can do the job - orchs, err = getJobOrchestrators(ctx, ls.LivepeerNode, jobReq.Capability, jobParams, jobReq.orchSearchTimeout, jobReq.orchSearchRespTimeout) + orchs, err = getJobOrchestrators(ctx, ls.LivepeerNode, jobReq.Capability, jobParams, jobReq.OrchSearchTimeout, jobReq.OrchSearchRespTimeout) if err != nil { return nil, errors.New(fmt.Sprintf("Unable to find orchestrators for capability %v err=%v", jobReq.Capability, err)) } @@ -541,7 +524,7 @@ func (ls *LivepeerServer) submitJob(ctx context.Context, w http.ResponseWriter, } } -func (ls *LivepeerServer) sendJobToOrch(ctx context.Context, r *http.Request, jobReq *JobRequest, signedReqHdr string, orchToken JobToken, route string, body []byte) (*http.Response, int, error) { +func (ls *LivepeerServer) sendJobToOrch(ctx context.Context, r *http.Request, jobReq *JobRequest, signedReqHdr string, orchToken core.JobToken, route string, body []byte) (*http.Response, int, error) { orchUrl := orchToken.ServiceAddr + route req, err := http.NewRequestWithContext(ctx, "POST", orchUrl, bytes.NewBuffer(body)) if err != nil { @@ -868,7 +851,7 @@ func (h *lphttp) setupOrchJob(ctx context.Context, r *http.Request, reserveCapac return &orchJob{Req: jobReq, Sender: sender, JobPrice: jobPrice, Details: &jobDetails}, nil } -func createPayment(ctx context.Context, jobReq *JobRequest, orchToken *JobToken, node *core.LivepeerNode) (string, error) { +func createPayment(ctx context.Context, jobReq *JobRequest, orchToken *core.JobToken, node *core.LivepeerNode) (string, error) { var payment *net.Payment createTickets := true clog.Infof(ctx, "creating payment for job request %s", jobReq.Capability) @@ -877,6 +860,8 @@ func createPayment(ctx context.Context, jobReq *JobRequest, orchToken *JobToken, sessionID := node.Sender.StartSession(*pmTicketParams(orchToken.TicketParams)) //setup balances and update Gateway balance to Orchestrator balance, log differences + //Orchestrator tracks balance paid and will not perform work if the balance it is + //has is not sufficient orchBal := big.NewRat(orchToken.Balance, 1) balance := node.Balances.Balance(orchAddr, core.ManifestID(jobReq.Capability)) if balance == nil { @@ -951,12 +936,12 @@ func createPayment(ctx context.Context, jobReq *JobRequest, orchToken *JobToken, senderParams := make([]*net.TicketSenderParams, len(tickets.SenderParams)) for i := 0; i < len(tickets.SenderParams); i++ { senderParams[i] = &net.TicketSenderParams{ - SenderNonce: orchToken.lastNonce + tickets.SenderParams[i].SenderNonce, + SenderNonce: orchToken.LastNonce + tickets.SenderParams[i].SenderNonce, Sig: tickets.SenderParams[i].Sig, } totalEV = totalEV.Add(totalEV, tickets.WinProbRat()) } - orchToken.lastNonce = tickets.SenderParams[len(tickets.SenderParams)-1].SenderNonce + 1 + orchToken.LastNonce = tickets.SenderParams[len(tickets.SenderParams)-1].SenderNonce + 1 payment.TicketSenderParams = senderParams ratPrice, _ := common.RatPriceInfo(payment.ExpectedPrice) @@ -985,7 +970,7 @@ func createPayment(ctx context.Context, jobReq *JobRequest, orchToken *JobToken, return base64.StdEncoding.EncodeToString(data), nil } -func updateGatewayBalance(node *core.LivepeerNode, orchToken JobToken, capability string, took time.Duration) *big.Rat { +func updateGatewayBalance(node *core.LivepeerNode, orchToken core.JobToken, capability string, took time.Duration) *big.Rat { orchAddr := ethcommon.BytesToAddress(orchToken.TicketParams.Recipient) // update for usage of compute orchPrice := big.NewRat(orchToken.Price.PricePerUnit, orchToken.Price.PixelsPerUnit) @@ -1042,14 +1027,14 @@ func getPaymentBalance(orch Orchestrator, sender ethcommon.Address, jobId string return senderBalance } -func verifyTokenCreds(ctx context.Context, orch Orchestrator, tokenCreds string) (*JobSender, error) { +func verifyTokenCreds(ctx context.Context, orch Orchestrator, tokenCreds string) (*core.JobSender, error) { buf, err := base64.StdEncoding.DecodeString(tokenCreds) if err != nil { glog.Error("Unable to base64-decode ", err) return nil, errSegEncoding } - var jobSender JobSender + var jobSender core.JobSender err = json.Unmarshal(buf, &jobSender) if err != nil { clog.Errorf(ctx, "Unable to parse the header text: ", err) @@ -1156,7 +1141,7 @@ func getOrchSearchTimeouts(ctx context.Context, searchTimeoutHdr, respTimeoutHdr return timeout, respTimeout } -func getJobOrchestrators(ctx context.Context, node *core.LivepeerNode, capability string, params JobParameters, timeout time.Duration, respTimeout time.Duration) ([]JobToken, error) { +func getJobOrchestrators(ctx context.Context, node *core.LivepeerNode, capability string, params JobParameters, timeout time.Duration, respTimeout time.Duration) ([]core.JobToken, error) { orchs := node.OrchestratorPool.GetInfos() //setup the GET request to get the Orchestrator tokens reqSender, err := getJobSender(ctx, node) @@ -1165,7 +1150,7 @@ func getJobOrchestrators(ctx context.Context, node *core.LivepeerNode, capabilit return nil, err } - getOrchJobToken := func(ctx context.Context, orchUrl *url.URL, reqSender JobSender, respTimeout time.Duration, tokenCh chan JobToken, errCh chan error) { + getOrchJobToken := func(ctx context.Context, orchUrl *url.URL, reqSender core.JobSender, respTimeout time.Duration, tokenCh chan core.JobToken, errCh chan error) { start := time.Now() tokenReq, err := http.NewRequestWithContext(ctx, "GET", orchUrl.String()+"/process/token", nil) reqSenderStr, _ := json.Marshal(reqSender) @@ -1199,7 +1184,7 @@ func getJobOrchestrators(ctx context.Context, node *core.LivepeerNode, capabilit errCh <- err return } - var jobToken JobToken + var jobToken core.JobToken err = json.Unmarshal(token, &jobToken) if err != nil { clog.Errorf(ctx, "Failed to unmarshal token from Orchestrator %v err=%v", orchUrl.String(), err) @@ -1210,11 +1195,11 @@ func getJobOrchestrators(ctx context.Context, node *core.LivepeerNode, capabilit tokenCh <- jobToken } - var jobTokens []JobToken + var jobTokens []core.JobToken timedOut := false nbResp := 0 numAvailableOrchs := node.OrchestratorPool.Size() - tokenCh := make(chan JobToken, numAvailableOrchs) + tokenCh := make(chan core.JobToken, numAvailableOrchs) errCh := make(chan error, numAvailableOrchs) tokensCtx, cancel := context.WithTimeout(clog.Clone(context.Background(), ctx), timeout) @@ -1250,7 +1235,7 @@ func getJobOrchestrators(ctx context.Context, node *core.LivepeerNode, capabilit return jobTokens, nil } -func getJobSender(ctx context.Context, node *core.LivepeerNode) (*JobSender, error) { +func getJobSender(ctx context.Context, node *core.LivepeerNode) (*core.JobSender, error) { gateway := node.OrchestratorPool.Broadcaster() orchReq, err := genOrchestratorReq(gateway, GetOrchestratorInfoParams{}) if err != nil { @@ -1258,17 +1243,17 @@ func getJobSender(ctx context.Context, node *core.LivepeerNode) (*JobSender, err return nil, err } addr := ethcommon.BytesToAddress(orchReq.Address) - jobSender := &JobSender{ + jobSender := &core.JobSender{ Addr: addr.Hex(), Sig: "0x" + hex.EncodeToString(orchReq.Sig), } return jobSender, nil } -func getToken(ctx context.Context, respTimeout time.Duration, orchUrl, capability, sender, senderSig string) (*JobToken, error) { +func getToken(ctx context.Context, respTimeout time.Duration, orchUrl, capability, sender, senderSig string) (*core.JobToken, error) { start := time.Now() tokenReq, err := http.NewRequestWithContext(ctx, "GET", orchUrl+"/process/token", nil) - jobSender := JobSender{Addr: sender, Sig: senderSig} + jobSender := core.JobSender{Addr: sender, Sig: senderSig} reqSenderStr, _ := json.Marshal(jobSender) tokenReq.Header.Set(jobEthAddressHdr, base64.StdEncoding.EncodeToString(reqSenderStr)) @@ -1298,7 +1283,7 @@ func getToken(ctx context.Context, respTimeout time.Duration, orchUrl, capabilit clog.Errorf(ctx, "Failed to read token from Orchestrator %v err=%v", orchUrl, err) return nil, err } - var jobToken JobToken + var jobToken core.JobToken err = json.Unmarshal(token, &jobToken) if err != nil { clog.Errorf(ctx, "Failed to unmarshal token from Orchestrator %v err=%v", orchUrl, err) diff --git a/server/job_rpc_test.go b/server/job_rpc_test.go index 84d3976887..f26ee30422 100644 --- a/server/job_rpc_test.go +++ b/server/job_rpc_test.go @@ -574,7 +574,7 @@ func TestGetJobToken_InvalidEthAddressHeader(t *testing.T) { } // Create a valid JobSender structure - js := &JobSender{ + js := &core.JobSender{ Addr: "0x0000000000000000000000000000000000000000", Sig: "0x000000000000000000000000000000000000000000000000000000000000000000", } @@ -603,7 +603,7 @@ func TestGetJobToken_MissingCapabilityHeader(t *testing.T) { } // Create a valid JobSender structure - js := &JobSender{ + js := &core.JobSender{ Addr: "0x0000000000000000000000000000000000000000", Sig: "0x000000000000000000000000000000000000000000000000000000000000000000", } @@ -645,7 +645,7 @@ func TestGetJobToken_NoCapacity(t *testing.T) { // Create a valid JobSender structure gateway := stubBroadcaster2() sig, _ := gateway.Sign([]byte(hexutil.Encode(gateway.Address().Bytes()))) - js := &JobSender{ + js := &core.JobSender{ Addr: hexutil.Encode(gateway.Address().Bytes()), Sig: hexutil.Encode(sig), } @@ -688,7 +688,7 @@ func TestGetJobToken_JobPriceInfoError(t *testing.T) { // Create a valid JobSender structure gateway := stubBroadcaster2() sig, _ := gateway.Sign([]byte(hexutil.Encode(gateway.Address().Bytes()))) - js := &JobSender{ + js := &core.JobSender{ Addr: hexutil.Encode(gateway.Address().Bytes()), Sig: hexutil.Encode(sig), } @@ -732,7 +732,7 @@ func TestGetJobToken_InsufficientReserve(t *testing.T) { // Create a valid JobSender structure gateway := stubBroadcaster2() sig, _ := gateway.Sign([]byte(hexutil.Encode(gateway.Address().Bytes()))) - js := &JobSender{ + js := &core.JobSender{ Addr: hexutil.Encode(gateway.Address().Bytes()), Sig: hexutil.Encode(sig), } @@ -783,7 +783,7 @@ func TestGetJobToken_TicketParamsError(t *testing.T) { // Create a valid JobSender structure gateway := stubBroadcaster2() sig, _ := gateway.Sign([]byte(hexutil.Encode(gateway.Address().Bytes()))) - js := &JobSender{ + js := &core.JobSender{ Addr: hexutil.Encode(gateway.Address().Bytes()), Sig: hexutil.Encode(sig), } @@ -847,7 +847,7 @@ func TestGetJobToken_Success(t *testing.T) { // Create a valid JobSender structure gateway := stubBroadcaster2() sig, _ := gateway.Sign([]byte(hexutil.Encode(gateway.Address().Bytes()))) - js := &JobSender{ + js := &core.JobSender{ Addr: hexutil.Encode(gateway.Address().Bytes()), Sig: hexutil.Encode(sig), } @@ -864,7 +864,7 @@ func TestGetJobToken_Success(t *testing.T) { resp := w.Result() assert.Equal(t, http.StatusOK, resp.StatusCode) - var token JobToken + var token core.JobToken body, _ := io.ReadAll(resp.Body) json.Unmarshal(body, &token) @@ -918,12 +918,12 @@ func TestCreatePayment(t *testing.T) { jobReq := JobRequest{ Capability: "test-payment-cap", } - sender := JobSender{ + sender := core.JobSender{ Addr: "0x1111111111111111111111111111111111111111", Sig: "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", } - orchTocken := JobToken{ + orchTocken := core.JobToken{ TicketParams: &net.TicketParams{ Recipient: ethcommon.HexToAddress("0x1111111111111111111111111111111111111111").Bytes(), FaceValue: big.NewInt(1000).Bytes(), @@ -1011,9 +1011,9 @@ func TestSubmitJob_OrchestratorSelectionParams(t *testing.T) { return } - token := &JobToken{ + token := &core.JobToken{ ServiceAddr: "http://" + r.Host, // Use the server's host as the service address - SenderAddress: &JobSender{ + SenderAddress: &core.JobSender{ Addr: "0x1234567890abcdef1234567890abcdef123456", Sig: "0x456", }, diff --git a/server/job_stream.go b/server/job_stream.go index 0fc2303a1d..95b5a42323 100644 --- a/server/job_stream.go +++ b/server/job_stream.go @@ -88,7 +88,7 @@ func (ls *LivepeerServer) StopStream() http.Handler { } stopJob.sign() - resp, code, err := ls.sendJobToOrch(ctx, r, stopJob.Job.Req, stopJob.SignedJobReq, streamInfoCopy.OrchToken.(JobToken), "/ai/stream/stop", streamInfoCopy.StreamRequest) + resp, code, err := ls.sendJobToOrch(ctx, r, stopJob.Job.Req, stopJob.SignedJobReq, *streamInfoCopy.OrchToken, "/ai/stream/stop", streamInfoCopy.StreamRequest) if err != nil { clog.Errorf(ctx, "Error sending job to orchestrator: %s", err) http.Error(w, err.Error(), code) @@ -124,21 +124,22 @@ func (ls *LivepeerServer) runStream(gatewayJob *gatewayJob) { firstProcessed := false for _, orch := range gatewayJob.Orchs { - stream.OrchToken = orch - stream.OrchUrl = orch.ServiceAddr - - ctx = clog.AddVal(ctx, "orch", ethcommon.Bytes2Hex(orch.TicketParams.Recipient)) - ctx = clog.AddVal(ctx, "orch_url", orch.ServiceAddr) clog.Infof(ctx, "Starting stream processing") //refresh the token if not first Orch to confirm capacity and new ticket params if firstProcessed { - orch, err := getToken(ctx, 3*time.Second, orch.ServiceAddr, gatewayJob.Job.Req.Capability, gatewayJob.Job.Req.Sender, gatewayJob.Job.Req.Sig) + newToken, err := getToken(ctx, 3*time.Second, orch.ServiceAddr, gatewayJob.Job.Req.Capability, gatewayJob.Job.Req.Sender, gatewayJob.Job.Req.Sig) if err != nil { clog.Errorf(ctx, "Error getting token for orch=%v err=%v", orch.ServiceAddr, err) continue } + stream.OrchToken = newToken + orch = *newToken } + stream.OrchToken = &orch + ctx = clog.AddVal(ctx, "orch", ethcommon.Bytes2Hex(orch.TicketParams.Recipient)) + ctx = clog.AddVal(ctx, "orch_url", orch.ServiceAddr) + //set request ID to persist from Gateway to Worker gatewayJob.Job.Req.ID = stream.StreamID err := gatewayJob.sign() @@ -239,15 +240,15 @@ func (ls *LivepeerServer) monitorStream(streamId string) { return case <-pmtTicker.C: // fetch new JobToken with each payment - token := stream.OrchToken.(JobToken) - updateGatewayBalance(ls.LivepeerNode, token, stream.Capability, dur) + token := stream.OrchToken + updateGatewayBalance(ls.LivepeerNode, *token, stream.Capability, dur) newToken, err := getToken(ctx, 3*time.Second, stream.OrchUrl, stream.Capability, jobSender.Addr, jobSender.Sig) if err != nil { clog.Errorf(ctx, "Error getting new token for %s: %v", stream.OrchUrl, err) continue } - stream.OrchToken = *newToken + stream.OrchToken = newToken // send the payment jobDetails := JobRequestDetails{StreamId: streamId} @@ -267,8 +268,8 @@ func (ls *LivepeerServer) monitorStream(streamId string) { clog.Errorf(ctx, "Error signing job, continuing monitoring: %v", err) continue } - orchToken := stream.OrchToken.(JobToken) - pmtHdr, err := createPayment(ctx, req, &orchToken, ls.LivepeerNode) + orchToken := stream.OrchToken + pmtHdr, err := createPayment(ctx, req, orchToken, ls.LivepeerNode) if err != nil { clog.Errorf(ctx, "Error processing stream payment for %s: %v", streamId, err) // Continue monitoring even if payment fails @@ -335,23 +336,6 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, req ctx = clog.AddVal(ctx, "stream", streamName) - //I think these are for mediamtx ingest - - //sourceID := formReq.FormValue("source_id") - //if sourceID == "" { - // return nil, http.StatusBadRequest, errors.New("missing source_id") - //} - //ctx = clog.AddVal(ctx, "source_id", sourceID) - //sourceType := formReq.FormValue("source_type") - //if sourceType == "" { - // return nil, http.StatusBadRequest, errors.New("missing source_type") - //} - //sourceTypeStr, err := media.MediamtxSourceTypeToString(sourceType) - //if err != nil { - // return nil, http.StatusBadRequest, errors.New("invalid source type") - //} - //ctx = clog.AddVal(ctx, "source_type", sourceType) - // If auth webhook is set and returns an output URL, this will be replaced outputURL := startReq.RtmpOutput @@ -570,6 +554,24 @@ func (ls *LivepeerServer) StartStreamRTMPIngest() http.Handler { } params := stream.Params.(aiRequestParams) + sourceID := r.FormValue("source_id") + if sourceID == "" { + http.Error(w, "missing source_id", http.StatusBadRequest) + return + } + ctx = clog.AddVal(ctx, "source_id", sourceID) + sourceType := r.FormValue("source_type") + if sourceType == "" { + http.Error(w, "missing source_type", http.StatusBadRequest) + return + } + + sourceTypeStr, err := media.MediamtxSourceTypeToString(sourceType) + if err != nil { + http.Error(w, "invalid source_type", http.StatusBadRequest) + return + } + ctx = clog.AddVal(ctx, "source_type", sourceType) //note that mediaMtxHost is the ip address of media mtx // media sends a post request in the runOnReady event setup in mediamtx.yml @@ -579,7 +581,7 @@ func (ls *LivepeerServer) StartStreamRTMPIngest() http.Handler { respondJsonError(ctx, w, err, http.StatusBadRequest) return } - mediaMTXClient := media.NewMediaMTXClient(mediaMtxHost, ls.mediaMTXApiPassword, "rtmp_ingest", "rtmp") + mediaMTXClient := media.NewMediaMTXClient(mediaMtxHost, ls.mediaMTXApiPassword, sourceID, sourceTypeStr) segmenterCtx, cancelSegmenter := context.WithCancel(clog.Clone(context.Background(), ctx)) // this function is called when the pipeline hits a fatal error, we kick the input connection to allow @@ -606,6 +608,7 @@ func (ls *LivepeerServer) StartStreamRTMPIngest() http.Handler { //orchCtx, orchCancel := context.WithCancel(ctx) // Kick off the RTMP pull and segmentation + clog.Infof(ctx, "Starting RTMP ingest from MediaMTX") go func() { ms := media.MediaSegmenter{Workdir: ls.LivepeerNode.WorkDir, MediaMTXClient: mediaMTXClient} //segmenter blocks until done From 9aacc0d67b2fc423e10b9d835bae944560f57778 Mon Sep 17 00:00:00 2001 From: Brad P Date: Thu, 4 Sep 2025 11:28:51 -0500 Subject: [PATCH 38/57] fix rtmp streaming and storing orch url --- server/job_stream.go | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/server/job_stream.go b/server/job_stream.go index 95b5a42323..5ad532cf43 100644 --- a/server/job_stream.go +++ b/server/job_stream.go @@ -137,6 +137,7 @@ func (ls *LivepeerServer) runStream(gatewayJob *gatewayJob) { } stream.OrchToken = &orch + stream.OrchUrl = orch.ServiceAddr ctx = clog.AddVal(ctx, "orch", ethcommon.Bytes2Hex(orch.TicketParams.Recipient)) ctx = clog.AddVal(ctx, "orch_url", orch.ServiceAddr) @@ -561,27 +562,23 @@ func (ls *LivepeerServer) StartStreamRTMPIngest() http.Handler { } ctx = clog.AddVal(ctx, "source_id", sourceID) sourceType := r.FormValue("source_type") + sourceType = strings.ToLower(sourceType) //normalize the source type so rtmpConn matches to rtmpconn if sourceType == "" { http.Error(w, "missing source_type", http.StatusBadRequest) return } - sourceTypeStr, err := media.MediamtxSourceTypeToString(sourceType) - if err != nil { - http.Error(w, "invalid source_type", http.StatusBadRequest) - return - } - ctx = clog.AddVal(ctx, "source_type", sourceType) - + clog.Infof(ctx, "RTMP ingest from MediaMTX connected sourceID=%s sourceType=%s", sourceID, sourceType) //note that mediaMtxHost is the ip address of media mtx - // media sends a post request in the runOnReady event setup in mediamtx.yml + // mediamtx sends a post request in the runOnReady event setup in mediamtx.yml // StartLiveVideo calls this remoteHost mediaMtxHost, err := getRemoteHost(r.RemoteAddr) if err != nil { respondJsonError(ctx, w, err, http.StatusBadRequest) return } - mediaMTXClient := media.NewMediaMTXClient(mediaMtxHost, ls.mediaMTXApiPassword, sourceID, sourceTypeStr) + mediaMTXInputURL := fmt.Sprintf("rtmp://%s/%s%s", mediaMtxHost, "", streamId) + mediaMTXClient := media.NewMediaMTXClient(mediaMtxHost, ls.mediaMTXApiPassword, sourceID, sourceType) segmenterCtx, cancelSegmenter := context.WithCancel(clog.Clone(context.Background(), ctx)) // this function is called when the pipeline hits a fatal error, we kick the input connection to allow @@ -601,12 +598,10 @@ func (ls *LivepeerServer) StartStreamRTMPIngest() http.Handler { } } + params.liveParams.localRTMPPrefix = mediaMTXInputURL params.liveParams.kickInput = kickInput stream.Params = params //update params used to kickInput - // Create a special parent context for orchestrator cancellation - //orchCtx, orchCancel := context.WithCancel(ctx) - // Kick off the RTMP pull and segmentation clog.Infof(ctx, "Starting RTMP ingest from MediaMTX") go func() { From 12f8895473b77afea2c7ba8ad9ad268892768bcc Mon Sep 17 00:00:00 2001 From: Brad P Date: Thu, 4 Sep 2025 15:56:33 -0500 Subject: [PATCH 39/57] add error logging to retrying stream --- server/job_stream.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server/job_stream.go b/server/job_stream.go index 5ad532cf43..862abc9018 100644 --- a/server/job_stream.go +++ b/server/job_stream.go @@ -192,7 +192,7 @@ func (ls *LivepeerServer) runStream(gatewayJob *gatewayJob) { break } firstProcessed = true - clog.Infof(ctx, "Retrying stream with a different orchestrator") + clog.Infof(ctx, "Retrying stream with a different orchestrator err=%v", err.Error()) // will swap, but first notify with the reason for the swap if err == nil { @@ -625,6 +625,9 @@ func (ls *LivepeerServer) StartStreamRTMPIngest() http.Handler { stream.CancelStream() //cleanupControl(ctx, params) }() + + //write response + w.WriteHeader(http.StatusOK) }) } @@ -676,7 +679,6 @@ func (ls *LivepeerServer) StartStreamWhipIngest(whipServer *media.WHIPServer) ht conn := whipServer.CreateWHIP(ctx, params.liveParams.segmentReader, whepURL, w, r) whipConn.SetWHIPConnection(conn) // might be nil if theres an error and thats okay }) - } func startStreamProcessing(ctx context.Context, streamInfo *core.StreamInfo) error { From 824682581382bc821b790218c997eaafa0aec671 Mon Sep 17 00:00:00 2001 From: Brad P Date: Fri, 5 Sep 2025 12:37:56 -0500 Subject: [PATCH 40/57] gateway job_stream refactor --- core/livepeernode.go | 37 +++++ server/ai_live_video.go | 159 +++++++++++++++++++--- server/ai_process.go | 6 + server/job_rpc.go | 28 ++-- server/job_stream.go | 295 +++++++++++++++++++++++++++------------- 5 files changed, 399 insertions(+), 126 deletions(-) diff --git a/core/livepeernode.go b/core/livepeernode.go index eb8e20469c..4720c6e850 100644 --- a/core/livepeernode.go +++ b/core/livepeernode.go @@ -10,6 +10,7 @@ orchestrator.go: Code that is called only when the node is in orchestrator mode. package core import ( + "context" "errors" "math/big" "math/rand" @@ -180,6 +181,42 @@ type LivePipeline struct { Pipeline string ControlPub *trickle.TricklePublisher StopControl func() + + StreamCtx context.Context + streamCancel context.CancelCauseFunc + streamParams interface{} + streamRequest []byte +} + +func (n *LivepeerNode) NewLivePipeline(requestID, streamID, pipeline string, streamParams interface{}, streamRequest []byte) *LivePipeline { + streamCtx, streamCancel := context.WithCancelCause(context.Background()) + n.LiveMu.Lock() + defer n.LiveMu.Unlock() + n.LivePipelines[streamID] = &LivePipeline{ + RequestID: requestID, + Pipeline: pipeline, + StreamCtx: streamCtx, + streamParams: streamParams, + streamCancel: streamCancel, + } + return n.LivePipelines[streamID] +} + +func (p *LivePipeline) StreamParams() interface{} { + return p.streamParams +} + +func (p *LivePipeline) UpdateStreamParams(newParams interface{}) { + p.streamParams = newParams +} + +func (p *LivePipeline) StreamRequest() []byte { + return p.streamRequest +} + +func (p *LivePipeline) StopStream(err error) { + p.StopControl() + p.streamCancel(err) DataWriter *media.SegmentWriter ReportUpdate func([]byte) } diff --git a/server/ai_live_video.go b/server/ai_live_video.go index aa7f3264d7..60d074ed2d 100644 --- a/server/ai_live_video.go +++ b/server/ai_live_video.go @@ -1,6 +1,7 @@ package server import ( + "bufio" "bytes" "context" "encoding/json" @@ -81,7 +82,7 @@ func startTricklePublish(ctx context.Context, url *url.URL, params aiRequestPara ctx, cancel := context.WithCancel(ctx) priceInfo := sess.OrchestratorInfo.PriceInfo var paymentProcessor *LivePaymentProcessor - if priceInfo != nil && priceInfo.PricePerUnit != 0 { + if priceInfo != nil && priceInfo.PricePerUnit != 0 && sess.OrchestratorInfo.AuthToken != nil { paymentSender := livePaymentSender{} sendPaymentFunc := func(inPixels int64) error { return paymentSender.SendPayment(context.Background(), &SegmentInfoSender{ @@ -199,23 +200,26 @@ func suspendOrchestrator(ctx context.Context, params aiRequestParams) { // If the ingest was closed, then do not suspend the orchestrator return } - sel, err := params.sessManager.getSelector(ctx, core.Capability_LiveVideoToVideo, params.liveParams.pipeline) - if err != nil { - clog.Warningf(ctx, "Error suspending orchestrator: %v", err) - return - } - if sel == nil || sel.suspender == nil || params.liveParams == nil || params.liveParams.sess == nil || params.liveParams.sess.OrchestratorInfo == nil { - clog.Warningf(ctx, "Error suspending orchestrator: selector or suspender is nil") - return + //live-video-to-video + if params.sessManager != nil { + sel, err := params.sessManager.getSelector(ctx, core.Capability_LiveVideoToVideo, params.liveParams.pipeline) + if err != nil { + clog.Warningf(ctx, "Error suspending orchestrator: %v", err) + return + } + if sel == nil || sel.suspender == nil || params.liveParams == nil || params.liveParams.sess == nil || params.liveParams.sess.OrchestratorInfo == nil { + clog.Warningf(ctx, "Error suspending orchestrator: selector or suspender is nil") + return + } + // Remove the session from the current pool + sel.Remove(params.liveParams.sess) + sel.warmPool.mu.Lock() + sel.warmPool.selector.Remove(params.liveParams.sess.BroadcastSession) + sel.warmPool.mu.Unlock() + // We do selection every 6 min, so it effectively means the Orchestrator won't be selected for the next 30 min (unless there is no other O available) + clog.Infof(ctx, "Suspending orchestrator %s with penalty %d", params.liveParams.sess.Transcoder(), aiLiveVideoToVideoPenalty) + sel.suspender.suspend(params.liveParams.sess.Transcoder(), aiLiveVideoToVideoPenalty) } - // Remove the session from the current pool - sel.Remove(params.liveParams.sess) - sel.warmPool.mu.Lock() - sel.warmPool.selector.Remove(params.liveParams.sess.BroadcastSession) - sel.warmPool.mu.Unlock() - // We do selection every 6 min, so it effectively means the Orchestrator won't be selected for the next 30 min (unless there is no other O available) - clog.Infof(ctx, "Suspending orchestrator %s with penalty %d", params.liveParams.sess.Transcoder(), aiLiveVideoToVideoPenalty) - sel.suspender.suspend(params.liveParams.sess.Transcoder(), aiLiveVideoToVideoPenalty) } func startTrickleSubscribe(ctx context.Context, url *url.URL, params aiRequestParams, sess *AISession) { @@ -785,6 +789,127 @@ func startEventsSubscribe(ctx context.Context, url *url.URL, params aiRequestPar }() } +func startDataSubscribe(ctx context.Context, url *url.URL, params aiRequestParams, sess *AISession) { + //only start DataSubscribe if enabled + if params.liveParams.dataWriter == nil { + return + } + + // subscribe to the outputs + subscriber, err := trickle.NewTrickleSubscriber(trickle.TrickleSubscriberConfig{ + URL: url.String(), + Ctx: ctx, + }) + if err != nil { + clog.Infof(ctx, "Failed to create data subscriber: %s", err) + return + } + + dataWriter := params.liveParams.dataWriter + + // read segments from trickle subscription + go func() { + defer dataWriter.Close() + + var err error + firstSegment := true + + retries := 0 + // we're trying to keep (retryPause x maxRetries) duration to fall within one output GOP length + const retryPause = 300 * time.Millisecond + const maxRetries = 5 + for { + select { + case <-ctx.Done(): + clog.Info(ctx, "data subscribe done") + return + default: + } + if !params.inputStreamExists() { + clog.Infof(ctx, "data subscribe stopping, input stream does not exist.") + break + } + var segment *http.Response + readBytes, readMessages := 0, 0 + clog.V(8).Infof(ctx, "data subscribe await") + segment, err = subscriber.Read() + if err != nil { + if errors.Is(err, trickle.EOS) || errors.Is(err, trickle.StreamNotFoundErr) { + stopProcessing(ctx, params, fmt.Errorf("data subscribe stopping, stream not found, err=%w", err)) + return + } + var sequenceNonexistent *trickle.SequenceNonexistent + if errors.As(err, &sequenceNonexistent) { + // stream exists but segment doesn't, so skip to leading edge + subscriber.SetSeq(sequenceNonexistent.Latest) + } + // TODO if not EOS then signal a new orchestrator is needed + err = fmt.Errorf("data subscribe error reading: %w", err) + clog.Infof(ctx, "%s", err) + if retries > maxRetries { + stopProcessing(ctx, params, errors.New("data subscribe stopping, retries exceeded")) + return + } + retries++ + params.liveParams.sendErrorEvent(err) + time.Sleep(retryPause) + continue + } + retries = 0 + seq := trickle.GetSeq(segment) + clog.V(8).Infof(ctx, "data subscribe received seq=%d", seq) + copyStartTime := time.Now() + + defer segment.Body.Close() + scanner := bufio.NewScanner(segment.Body) + for scanner.Scan() { + writer, err := dataWriter.Next() + if err != nil { + if err != io.EOF { + stopProcessing(ctx, params, fmt.Errorf("data subscribe could not get next: %w", err)) + } + return + } + n, err := writer.Write(scanner.Bytes()) + if err != nil { + stopProcessing(ctx, params, fmt.Errorf("data subscribe could not write: %w", err)) + } + readBytes += n + readMessages += 1 + } + if err := scanner.Err(); err != nil { + clog.InfofErr(ctx, "data subscribe error reading seq=%d", seq, err) + subscriber.SetSeq(seq) + retries++ + continue + } + + if firstSegment { + firstSegment = false + delayMs := time.Since(params.liveParams.startTime).Milliseconds() + if monitor.Enabled { + //monitor.AIFirstSegmentDelay(delayMs, params.liveParams.sess.OrchestratorInfo) + monitor.SendQueueEventAsync("stream_trace", map[string]interface{}{ + "type": "gateway_receive_first_data_segment", + "timestamp": time.Now().UnixMilli(), + "stream_id": params.liveParams.streamID, + "pipeline_id": params.liveParams.pipelineID, + "request_id": params.liveParams.requestID, + "orchestrator_info": map[string]interface{}{ + "address": sess.Address(), + "url": sess.Transcoder(), + }, + }) + } + + clog.V(common.VERBOSE).Infof(ctx, "First Data Segment delay=%dms streamID=%s", delayMs, params.liveParams.streamID) + } + + clog.V(8).Info(ctx, "data subscribe read completed", "seq", seq, "bytes", humanize.Bytes(uint64(readBytes)), "messages", readMessages, "took", time.Since(copyStartTime)) + } + }() +} + func (a aiRequestParams) inputStreamExists() bool { if a.node == nil { return false diff --git a/server/ai_process.go b/server/ai_process.go index a0034e57a1..464a6e96a3 100644 --- a/server/ai_process.go +++ b/server/ai_process.go @@ -132,6 +132,12 @@ type liveRequestParams struct { // when the write for the last segment started lastSegmentTime time.Time + + orchPublishUrl string + orchSubscribeUrl string + orchControlUrl string + orchEventsUrl string + orchDataUrl string } // CalculateTextToImageLatencyScore computes the time taken per pixel for an text-to-image request. diff --git a/server/job_rpc.go b/server/job_rpc.go index 3ad8db720a..0932345e6d 100644 --- a/server/job_rpc.go +++ b/server/job_rpc.go @@ -289,7 +289,7 @@ func (h *lphttp) GetJobToken(w http.ResponseWriter, r *http.Request) { json.NewEncoder(w).Encode(jobToken) } -func (ls *LivepeerServer) setupGatewayJob(ctx context.Context, r *http.Request) (*gatewayJob, error) { +func (ls *LivepeerServer) setupGatewayJob(ctx context.Context, r *http.Request, skipOrchSearch bool) (*gatewayJob, error) { var orchs []core.JobToken @@ -310,18 +310,22 @@ func (ls *LivepeerServer) setupGatewayJob(ctx context.Context, r *http.Request) return nil, errors.New(fmt.Sprintf("Unable to unmarshal job parameters err=%v", err)) } - searchTimeout, respTimeout := getOrchSearchTimeouts(ctx, r.Header.Get(jobOrchSearchTimeoutHdr), r.Header.Get(jobOrchSearchRespTimeoutHdr)) - jobReq.OrchSearchTimeout = searchTimeout - jobReq.OrchSearchRespTimeout = respTimeout + // get list of Orchestrators that can do the job if needed + // (e.g. stop requests don't need new list of orchestrators) + if !skipOrchSearch { + searchTimeout, respTimeout := getOrchSearchTimeouts(ctx, r.Header.Get(jobOrchSearchTimeoutHdr), r.Header.Get(jobOrchSearchRespTimeoutHdr)) + jobReq.OrchSearchTimeout = searchTimeout + jobReq.OrchSearchRespTimeout = respTimeout - //get pool of Orchestrators that can do the job - orchs, err = getJobOrchestrators(ctx, ls.LivepeerNode, jobReq.Capability, jobParams, jobReq.OrchSearchTimeout, jobReq.OrchSearchRespTimeout) - if err != nil { - return nil, errors.New(fmt.Sprintf("Unable to find orchestrators for capability %v err=%v", jobReq.Capability, err)) - } + //get pool of Orchestrators that can do the job + orchs, err = getJobOrchestrators(ctx, ls.LivepeerNode, jobReq.Capability, jobParams, jobReq.OrchSearchTimeout, jobReq.OrchSearchRespTimeout) + if err != nil { + return nil, errors.New(fmt.Sprintf("Unable to find orchestrators for capability %v err=%v", jobReq.Capability, err)) + } - if len(orchs) == 0 { - return nil, errors.New(fmt.Sprintf("No orchestrators found for capability %v", jobReq.Capability)) + if len(orchs) == 0 { + return nil, errors.New(fmt.Sprintf("No orchestrators found for capability %v", jobReq.Capability)) + } } job := orchJob{Req: jobReq, @@ -360,7 +364,7 @@ func (ls *LivepeerServer) SubmitJob() http.Handler { func (ls *LivepeerServer) submitJob(ctx context.Context, w http.ResponseWriter, r *http.Request) { - gatewayJob, err := ls.setupGatewayJob(ctx, r) + gatewayJob, err := ls.setupGatewayJob(ctx, r, false) if err != nil { clog.Errorf(ctx, "Error setting up job: %s", err) http.Error(w, err.Error(), http.StatusBadRequest) diff --git a/server/job_stream.go b/server/job_stream.go index 862abc9018..b729fffde4 100644 --- a/server/job_stream.go +++ b/server/job_stream.go @@ -12,6 +12,7 @@ import ( "net/http" "os" "strings" + "sync" "time" ethcommon "github.com/ethereum/go-ethereum/common" @@ -21,6 +22,7 @@ import ( "github.com/livepeer/go-livepeer/core" "github.com/livepeer/go-livepeer/media" "github.com/livepeer/go-livepeer/monitor" + "github.com/livepeer/go-livepeer/net" "github.com/livepeer/go-livepeer/trickle" "github.com/livepeer/go-tools/drivers" ) @@ -38,14 +40,14 @@ func (ls *LivepeerServer) StartStream() http.Handler { corsHeaders(w, r.Method) //verify request, get orchestrators available and sign request - gatewayJob, err := ls.setupGatewayJob(ctx, r) + gatewayJob, err := ls.setupGatewayJob(ctx, r, false) if err != nil { clog.Errorf(ctx, "Error setting up job: %s", err) http.Error(w, err.Error(), http.StatusBadRequest) return } - streamUrls, code, err := ls.setupStream(ctx, r, gatewayJob.Job.Req) + streamUrls, code, err := ls.setupStream(ctx, r, gatewayJob) if err != nil { clog.Errorf(ctx, "Error setting up stream: %s", err) http.Error(w, err.Error(), code) @@ -74,51 +76,75 @@ func (ls *LivepeerServer) StopStream() http.Handler { ctx := r.Context() streamId := r.PathValue("streamId") - if streamInfo, exists := ls.LivepeerNode.ExternalCapabilities.Streams[streamId]; exists { - // Copy streamInfo before deletion - streamInfoCopy := *streamInfo - //remove the stream - ls.LivepeerNode.ExternalCapabilities.RemoveStream(streamId) + stream, exists := ls.LivepeerNode.LivePipelines[streamId] + if !exists { + http.Error(w, "Stream not found", http.StatusNotFound) + return + } - stopJob, err := ls.setupGatewayJob(ctx, r) - if err != nil { - clog.Errorf(ctx, "Error setting up stop job: %s", err) - http.Error(w, err.Error(), http.StatusBadRequest) - return - } + params, err := getStreamRequestParams(stream) + if err != nil { + clog.Errorf(ctx, "Error getting stream request params: %s", err) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } - stopJob.sign() - resp, code, err := ls.sendJobToOrch(ctx, r, stopJob.Job.Req, stopJob.SignedJobReq, *streamInfoCopy.OrchToken, "/ai/stream/stop", streamInfoCopy.StreamRequest) - if err != nil { - clog.Errorf(ctx, "Error sending job to orchestrator: %s", err) - http.Error(w, err.Error(), code) - return - } + stream.StopStream(nil) + delete(ls.LivepeerNode.LivePipelines, streamId) - w.WriteHeader(http.StatusOK) - io.Copy(w, resp.Body) + stopJob, err := ls.setupGatewayJob(ctx, r, true) + if err != nil { + clog.Errorf(ctx, "Error setting up stop job: %s", err) + http.Error(w, err.Error(), http.StatusBadRequest) return } + stopJob.sign() //no changes to make, sign job - //no stream exists - w.WriteHeader(http.StatusNoContent) - return + token, err := sessionToToken(params.liveParams.sess) + if err != nil { + clog.Errorf(ctx, "Error converting session to token: %s", err) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + body, err := io.ReadAll(r.Body) + if err != nil { + clog.Errorf(ctx, "Error reading request body: %s", err) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + defer r.Body.Close() + + resp, code, err := ls.sendJobToOrch(ctx, r, stopJob.Job.Req, stopJob.SignedJobReq, token, "/ai/stream/stop", body) + if err != nil { + clog.Errorf(ctx, "Error sending job to orchestrator: %s", err) + http.Error(w, err.Error(), code) + return + } + + w.WriteHeader(http.StatusOK) + io.Copy(w, resp.Body) + return }) } func (ls *LivepeerServer) runStream(gatewayJob *gatewayJob) { streamID := gatewayJob.Job.Req.ID - stream, exists := ls.LivepeerNode.ExternalCapabilities.Streams[streamID] + stream, exists := ls.LivepeerNode.LivePipelines[streamID] if !exists { glog.Errorf("Stream %s not found", streamID) return } - params := stream.Params.(aiRequestParams) //this context passes to all channels that will close when stream is canceled ctx := stream.StreamCtx ctx = clog.AddVal(ctx, "stream_id", streamID) + params, err := getStreamRequestParams(stream) + if err != nil { + clog.Errorf(ctx, "Error getting stream request params: %s", err) + return + } + //monitor for lots of fast swaps, likely something wrong with request orchSwapper := NewOrchestratorSwapper(params) @@ -132,39 +158,43 @@ func (ls *LivepeerServer) runStream(gatewayJob *gatewayJob) { clog.Errorf(ctx, "Error getting token for orch=%v err=%v", orch.ServiceAddr, err) continue } - stream.OrchToken = newToken orch = *newToken } - stream.OrchToken = &orch - stream.OrchUrl = orch.ServiceAddr + orchSession, err := tokenToAISession(orch) + if err != nil { + clog.Errorf(ctx, "Error converting token to AISession: %v", err) + continue + } + params.liveParams.sess = &orchSession + ctx = clog.AddVal(ctx, "orch", ethcommon.Bytes2Hex(orch.TicketParams.Recipient)) ctx = clog.AddVal(ctx, "orch_url", orch.ServiceAddr) //set request ID to persist from Gateway to Worker - gatewayJob.Job.Req.ID = stream.StreamID - err := gatewayJob.sign() + gatewayJob.Job.Req.ID = params.liveParams.streamID + err = gatewayJob.sign() if err != nil { clog.Errorf(ctx, "Error signing job, exiting stream processing request: %v", err) - stream.CancelStream() + stream.StopStream(err) return } - orchResp, _, err := ls.sendJobToOrch(ctx, nil, gatewayJob.Job.Req, gatewayJob.SignedJobReq, orch, "/ai/stream/start", stream.StreamRequest) + orchResp, _, err := ls.sendJobToOrch(ctx, nil, gatewayJob.Job.Req, gatewayJob.SignedJobReq, orch, "/ai/stream/start", stream.StreamRequest()) if err != nil { clog.Errorf(ctx, "job not able to be processed by Orchestrator %v err=%v ", orch.ServiceAddr, err.Error()) continue } - stream.OrchPublishUrl = orchResp.Header.Get("X-Publish-Url") - stream.OrchSubscribeUrl = orchResp.Header.Get("X-Subscribe-Url") - stream.OrchControlUrl = orchResp.Header.Get("X-Control-Url") - stream.OrchEventsUrl = orchResp.Header.Get("X-Events-Url") - stream.OrchDataUrl = orchResp.Header.Get("X-Data-Url") + params.liveParams.orchPublishUrl = orchResp.Header.Get("X-Publish-Url") + params.liveParams.orchSubscribeUrl = orchResp.Header.Get("X-Subscribe-Url") + params.liveParams.orchControlUrl = orchResp.Header.Get("X-Control-Url") + params.liveParams.orchEventsUrl = orchResp.Header.Get("X-Events-Url") + params.liveParams.orchDataUrl = orchResp.Header.Get("X-Data-Url") perOrchCtx, perOrchCancel := context.WithCancelCause(ctx) params.liveParams.kickOrch = perOrchCancel - stream.Params = params //update params used to kickOrch (perOrchCancel) - if err = startStreamProcessing(perOrchCtx, stream); err != nil { + stream.UpdateStreamParams(params) //update params used to kickOrch (perOrchCancel) and urls + if err = startStreamProcessing(perOrchCtx, stream, params); err != nil { clog.Errorf(ctx, "Error starting processing: %s", err) perOrchCancel(err) break @@ -214,12 +244,16 @@ func (ls *LivepeerServer) monitorStream(streamId string) { ctx := context.Background() ctx = clog.AddVal(ctx, "stream_id", streamId) - stream, exists := ls.LivepeerNode.ExternalCapabilities.Streams[streamId] + stream, exists := ls.LivepeerNode.LivePipelines[streamId] if !exists { clog.Errorf(ctx, "Stream %s not found", streamId) return } - params := stream.Params.(aiRequestParams) + params, err := getStreamRequestParams(stream) + if err != nil { + clog.Errorf(ctx, "Error getting stream request params: %v", err) + return + } ctx = clog.AddVal(ctx, "request_id", params.liveParams.requestID) @@ -227,6 +261,7 @@ func (ls *LivepeerServer) monitorStream(streamId string) { dur := 50 * time.Second pmtTicker := time.NewTicker(dur) defer pmtTicker.Stop() + //setup sender jobSender, err := getJobSender(ctx, ls.LivepeerNode) if err != nil { clog.Errorf(ctx, "Error getting job sender: %v", err) @@ -240,16 +275,34 @@ func (ls *LivepeerServer) monitorStream(streamId string) { ls.LivepeerNode.ExternalCapabilities.RemoveStream(streamId) return case <-pmtTicker.C: - // fetch new JobToken with each payment - token := stream.OrchToken - updateGatewayBalance(ls.LivepeerNode, *token, stream.Capability, dur) + if !params.inputStreamExists() { + clog.Infof(ctx, "Input stream does not exist for stream %s, ending monitoring", streamId) + return + } + params, err = getStreamRequestParams(stream) + if err != nil { + clog.Errorf(ctx, "Error getting stream request params: %v", err) + continue + } + token, err := sessionToToken(params.liveParams.sess) + if err != nil { + clog.Errorf(ctx, "Error getting token for session: %v", err) + continue + } - newToken, err := getToken(ctx, 3*time.Second, stream.OrchUrl, stream.Capability, jobSender.Addr, jobSender.Sig) + // fetch new JobToken with each payment + newToken, err := getToken(ctx, 3*time.Second, token.ServiceAddr, stream.Pipeline, jobSender.Addr, jobSender.Sig) if err != nil { - clog.Errorf(ctx, "Error getting new token for %s: %v", stream.OrchUrl, err) + clog.Errorf(ctx, "Error getting new token for %s: %v", token.ServiceAddr, err) continue } - stream.OrchToken = newToken + newSess, err := tokenToAISession(*newToken) + if err != nil { + clog.Errorf(ctx, "Error converting token to AI session: %v", err) + continue + } + params.liveParams.sess = &newSess + stream.UpdateStreamParams(params) // send the payment jobDetails := JobRequestDetails{StreamId: streamId} @@ -258,7 +311,7 @@ func (ls *LivepeerServer) monitorStream(streamId string) { clog.Errorf(ctx, "Error marshalling job details: %v", err) continue } - req := &JobRequest{Request: string(jobDetailsStr), Parameters: "{}", Capability: stream.Capability, + req := &JobRequest{Request: string(jobDetailsStr), Parameters: "{}", Capability: stream.Pipeline, Sender: ls.LivepeerNode.OrchestratorPool.Broadcaster().Address().Hex(), Timeout: 60, } @@ -269,8 +322,8 @@ func (ls *LivepeerServer) monitorStream(streamId string) { clog.Errorf(ctx, "Error signing job, continuing monitoring: %v", err) continue } - orchToken := stream.OrchToken - pmtHdr, err := createPayment(ctx, req, orchToken, ls.LivepeerNode) + + pmtHdr, err := createPayment(ctx, req, newToken, ls.LivepeerNode) if err != nil { clog.Errorf(ctx, "Error processing stream payment for %s: %v", streamId, err) // Continue monitoring even if payment fails @@ -281,7 +334,7 @@ func (ls *LivepeerServer) monitorStream(streamId string) { //send the payment, update the stream with the refreshed token clog.Infof(ctx, "Sending stream payment for %s", streamId) - statusCode, err := ls.sendPayment(ctx, stream.OrchUrl+"/ai/stream/payment", stream.Capability, job.SignedJobReq, pmtHdr) + statusCode, err := ls.sendPayment(ctx, token.ServiceAddr+"/ai/stream/payment", stream.Pipeline, job.SignedJobReq, pmtHdr) if err != nil { clog.Errorf(ctx, "Error sending stream payment for %s: %v", streamId, err) continue @@ -312,7 +365,11 @@ type StreamUrls struct { DataUrl string `json:"data_url"` } -func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, req *JobRequest) (*StreamUrls, int, error) { +func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, job *gatewayJob) (*StreamUrls, int, error) { + if job == nil { + return nil, http.StatusBadRequest, errors.New("invalid job") + } + requestID := string(core.RandomManifestID()) ctx = clog.AddVal(ctx, "request_id", requestID) @@ -349,7 +406,7 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, req } // if auth webhook returns pipeline config these will be replaced - pipeline := req.Capability //streamParamsJson["pipeline"].(string) + pipeline := job.Job.Req.Capability rawParams := startReq.Params streamID := startReq.StreamId @@ -461,7 +518,7 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, req // Count `ai_live_attempts` after successful parameters validation clog.V(common.VERBOSE).Infof(ctx, "AI Live video attempt") if monitor.Enabled { - monitor.AILiveVideoAttempt(req.Capability) + monitor.AILiveVideoAttempt(job.Job.Req.Capability) } sendErrorEvent := LiveErrorEventSender(ctx, streamID, map[string]string{ @@ -480,15 +537,16 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, req // kickInput will close the whip connection // localRTMPPrefix set by ENV variable LIVE_AI_PLAYBACK_HOST ssr := media.NewSwitchableSegmentReader() //this converts ingest to segments to send to Orchestrator + params := aiRequestParams{ node: ls.LivepeerNode, os: drivers.NodeStorage.NewSession(requestID), - sessManager: ls.AISessionManager, + sessManager: nil, liveParams: &liveRequestParams{ segmentReader: ssr, rtmpOutputs: rtmpOutputs, - stream: streamName, + stream: streamID, //live video to video uses stream name, byoc combines to one id paymentProcessInterval: ls.livePaymentInterval, outSegmentTimeout: ls.outSegmentTimeout, requestID: requestID, @@ -496,12 +554,13 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, req pipelineID: pipelineID, pipeline: pipeline, sendErrorEvent: sendErrorEvent, + manifestID: pipeline, //byoc uses one balance per capability name }, } //create a dataWriter for data channel if enabled var jobParams JobParameters - if err = json.Unmarshal([]byte(req.Parameters), &jobParams); err != nil { + if err = json.Unmarshal([]byte(job.Job.Req.Parameters), &jobParams); err != nil { return nil, http.StatusBadRequest, errors.New("invalid job parameters") } if jobParams.EnableDataOutput { @@ -509,20 +568,16 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, req } //check if stream exists - _, exists := ls.LivepeerNode.ExternalCapabilities.Streams[streamID] - if exists { + if params.inputStreamExists() { return nil, http.StatusBadRequest, fmt.Errorf("stream already exists: %s", streamID) } clog.Infof(ctx, "stream setup videoIngress=%v videoEgress=%v dataOutput=%v", jobParams.EnableVideoIngress, jobParams.EnableVideoEgress, jobParams.EnableDataOutput) //save the stream setup - _, err = ls.LivepeerNode.ExternalCapabilities.AddStream(streamID, pipeline, params, bodyBytes) - if err != nil { - return nil, http.StatusBadRequest, err - } + ls.LivepeerNode.NewLivePipeline(requestID, streamID, pipeline, params, bodyBytes) //track the pipeline for cancellation - req.ID = streamID + job.Job.Req.ID = streamID streamUrls := StreamUrls{ StreamId: streamID, WhipUrl: whipURL, @@ -542,19 +597,23 @@ func (ls *LivepeerServer) StartStreamRTMPIngest() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { remoteAddr := getRemoteAddr(r) ctx := clog.AddVal(context.Background(), clog.ClientIP, remoteAddr) - requestID := string(core.RandomManifestID()) - ctx = clog.AddVal(ctx, "request_id", requestID) streamId := r.PathValue("streamId") ctx = clog.AddVal(ctx, "stream_id", streamId) - stream, ok := ls.LivepeerNode.ExternalCapabilities.Streams[streamId] + stream, ok := ls.LivepeerNode.LivePipelines[streamId] if !ok { respondJsonError(ctx, w, fmt.Errorf("stream not found: %s", streamId), http.StatusNotFound) return } - params := stream.Params.(aiRequestParams) + params, err := getStreamRequestParams(stream) + if err != nil { + respondJsonError(ctx, w, err, http.StatusBadRequest) + return + } + + //set source ID and source type needed for mediamtx client control api sourceID := r.FormValue("source_id") if sourceID == "" { http.Error(w, "missing source_id", http.StatusBadRequest) @@ -600,7 +659,7 @@ func (ls *LivepeerServer) StartStreamRTMPIngest() http.Handler { params.liveParams.localRTMPPrefix = mediaMTXInputURL params.liveParams.kickInput = kickInput - stream.Params = params //update params used to kickInput + stream.UpdateStreamParams(params) //add kickInput to stream params // Kick off the RTMP pull and segmentation clog.Infof(ctx, "Starting RTMP ingest from MediaMTX") @@ -623,7 +682,7 @@ func (ls *LivepeerServer) StartStreamRTMPIngest() http.Handler { }) params.liveParams.segmentReader.Close() - stream.CancelStream() //cleanupControl(ctx, params) + stream.StopStream(nil) }() //write response @@ -639,13 +698,17 @@ func (ls *LivepeerServer) StartStreamWhipIngest(whipServer *media.WHIPServer) ht streamId := r.PathValue("streamId") ctx = clog.AddVal(ctx, "stream_id", streamId) - stream, ok := ls.LivepeerNode.ExternalCapabilities.Streams[streamId] + stream, ok := ls.LivepeerNode.LivePipelines[streamId] if !ok { respondJsonError(ctx, w, fmt.Errorf("stream not found: %s", streamId), http.StatusNotFound) return } - params := stream.Params.(aiRequestParams) + params, err := getStreamRequestParams(stream) + if err != nil { + respondJsonError(ctx, w, err, http.StatusBadRequest) + return + } whipConn := media.NewWHIPConnection() whepURL := generateWhepUrl(streamId, params.liveParams.requestID) @@ -661,16 +724,16 @@ func (ls *LivepeerServer) StartStreamWhipIngest(whipServer *media.WHIPServer) ht whipConn.Close() } params.liveParams.kickInput = kickInput - stream.Params = params //update params used to kickInput + stream.UpdateStreamParams(params) //add kickInput to stream params //wait for the WHIP connection to close and then cleanup go func() { statsContext, statsCancel := context.WithCancel(ctx) defer statsCancel() - go runStats(statsContext, whipConn, streamId, stream.Capability, params.liveParams.requestID) + go runStats(statsContext, whipConn, streamId, stream.Pipeline, params.liveParams.requestID) whipConn.AwaitClose() - stream.CancelStream() //cleanupControl(ctx, params) + stream.StopStream(nil) params.liveParams.segmentReader.Close() params.liveParams.kickOrch(errors.New("whip ingest disconnected")) clog.Info(ctx, "Live cleaned up") @@ -681,54 +744,53 @@ func (ls *LivepeerServer) StartStreamWhipIngest(whipServer *media.WHIPServer) ht }) } -func startStreamProcessing(ctx context.Context, streamInfo *core.StreamInfo) error { +func startStreamProcessing(ctx context.Context, stream *core.LivePipeline, params aiRequestParams) error { var channels []string - //required channels - control, err := common.AppendHostname(streamInfo.OrchControlUrl, streamInfo.OrchUrl) + control, err := common.AppendHostname(params.liveParams.orchControlUrl, params.liveParams.sess.BroadcastSession.Transcoder()) if err != nil { return fmt.Errorf("invalid control URL: %w", err) } - events, err := common.AppendHostname(streamInfo.OrchEventsUrl, streamInfo.OrchUrl) + events, err := common.AppendHostname(params.liveParams.orchEventsUrl, params.liveParams.sess.BroadcastSession.Transcoder()) if err != nil { return fmt.Errorf("invalid events URL: %w", err) } channels = append(channels, control.String()) channels = append(channels, events.String()) - startStreamControlPublish(ctx, control, streamInfo) - startStreamEventsSubscribe(ctx, events, streamInfo) + startControlPublish(ctx, control, params) + startEventsSubscribe(ctx, events, params, params.liveParams.sess) //Optional channels - if streamInfo.OrchPublishUrl != "" { + if params.liveParams.orchPublishUrl != "" { clog.Infof(ctx, "Starting video ingress publisher") - pub, err := common.AppendHostname(streamInfo.OrchPublishUrl, streamInfo.OrchUrl) + pub, err := common.AppendHostname(params.liveParams.orchPublishUrl, params.liveParams.sess.BroadcastSession.Transcoder()) if err != nil { return fmt.Errorf("invalid publish URL: %w", err) } channels = append(channels, pub.String()) - startStreamTricklePublish(ctx, pub, streamInfo) + startTricklePublish(ctx, pub, params, params.liveParams.sess) } - if streamInfo.OrchSubscribeUrl != "" { + if params.liveParams.orchSubscribeUrl != "" { clog.Infof(ctx, "Starting video egress subscriber") - sub, err := common.AppendHostname(streamInfo.OrchSubscribeUrl, streamInfo.OrchUrl) + sub, err := common.AppendHostname(params.liveParams.orchSubscribeUrl, params.liveParams.sess.BroadcastSession.Transcoder()) if err != nil { return fmt.Errorf("invalid subscribe URL: %w", err) } channels = append(channels, sub.String()) - startStreamTrickleSubscribe(ctx, sub, streamInfo) + startTrickleSubscribe(ctx, sub, params, params.liveParams.sess) } - if streamInfo.OrchDataUrl != "" { + if params.liveParams.orchDataUrl != "" { clog.Infof(ctx, "Starting data channel subscriber") - data, err := common.AppendHostname(streamInfo.OrchDataUrl, streamInfo.OrchUrl) + data, err := common.AppendHostname(params.liveParams.orchDataUrl, params.liveParams.sess.BroadcastSession.Transcoder()) if err != nil { return fmt.Errorf("invalid data URL: %w", err) } - streamInfo.Params.(aiRequestParams).liveParams.manifestID = streamInfo.Capability + params.liveParams.manifestID = stream.Pipeline - startStreamDataSubscribe(ctx, data, streamInfo) + startDataSubscribe(ctx, data, params, params.liveParams.sess) } return nil @@ -746,12 +808,16 @@ func (ls *LivepeerServer) GetStreamData() http.Handler { ctx = clog.AddVal(ctx, "stream", streamId) // Get the live pipeline for this stream - stream, exists := ls.LivepeerNode.ExternalCapabilities.Streams[streamId] + stream, exists := ls.LivepeerNode.LivePipelines[streamId] if !exists { http.Error(w, "Stream not found", http.StatusNotFound) return } - params := stream.Params.(aiRequestParams) + params, err := getStreamRequestParams(stream) + if err != nil { + respondJsonError(ctx, w, err, http.StatusBadRequest) + return + } // Get the data reading buffer if params.liveParams.dataWriter == nil { http.Error(w, "Stream data not available", http.StatusServiceUnavailable) @@ -814,7 +880,7 @@ func (ls *LivepeerServer) UpdateStream() http.Handler { http.Error(w, "Missing stream name", http.StatusBadRequest) return } - stream, ok := ls.LivepeerNode.ExternalCapabilities.Streams[streamId] + stream, ok := ls.LivepeerNode.LivePipelines[streamId] if !ok { // Stream not found http.Error(w, "Stream not found", http.StatusNotFound) @@ -830,7 +896,7 @@ func (ls *LivepeerServer) UpdateStream() http.Handler { } params := string(data) - stream.JobParams = params + stream.Params = params controlPub := stream.ControlPub if controlPub == nil { @@ -1189,3 +1255,38 @@ func (h *lphttp) ProcessStreamPayment(w http.ResponseWriter, r *http.Request) { w.Header().Set(jobPaymentBalanceHdr, capBal.FloatString(0)) w.WriteHeader(http.StatusOK) } + +func tokenToAISession(token core.JobToken) (AISession, error) { + var session BroadcastSession + + // Initialize the lock to avoid nil pointer dereference in methods + // like (*BroadcastSession).Transcoder() which acquire RLock() + session.lock = &sync.RWMutex{} + + orchInfo := net.OrchestratorInfo{Transcoder: token.ServiceAddr, TicketParams: token.TicketParams, PriceInfo: token.Price} + orchInfo.Transcoder = token.ServiceAddr + if token.SenderAddress != nil { + orchInfo.Address = ethcommon.Hex2Bytes(token.SenderAddress.Addr) + } + session.OrchestratorInfo = &orchInfo + + return AISession{BroadcastSession: &session}, nil +} + +func sessionToToken(session *AISession) (core.JobToken, error) { + var token core.JobToken + + token.ServiceAddr = session.OrchestratorInfo.Transcoder + token.TicketParams = session.OrchestratorInfo.TicketParams + token.Price = session.OrchestratorInfo.PriceInfo + return token, nil +} + +func getStreamRequestParams(stream *core.LivePipeline) (aiRequestParams, error) { + streamParams := stream.StreamParams() + params, ok := streamParams.(aiRequestParams) + if !ok { + return aiRequestParams{}, fmt.Errorf("failed to cast stream params to aiRequestParams") + } + return params, nil +} From 358cea25575da907eeaa57e063d1c624185907af Mon Sep 17 00:00:00 2001 From: Brad P Date: Fri, 5 Sep 2025 12:38:39 -0500 Subject: [PATCH 41/57] remove job_trickle.go --- server/job_trickle.go | 745 ------------------------------------------ 1 file changed, 745 deletions(-) delete mode 100644 server/job_trickle.go diff --git a/server/job_trickle.go b/server/job_trickle.go deleted file mode 100644 index 4b683624ce..0000000000 --- a/server/job_trickle.go +++ /dev/null @@ -1,745 +0,0 @@ -package server - -import ( - "bufio" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "net/url" - "os/exec" - "strings" - "sync" - "syscall" - "time" - - "github.com/dustin/go-humanize" - "github.com/livepeer/go-livepeer/clog" - "github.com/livepeer/go-livepeer/common" - "github.com/livepeer/go-livepeer/core" - "github.com/livepeer/go-livepeer/media" - "github.com/livepeer/go-livepeer/monitor" - "github.com/livepeer/go-livepeer/trickle" -) - -func startStreamTricklePublish(ctx context.Context, url *url.URL, streamInfo *core.StreamInfo) { - ctx = clog.AddVal(ctx, "url", url.Redacted()) - params := streamInfo.Params.(aiRequestParams) - - publisher, err := trickle.NewTricklePublisher(url.String()) - if err != nil { - stopProcessing(ctx, params, fmt.Errorf("trickle publish init err: %w", err)) - return - } - - // Start payments which probes a segment every "paymentProcessInterval" and sends a payment - ctx, cancel := context.WithCancel(ctx) - //byoc sets as context values - orchAddr := clog.GetVal(ctx, "orch") - orchUrl := clog.GetVal(ctx, "orch_url") - - slowOrchChecker := &SlowOrchChecker{} - - firstSegment := true - - params.liveParams.segmentReader.SwitchReader(func(reader media.CloneableReader) { - // check for end of stream - if _, eos := reader.(*media.EOSReader); eos { - if err := publisher.Close(); err != nil { - clog.Infof(ctx, "Error closing trickle publisher. err=%v", err) - } - cancel() - return - } - thisSeq, atMax := slowOrchChecker.BeginSegment() - if atMax { - clog.Infof(ctx, "Orchestrator is slow - terminating") - streamInfo.ExcludeOrch(orchUrl) //suspendOrchestrator(ctx, params) - cancel() - stopProcessing(ctx, params, errors.New("orchestrator is slow")) - return - } - go func(seq int) { - defer slowOrchChecker.EndSegment() - var r io.Reader = reader - - clog.V(8).Infof(ctx, "trickle publish writing data seq=%d", seq) - segment, err := publisher.Next() - if err != nil { - clog.Infof(ctx, "error getting next publish handle; dropping segment err=%v", err) - params.liveParams.sendErrorEvent(fmt.Errorf("Missing next handle %v", err)) - return - } - for { - select { - case <-ctx.Done(): - clog.Info(ctx, "trickle publish done") - return - default: - } - - startTime := time.Now() - currentSeq := slowOrchChecker.GetCount() - if seq != currentSeq { - clog.Infof(ctx, "Next segment has already started; skipping this one seq=%d currentSeq=%d", seq, currentSeq) - params.liveParams.sendErrorEvent(fmt.Errorf("Next segment has started")) - segment.Close() - return - } - params.liveParams.mu.Lock() - params.liveParams.lastSegmentTime = startTime - params.liveParams.mu.Unlock() - logToDisk(ctx, reader, params.node.WorkDir, params.liveParams.requestID, seq) - n, err := segment.Write(r) - if err == nil { - // no error, all done, let's leave - if monitor.Enabled && firstSegment { - firstSegment = false - monitor.SendQueueEventAsync("stream_trace", map[string]interface{}{ - "type": "gateway_send_first_ingest_segment", - "timestamp": time.Now().UnixMilli(), - "stream_id": params.liveParams.streamID, - "pipeline_id": params.liveParams.pipelineID, - "request_id": params.liveParams.requestID, - "orchestrator_info": map[string]interface{}{ - "address": orchAddr, - "url": orchUrl, - }, - }) - } - clog.Info(ctx, "trickle publish complete", "wrote", humanize.Bytes(uint64(n)), "seq", seq, "took", time.Since(startTime)) - return - } - if errors.Is(err, trickle.StreamNotFoundErr) { - stopProcessing(ctx, params, errors.New("stream no longer exists on orchestrator; terminating")) - return - } - // Retry segment only if nothing has been sent yet - // and the next segment has not yet started - // otherwise drop - if n > 0 { - clog.Infof(ctx, "Error publishing segment; dropping remainder wrote=%d err=%v", n, err) - params.liveParams.sendErrorEvent(fmt.Errorf("Error publishing, wrote %d dropping %v", n, err)) - segment.Close() - return - } - clog.Infof(ctx, "Error publishing segment before writing; retrying err=%v", err) - // Clone in case read head was incremented somewhere, which cloning resets - r = reader.Clone() - time.Sleep(250 * time.Millisecond) - } - }(thisSeq) - }) - clog.Infof(ctx, "trickle pub") -} - -func startStreamTrickleSubscribe(ctx context.Context, url *url.URL, streamInfo *core.StreamInfo) { - // subscribe to inference outputs and send them into the world - params := streamInfo.Params.(aiRequestParams) - orchAddr := clog.GetVal(ctx, "orch") - orchUrl := clog.GetVal(ctx, "orch_url") - subscriber, err := trickle.NewTrickleSubscriber(trickle.TrickleSubscriberConfig{ - URL: url.String(), - Ctx: ctx, - }) - if err != nil { - stopProcessing(ctx, params, fmt.Errorf("trickle subscription init failed: %w", err)) - return - } - - ctx = clog.AddVal(ctx, "url", url.Redacted()) - - // Set up output buffers and ffmpeg processes - rbc := media.RingBufferConfig{BufferLen: 5_000_000} // 5 MB, 20-30 seconds at current rates - outWriter, err := media.NewRingBuffer(&rbc) - if err != nil { - stopProcessing(ctx, params, fmt.Errorf("ringbuffer init failed: %w", err)) - return - } - // Launch ffmpeg for each configured RTMP output - for _, outURL := range params.liveParams.rtmpOutputs { - go ffmpegStreamOutput(ctx, outURL, outWriter, streamInfo) - } - - // watchdog that gets reset on every segment to catch output stalls - segmentTimeout := params.liveParams.outSegmentTimeout - if segmentTimeout <= 0 { - segmentTimeout = 30 * time.Second - } - segmentTicker := time.NewTicker(segmentTimeout) - - // read segments from trickle subscription - go func() { - defer outWriter.Close() - defer segmentTicker.Stop() - - var err error - firstSegment := true - var segmentsReceived int64 - - retries := 0 - // we're trying to keep (retryPause x maxRetries) duration to fall within one output GOP length - const retryPause = 300 * time.Millisecond - const maxRetries = 5 - for { - select { - case <-ctx.Done(): - clog.Info(ctx, "trickle subscribe done") - return - default: - } - if streamInfo != nil && !streamInfo.IsActive() { - clog.Infof(ctx, "trickle subscribe stopping, input stream does not exist.") - break - } - segmentTicker.Reset(segmentTimeout) // reset ticker on each iteration. - var segment *http.Response - clog.V(8).Infof(ctx, "trickle subscribe read data await") - segment, err = subscriber.Read() - if err != nil { - if errors.Is(err, trickle.EOS) || errors.Is(err, trickle.StreamNotFoundErr) { - stopProcessing(ctx, params, fmt.Errorf("trickle subscribe stopping, stream not found, err=%w", err)) - return - } - var sequenceNonexistent *trickle.SequenceNonexistent - if errors.As(err, &sequenceNonexistent) { - // stream exists but segment doesn't, so skip to leading edge - subscriber.SetSeq(sequenceNonexistent.Latest) - } - // TODO if not EOS then signal a new orchestrator is needed - err = fmt.Errorf("trickle subscribe error reading: %w", err) - clog.Infof(ctx, "%s", err) - if retries > maxRetries { - stopProcessing(ctx, params, errors.New("trickle subscribe stopping, retries exceeded")) - return - } - retries++ - params.liveParams.sendErrorEvent(err) - time.Sleep(retryPause) - continue - } - retries = 0 - seq := trickle.GetSeq(segment) - clog.V(8).Infof(ctx, "trickle subscribe read data received seq=%d", seq) - copyStartTime := time.Now() - - n, err := copySegment(ctx, segment, outWriter, seq, params) - if err != nil { - if errors.Is(err, context.Canceled) { - clog.Info(ctx, "trickle subscribe stopping - context canceled") - return - } - // Check whether the client has sent data recently. - // TODO ensure the threshold is some multiple of LIVE_AI_MIN_SEG_DUR - params.liveParams.mu.Lock() - lastSegmentTime := params.liveParams.lastSegmentTime - params.liveParams.mu.Unlock() - segmentAge := time.Since(lastSegmentTime) - maxSegmentDelay := params.liveParams.outSegmentTimeout / 2 - if segmentAge < maxSegmentDelay && streamInfo != nil && streamInfo.IsActive() { - // we have some recent input but no output from orch, so kick - streamInfo.ExcludeOrch(orchUrl) //suspendOrchestrator(ctx, params) - stopProcessing(ctx, params, fmt.Errorf("trickle subscribe error, swapping: %w", err)) - return - } - clog.InfofErr(ctx, "trickle subscribe error copying segment seq=%d", seq, err) - subscriber.SetSeq(seq) - retries++ - continue - } - if firstSegment { - firstSegment = false - delayMs := time.Since(params.liveParams.startTime).Milliseconds() - if monitor.Enabled { - //monitor.AIFirstSegmentDelay(delayMs, streamInfo) //update this to take the address and url as strings - monitor.SendQueueEventAsync("stream_trace", map[string]interface{}{ - "type": "gateway_receive_first_processed_segment", - "timestamp": time.Now().UnixMilli(), - "stream_id": params.liveParams.streamID, - "pipeline_id": params.liveParams.pipelineID, - "request_id": params.liveParams.requestID, - "orchestrator_info": map[string]interface{}{ - "address": orchAddr, - "url": orchUrl, - }, - }) - } - clog.V(common.VERBOSE).Infof(ctx, "First Segment delay=%dms streamID=%s", delayMs, params.liveParams.streamID) - } - segmentsReceived += 1 - if segmentsReceived == 3 && monitor.Enabled { - // We assume that after receiving 3 segments, the runner started successfully - // and we should be able to start the playback - monitor.SendQueueEventAsync("stream_trace", map[string]interface{}{ - "type": "gateway_receive_few_processed_segments", - "timestamp": time.Now().UnixMilli(), - "stream_id": params.liveParams.streamID, - "pipeline_id": params.liveParams.pipelineID, - "request_id": params.liveParams.requestID, - "orchestrator_info": map[string]interface{}{ - "address": orchAddr, - "url": orchUrl, - }, - }) - - } - clog.V(8).Info(ctx, "trickle subscribe read data completed", "seq", seq, "bytes", humanize.Bytes(uint64(n)), "took", time.Since(copyStartTime)) - } - }() - - // watchdog: fires if orch does not produce segments for too long - go func() { - for { - select { - case <-segmentTicker.C: - // check whether this timeout is due to missing input - // only suspend orchestrator if there is recent input - // ( no input == no output, so don't suspend for that ) - params.liveParams.mu.Lock() - lastInputSegmentTime := params.liveParams.lastSegmentTime - params.liveParams.mu.Unlock() - lastInputSegmentAge := time.Since(lastInputSegmentTime) - hasRecentInput := lastInputSegmentAge < segmentTimeout/2 - if hasRecentInput && streamInfo != nil && streamInfo.IsActive() { - // abandon the orchestrator - streamInfo.ExcludeOrch(orchUrl) - stopProcessing(ctx, params, fmt.Errorf("timeout waiting for segments")) - segmentTicker.Stop() - return - } - } - } - }() - -} - -func startStreamControlPublish(ctx context.Context, control *url.URL, streamInfo *core.StreamInfo) { - params := streamInfo.Params.(aiRequestParams) - controlPub, err := trickle.NewTricklePublisher(control.String()) - if err != nil { - stopProcessing(ctx, params, fmt.Errorf("error starting control publisher, err=%w", err)) - return - } - params.node.LiveMu.Lock() - defer params.node.LiveMu.Unlock() - - ticker := time.NewTicker(10 * time.Second) - done := make(chan bool, 1) - once := sync.Once{} - stop := func() { - once.Do(func() { - ticker.Stop() - done <- true - }) - } - - //sess, exists := params.node.LivePipelines[stream] - //if !exists || sess.RequestID != params.liveParams.requestID { - // stopProcessing(ctx, params, fmt.Errorf("control session did not exist")) - // return - //} - //if sess.ControlPub != nil { - // // clean up from existing orchestrator - // go sess.ControlPub.Close() - //} - streamInfo.ControlPub = controlPub - streamInfo.StopControl = stop - - if monitor.Enabled { - monitorCurrentLiveSessions(params.node.LivePipelines) - } - - // Send any cached control params in a goroutine outside the lock. - msg := streamInfo.JobParams - go func() { - if msg == "" { - return - } - var err error - for i := 0; i < 3; i++ { - err = controlPub.Write(strings.NewReader(msg)) - if err == nil { - return - } - time.Sleep(100 * time.Millisecond) - } - stopProcessing(ctx, params, fmt.Errorf("control write failed: %w", err)) - }() - - // send a keepalive periodically to keep both ends of the connection alive - go func() { - for { - select { - case <-ticker.C: - const msg = `{"keep":"alive"}` - err := controlPub.Write(strings.NewReader(msg)) - if err == trickle.StreamNotFoundErr { - // the channel doesn't exist anymore, so stop - stop() - stopProcessing(ctx, params, errors.New("control channel does not exist")) - continue // loop back to consume the `done` chan - } - // if there was another type of error, we'll just retry anyway - case <-done: - return - case <-ctx.Done(): - stop() - } - } - }() -} - -func startStreamDataSubscribe(ctx context.Context, url *url.URL, streamInfo *core.StreamInfo) { - //only start DataSubscribe if enabled - params := streamInfo.Params.(aiRequestParams) - orchAddr := clog.GetVal(ctx, "orch") - orchUrl := clog.GetVal(ctx, "orch_url") - if params.liveParams.dataWriter == nil { - return - } - - // subscribe to the outputs - subscriber, err := trickle.NewTrickleSubscriber(trickle.TrickleSubscriberConfig{ - URL: url.String(), - Ctx: ctx, - }) - if err != nil { - clog.Infof(ctx, "Failed to create data subscriber: %s", err) - return - } - - dataWriter := params.liveParams.dataWriter - - // read segments from trickle subscription - go func() { - defer dataWriter.Close() - - var err error - firstSegment := true - - retries := 0 - // we're trying to keep (retryPause x maxRetries) duration to fall within one output GOP length - const retryPause = 300 * time.Millisecond - const maxRetries = 5 - for { - select { - case <-ctx.Done(): - clog.Info(ctx, "data subscribe done") - return - default: - } - if streamInfo != nil && !streamInfo.IsActive() { - clog.Infof(ctx, "data subscribe stopping, input stream does not exist.") - break - } - var segment *http.Response - readBytes, readMessages := 0, 0 - clog.V(8).Infof(ctx, "data subscribe await") - segment, err = subscriber.Read() - if err != nil { - if errors.Is(err, trickle.EOS) || errors.Is(err, trickle.StreamNotFoundErr) { - stopProcessing(ctx, params, fmt.Errorf("data subscribe stopping, stream not found, err=%w", err)) - return - } - var sequenceNonexistent *trickle.SequenceNonexistent - if errors.As(err, &sequenceNonexistent) { - // stream exists but segment doesn't, so skip to leading edge - subscriber.SetSeq(sequenceNonexistent.Latest) - } - // TODO if not EOS then signal a new orchestrator is needed - err = fmt.Errorf("data subscribe error reading: %w", err) - clog.Infof(ctx, "%s", err) - if retries > maxRetries { - stopProcessing(ctx, params, errors.New("data subscribe stopping, retries exceeded")) - return - } - retries++ - params.liveParams.sendErrorEvent(err) - time.Sleep(retryPause) - continue - } - retries = 0 - seq := trickle.GetSeq(segment) - clog.V(8).Infof(ctx, "data subscribe received seq=%d", seq) - copyStartTime := time.Now() - - defer segment.Body.Close() - scanner := bufio.NewScanner(segment.Body) - for scanner.Scan() { - writer, err := dataWriter.Next() - if err != nil { - if err != io.EOF { - stopProcessing(ctx, params, fmt.Errorf("data subscribe could not get next: %w", err)) - } - return - } - n, err := writer.Write(scanner.Bytes()) - if err != nil { - stopProcessing(ctx, params, fmt.Errorf("data subscribe could not write: %w", err)) - } - readBytes += n - readMessages += 1 - } - if err := scanner.Err(); err != nil { - clog.InfofErr(ctx, "data subscribe error reading seq=%d", seq, err) - subscriber.SetSeq(seq) - retries++ - continue - } - - if firstSegment { - firstSegment = false - delayMs := time.Since(params.liveParams.startTime).Milliseconds() - if monitor.Enabled { - //monitor.AIFirstSegmentDelay(delayMs, params.liveParams.sess.OrchestratorInfo) - monitor.SendQueueEventAsync("stream_trace", map[string]interface{}{ - "type": "gateway_receive_first_data_segment", - "timestamp": time.Now().UnixMilli(), - "stream_id": params.liveParams.streamID, - "pipeline_id": params.liveParams.pipelineID, - "request_id": params.liveParams.requestID, - "orchestrator_info": map[string]interface{}{ - "address": orchAddr, - "url": orchUrl, - }, - }) - } - - clog.V(common.VERBOSE).Infof(ctx, "First Data Segment delay=%dms streamID=%s", delayMs, params.liveParams.streamID) - } - - clog.V(8).Info(ctx, "data subscribe read completed", "seq", seq, "bytes", humanize.Bytes(uint64(readBytes)), "messages", readMessages, "took", time.Since(copyStartTime)) - } - }() -} - -func startStreamEventsSubscribe(ctx context.Context, url *url.URL, streamInfo *core.StreamInfo) { - params := streamInfo.Params.(aiRequestParams) - subscriber, err := trickle.NewTrickleSubscriber(trickle.TrickleSubscriberConfig{ - URL: url.String(), - Ctx: ctx, - }) - if err != nil { - stopProcessing(ctx, params, fmt.Errorf("event sub init failed: %w", err)) - return - } - stream := params.liveParams.stream - streamId := params.liveParams.streamID - - // vars to check events periodically to ensure liveness - var ( - eventCheckInterval = 10 * time.Second - maxEventGap = 30 * time.Second - eventTicker = time.NewTicker(eventCheckInterval) - eventsDone = make(chan bool) - // remaining vars in this block must be protected by mutex - lastEventMu = &sync.Mutex{} - lastEvent = time.Now() - ) - - clog.Infof(ctx, "Starting event subscription for URL: %s", url.String()) - - go func() { - defer time.AfterFunc(clearStreamDelay, func() { - StreamStatusStore.Clear(streamId) - GatewayStatus.Clear(streamId) - }) - defer func() { - eventTicker.Stop() - eventsDone <- true - }() - const maxRetries = 5 - const retryPause = 300 * time.Millisecond - retries := 0 - for { - select { - case <-ctx.Done(): - clog.Info(ctx, "event subscription done") - return - default: - } - clog.Infof(ctx, "Reading from event subscription for URL: %s", url.String()) - segment, err := subscriber.Read() - if err == nil { - retries = 0 - } else { - // handle errors from event read - if errors.Is(err, trickle.EOS) || errors.Is(err, trickle.StreamNotFoundErr) { - clog.Infof(ctx, "Stopping subscription due to %s", err) - return - } - var seqErr *trickle.SequenceNonexistent - if errors.As(err, &seqErr) { - // stream exists but segment doesn't, so skip to leading edge - subscriber.SetSeq(seqErr.Latest) - } - if retries > maxRetries { - stopProcessing(ctx, params, fmt.Errorf("too many errors reading events; stopping subscription, err=%w", err)) - return - } - clog.Infof(ctx, "Error reading events subscription: err=%v retry=%d", err, retries) - retries++ - time.Sleep(retryPause) - continue - } - - body, err := io.ReadAll(segment.Body) - segment.Body.Close() - - if err != nil { - clog.Infof(ctx, "Error reading events subscription body: %s", err) - continue - } - - var eventWrapper struct { - QueueEventType string `json:"queue_event_type"` - Event map[string]interface{} `json:"event"` - } - if err := json.Unmarshal(body, &eventWrapper); err != nil { - clog.Infof(ctx, "Failed to parse JSON from events subscription: %s", err) - continue - } - - event := eventWrapper.Event - queueEventType := eventWrapper.QueueEventType - if event == nil { - // revert this once push to prod -- If no "event" field found, treat the entire body as the event - event = make(map[string]interface{}) - if err := json.Unmarshal(body, &event); err != nil { - clog.Infof(ctx, "Failed to parse JSON as direct event: %s", err) - continue - } - queueEventType = "ai_stream_events" - } - - event["stream_id"] = streamId - event["request_id"] = params.liveParams.requestID - event["pipeline_id"] = params.liveParams.pipelineID - event["orchestrator_info"] = map[string]interface{}{ - "address": clog.GetVal(ctx, "orch"), - "url": clog.GetVal(ctx, "orch_url"), - } - - clog.V(8).Infof(ctx, "Received event for seq=%d event=%+v", trickle.GetSeq(segment), event) - - // record the event time - lastEventMu.Lock() - lastEvent = time.Now() - lastEventMu.Unlock() - - eventType, ok := event["type"].(string) - if !ok { - eventType = "unknown" - clog.Warningf(ctx, "Received event without a type stream=%s event=%+v", stream, event) - } - - if eventType == "status" { - queueEventType = "ai_stream_status" - // The large logs and params fields are only sent once and then cleared to save bandwidth. So coalesce the - // incoming status with the last non-null value that we received on such fields for the status API. - lastStreamStatus, _ := StreamStatusStore.Get(streamId) - - // Check if inference_status exists in both current and last status - inferenceStatus, hasInference := event["inference_status"].(map[string]interface{}) - lastInferenceStatus, hasLastInference := lastStreamStatus["inference_status"].(map[string]interface{}) - - if hasInference { - if logs, ok := inferenceStatus["last_restart_logs"]; !ok || logs == nil { - if hasLastInference { - inferenceStatus["last_restart_logs"] = lastInferenceStatus["last_restart_logs"] - } - } - if params, ok := inferenceStatus["last_params"]; !ok || params == nil { - if hasLastInference { - inferenceStatus["last_params"] = lastInferenceStatus["last_params"] - } - } - } - - StreamStatusStore.Store(streamId, event) - } - - monitor.SendQueueEventAsync(queueEventType, event) - } - }() - - // Use events as a heartbeat of sorts: - // if no events arrive for too long, abort the job - go func() { - for { - select { - case <-eventTicker.C: - lastEventMu.Lock() - eventTime := lastEvent - lastEventMu.Unlock() - if time.Now().Sub(eventTime) > maxEventGap { - stopProcessing(ctx, params, fmt.Errorf("timeout waiting for events")) - eventTicker.Stop() - return - } - case <-eventsDone: - return - } - } - }() -} - -func ffmpegStreamOutput(ctx context.Context, outputUrl string, outWriter *media.RingBuffer, streamInfo *core.StreamInfo) { - // Clone the context since we can call this function multiple times - // Adding rtmpOut val multiple times to the same context will just stomp over old ones - ctx = clog.Clone(ctx, ctx) - ctx = clog.AddVal(ctx, "rtmpOut", outputUrl) - params := streamInfo.Params.(aiRequestParams) - defer func() { - if rec := recover(); rec != nil { - // panicked, so shut down the stream and handle it - err, ok := rec.(error) - if !ok { - err = errors.New("unknown error") - } - stopProcessing(ctx, params, fmt.Errorf("ffmpeg panic: %w", err)) - } - }() - for { - clog.V(6).Infof(ctx, "Starting output rtmp") - if streamInfo != nil && !streamInfo.IsActive() { - clog.Errorf(ctx, "Stopping output rtmp stream, input stream does not exist.") - break - } - - // we receive opus by default, but re-encode to AAC for non-local outputs - acodec := "copy" - if !strings.Contains(outputUrl, params.liveParams.localRTMPPrefix) { - acodec = "libfdk_aac" - } - - cmd := exec.CommandContext(ctx, "ffmpeg", - "-analyzeduration", "2500000", // 2.5 seconds - "-i", "pipe:0", - "-c:a", acodec, - "-c:v", "copy", - "-f", "flv", - outputUrl, - ) - // Change Cancel function to send a SIGTERM instead of SIGKILL. Still send a SIGKILL after 5s (WaitDelay) if it's stuck. - cmd.Cancel = func() error { - return cmd.Process.Signal(syscall.SIGTERM) - } - cmd.WaitDelay = 5 * time.Second - cmd.Stdin = outWriter.MakeReader() // start at leading edge of output for each retry - output, err := cmd.CombinedOutput() - clog.Infof(ctx, "Process err=%v output: %s", err, output) - - select { - case <-ctx.Done(): - clog.Info(ctx, "Context done, stopping rtmp output") - return // Returns context.Canceled or context.DeadlineExceeded - default: - // Context is still active, continue with normal processing - } - - time.Sleep(5 * time.Second) - } -} From ec00c5e2b9e93da878d4b72ee6336ff885f681aa Mon Sep 17 00:00:00 2001 From: Brad P Date: Fri, 5 Sep 2025 12:44:04 -0500 Subject: [PATCH 42/57] slim down StreamInfo in external capabilities used by Orchestrator --- core/external_capabilities.go | 38 ++++------------------------------- server/job_stream.go | 5 ++--- 2 files changed, 6 insertions(+), 37 deletions(-) diff --git a/core/external_capabilities.go b/core/external_capabilities.go index 602af447df..314b942b6c 100644 --- a/core/external_capabilities.go +++ b/core/external_capabilities.go @@ -10,7 +10,6 @@ import ( ethcommon "github.com/ethereum/go-ethereum/common" "github.com/golang/glog" - "github.com/livepeer/go-livepeer/media" "github.com/livepeer/go-livepeer/net" "github.com/livepeer/go-livepeer/trickle" ) @@ -47,29 +46,16 @@ type ExternalCapability struct { type StreamInfo struct { StreamID string Capability string - //Gateway fields - StreamRequest []byte - ExcludeOrchs []string - OrchToken *JobToken - OrchUrl string - OrchPublishUrl string - OrchSubscribeUrl string - OrchControlUrl string - OrchEventsUrl string - OrchDataUrl string - ControlPub *trickle.TricklePublisher - StopControl func() //Orchestrator fields Sender ethcommon.Address + StreamRequest []byte pubChannel *trickle.TrickleLocalPublisher subChannel *trickle.TrickleLocalPublisher controlChannel *trickle.TrickleLocalPublisher eventsChannel *trickle.TrickleLocalPublisher dataChannel *trickle.TrickleLocalPublisher //Stream fields - Params interface{} - DataWriter *media.SegmentWriter JobParams string StreamCtx context.Context CancelStream context.CancelFunc @@ -82,19 +68,13 @@ func (sd *StreamInfo) IsActive() bool { return false } - if sd.controlChannel == nil && sd.ControlPub == nil { + if sd.controlChannel == nil { return false } return true } -func (sd *StreamInfo) ExcludeOrch(orchUrl string) { - sd.sdm.Lock() - defer sd.sdm.Unlock() - sd.ExcludeOrchs = append(sd.ExcludeOrchs, orchUrl) -} - func (sd *StreamInfo) UpdateParams(params string) { sd.sdm.Lock() defer sd.sdm.Unlock() @@ -123,7 +103,7 @@ func NewExternalCapabilities() *ExternalCapabilities { } } -func (extCaps *ExternalCapabilities) AddStream(streamID string, pipeline string, params interface{}, streamReq []byte) (*StreamInfo, error) { +func (extCaps *ExternalCapabilities) AddStream(streamID string, capability string, streamReq []byte) (*StreamInfo, error) { extCaps.capm.Lock() defer extCaps.capm.Unlock() _, ok := extCaps.Streams[streamID] @@ -135,8 +115,7 @@ func (extCaps *ExternalCapabilities) AddStream(streamID string, pipeline string, ctx, cancel := context.WithCancel(context.Background()) stream := StreamInfo{ StreamID: streamID, - Capability: pipeline, - Params: params, // Store the interface value directly, not a pointer to it + Capability: capability, StreamRequest: streamReq, StreamCtx: ctx, CancelStream: cancel, @@ -147,15 +126,6 @@ func (extCaps *ExternalCapabilities) AddStream(streamID string, pipeline string, go func() { <-ctx.Done() - //gateway channels shutdown - if stream.DataWriter != nil { - stream.DataWriter.Close() - } - if stream.ControlPub != nil { - stream.StopControl() - stream.ControlPub.Close() - } - //orchestrator channels shutdown if stream.pubChannel != nil { stream.pubChannel.Close() diff --git a/server/job_stream.go b/server/job_stream.go index b729fffde4..54db89c589 100644 --- a/server/job_stream.go +++ b/server/job_stream.go @@ -3,7 +3,6 @@ package server import ( "bytes" "context" - "encoding/base64" "encoding/json" "errors" "fmt" @@ -1042,7 +1041,7 @@ func (h *lphttp) StartStream(w http.ResponseWriter, r *http.Request) { w.Header().Set("X-Data-Url", dataUrl) } - reqBodyForRunner["request"] = base64.StdEncoding.EncodeToString(body) + reqBodyForRunner["request"] = string(body) reqBodyBytes, err := json.Marshal(reqBodyForRunner) if err != nil { clog.Errorf(ctx, "Failed to marshal request body err=%v", err) @@ -1089,7 +1088,7 @@ func (h *lphttp) StartStream(w http.ResponseWriter, r *http.Request) { clog.V(common.SHORT).Infof(ctx, "stream start processed successfully took=%v balance=%v", time.Since(start), getPaymentBalance(orch, orchJob.Sender, orchJob.Req.Capability).FloatString(0)) //setup the stream - stream, err := h.node.ExternalCapabilities.AddStream(orchJob.Req.ID, orchJob.Req.Capability, orchJob.Req, respBody) + stream, err := h.node.ExternalCapabilities.AddStream(orchJob.Req.ID, orchJob.Req.Capability, reqBodyBytes) if err != nil { clog.Errorf(ctx, "Error adding stream to external capabilities: %v", err) respondWithError(w, "Error adding stream to external capabilities", http.StatusInternalServerError) From 85e5c25dfa624375acad06799f9afeb8a53e62b7 Mon Sep 17 00:00:00 2001 From: Brad P Date: Fri, 5 Sep 2025 16:04:23 -0500 Subject: [PATCH 43/57] get new token for stop request --- server/job_stream.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/job_stream.go b/server/job_stream.go index 54db89c589..f37232d347 100644 --- a/server/job_stream.go +++ b/server/job_stream.go @@ -100,6 +100,7 @@ func (ls *LivepeerServer) StopStream() http.Handler { stopJob.sign() //no changes to make, sign job token, err := sessionToToken(params.liveParams.sess) + newToken, err := getToken(ctx, 3*time.Second, token.ServiceAddr, stopJob.Job.Req.Capability, stopJob.Job.Req.Sender, stopJob.Job.Req.Sig) if err != nil { clog.Errorf(ctx, "Error converting session to token: %s", err) http.Error(w, err.Error(), http.StatusBadRequest) @@ -114,7 +115,7 @@ func (ls *LivepeerServer) StopStream() http.Handler { } defer r.Body.Close() - resp, code, err := ls.sendJobToOrch(ctx, r, stopJob.Job.Req, stopJob.SignedJobReq, token, "/ai/stream/stop", body) + resp, code, err := ls.sendJobToOrch(ctx, r, stopJob.Job.Req, stopJob.SignedJobReq, *newToken, "/ai/stream/stop", body) if err != nil { clog.Errorf(ctx, "Error sending job to orchestrator: %s", err) http.Error(w, err.Error(), code) @@ -899,7 +900,7 @@ func (ls *LivepeerServer) UpdateStream() http.Handler { controlPub := stream.ControlPub if controlPub == nil { - clog.Info(ctx, "No orchestrator available, caching params", "stream", stream, "params", params) + clog.Info(ctx, "No orchestrator available, caching params", "stream", streamId, "params", params) return } From 9182ef87aa3332167bcf99f466372cc11827b02a Mon Sep 17 00:00:00 2001 From: Brad P Date: Fri, 5 Sep 2025 16:14:45 -0500 Subject: [PATCH 44/57] some cleanup --- server/job_stream.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/job_stream.go b/server/job_stream.go index f37232d347..b36701b4bd 100644 --- a/server/job_stream.go +++ b/server/job_stream.go @@ -207,7 +207,7 @@ func (ls *LivepeerServer) runStream(gatewayJob *gatewayJob) { // or if passing `nil` as a CancelCause err = nil } - if !ls.LivepeerNode.ExternalCapabilities.StreamExists(streamID) { + if !params.inputStreamExists() { clog.Info(ctx, "No stream exists, skipping orchestrator swap") break } @@ -272,7 +272,7 @@ func (ls *LivepeerServer) monitorStream(streamId string) { select { case <-stream.StreamCtx.Done(): clog.Infof(ctx, "Stream %s stopped, ending monitoring", streamId) - ls.LivepeerNode.ExternalCapabilities.RemoveStream(streamId) + delete(ls.LivepeerNode.LivePipelines, streamId) return case <-pmtTicker.C: if !params.inputStreamExists() { @@ -291,6 +291,7 @@ func (ls *LivepeerServer) monitorStream(streamId string) { } // fetch new JobToken with each payment + // update the session for the LivePipeline with new token newToken, err := getToken(ctx, 3*time.Second, token.ServiceAddr, stream.Pipeline, jobSender.Addr, jobSender.Sig) if err != nil { clog.Errorf(ctx, "Error getting new token for %s: %v", token.ServiceAddr, err) From 55666ef449ed378047d4e07c4fee08543ea44468 Mon Sep 17 00:00:00 2001 From: Brad P Date: Tue, 9 Sep 2025 15:47:28 -0500 Subject: [PATCH 45/57] fix merge update --- core/livepeernode.go | 4 ++-- server/job_stream.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/livepeernode.go b/core/livepeernode.go index 4720c6e850..ce41d1726d 100644 --- a/core/livepeernode.go +++ b/core/livepeernode.go @@ -181,6 +181,8 @@ type LivePipeline struct { Pipeline string ControlPub *trickle.TricklePublisher StopControl func() + ReportUpdate func([]byte) + DataWriter *media.SegmentWriter StreamCtx context.Context streamCancel context.CancelCauseFunc @@ -217,8 +219,6 @@ func (p *LivePipeline) StreamRequest() []byte { func (p *LivePipeline) StopStream(err error) { p.StopControl() p.streamCancel(err) - DataWriter *media.SegmentWriter - ReportUpdate func([]byte) } // NewLivepeerNode creates a new Livepeer Node. Eth can be nil. diff --git a/server/job_stream.go b/server/job_stream.go index b36701b4bd..a6dc05b16f 100644 --- a/server/job_stream.go +++ b/server/job_stream.go @@ -897,7 +897,7 @@ func (ls *LivepeerServer) UpdateStream() http.Handler { } params := string(data) - stream.Params = params + stream.Params = data controlPub := stream.ControlPub if controlPub == nil { From ddf5639fbd462be55997b5918e1a43b753f9bf51 Mon Sep 17 00:00:00 2001 From: Victor Elias Date: Thu, 11 Sep 2025 00:35:44 -0300 Subject: [PATCH 46/57] Add sdxl and faceid docker containers (#3738) * Add SDXL and SDXL FaceID streamdiffusion images Co-authored-by: victorgelias * Refactor: Clarify docker image descriptions Co-authored-by: victorgelias --------- Co-authored-by: Cursor Agent --- ai/worker/docker.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ai/worker/docker.go b/ai/worker/docker.go index f7a9a3393b..c061c188b5 100644 --- a/ai/worker/docker.go +++ b/ai/worker/docker.go @@ -68,8 +68,12 @@ var pipelineToImage = map[string]string{ } var livePipelineToImage = map[string]string{ "streamdiffusion": "livepeer/ai-runner:live-app-streamdiffusion", - // streamdiffusion-sd15 is a utility image that uses a SD1.5 model on the default config of the pipeline. Optimizes startup time. + // streamdiffusion-sd15 is a utility image that uses an SD1.5 model on the default config of the pipeline. Optimizes startup time. "streamdiffusion-sd15": "livepeer/ai-runner:live-app-streamdiffusion-sd15", + // streamdiffusion-sdxl is a utility image that uses an SDXL model on the default config of the pipeline. Optimizes startup time. + "streamdiffusion-sdxl": "livepeer/ai-runner:live-app-streamdiffusion-sdxl", + // streamdiffusion-sdxl-faceid is a utility image that uses an SDXL model with a FaceID IP Adapter on the default config of the pipeline. Optimizes startup time. + "streamdiffusion-sdxl-faceid": "livepeer/ai-runner:live-app-streamdiffusion-sdxl-faceid", "comfyui": "livepeer/ai-runner:live-app-comfyui", "segment_anything_2": "livepeer/ai-runner:live-app-segment_anything_2", "noop": "livepeer/ai-runner:live-app-noop", From f79d2fe9df04dac08317c8e1819a1c310810d97b Mon Sep 17 00:00:00 2001 From: Brad P Date: Fri, 12 Sep 2025 13:26:01 -0500 Subject: [PATCH 47/57] error handling, logging, lock stream info when checking active --- core/ai_orchestrator.go | 2 +- core/external_capabilities.go | 22 +++++++++++++++++----- server/job_stream.go | 5 +++++ 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/core/ai_orchestrator.go b/core/ai_orchestrator.go index ddd81e90e1..9f69138ccc 100644 --- a/core/ai_orchestrator.go +++ b/core/ai_orchestrator.go @@ -1203,7 +1203,7 @@ func (orch *orchestrator) JobPriceInfo(sender ethcommon.Address, jobCapability s //ensure price numerator and denominator can be int64 jobPrice, err = common.PriceToInt64(jobPrice) if err != nil { - return nil, err + return nil, fmt.Errorf("invalid job price: %w", err) } return &net.PriceInfo{ diff --git a/core/external_capabilities.go b/core/external_capabilities.go index 314b942b6c..69c6ddafde 100644 --- a/core/external_capabilities.go +++ b/core/external_capabilities.go @@ -64,6 +64,8 @@ type StreamInfo struct { } func (sd *StreamInfo) IsActive() bool { + sd.sdm.Lock() + defer sd.sdm.Unlock() if sd.StreamCtx.Err() != nil { return false } @@ -128,19 +130,29 @@ func (extCaps *ExternalCapabilities) AddStream(streamID string, capability strin //orchestrator channels shutdown if stream.pubChannel != nil { - stream.pubChannel.Close() + if err := stream.pubChannel.Close(); err != nil { + glog.Errorf("error closing pubChannel for stream=%s: %v", streamID, err) + } } if stream.subChannel != nil { - stream.subChannel.Close() + if err := stream.subChannel.Close(); err != nil { + glog.Errorf("error closing subChannel for stream=%s: %v", streamID, err) + } } if stream.controlChannel != nil { - stream.controlChannel.Close() + if err := stream.controlChannel.Close(); err != nil { + glog.Errorf("error closing controlChannel for stream=%s: %v", streamID, err) + } } if stream.eventsChannel != nil { - stream.eventsChannel.Close() + if err := stream.eventsChannel.Close(); err != nil { + glog.Errorf("error closing eventsChannel for stream=%s: %v", streamID, err) + } } if stream.dataChannel != nil { - stream.dataChannel.Close() + if err := stream.dataChannel.Close(); err != nil { + glog.Errorf("error closing dataChannel for stream=%s: %v", streamID, err) + } } return }() diff --git a/server/job_stream.go b/server/job_stream.go index a6dc05b16f..456c35153a 100644 --- a/server/job_stream.go +++ b/server/job_stream.go @@ -100,6 +100,11 @@ func (ls *LivepeerServer) StopStream() http.Handler { stopJob.sign() //no changes to make, sign job token, err := sessionToToken(params.liveParams.sess) + if err != nil { + clog.Errorf(ctx, "Error converting session to token: %s", err) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } newToken, err := getToken(ctx, 3*time.Second, token.ServiceAddr, stopJob.Job.Req.Capability, stopJob.Job.Req.Sender, stopJob.Job.Req.Sig) if err != nil { clog.Errorf(ctx, "Error converting session to token: %s", err) From 30af12354034e7ccdc6e543e286308d5c27948eb Mon Sep 17 00:00:00 2001 From: Brad P Date: Fri, 12 Sep 2025 13:34:38 -0500 Subject: [PATCH 48/57] create get new token timeout --- server/job_stream.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/server/job_stream.go b/server/job_stream.go index 456c35153a..b4332356d1 100644 --- a/server/job_stream.go +++ b/server/job_stream.go @@ -26,6 +26,8 @@ import ( "github.com/livepeer/go-tools/drivers" ) +var getNewTokenTimeout = 3 * time.Second + func (ls *LivepeerServer) StartStream() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method == http.MethodOptions { @@ -105,7 +107,7 @@ func (ls *LivepeerServer) StopStream() http.Handler { http.Error(w, err.Error(), http.StatusBadRequest) return } - newToken, err := getToken(ctx, 3*time.Second, token.ServiceAddr, stopJob.Job.Req.Capability, stopJob.Job.Req.Sender, stopJob.Job.Req.Sig) + newToken, err := getToken(ctx, getNewTokenTimeout, token.ServiceAddr, stopJob.Job.Req.Capability, stopJob.Job.Req.Sender, stopJob.Job.Req.Sig) if err != nil { clog.Errorf(ctx, "Error converting session to token: %s", err) http.Error(w, err.Error(), http.StatusBadRequest) @@ -158,7 +160,7 @@ func (ls *LivepeerServer) runStream(gatewayJob *gatewayJob) { clog.Infof(ctx, "Starting stream processing") //refresh the token if not first Orch to confirm capacity and new ticket params if firstProcessed { - newToken, err := getToken(ctx, 3*time.Second, orch.ServiceAddr, gatewayJob.Job.Req.Capability, gatewayJob.Job.Req.Sender, gatewayJob.Job.Req.Sig) + newToken, err := getToken(ctx, getNewTokenTimeout, orch.ServiceAddr, gatewayJob.Job.Req.Capability, gatewayJob.Job.Req.Sender, gatewayJob.Job.Req.Sig) if err != nil { clog.Errorf(ctx, "Error getting token for orch=%v err=%v", orch.ServiceAddr, err) continue @@ -297,7 +299,7 @@ func (ls *LivepeerServer) monitorStream(streamId string) { // fetch new JobToken with each payment // update the session for the LivePipeline with new token - newToken, err := getToken(ctx, 3*time.Second, token.ServiceAddr, stream.Pipeline, jobSender.Addr, jobSender.Sig) + newToken, err := getToken(ctx, getNewTokenTimeout, token.ServiceAddr, stream.Pipeline, jobSender.Addr, jobSender.Sig) if err != nil { clog.Errorf(ctx, "Error getting new token for %s: %v", token.ServiceAddr, err) continue From d648a5933f9f1fb03d588d875e5f53bc12ae3cfd Mon Sep 17 00:00:00 2001 From: Brad P Date: Fri, 12 Sep 2025 13:41:49 -0500 Subject: [PATCH 49/57] add retries to getToken --- server/job_rpc.go | 60 +++++++++++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 23 deletions(-) diff --git a/server/job_rpc.go b/server/job_rpc.go index 0932345e6d..168c49139f 100644 --- a/server/job_rpc.go +++ b/server/job_rpc.go @@ -1267,32 +1267,46 @@ func getToken(ctx context.Context, respTimeout time.Duration, orchUrl, capabilit return nil, err } - resp, err := sendJobReqWithTimeout(tokenReq, respTimeout) - if err != nil { - clog.Errorf(ctx, "failed to get token from Orchestrator err=%v", err) - return nil, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - clog.Errorf(ctx, "Failed to get token from Orchestrator %v err=%v", orchUrl, err) - return nil, fmt.Errorf("failed to get token from Orchestrator") - } - - latency := time.Since(start) - clog.V(common.DEBUG).Infof(ctx, "Received job token from uri=%v, latency=%v", orchUrl, latency) + var resp *http.Response + var token []byte + var jobToken core.JobToken + var attempt int + var backoff time.Duration = 100 * time.Millisecond + deadline := time.Now().Add(respTimeout) - token, err := io.ReadAll(resp.Body) - if err != nil { - clog.Errorf(ctx, "Failed to read token from Orchestrator %v err=%v", orchUrl, err) - return nil, err + for attempt = 0; attempt < 3; attempt++ { + resp, err = sendJobReqWithTimeout(tokenReq, respTimeout) + if err != nil { + clog.Errorf(ctx, "failed to get token from Orchestrator (attempt %d) err=%v", attempt+1, err) + } else if resp.StatusCode != http.StatusOK { + clog.Errorf(ctx, "Failed to get token from Orchestrator %v status=%v (attempt %d)", orchUrl, resp.StatusCode, attempt+1) + } else { + defer resp.Body.Close() + latency := time.Since(start) + clog.V(common.DEBUG).Infof(ctx, "Received job token from uri=%v, latency=%v", orchUrl, latency) + token, err = io.ReadAll(resp.Body) + if err != nil { + clog.Errorf(ctx, "Failed to read token from Orchestrator %v err=%v", orchUrl, err) + } else { + err = json.Unmarshal(token, &jobToken) + if err != nil { + clog.Errorf(ctx, "Failed to unmarshal token from Orchestrator %v err=%v", orchUrl, err) + } else { + return &jobToken, nil + } + } + } + // If not last attempt and time remains, backoff + if time.Now().Add(backoff).Before(deadline) && attempt < 2 { + time.Sleep(backoff) + backoff *= 2 + } else { + break + } } - var jobToken core.JobToken - err = json.Unmarshal(token, &jobToken) + // All attempts failed if err != nil { - clog.Errorf(ctx, "Failed to unmarshal token from Orchestrator %v err=%v", orchUrl, err) return nil, err } - - return &jobToken, nil + return nil, fmt.Errorf("failed to get token from Orchestrator after %d attempts", attempt) } From da54cbfcb35d78e8fce2bd9f8a4c804618fc661d Mon Sep 17 00:00:00 2001 From: Brad P Date: Fri, 12 Sep 2025 13:51:36 -0500 Subject: [PATCH 50/57] add logging end response for too large body --- server/job_stream.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/server/job_stream.go b/server/job_stream.go index b4332356d1..f4d8d1cf4b 100644 --- a/server/job_stream.go +++ b/server/job_stream.go @@ -48,6 +48,8 @@ func (ls *LivepeerServer) StartStream() http.Handler { return } + //setup body size limit, will error if too large + r.Body = http.MaxBytesReader(w, r.Body, 10<<20) streamUrls, code, err := ls.setupStream(ctx, r, gatewayJob) if err != nil { clog.Errorf(ctx, "Error setting up stream: %s", err) @@ -383,9 +385,15 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, job // Setup request body to be able to preserve for retries // Read the entire body first with 10MB limit - bodyBytes, err := io.ReadAll(io.LimitReader(r.Body, 10<<20)) + bodyBytes, err := io.ReadAll(r.Body) if err != nil { - return nil, http.StatusBadRequest, err + if errors.As(err, http.MaxBytesError{}) { + clog.Warningf(ctx, "Request body too large (over 10MB)") + return nil, http.StatusRequestEntityTooLarge, fmt.Errorf("request body too large (max 10MB)") + } else { + clog.Errorf(ctx, "Error reading request body: %v", err) + return nil, http.StatusBadRequest, fmt.Errorf("error reading request body: %w", err) + } } r.Body.Close() From dcc98b1e7a3b591f82cc8ecf58b267b19a4b302d Mon Sep 17 00:00:00 2001 From: Brad P Date: Fri, 12 Sep 2025 15:59:01 -0500 Subject: [PATCH 51/57] move payment processing to separate functions: orch->processPayment, gateway->sendPaymentForStream --- server/job_rpc.go | 46 ++++++++------- server/job_stream.go | 131 +++++++++++++++++++++++-------------------- 2 files changed, 98 insertions(+), 79 deletions(-) diff --git a/server/job_rpc.go b/server/job_rpc.go index 168c49139f..b9f7889d11 100644 --- a/server/job_rpc.go +++ b/server/job_rpc.go @@ -805,6 +805,7 @@ func (h *lphttp) setupOrchJob(ctx context.Context, r *http.Request, reserveCapac } sender := ethcommon.HexToAddress(jobReq.Sender) + jobPrice, err := orch.JobPriceInfo(sender, jobReq.Capability) if err != nil { return nil, errors.New("Could not get job price") @@ -813,29 +814,15 @@ func (h *lphttp) setupOrchJob(ctx context.Context, r *http.Request, reserveCapac //no payment included, confirm if balance remains jobPriceRat := big.NewRat(jobPrice.PricePerUnit, jobPrice.PixelsPerUnit) - var payment net.Payment orchBal := big.NewRat(0, 1) // if price is 0, no payment required if jobPriceRat.Cmp(big.NewRat(0, 1)) > 0 { - // get payment information - paymentHdr := r.Header.Get(jobPaymentHeaderHdr) minBal := new(big.Rat).Mul(jobPriceRat, big.NewRat(60, 1)) //minimum 1 minute balance - if paymentHdr != "" { - payment, err = getPayment(paymentHdr) - if err != nil { - clog.Errorf(ctx, "job payment invalid: %v", err) - return nil, errPaymentError - } - - if err := orch.ProcessPayment(ctx, payment, core.ManifestID(jobReq.Capability)); err != nil { - orch.FreeExternalCapabilityCapacity(jobReq.Capability) - clog.Errorf(ctx, "Error processing payment: %v", err) - return nil, errPaymentError - } - //update balance for payment - orchBal = getPaymentBalance(orch, sender, jobReq.Capability) - } else { - orchBal = getPaymentBalance(orch, sender, jobReq.Capability) + //process payment if included + orchBal, pmtErr := processPayment(ctx, orch, sender, jobReq.Capability, r.Header.Get(jobPaymentHeaderHdr)) + if pmtErr != nil { + //log if there are payment errors but continue, balance will runout and clean up + clog.Infof(ctx, "job payment error: %v", pmtErr) } if orchBal.Cmp(minBal) < 0 { @@ -855,6 +842,27 @@ func (h *lphttp) setupOrchJob(ctx context.Context, r *http.Request, reserveCapac return &orchJob{Req: jobReq, Sender: sender, JobPrice: jobPrice, Details: &jobDetails}, nil } +// process payment and return balance +func processPayment(ctx context.Context, orch Orchestrator, sender ethcommon.Address, capability string, paymentHdr string) (*big.Rat, error) { + if paymentHdr != "" { + payment, err := getPayment(paymentHdr) + if err != nil { + clog.Errorf(ctx, "job payment invalid: %v", err) + return nil, errPaymentError + } + + if err := orch.ProcessPayment(ctx, payment, core.ManifestID(capability)); err != nil { + orch.FreeExternalCapabilityCapacity(capability) + clog.Errorf(ctx, "Error processing payment: %v", err) + return nil, errPaymentError + } + } + orchBal := getPaymentBalance(orch, sender, capability) + + return orchBal, nil + +} + func createPayment(ctx context.Context, jobReq *JobRequest, orchToken *core.JobToken, node *core.LivepeerNode) (string, error) { var payment *net.Payment createTickets := true diff --git a/server/job_stream.go b/server/job_stream.go index f4d8d1cf4b..61f8f58945 100644 --- a/server/job_stream.go +++ b/server/job_stream.go @@ -288,73 +288,84 @@ func (ls *LivepeerServer) monitorStream(streamId string) { clog.Infof(ctx, "Input stream does not exist for stream %s, ending monitoring", streamId) return } - params, err = getStreamRequestParams(stream) - if err != nil { - clog.Errorf(ctx, "Error getting stream request params: %v", err) - continue - } - token, err := sessionToToken(params.liveParams.sess) - if err != nil { - clog.Errorf(ctx, "Error getting token for session: %v", err) - continue - } - // fetch new JobToken with each payment - // update the session for the LivePipeline with new token - newToken, err := getToken(ctx, getNewTokenTimeout, token.ServiceAddr, stream.Pipeline, jobSender.Addr, jobSender.Sig) - if err != nil { - clog.Errorf(ctx, "Error getting new token for %s: %v", token.ServiceAddr, err) - continue - } - newSess, err := tokenToAISession(*newToken) + err := ls.sendPaymentForStream(ctx, stream, jobSender) if err != nil { - clog.Errorf(ctx, "Error converting token to AI session: %v", err) - continue + clog.Errorf(ctx, "Error sending payment for stream %s: %v", streamId, err) } - params.liveParams.sess = &newSess - stream.UpdateStreamParams(params) + } + } +} - // send the payment - jobDetails := JobRequestDetails{StreamId: streamId} - jobDetailsStr, err := json.Marshal(jobDetails) - if err != nil { - clog.Errorf(ctx, "Error marshalling job details: %v", err) - continue - } - req := &JobRequest{Request: string(jobDetailsStr), Parameters: "{}", Capability: stream.Pipeline, - Sender: ls.LivepeerNode.OrchestratorPool.Broadcaster().Address().Hex(), - Timeout: 60, - } - //sign the request - job := gatewayJob{Job: &orchJob{Req: req}, node: ls.LivepeerNode} - err = job.sign() - if err != nil { - clog.Errorf(ctx, "Error signing job, continuing monitoring: %v", err) - continue - } +func (ls *LivepeerServer) sendPaymentForStream(ctx context.Context, stream *core.LivePipeline, jobSender *core.JobSender) error { + params, err := getStreamRequestParams(stream) + if err != nil { + clog.Errorf(ctx, "Error getting stream request params: %v", err) + return err + } + token, err := sessionToToken(params.liveParams.sess) + if err != nil { + clog.Errorf(ctx, "Error getting token for session: %v", err) + return err + } - pmtHdr, err := createPayment(ctx, req, newToken, ls.LivepeerNode) - if err != nil { - clog.Errorf(ctx, "Error processing stream payment for %s: %v", streamId, err) - // Continue monitoring even if payment fails - } - if pmtHdr == "" { - continue - } + // fetch new JobToken with each payment + // update the session for the LivePipeline with new token + newToken, err := getToken(ctx, getNewTokenTimeout, token.ServiceAddr, stream.Pipeline, jobSender.Addr, jobSender.Sig) + if err != nil { + clog.Errorf(ctx, "Error getting new token for %s: %v", token.ServiceAddr, err) + return err + } + newSess, err := tokenToAISession(*newToken) + if err != nil { + clog.Errorf(ctx, "Error converting token to AI session: %v", err) + return err + } + params.liveParams.sess = &newSess + stream.UpdateStreamParams(params) - //send the payment, update the stream with the refreshed token - clog.Infof(ctx, "Sending stream payment for %s", streamId) - statusCode, err := ls.sendPayment(ctx, token.ServiceAddr+"/ai/stream/payment", stream.Pipeline, job.SignedJobReq, pmtHdr) - if err != nil { - clog.Errorf(ctx, "Error sending stream payment for %s: %v", streamId, err) - continue - } - if statusCode != http.StatusOK { - clog.Errorf(ctx, "Unexpected status code %d received for %s", statusCode, streamId) - continue - } - } + // send the payment + streamID := params.liveParams.streamID + jobDetails := JobRequestDetails{StreamId: streamID} + jobDetailsStr, err := json.Marshal(jobDetails) + if err != nil { + clog.Errorf(ctx, "Error marshalling job details: %v", err) + return err } + req := &JobRequest{Request: string(jobDetailsStr), Parameters: "{}", Capability: stream.Pipeline, + Sender: jobSender.Addr, + Timeout: 60, + } + //sign the request + job := gatewayJob{Job: &orchJob{Req: req}, node: ls.LivepeerNode} + err = job.sign() + if err != nil { + clog.Errorf(ctx, "Error signing job, continuing monitoring: %v", err) + return err + } + + pmtHdr, err := createPayment(ctx, req, newToken, ls.LivepeerNode) + if err != nil { + clog.Errorf(ctx, "Error processing stream payment for %s: %v", streamID, err) + // Continue monitoring even if payment fails + } + if pmtHdr == "" { + return errors.New("empty payment header") + } + + //send the payment, update the stream with the refreshed token + clog.Infof(ctx, "Sending stream payment for %s", streamID) + statusCode, err := ls.sendPayment(ctx, token.ServiceAddr+"/ai/stream/payment", stream.Pipeline, job.SignedJobReq, pmtHdr) + if err != nil { + clog.Errorf(ctx, "Error sending stream payment for %s: %v", streamID, err) + return err + } + if statusCode != http.StatusOK { + clog.Errorf(ctx, "Unexpected status code %d received for %s", statusCode, streamID) + return errors.New("unexpected status code") + } + + return nil } type StartRequest struct { From 5881ea988f85a92e2d1fd73310676e9992f3b5fc Mon Sep 17 00:00:00 2001 From: Brad P Date: Tue, 16 Sep 2025 13:14:42 -0500 Subject: [PATCH 52/57] fix error check, reduce segment writer history, fix stop stream --- server/job_stream.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/server/job_stream.go b/server/job_stream.go index 61f8f58945..b8f94fb467 100644 --- a/server/job_stream.go +++ b/server/job_stream.go @@ -102,6 +102,12 @@ func (ls *LivepeerServer) StopStream() http.Handler { return } stopJob.sign() //no changes to make, sign job + //setup sender + jobSender, err := getJobSender(ctx, ls.LivepeerNode) + if err != nil { + clog.Errorf(ctx, "Error getting job sender: %v", err) + return + } token, err := sessionToToken(params.liveParams.sess) if err != nil { @@ -109,7 +115,7 @@ func (ls *LivepeerServer) StopStream() http.Handler { http.Error(w, err.Error(), http.StatusBadRequest) return } - newToken, err := getToken(ctx, getNewTokenTimeout, token.ServiceAddr, stopJob.Job.Req.Capability, stopJob.Job.Req.Sender, stopJob.Job.Req.Sig) + newToken, err := getToken(ctx, getNewTokenTimeout, token.ServiceAddr, stopJob.Job.Req.Capability, jobSender.Addr, jobSender.Sig) if err != nil { clog.Errorf(ctx, "Error converting session to token: %s", err) http.Error(w, err.Error(), http.StatusBadRequest) @@ -350,7 +356,8 @@ func (ls *LivepeerServer) sendPaymentForStream(ctx context.Context, stream *core // Continue monitoring even if payment fails } if pmtHdr == "" { - return errors.New("empty payment header") + // This is no payment required, error logged above + return nil } //send the payment, update the stream with the refreshed token @@ -398,9 +405,9 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, job // Read the entire body first with 10MB limit bodyBytes, err := io.ReadAll(r.Body) if err != nil { - if errors.As(err, http.MaxBytesError{}) { + if maxErr, ok := err.(*http.MaxBytesError); ok { clog.Warningf(ctx, "Request body too large (over 10MB)") - return nil, http.StatusRequestEntityTooLarge, fmt.Errorf("request body too large (max 10MB)") + return nil, http.StatusRequestEntityTooLarge, fmt.Errorf("request body too large (max %d bytes)", maxErr.Limit) } else { clog.Errorf(ctx, "Error reading request body: %v", err) return nil, http.StatusBadRequest, fmt.Errorf("error reading request body: %w", err) @@ -591,7 +598,7 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, job return nil, http.StatusBadRequest, errors.New("invalid job parameters") } if jobParams.EnableDataOutput { - params.liveParams.dataWriter = media.NewSegmentWriter(5) + params.liveParams.dataWriter = media.NewSegmentWriter(1) } //check if stream exists @@ -885,8 +892,9 @@ func (ls *LivepeerServer) GetStreamData() http.Handler { clog.Errorf(ctx, "Error reading from ring buffer: %v", err) return } - + start := time.Now() data, err := io.ReadAll(reader) + clog.V(6).Infof(ctx, "SSE data read took %v", time.Since(start)) fmt.Fprintf(w, "data: %s\n\n", data) flusher.Flush() } From 6fb53081dd46a4fa83c24b0824ca6fcb3ef6ea15 Mon Sep 17 00:00:00 2001 From: Brad P Date: Tue, 16 Sep 2025 17:27:05 -0500 Subject: [PATCH 53/57] close data segment writer after right and put segment writer back to 5 segment history --- server/ai_live_video.go | 3 +++ server/job_stream.go | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/server/ai_live_video.go b/server/ai_live_video.go index 60d074ed2d..bf1ad61eaa 100644 --- a/server/ai_live_video.go +++ b/server/ai_live_video.go @@ -864,6 +864,7 @@ func startDataSubscribe(ctx context.Context, url *url.URL, params aiRequestParam scanner := bufio.NewScanner(segment.Body) for scanner.Scan() { writer, err := dataWriter.Next() + clog.V(8).Infof(ctx, "data subscribe writing seq=%d", seq) if err != nil { if err != io.EOF { stopProcessing(ctx, params, fmt.Errorf("data subscribe could not get next: %w", err)) @@ -876,6 +877,8 @@ func startDataSubscribe(ctx context.Context, url *url.URL, params aiRequestParam } readBytes += n readMessages += 1 + + writer.Close() } if err := scanner.Err(); err != nil { clog.InfofErr(ctx, "data subscribe error reading seq=%d", seq, err) diff --git a/server/job_stream.go b/server/job_stream.go index b8f94fb467..00d09db7fa 100644 --- a/server/job_stream.go +++ b/server/job_stream.go @@ -598,7 +598,7 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, job return nil, http.StatusBadRequest, errors.New("invalid job parameters") } if jobParams.EnableDataOutput { - params.liveParams.dataWriter = media.NewSegmentWriter(1) + params.liveParams.dataWriter = media.NewSegmentWriter(5) } //check if stream exists From 1ca67c4504903554c37464df7b2c5c3e8e4ea0ed Mon Sep 17 00:00:00 2001 From: Brad P Date: Fri, 19 Sep 2025 18:01:13 -0500 Subject: [PATCH 54/57] tests tests tests --- common/testutil.go | 15 +- core/livepeernode.go | 23 +- server/job_rpc.go | 12 + server/job_rpc_test.go | 240 +++++- server/job_stream.go | 117 ++- server/job_stream_test.go | 1597 +++++++++++++++++++++++++++++++++++++ 6 files changed, 1924 insertions(+), 80 deletions(-) create mode 100644 server/job_stream_test.go diff --git a/common/testutil.go b/common/testutil.go index a3d98a6a9b..4ac044f1d1 100644 --- a/common/testutil.go +++ b/common/testutil.go @@ -90,12 +90,23 @@ func IgnoreRoutines() []goleak.Option { "github.com/livepeer/go-livepeer/server.(*LivepeerServer).HandlePush.func1", "github.com/rjeczalik/notify.(*nonrecursiveTree).dispatch", "github.com/rjeczalik/notify.(*nonrecursiveTree).internal", "github.com/livepeer/lpms/stream.NewBasicRTMPVideoStream.func1", "github.com/patrickmn/go-cache.(*janitor).Run", "github.com/golang/glog.(*fileSink).flushDaemon", "github.com/livepeer/go-livepeer/core.(*LivepeerNode).transcodeFrames.func2", "github.com/ipfs/go-log/writer.(*MirrorWriter).logRoutine", - "github.com/livepeer/go-livepeer/core.(*Balances).StartCleanup", + "github.com/livepeer/go-livepeer/core.(*Balances).StartCleanup", "github.com/livepeer/go-livepeer/server.startTrickleSubscribe.func2", "github.com/livepeer/go-livepeer/server.startTrickleSubscribe", + "net/http.(*persistConn).writeLoop", "net/http.(*persistConn).readLoop", "io.(*pipe).read", + "github.com/livepeer/go-livepeer/media.gatherIncomingTracks", } - res := make([]goleak.Option, 0, len(funcs2ignore)) + // Functions that might have other functions on top of their stack (like time.Sleep) + // These need to be ignored with IgnoreAnyFunction instead of IgnoreTopFunction + funcsAnyIgnore := []string{ + "github.com/livepeer/go-livepeer/server.ffmpegOutput", + } + + res := make([]goleak.Option, 0, len(funcs2ignore)+len(funcsAnyIgnore)) for _, f := range funcs2ignore { res = append(res, goleak.IgnoreTopFunction(f)) } + for _, f := range funcsAnyIgnore { + res = append(res, goleak.IgnoreAnyFunction(f)) + } return res } diff --git a/core/livepeernode.go b/core/livepeernode.go index ce41d1726d..dd1d26ce42 100644 --- a/core/livepeernode.go +++ b/core/livepeernode.go @@ -195,15 +195,23 @@ func (n *LivepeerNode) NewLivePipeline(requestID, streamID, pipeline string, str n.LiveMu.Lock() defer n.LiveMu.Unlock() n.LivePipelines[streamID] = &LivePipeline{ - RequestID: requestID, - Pipeline: pipeline, - StreamCtx: streamCtx, - streamParams: streamParams, - streamCancel: streamCancel, + RequestID: requestID, + StreamID: streamID, + Pipeline: pipeline, + StreamCtx: streamCtx, + streamParams: streamParams, + streamCancel: streamCancel, + streamRequest: streamRequest, } return n.LivePipelines[streamID] } +func (n *LivepeerNode) RemoveLivePipeline(streamID string) { + n.LiveMu.Lock() + defer n.LiveMu.Unlock() + delete(n.LivePipelines, streamID) +} + func (p *LivePipeline) StreamParams() interface{} { return p.streamParams } @@ -217,7 +225,10 @@ func (p *LivePipeline) StreamRequest() []byte { } func (p *LivePipeline) StopStream(err error) { - p.StopControl() + if p.StopControl != nil { + p.StopControl() + } + p.streamCancel(err) } diff --git a/server/job_rpc.go b/server/job_rpc.go index b9f7889d11..eeb119cc1c 100644 --- a/server/job_rpc.go +++ b/server/job_rpc.go @@ -864,10 +864,22 @@ func processPayment(ctx context.Context, orch Orchestrator, sender ethcommon.Add } func createPayment(ctx context.Context, jobReq *JobRequest, orchToken *core.JobToken, node *core.LivepeerNode) (string, error) { + if orchToken == nil { + return "", errors.New("orchestrator token is nil, cannot create payment") + } + //if no sender or ticket params, no payment + if node.Sender == nil { + return "", errors.New("no ticket sender available, cannot create payment") + } + if orchToken.TicketParams == nil { + return "", errors.New("no ticket params available, cannot create payment") + } + var payment *net.Payment createTickets := true clog.Infof(ctx, "creating payment for job request %s", jobReq.Capability) sender := ethcommon.HexToAddress(jobReq.Sender) + orchAddr := ethcommon.BytesToAddress(orchToken.TicketParams.Recipient) sessionID := node.Sender.StartSession(*pmTicketParams(orchToken.TicketParams)) diff --git a/server/job_rpc_test.go b/server/job_rpc_test.go index f26ee30422..db842ad5e0 100644 --- a/server/job_rpc_test.go +++ b/server/job_rpc_test.go @@ -13,6 +13,7 @@ import ( "net/http/httptest" "net/url" "slices" + "sync" "testing" "time" @@ -54,6 +55,7 @@ type mockJobOrchestrator struct { reserveCapacity func(string) error getUrlForCapability func(string) string balance func(ethcommon.Address, core.ManifestID) *big.Rat + processPayment func(context.Context, net.Payment, core.ManifestID) error debitFees func(ethcommon.Address, core.ManifestID, *net.PriceInfo, int64) freeCapacity func(string) error jobPriceInfo func(ethcommon.Address, string) (*net.PriceInfo, error) @@ -110,6 +112,9 @@ func (r *mockJobOrchestrator) StreamIDs(jobID string) ([]core.StreamID, error) { } func (r *mockJobOrchestrator) ProcessPayment(ctx context.Context, payment net.Payment, manifestID core.ManifestID) error { + if r.processPayment != nil { + return r.processPayment(ctx, payment, manifestID) + } return nil } @@ -130,6 +135,9 @@ func (r *mockJobOrchestrator) SufficientBalance(addr ethcommon.Address, manifest } func (r *mockJobOrchestrator) DebitFees(addr ethcommon.Address, manifestID core.ManifestID, price *net.PriceInfo, pixels int64) { + if r.debitFees != nil { + r.debitFees(addr, manifestID, price, pixels) + } } func (r *mockJobOrchestrator) Balance(addr ethcommon.Address, manifestID core.ManifestID) *big.Rat { @@ -332,13 +340,14 @@ func (s *stubJobOrchestratorPool) SizeWith(scorePred common.ScorePred) int { return count } func (s *stubJobOrchestratorPool) Broadcaster() common.Broadcaster { - return core.NewBroadcaster(s.node) + return stubBroadcaster2() } func mockJobLivepeerNode() *core.LivepeerNode { node, _ := core.NewLivepeerNode(nil, "/tmp/thisdirisnotactuallyusedinthistest", nil) node.NodeType = core.OrchestratorNode node.OrchSecret = "verbigsecret" + node.LiveMu = &sync.RWMutex{} return node } @@ -912,7 +921,7 @@ func TestCreatePayment(t *testing.T) { mockSender.On("StartSession", mock.Anything).Return("foo").Times(4) node.Sender = &mockSender - node.Balances = core.NewAddressBalances(10) + node.Balances = core.NewAddressBalances(1 * time.Second) defer node.Balances.StopCleanup() jobReq := JobRequest{ @@ -976,6 +985,51 @@ func TestCreatePayment(t *testing.T) { assert.Equal(t, 600, len(pmTickets.TicketSenderParams)) } +func createTestPayment(capability string) (string, error) { + ctx := context.TODO() + node, _ := core.NewLivepeerNode(nil, "/tmp/thisdirisnotactuallyusedinthistest", nil) + mockSender := pm.MockSender{} + mockSender.On("StartSession", mock.Anything).Return("foo").Times(4) + mockSender.On("CreateTicketBatch", "foo", 1).Return(mockTicketBatch(1), nil).Once() + node.Sender = &mockSender + + node.Balances = core.NewAddressBalances(1 * time.Second) + defer node.Balances.StopCleanup() + + jobReq := JobRequest{ + Capability: capability, + Timeout: 1, + } + sender := core.JobSender{ + Addr: "0x1111111111111111111111111111111111111111", + Sig: "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", + } + + orchTocken := core.JobToken{ + TicketParams: &net.TicketParams{ + Recipient: ethcommon.HexToAddress("0x1111111111111111111111111111111111111111").Bytes(), + FaceValue: big.NewInt(1000).Bytes(), + WinProb: big.NewInt(1).Bytes(), + RecipientRandHash: []byte("hash"), + Seed: big.NewInt(1234).Bytes(), + ExpirationBlock: big.NewInt(100000).Bytes(), + }, + SenderAddress: &sender, + Balance: 0, + Price: &net.PriceInfo{ + PricePerUnit: 10, + PixelsPerUnit: 1, + }, + } + + pmt, err := createPayment(ctx, &jobReq, &orchTocken, node) + if err != nil { + return "", err + } + + return pmt, nil +} + func mockTicketBatch(count int) *pm.TicketBatch { senderParams := make([]*pm.TicketSenderParams, count) for i := 0; i < count; i++ { @@ -994,7 +1048,7 @@ func mockTicketBatch(count int) *pm.TicketBatch { ExpirationBlock: big.NewInt(1000), }, TicketExpirationParams: &pm.TicketExpirationParams{}, - Sender: pm.RandAddress(), + Sender: ethcommon.HexToAddress("0x1111111111111111111111111111111111111111"), SenderParams: senderParams, } } @@ -1004,33 +1058,9 @@ func TestSubmitJob_OrchestratorSelectionParams(t *testing.T) { mockServers := make([]*httptest.Server, 5) orchURLs := make([]string, 5) - // Create a handler that returns a valid job token - tokenHandler := func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/process/token" { - http.NotFound(w, r) - return - } - - token := &core.JobToken{ - ServiceAddr: "http://" + r.Host, // Use the server's host as the service address - SenderAddress: &core.JobSender{ - Addr: "0x1234567890abcdef1234567890abcdef123456", - Sig: "0x456", - }, - TicketParams: nil, - Price: &net.PriceInfo{ - PricePerUnit: 100, - PixelsPerUnit: 1, - }, - } - - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(token) - } - // Start HTTP test servers for i := 0; i < 5; i++ { - server := httptest.NewServer(http.HandlerFunc(tokenHandler)) + server := httptest.NewServer(http.HandlerFunc(orchTokenHandler)) mockServers[i] = server orchURLs[i] = server.URL t.Logf("Mock server %d started at %s", i, orchURLs[i]) @@ -1137,3 +1167,157 @@ func TestSubmitJob_OrchestratorSelectionParams(t *testing.T) { } } + +func TestProcessPayment(t *testing.T) { + + ctx := context.Background() + sender := ethcommon.HexToAddress("0x1111111111111111111111111111111111111111") + + cases := []struct { + name string + capability string + expectDelta bool + }{ + {"empty header", "testcap", false}, + {"empty capability", "", false}, + {"random capability", "randomcap", false}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // Simulate a mutable balance for the test + testBalance := big.NewRat(100, 1) + balanceCalled := 0 + paymentCalled := 0 + orch := newMockJobOrchestrator() + orch.balance = func(addr ethcommon.Address, manifestID core.ManifestID) *big.Rat { + balanceCalled++ + return new(big.Rat).Set(testBalance) + } + orch.processPayment = func(ctx context.Context, payment net.Payment, manifestID core.ManifestID) error { + paymentCalled++ + // Simulate payment by increasing balance + testBalance = testBalance.Add(testBalance, big.NewRat(50, 1)) + return nil + } + + testPmtHdr, err := createTestPayment(tc.capability) + if err != nil { + t.Fatalf("Failed to create test payment: %v", err) + } + + before := orch.Balance(sender, core.ManifestID(tc.capability)).FloatString(0) + bal, err := processPayment(ctx, orch, sender, tc.capability, testPmtHdr) + after := orch.Balance(sender, core.ManifestID(tc.capability)).FloatString(0) + t.Logf("Balance before: %s, after: %s", before, after) + assert.NoError(t, err) + assert.NotNil(t, bal) + if testPmtHdr != "" { + assert.NotEqual(t, before, after, "Balance should change if payment header is not empty") + assert.Equal(t, 1, paymentCalled, "ProcessPayment should be called once for non-empty header") + } else { + assert.Equal(t, before, after, "Balance should not change if payment header is empty") + assert.Equal(t, 0, paymentCalled, "ProcessPayment should not be called for empty header") + } + }) + } +} + +func TestSetupGatewayJob(t *testing.T) { + // Prepare a JobRequest with valid fields + jobDetails := JobRequestDetails{StreamId: "test-stream"} + jobParams := JobParameters{ + Orchestrators: JobOrchestratorsFilter{}, + EnableVideoIngress: true, + EnableVideoEgress: true, + EnableDataOutput: true, + } + jobReq := JobRequest{ + ID: "job-1", + Request: marshalToString(t, jobDetails), + Parameters: marshalToString(t, jobParams), + Capability: "test-capability", + Timeout: 10, + } + jobReqB, err := json.Marshal(jobReq) + assert.NoError(t, err) + jobReqB64 := base64.StdEncoding.EncodeToString(jobReqB) + + // Setup a minimal LivepeerServer with a stub OrchestratorPool + server := httptest.NewServer(http.HandlerFunc(orchTokenHandler)) + defer server.Close() + node := mockJobLivepeerNode() + + node.OrchestratorPool = newStubOrchestratorPool(node, []string{server.URL}) + ls := &LivepeerServer{LivepeerNode: node} + + req := httptest.NewRequest(http.MethodPost, "/", nil) + req.Header.Set(jobRequestHdr, jobReqB64) + + // Should succeed + gatewayJob, err := ls.setupGatewayJob(context.Background(), req, false) + assert.NoError(t, err) + assert.NotNil(t, gatewayJob) + assert.Equal(t, "test-capability", gatewayJob.Job.Req.Capability) + assert.Equal(t, "test-stream", gatewayJob.Job.Details.StreamId) + assert.Equal(t, 10, gatewayJob.Job.Req.Timeout) + assert.Equal(t, 1, len(gatewayJob.Orchs)) + + //test signing request + assert.Empty(t, gatewayJob.SignedJobReq) + gatewayJob.sign() + assert.NotEmpty(t, gatewayJob.SignedJobReq) + + // Should fail with invalid base64 + req.Header.Set(jobRequestHdr, "not-base64") + gatewayJob, err = ls.setupGatewayJob(context.Background(), req, false) + assert.Error(t, err) + assert.Nil(t, gatewayJob) + + // Should fail with missing orchestrators (simulate getJobOrchestrators returns empty) + req.Header.Set(jobRequestHdr, jobReqB64) + ls.LivepeerNode.OrchestratorPool = newStubOrchestratorPool(node, []string{}) + gatewayJob, err = ls.setupGatewayJob(context.Background(), req, false) + assert.Error(t, err) + assert.Nil(t, gatewayJob) +} + +// marshalToString is a helper to marshal a struct to a JSON string +func marshalToString(t *testing.T, v interface{}) string { + b, err := json.Marshal(v) + if err != nil { + t.Fatalf("marshalToString failed: %v", err) + } + return string(b) +} + +func orchTokenHandler(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/process/token" { + http.NotFound(w, r) + return + } + + token := createMockJobToken("http://" + r.Host) + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(token) + +} + +func createMockJobToken(hostUrl string) *core.JobToken { + return &core.JobToken{ + ServiceAddr: hostUrl, + SenderAddress: &core.JobSender{ + Addr: "0x1234567890abcdef1234567890abcdef123456", + Sig: "0x456", + }, + TicketParams: &net.TicketParams{ + Recipient: ethcommon.HexToAddress("0x1111111111111111111111111111111111111111").Bytes(), + FaceValue: big.NewInt(1000).Bytes(), + }, + Price: &net.PriceInfo{ + PricePerUnit: 100, + PixelsPerUnit: 1, + }, + } +} diff --git a/server/job_stream.go b/server/job_stream.go index 00d09db7fa..6a7f1918cc 100644 --- a/server/job_stream.go +++ b/server/job_stream.go @@ -15,6 +15,7 @@ import ( "time" ethcommon "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/golang/glog" "github.com/livepeer/go-livepeer/clog" "github.com/livepeer/go-livepeer/common" @@ -183,7 +184,7 @@ func (ls *LivepeerServer) runStream(gatewayJob *gatewayJob) { } params.liveParams.sess = &orchSession - ctx = clog.AddVal(ctx, "orch", ethcommon.Bytes2Hex(orch.TicketParams.Recipient)) + ctx = clog.AddVal(ctx, "orch", hexutil.Encode(orch.TicketParams.Recipient)) ctx = clog.AddVal(ctx, "orch_url", orch.ServiceAddr) //set request ID to persist from Gateway to Worker @@ -200,6 +201,8 @@ func (ls *LivepeerServer) runStream(gatewayJob *gatewayJob) { continue } + GatewayStatus.StoreKey(streamID, "orchestrator", orch.ServiceAddr) + params.liveParams.orchPublishUrl = orchResp.Header.Get("X-Publish-Url") params.liveParams.orchSubscribeUrl = orchResp.Header.Get("X-Subscribe-Url") params.liveParams.orchControlUrl = orchResp.Header.Get("X-Control-Url") @@ -237,12 +240,13 @@ func (ls *LivepeerServer) runStream(gatewayJob *gatewayJob) { break } firstProcessed = true - clog.Infof(ctx, "Retrying stream with a different orchestrator err=%v", err.Error()) - // will swap, but first notify with the reason for the swap if err == nil { err = errors.New("unknown swap reason") } + + clog.Infof(ctx, "Retrying stream with a different orchestrator err=%v", err.Error()) + params.liveParams.sendErrorEvent(err) //if there is ingress input then force off @@ -287,7 +291,7 @@ func (ls *LivepeerServer) monitorStream(streamId string) { select { case <-stream.StreamCtx.Done(): clog.Infof(ctx, "Stream %s stopped, ending monitoring", streamId) - delete(ls.LivepeerNode.LivePipelines, streamId) + ls.LivepeerNode.RemoveLivePipeline(streamId) return case <-pmtTicker.C: if !params.inputStreamExists() { @@ -350,26 +354,28 @@ func (ls *LivepeerServer) sendPaymentForStream(ctx context.Context, stream *core return err } - pmtHdr, err := createPayment(ctx, req, newToken, ls.LivepeerNode) - if err != nil { - clog.Errorf(ctx, "Error processing stream payment for %s: %v", streamID, err) - // Continue monitoring even if payment fails - } - if pmtHdr == "" { - // This is no payment required, error logged above - return nil - } + if newSess.OrchestratorInfo.PriceInfo.PricePerUnit > 0 { + pmtHdr, err := createPayment(ctx, req, newToken, ls.LivepeerNode) + if err != nil { + clog.Errorf(ctx, "Error processing stream payment for %s: %v", streamID, err) + // Continue monitoring even if payment fails + } + if pmtHdr == "" { + // This is no payment required, error logged above + return nil + } - //send the payment, update the stream with the refreshed token - clog.Infof(ctx, "Sending stream payment for %s", streamID) - statusCode, err := ls.sendPayment(ctx, token.ServiceAddr+"/ai/stream/payment", stream.Pipeline, job.SignedJobReq, pmtHdr) - if err != nil { - clog.Errorf(ctx, "Error sending stream payment for %s: %v", streamID, err) - return err - } - if statusCode != http.StatusOK { - clog.Errorf(ctx, "Unexpected status code %d received for %s", statusCode, streamID) - return errors.New("unexpected status code") + //send the payment, update the stream with the refreshed token + clog.Infof(ctx, "Sending stream payment for %s", streamID) + statusCode, err := ls.sendPayment(ctx, token.ServiceAddr+"/ai/stream/payment", stream.Pipeline, job.SignedJobReq, pmtHdr) + if err != nil { + clog.Errorf(ctx, "Error sending stream payment for %s: %v", streamID, err) + return err + } + if statusCode != http.StatusOK { + clog.Errorf(ctx, "Unexpected status code %d received for %s", statusCode, streamID) + return errors.New("unexpected status code") + } } return nil @@ -473,12 +479,26 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, job mediaMTXOutputURL := mediaMTXInputURL + "-out" mediaMTXOutputAlias := fmt.Sprintf("%s-%s-out", mediaMTXInputURL, requestID) - whepURL := generateWhepUrl(streamID, requestID) - whipURL := fmt.Sprintf("https://%s/ai/stream/%s/%s", ls.LivepeerNode.GatewayHost, streamID, "whip") - rtmpURL := mediaMTXInputURL + var ( + whipURL string + rtmpURL string + whepURL string + dataURL string + ) + updateURL := fmt.Sprintf("https://%s/ai/stream/%s/%s", ls.LivepeerNode.GatewayHost, streamID, "update") statusURL := fmt.Sprintf("https://%s/ai/stream/%s/%s", ls.LivepeerNode.GatewayHost, streamID, "status") - dataURL := fmt.Sprintf("https://%s/ai/stream/%s/%s", ls.LivepeerNode.GatewayHost, streamID, "data") + + if job.Job.Params.EnableVideoIngress { + whipURL = fmt.Sprintf("https://%s/ai/stream/%s/%s", ls.LivepeerNode.GatewayHost, streamID, "whip") + rtmpURL = mediaMTXInputURL + } + if job.Job.Params.EnableVideoEgress { + whepURL = generateWhepUrl(streamID, requestID) + } + if job.Job.Params.EnableDataOutput { + dataURL = fmt.Sprintf("https://%s/ai/stream/%s/%s", ls.LivepeerNode.GatewayHost, streamID, "data") + } //if set this will overwrite settings above if LiveAIAuthWebhookURL != nil { @@ -525,12 +545,15 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, job // collect all RTMP outputs var rtmpOutputs []string - if outputURL != "" { - rtmpOutputs = append(rtmpOutputs, outputURL) - } - if mediaMTXOutputURL != "" { - rtmpOutputs = append(rtmpOutputs, mediaMTXOutputURL, mediaMTXOutputAlias) + if job.Job.Params.EnableVideoEgress { + if outputURL != "" { + rtmpOutputs = append(rtmpOutputs, outputURL) + } + if mediaMTXOutputURL != "" { + rtmpOutputs = append(rtmpOutputs, mediaMTXOutputURL, mediaMTXOutputAlias) + } } + clog.Info(ctx, "RTMP outputs", "destinations", rtmpOutputs) // Clear any previous gateway status @@ -593,11 +616,7 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, job } //create a dataWriter for data channel if enabled - var jobParams JobParameters - if err = json.Unmarshal([]byte(job.Job.Req.Parameters), &jobParams); err != nil { - return nil, http.StatusBadRequest, errors.New("invalid job parameters") - } - if jobParams.EnableDataOutput { + if job.Job.Params.EnableDataOutput { params.liveParams.dataWriter = media.NewSegmentWriter(5) } @@ -606,7 +625,7 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, job return nil, http.StatusBadRequest, fmt.Errorf("stream already exists: %s", streamID) } - clog.Infof(ctx, "stream setup videoIngress=%v videoEgress=%v dataOutput=%v", jobParams.EnableVideoIngress, jobParams.EnableVideoEgress, jobParams.EnableDataOutput) + clog.Infof(ctx, "stream setup videoIngress=%v videoEgress=%v dataOutput=%v", job.Job.Params.EnableVideoIngress, job.Job.Params.EnableVideoEgress, job.Job.Params.EnableDataOutput) //save the stream setup ls.LivepeerNode.NewLivePipeline(requestID, streamID, pipeline, params, bodyBytes) //track the pipeline for cancellation @@ -767,19 +786,24 @@ func (ls *LivepeerServer) StartStreamWhipIngest(whipServer *media.WHIPServer) ht go runStats(statsContext, whipConn, streamId, stream.Pipeline, params.liveParams.requestID) whipConn.AwaitClose() - stream.StopStream(nil) params.liveParams.segmentReader.Close() params.liveParams.kickOrch(errors.New("whip ingest disconnected")) + stream.StopStream(nil) clog.Info(ctx, "Live cleaned up") }() + if whipServer == nil { + respondJsonError(ctx, w, fmt.Errorf("whip server not configured"), http.StatusInternalServerError) + whipConn.Close() + return + } + conn := whipServer.CreateWHIP(ctx, params.liveParams.segmentReader, whepURL, w, r) whipConn.SetWHIPConnection(conn) // might be nil if theres an error and thats okay }) } func startStreamProcessing(ctx context.Context, stream *core.LivePipeline, params aiRequestParams) error { - var channels []string //required channels control, err := common.AppendHostname(params.liveParams.orchControlUrl, params.liveParams.sess.BroadcastSession.Transcoder()) if err != nil { @@ -790,8 +814,6 @@ func startStreamProcessing(ctx context.Context, stream *core.LivePipeline, param return fmt.Errorf("invalid events URL: %w", err) } - channels = append(channels, control.String()) - channels = append(channels, events.String()) startControlPublish(ctx, control, params) startEventsSubscribe(ctx, events, params, params.liveParams.sess) @@ -802,7 +824,6 @@ func startStreamProcessing(ctx context.Context, stream *core.LivePipeline, param if err != nil { return fmt.Errorf("invalid publish URL: %w", err) } - channels = append(channels, pub.String()) startTricklePublish(ctx, pub, params, params.liveParams.sess) } @@ -812,7 +833,6 @@ func startStreamProcessing(ctx context.Context, stream *core.LivePipeline, param if err != nil { return fmt.Errorf("invalid subscribe URL: %w", err) } - channels = append(channels, sub.String()) startTrickleSubscribe(ctx, sub, params, params.liveParams.sess) } @@ -1298,6 +1318,11 @@ func tokenToAISession(token core.JobToken) (AISession, error) { // like (*BroadcastSession).Transcoder() which acquire RLock() session.lock = &sync.RWMutex{} + //default to zero price if its nil, Orchestrator will reject stream if charging a price above zero + if token.Price == nil { + token.Price = &net.PriceInfo{} + } + orchInfo := net.OrchestratorInfo{Transcoder: token.ServiceAddr, TicketParams: token.TicketParams, PriceInfo: token.Price} orchInfo.Transcoder = token.ServiceAddr if token.SenderAddress != nil { @@ -1318,6 +1343,10 @@ func sessionToToken(session *AISession) (core.JobToken, error) { } func getStreamRequestParams(stream *core.LivePipeline) (aiRequestParams, error) { + if stream == nil { + return aiRequestParams{}, fmt.Errorf("stream is nil") + } + streamParams := stream.StreamParams() params, ok := streamParams.(aiRequestParams) if !ok { diff --git a/server/job_stream_test.go b/server/job_stream_test.go new file mode 100644 index 0000000000..ef9600826c --- /dev/null +++ b/server/job_stream_test.go @@ -0,0 +1,1597 @@ +package server + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync" + "testing" + "time" + + "github.com/livepeer/go-livepeer/core" + "github.com/livepeer/go-livepeer/media" + "github.com/livepeer/go-livepeer/pm" + "github.com/livepeer/go-livepeer/trickle" + "github.com/livepeer/go-tools/drivers" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +var stubOrchServerUrl string + +// testOrch wraps mockOrchestrator to override a few methods needed by lphttp in tests +type testStreamOrch struct { + *mockOrchestrator + svc *url.URL + capURL string +} + +func (o *testStreamOrch) ServiceURI() *url.URL { return o.svc } +func (o *testStreamOrch) GetUrlForCapability(capability string) string { return o.capURL } + +// streamingResponseWriter implements http.ResponseWriter for streaming responses +type streamingResponseWriter struct { + pipe *io.PipeWriter + headers http.Header + status int +} + +func (w *streamingResponseWriter) Header() http.Header { + return w.headers +} + +func (w *streamingResponseWriter) Write(data []byte) (int, error) { + return w.pipe.Write(data) +} + +func (w *streamingResponseWriter) WriteHeader(statusCode int) { + w.status = statusCode +} + +// Helper: base64-encoded JobRequest with JobParameters (Enable all true, test-capability name) +func base64TestJobRequest(timeout int, enableVideoIngress, enableVideoEgress, enableDataOutput bool) string { + params := JobParameters{ + EnableVideoIngress: enableVideoIngress, + EnableVideoEgress: enableVideoEgress, + EnableDataOutput: enableDataOutput, + } + paramsStr, _ := json.Marshal(params) + + jr := JobRequest{ + Capability: "test-capability", + Parameters: string(paramsStr), + Request: "{}", + Timeout: timeout, + } + + b, _ := json.Marshal(jr) + + return base64.StdEncoding.EncodeToString(b) +} + +func orchAIStreamStartHandler(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/ai/stream/start" { + http.NotFound(w, r) + return + } + + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Publish-Url", fmt.Sprintf("%s%s%s", stubOrchServerUrl, TrickleHTTPPath, "test-stream")) + w.Header().Set("X-Subscribe-Url", fmt.Sprintf("%s%s%s", stubOrchServerUrl, TrickleHTTPPath, "test-stream-out")) + w.Header().Set("X-Control-Url", fmt.Sprintf("%s%s%s", stubOrchServerUrl, TrickleHTTPPath, "test-stream-control")) + w.Header().Set("X-Events-Url", fmt.Sprintf("%s%s%s", stubOrchServerUrl, TrickleHTTPPath, "test-stream-events")) + w.Header().Set("X-Data-Url", fmt.Sprintf("%s%s%s", stubOrchServerUrl, TrickleHTTPPath, "test-stream-data")) + w.WriteHeader(http.StatusOK) +} + +func orchCapabilityUrlHandler(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) +} + +func TestStartStream_MaxBodyLimit(t *testing.T) { + // Setup server with minimal dependencies + node := mockJobLivepeerNode() + server := httptest.NewServer(http.HandlerFunc(orchTokenHandler)) + defer server.Close() + node.OrchestratorPool = newStubOrchestratorPool(node, []string{server.URL}) + + // Set up mock sender to prevent nil pointer dereference + mockSender := pm.MockSender{} + mockSender.On("StartSession", mock.Anything).Return("foo") + mockSender.On("CreateTicketBatch", mock.Anything, mock.Anything).Return(mockTicketBatch(10), nil) + node.Sender = &mockSender + node.Balances = core.NewAddressBalances(10) + defer node.Balances.StopCleanup() + + ls := &LivepeerServer{LivepeerNode: node} + + // Prepare a valid job request header + jobDetails := JobRequestDetails{StreamId: "test-stream"} + jobParams := JobParameters{EnableVideoIngress: true, EnableVideoEgress: true, EnableDataOutput: true} + jobReq := JobRequest{ + ID: "job-1", + Request: marshalToString(t, jobDetails), + Parameters: marshalToString(t, jobParams), + Capability: "test-capability", + Timeout: 10, + } + jobReqB, err := json.Marshal(jobReq) + assert.NoError(t, err) + jobReqB64 := base64.StdEncoding.EncodeToString(jobReqB) + + // Create a body over 10MB + bigBody := bytes.Repeat([]byte("a"), 10<<20+1) // 10MB + 1 byte + req := httptest.NewRequest(http.MethodPost, "/ai/stream/start", bytes.NewReader(bigBody)) + req.Header.Set(jobRequestHdr, jobReqB64) + + w := httptest.NewRecorder() + handler := ls.StartStream() + handler.ServeHTTP(w, req) + + assert.Equal(t, http.StatusRequestEntityTooLarge, w.Code) +} + +func TestStreamStart_SetupStream(t *testing.T) { + node := mockJobLivepeerNode() + server := httptest.NewServer(http.HandlerFunc(orchTokenHandler)) + defer server.Close() + node.OrchestratorPool = newStubOrchestratorPool(node, []string{server.URL}) + + // Set up mock sender to prevent nil pointer dereference + mockSender := pm.MockSender{} + mockSender.On("StartSession", mock.Anything).Return("foo") + mockSender.On("CreateTicketBatch", mock.Anything, mock.Anything).Return(mockTicketBatch(10), nil) + node.Sender = &mockSender + node.Balances = core.NewAddressBalances(10) + defer node.Balances.StopCleanup() + + ls := &LivepeerServer{LivepeerNode: node} + drivers.NodeStorage = drivers.NewMemoryDriver(nil) + + // Prepare a valid gatewayJob + jobParams := JobParameters{EnableVideoIngress: true, EnableVideoEgress: true, EnableDataOutput: true} + paramsStr := marshalToString(t, jobParams) + jobReq := &JobRequest{ + Capability: "test-capability", + Parameters: paramsStr, + Timeout: 10, + } + orchJob := &orchJob{Req: jobReq, Params: &jobParams} + gatewayJob := &gatewayJob{Job: orchJob} + + // Prepare a valid StartRequest body + startReq := StartRequest{ + Stream: "teststream", + RtmpOutput: "rtmp://output", + StreamId: "streamid", + Params: "{}", + } + body, _ := json.Marshal(startReq) + req := httptest.NewRequest(http.MethodPost, "/ai/stream/start", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + + urls, code, err := ls.setupStream(context.Background(), req, gatewayJob) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, code) + assert.NotNil(t, urls) + assert.Equal(t, "teststream-streamid", urls.StreamId) + //confirm all urls populated + assert.NotEmpty(t, urls.WhipUrl) + assert.NotEmpty(t, urls.RtmpUrl) + assert.NotEmpty(t, urls.WhepUrl) + assert.NotEmpty(t, urls.RtmpOutputUrl) + assert.Contains(t, urls.RtmpOutputUrl, "rtmp://output") + assert.NotEmpty(t, urls.DataUrl) + assert.NotEmpty(t, urls.StatusUrl) + assert.NotEmpty(t, urls.UpdateUrl) + + //confirm LivePipeline created + stream, ok := ls.LivepeerNode.LivePipelines[urls.StreamId] + assert.True(t, ok) + assert.NotNil(t, stream) + assert.Equal(t, urls.StreamId, stream.StreamID) + assert.Equal(t, stream.StreamRequest(), body) + params := stream.StreamParams() + _, checkParamsType := params.(aiRequestParams) + assert.True(t, checkParamsType) + + //test with no data output + jobParams = JobParameters{EnableVideoIngress: true, EnableVideoEgress: true, EnableDataOutput: false} + paramsStr = marshalToString(t, jobParams) + jobReq.Parameters = paramsStr + gatewayJob.Job.Params = &jobParams + req.Body = io.NopCloser(bytes.NewReader(body)) + urls, code, err = ls.setupStream(context.Background(), req, gatewayJob) + assert.Empty(t, urls.DataUrl) + + //test with no video ingress + jobParams = JobParameters{EnableVideoIngress: false, EnableVideoEgress: true, EnableDataOutput: true} + paramsStr = marshalToString(t, jobParams) + jobReq.Parameters = paramsStr + gatewayJob.Job.Params = &jobParams + req.Body = io.NopCloser(bytes.NewReader(body)) + urls, code, err = ls.setupStream(context.Background(), req, gatewayJob) + assert.Empty(t, urls.WhipUrl) + assert.Empty(t, urls.RtmpUrl) + + //test with no video egress + jobParams = JobParameters{EnableVideoIngress: true, EnableVideoEgress: false, EnableDataOutput: true} + paramsStr = marshalToString(t, jobParams) + jobReq.Parameters = paramsStr + gatewayJob.Job.Params = &jobParams + req.Body = io.NopCloser(bytes.NewReader(body)) + urls, code, err = ls.setupStream(context.Background(), req, gatewayJob) + assert.Empty(t, urls.WhepUrl) + assert.Empty(t, urls.RtmpOutputUrl) + + // Test with nil job + urls, code, err = ls.setupStream(context.Background(), req, nil) + assert.Error(t, err) + assert.Equal(t, http.StatusBadRequest, code) + assert.Nil(t, urls) + + // Test with invalid JSON body + badReq := httptest.NewRequest(http.MethodPost, "/ai/stream/start", bytes.NewReader([]byte("notjson"))) + badReq.Header.Set("Content-Type", "application/json") + urls, code, err = ls.setupStream(context.Background(), badReq, gatewayJob) + assert.Error(t, err) + assert.Equal(t, http.StatusBadRequest, code) + assert.Nil(t, urls) + + // Test with stream name ending in -out (should return nil, 0, nil) + outReq := StartRequest{ + Stream: "teststream-out", + RtmpOutput: "rtmp://output", + StreamId: "streamid", + Params: "{}", + } + outBody, _ := json.Marshal(outReq) + outReqHTTP := httptest.NewRequest(http.MethodPost, "/ai/stream/start", bytes.NewReader(outBody)) + outReqHTTP.Header.Set("Content-Type", "application/json") + urls, code, err = ls.setupStream(context.Background(), outReqHTTP, gatewayJob) + assert.NoError(t, err) + assert.Equal(t, 0, code) + assert.Nil(t, urls) +} + +func TestRunStream_RunAndCancelStream(t *testing.T) { + node := mockJobLivepeerNode() + + // Set up an lphttp-based orchestrator test server with trickle endpoints + mux := http.NewServeMux() + mockOrch := &mockOrchestrator{} + mockOrch.On("VerifySig", mock.Anything, mock.Anything, mock.Anything).Return(true) + mockOrch.On("DebitFees", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return() + + lp := &lphttp{orchestrator: nil, transRPC: mux, node: node} + // Configure trickle server on the mux (imitate production trickle endpoints) + lp.trickleSrv = trickle.ConfigureServer(trickle.TrickleServerConfig{ + Mux: mux, + BasePath: TrickleHTTPPath, + Autocreate: true, + }) + // Register orchestrator endpoints used by runStream path + mux.HandleFunc("/ai/stream/start", lp.StartStream) + mux.HandleFunc("/ai/stream/stop", lp.StopStream) + mux.HandleFunc("/process/token", orchTokenHandler) + + server := httptest.NewServer(lp) + defer server.Close() + stubOrchServerUrl = server.URL + + // Configure mock orchestrator behavior expected by lphttp handlers + parsedURL, _ := url.Parse(server.URL) + capabilitySrv := httptest.NewServer(http.HandlerFunc(orchCapabilityUrlHandler)) + defer capabilitySrv.Close() + + // attach our orchestrator implementation to lphttp + lp.orchestrator = &testStreamOrch{mockOrchestrator: mockOrch, svc: parsedURL, capURL: capabilitySrv.URL} + + // Prepare a gatewayJob with a dummy orchestrator token + jobReq := &JobRequest{ + ID: "test-stream", + Capability: "test-capability", + Timeout: 10, + Request: "{}", + } + jobParams := JobParameters{EnableVideoIngress: true, EnableVideoEgress: true, EnableDataOutput: true} + paramsStr := marshalToString(t, jobParams) + jobReq.Parameters = paramsStr + + orchToken := createMockJobToken(server.URL) + orchJob := &orchJob{Req: jobReq, Params: &jobParams} + gatewayJob := &gatewayJob{Job: orchJob, Orchs: []core.JobToken{*orchToken}, node: node} + + // Setup a LivepeerServer and a mock pipeline + ls := &LivepeerServer{LivepeerNode: node} + ls.LivepeerNode.OrchestratorPool = newStubOrchestratorPool(ls.LivepeerNode, []string{server.URL}) + drivers.NodeStorage = drivers.NewMemoryDriver(nil) + mockSender := pm.MockSender{} + mockSender.On("StartSession", mock.Anything).Return("foo").Times(4) + mockSender.On("CreateTicketBatch", "foo", orchJob.Req.Timeout).Return(mockTicketBatch(orchJob.Req.Timeout), nil).Once() + node.Sender = &mockSender + node.Balances = core.NewAddressBalances(10) + defer node.Balances.StopCleanup() + + //now sign job and create a sig for the sender to include + gatewayJob.sign() + sender, err := getJobSender(context.TODO(), node) + assert.NoError(t, err) + orchJob.Req.Sender = sender.Addr + orchJob.Req.Sig = sender.Sig + // Minimal aiRequestParams and liveRequestParams + params := aiRequestParams{ + liveParams: &liveRequestParams{ + requestID: "req-1", + stream: "test-stream", + streamID: "test-stream", + sendErrorEvent: func(err error) {}, + segmentReader: media.NewSwitchableSegmentReader(), + }, + node: node, + } + + ls.LivepeerNode.NewLivePipeline("req-1", "test-stream", "test-capability", params, nil) + + // Cancel the stream after a short delay to simulate shutdown + done := make(chan struct{}) + go func() { + time.Sleep(100 * time.Millisecond) + stream := node.LivePipelines["test-stream"] + if stream != nil { + params, _ := getStreamRequestParams(stream) + if params.liveParams.kickOrch != nil { + params.liveParams.kickOrch(errors.New("test cancel")) + } + + stream.StopStream(nil) + } + close(done) + }() + + // Should not panic and should clean up + var wg sync.WaitGroup + wg.Add(2) + go func() { defer wg.Done(); ls.runStream(gatewayJob) }() + go func() { defer wg.Done(); ls.monitorStream(gatewayJob.Job.Req.ID) }() + <-done + // Wait for both goroutines to finish before asserting + wg.Wait() + // After cancel, the stream should be removed from LivePipelines + _, exists := node.LivePipelines["test-stream"] + assert.False(t, exists) +} + +// Test StartStream handler +func TestStartStreamHandler(t *testing.T) { + node := mockJobLivepeerNode() + + // Set up an lphttp-based orchestrator test server with trickle endpoints + mux := http.NewServeMux() + ls := &LivepeerServer{ + LivepeerNode: node, + } + mockSender := pm.MockSender{} + mockSender.On("StartSession", mock.Anything).Return("foo") + mockSender.On("CreateTicketBatch", mock.Anything, mock.Anything).Return(mockTicketBatch(10), nil) + node.Sender = &mockSender + node.Balances = core.NewAddressBalances(1 * time.Second) + defer node.Balances.StopCleanup() + //setup Orch server stub + mux.HandleFunc("/process/token", orchTokenHandler) + mux.HandleFunc("/ai/stream/start", orchAIStreamStartHandler) + server := httptest.NewServer(mux) + defer server.Close() + ls.LivepeerNode.OrchestratorPool = newStubOrchestratorPool(ls.LivepeerNode, []string{server.URL}) + drivers.NodeStorage = drivers.NewMemoryDriver(nil) + // Prepare a valid StartRequest body + startReq := StartRequest{ + Stream: "teststream", + RtmpOutput: "rtmp://output", + StreamId: "streamid", + Params: "{}", + } + body, _ := json.Marshal(startReq) + req := httptest.NewRequest(http.MethodPost, "/ai/stream/start", bytes.NewBuffer(body)) + req.Header.Set("Content-Type", "application/json") + + req.Header.Set("Livepeer", base64TestJobRequest(10, true, true, true)) + + w := httptest.NewRecorder() + + handler := ls.StartStream() + handler.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + body = w.Body.Bytes() + var streamUrls StreamUrls + err := json.Unmarshal(body, &streamUrls) + assert.NoError(t, err) + stream, exits := ls.LivepeerNode.LivePipelines[streamUrls.StreamId] + assert.True(t, exits) + assert.NotNil(t, stream) + assert.Equal(t, streamUrls.StreamId, stream.StreamID) + params := stream.StreamParams() + streamParams, checkParamsType := params.(aiRequestParams) + assert.True(t, checkParamsType) + //wrap up processing + time.Sleep(100 * time.Millisecond) + streamParams.liveParams.kickOrch(errors.New("test error")) + stream.StopStream(nil) +} + +// Test StopStream handler +func TestStopStreamHandler(t *testing.T) { + t.Run("StreamNotFound", func(t *testing.T) { + // Test case 1: Stream doesn't exist - should return 404 + ls := &LivepeerServer{LivepeerNode: &core.LivepeerNode{LivePipelines: map[string]*core.LivePipeline{}}} + req := httptest.NewRequest(http.MethodPost, "/ai/stream/{streamId}/stop", nil) + req.SetPathValue("streamId", "non-existent-stream") + w := httptest.NewRecorder() + + handler := ls.StopStream() + handler.ServeHTTP(w, req) + + assert.Equal(t, http.StatusNotFound, w.Code) + assert.Contains(t, w.Body.String(), "Stream not found") + }) + + t.Run("StreamExistsAndStopsSuccessfully", func(t *testing.T) { + // Test case 2: Stream exists - should stop stream and attempt to send request to orchestrator + node := mockJobLivepeerNode() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Mock orchestrator response handlers + switch r.URL.Path { + case "/process/token": + orchTokenHandler(w, r) + case "/ai/stream/stop": + // Mock successful stop response from orchestrator + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"status": "stopped"}`)) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + node.OrchestratorPool = newStubOrchestratorPool(node, []string{server.URL}) + ls := &LivepeerServer{LivepeerNode: node} + drivers.NodeStorage = drivers.NewMemoryDriver(nil) + mockSender := pm.MockSender{} + mockSender.On("StartSession", mock.Anything).Return("foo").Times(4) + mockSender.On("CreateTicketBatch", "foo", 10).Return(mockTicketBatch(10), nil).Once() + node.Sender = &mockSender + node.Balances = core.NewAddressBalances(10) + defer node.Balances.StopCleanup() + // Create a stream to stop + streamID := "test-stream-to-stop" + + // Create minimal AI session with properly formatted URL + token := createMockJobToken(server.URL) + + sess, err := tokenToAISession(*token) + + // Create stream parameters + params := aiRequestParams{ + liveParams: &liveRequestParams{ + requestID: "req-1", + sess: &sess, + stream: streamID, + streamID: streamID, + sendErrorEvent: func(err error) {}, + segmentReader: media.NewSwitchableSegmentReader(), + }, + node: node, + } + + // Add the stream to LivePipelines + stream := node.NewLivePipeline("req-1", streamID, "test-capability", params, nil) + assert.NotNil(t, stream) + + // Verify stream exists before stopping + _, exists := ls.LivepeerNode.LivePipelines[streamID] + assert.True(t, exists, "Stream should exist before stopping") + + // Create stop request with proper job header + jobParams := JobParameters{EnableVideoIngress: true, EnableVideoEgress: true, EnableDataOutput: true} + jobDetails := JobRequestDetails{StreamId: streamID} + jobReq := JobRequest{ + ID: streamID, + Request: marshalToString(t, jobDetails), + Capability: "test-capability", + Parameters: marshalToString(t, jobParams), + Timeout: 10, + } + jobReqB, err := json.Marshal(jobReq) + assert.NoError(t, err) + jobReqB64 := base64.StdEncoding.EncodeToString(jobReqB) + + req := httptest.NewRequest(http.MethodPost, "/ai/stream/{streamId}/stop", strings.NewReader(`{"reason": "test stop"}`)) + req.SetPathValue("streamId", streamID) + req.Header.Set("Content-Type", "application/json") + req.Header.Set(jobRequestHdr, jobReqB64) + + w := httptest.NewRecorder() + + handler := ls.StopStream() + handler.ServeHTTP(w, req) + + // The response might vary depending on orchestrator communication success + // The important thing is that the stream is removed regardless + assert.Contains(t, []int{http.StatusOK, http.StatusInternalServerError, http.StatusBadRequest}, w.Code, + "Should return valid HTTP status") + + // Verify stream was removed from LivePipelines (this should always happen) + _, exists = ls.LivepeerNode.LivePipelines[streamID] + assert.False(t, exists, "Stream should be removed after stopping") + }) + + t.Run("StreamExistsButOrchestratorError", func(t *testing.T) { + // Test case 3: Stream exists but orchestrator returns error + node := mockJobLivepeerNode() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/process/token": + orchTokenHandler(w, r) + case "/ai/stream/stop": + // Mock orchestrator error + http.Error(w, "Orchestrator error", http.StatusInternalServerError) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + node.OrchestratorPool = newStubOrchestratorPool(node, []string{server.URL}) + ls := &LivepeerServer{LivepeerNode: node} + drivers.NodeStorage = drivers.NewMemoryDriver(nil) + mockSender := pm.MockSender{} + mockSender.On("StartSession", mock.Anything).Return("foo").Times(4) + mockSender.On("CreateTicketBatch", "foo", 10).Return(mockTicketBatch(10), nil).Once() + node.Sender = &mockSender + node.Balances = core.NewAddressBalances(10) + defer node.Balances.StopCleanup() + streamID := "test-stream-orch-error" + + // Create minimal AI session + token := createMockJobToken(server.URL) + sess, err := tokenToAISession(*token) + assert.NoError(t, err) + + params := aiRequestParams{ + liveParams: &liveRequestParams{ + requestID: "req-1", + sess: &sess, + stream: streamID, + streamID: streamID, + sendErrorEvent: func(err error) {}, + segmentReader: media.NewSwitchableSegmentReader(), + }, + node: node, + } + + // Add the stream + stream := node.NewLivePipeline("req-1", streamID, "test-capability", params, nil) + assert.NotNil(t, stream) + + // Create stop request + jobParams := JobParameters{EnableVideoIngress: true, EnableVideoEgress: true, EnableDataOutput: true} + jobDetails := JobRequestDetails{StreamId: streamID} + jobReq := JobRequest{ + ID: streamID, + Request: marshalToString(t, jobDetails), + Capability: "test-capability", + Parameters: marshalToString(t, jobParams), + Timeout: 10, + } + jobReqB, err := json.Marshal(jobReq) + assert.NoError(t, err) + jobReqB64 := base64.StdEncoding.EncodeToString(jobReqB) + + req := httptest.NewRequest(http.MethodPost, "/ai/stream/{streamId}/stop", nil) + req.SetPathValue("streamId", streamID) + req.Header.Set(jobRequestHdr, jobReqB64) + + w := httptest.NewRecorder() + + handler := ls.StopStream() + handler.ServeHTTP(w, req) + + // Returns 200 OK because Gateway removed the stream. If the Orchestrator errors, it will return + // the error in the response body + assert.Equal(t, http.StatusOK, w.Code) + + // Stream should still be removed even if orchestrator returns error + _, exists := ls.LivepeerNode.LivePipelines[streamID] + assert.False(t, exists, "Stream should be removed even on orchestrator error") + }) +} + +// Test StartStreamRTMPIngest handler +func TestStartStreamRTMPIngestHandler(t *testing.T) { + // Setup mock MediaMTX server on port 9997 before starting the test + mockMediaMTXServer := createMockMediaMTXServer(t) + defer mockMediaMTXServer.Close() + + node := mockJobLivepeerNode() + node.WorkDir = t.TempDir() + server := httptest.NewServer(http.HandlerFunc(orchTokenHandler)) + defer server.Close() + node.OrchestratorPool = newStubOrchestratorPool(node, []string{server.URL}) + + ls := &LivepeerServer{ + LivepeerNode: node, + mediaMTXApiPassword: "test-password", + } + drivers.NodeStorage = drivers.NewMemoryDriver(nil) + + // Prepare a valid gatewayJob + jobParams := JobParameters{EnableVideoIngress: true, EnableVideoEgress: true, EnableDataOutput: true} + paramsStr := marshalToString(t, jobParams) + jobReq := &JobRequest{ + Capability: "test-capability", + Parameters: paramsStr, + Timeout: 10, + } + orchJob := &orchJob{Req: jobReq, Params: &jobParams} + gatewayJob := &gatewayJob{Job: orchJob} + + // Prepare a valid StartRequest body + startReq := StartRequest{ + Stream: "teststream", + RtmpOutput: "rtmp://output", + StreamId: "streamid", + Params: "{}", + } + body, _ := json.Marshal(startReq) + req := httptest.NewRequest(http.MethodPost, "/ai/stream/start", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + + urls, code, err := ls.setupStream(context.Background(), req, gatewayJob) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, code) + assert.NotNil(t, urls) + assert.Equal(t, "teststream-streamid", urls.StreamId) //combination of stream name (Stream) and id (StreamId) + + stream, ok := ls.LivepeerNode.LivePipelines[urls.StreamId] + assert.True(t, ok) + assert.NotNil(t, stream) + + params, err := getStreamRequestParams(stream) + assert.NoError(t, err) + + //these should be empty/nil before rtmp ingest starts + assert.Empty(t, params.liveParams.localRTMPPrefix) + assert.Nil(t, params.liveParams.kickInput) + + rtmpReq := httptest.NewRequest(http.MethodPost, "/ai/stream/{streamId}/rtmp", nil) + rtmpReq.SetPathValue("streamId", "teststream-streamid") + w := httptest.NewRecorder() + + handler := ls.StartStreamRTMPIngest() + handler.ServeHTTP(w, rtmpReq) + // Missing source_id and source_type + assert.Equal(t, http.StatusBadRequest, w.Code) + + // Now provide valid form data + formData := url.Values{} + formData.Set("source_id", "testsourceid") + formData.Set("source_type", "rtmpconn") + rtmpReq = httptest.NewRequest(http.MethodPost, "/ai/stream/{streamId}/rtmp", strings.NewReader(formData.Encode())) + rtmpReq.SetPathValue("streamId", "teststream-streamid") + // Use localhost as the remote addr to simulate MediaMTX + rtmpReq.RemoteAddr = "127.0.0.1:1935" + + rtmpReq.Header.Set("Content-Type", "application/x-www-form-urlencoded") + w = httptest.NewRecorder() + handler.ServeHTTP(w, rtmpReq) + assert.Equal(t, http.StatusOK, w.Code) + + // Verify that the stream parameters were updated correctly + newParams, _ := getStreamRequestParams(stream) + assert.NotNil(t, newParams.liveParams.kickInput) + assert.NotEmpty(t, newParams.liveParams.localRTMPPrefix) + + // Stop the stream to cleanup + newParams.liveParams.kickInput(errors.New("test error")) + stream.StopStream(nil) +} + +// Test StartStreamWhipIngest handler +func TestStartStreamWhipIngestHandler(t *testing.T) { + node := mockJobLivepeerNode() + node.WorkDir = t.TempDir() + server := httptest.NewServer(http.HandlerFunc(orchTokenHandler)) + defer server.Close() + node.OrchestratorPool = newStubOrchestratorPool(node, []string{server.URL}) + ls := &LivepeerServer{LivepeerNode: node} + drivers.NodeStorage = drivers.NewMemoryDriver(nil) + + // Prepare a valid gatewayJob + jobParams := JobParameters{EnableVideoIngress: true, EnableVideoEgress: true, EnableDataOutput: true} + paramsStr := marshalToString(t, jobParams) + jobReq := &JobRequest{ + Capability: "test-capability", + Parameters: paramsStr, + Timeout: 10, + } + orchJob := &orchJob{Req: jobReq, Params: &jobParams} + gatewayJob := &gatewayJob{Job: orchJob} + + // Prepare a valid StartRequest body for /ai/stream/start + startReq := StartRequest{ + Stream: "teststream", + RtmpOutput: "rtmp://output", + StreamId: "streamid", + Params: "{}", + } + body, _ := json.Marshal(startReq) + req := httptest.NewRequest(http.MethodPost, "/ai/stream/start", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + + urls, code, err := ls.setupStream(context.Background(), req, gatewayJob) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, code) + assert.NotNil(t, urls) + assert.Equal(t, "teststream-streamid", urls.StreamId) //combination of stream name (Stream) and id (StreamId) + + stream, ok := ls.LivepeerNode.LivePipelines[urls.StreamId] + assert.True(t, ok) + assert.NotNil(t, stream) + + params, err := getStreamRequestParams(stream) + assert.NoError(t, err) + + //these should be empty/nil before whip ingest starts + assert.Empty(t, params.liveParams.localRTMPPrefix) + assert.Nil(t, params.liveParams.kickInput) + + // whipServer is required, using nil will test setup up to initializing the WHIP connection + whipServer := media.NewWHIPServer() + handler := ls.StartStreamWhipIngest(whipServer) + + // SDP offer for WHIP with H.264 video and Opus audio + sdpOffer := `v=0 +o=- 123456789 2 IN IP4 127.0.0.1 +s=- +t=0 0 +a=group:BUNDLE 0 1 +a=msid-semantic: WMS stream +m=video 9 UDP/TLS/RTP/SAVPF 96 +c=IN IP4 0.0.0.0 +a=rtcp:9 IN IP4 0.0.0.0 +a=ice-ufrag:abcd +a=ice-pwd:abcdefghijklmnopqrstuvwxyz123456 +a=fingerprint:sha-256 00:11:22:33:44:55:66:77:88:99:AA:BB:CC:DD:EE:FF:00:11:22:33:44:55:66:77:88:99:AA:BB:CC:DD:EE:FF +a=setup:actpass +a=mid:0 +a=extmap:1 urn:ietf:params:rtp-hdrext:sdes:mid +a=extmap:2 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id +a=extmap:3 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id +a=sendonly +a=msid:stream video +a=rtcp-mux +a=rtpmap:96 H264/90000 +a=rtcp-fb:96 goog-remb +a=rtcp-fb:96 transport-cc +a=rtcp-fb:96 ccm fir +a=rtcp-fb:96 nack +a=rtcp-fb:96 nack pli +a=fmtp:96 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f +m=audio 9 UDP/TLS/RTP/SAVPF 111 +c=IN IP4 0.0.0.0 +a=rtcp:9 IN IP4 0.0.0.0 +a=ice-ufrag:abcd +a=ice-pwd:abcdefghijklmnopqrstuvwxyz123456 +a=fingerprint:sha-256 00:11:22:33:44:55:66:77:88:99:AA:BB:CC:DD:EE:FF:00:11:22:33:44:55:66:77:88:99:AA:BB:CC:DD:EE:FF +a=setup:actpass +a=mid:1 +a=extmap:1 urn:ietf:params:rtp-hdrext:sdes:mid +a=sendonly +a=msid:stream audio +a=rtcp-mux +a=rtpmap:111 opus/48000/2 +a=rtcp-fb:111 transport-cc +a=fmtp:111 minptime=10;useinbandfec=1 +` + + whipReq := httptest.NewRequest(http.MethodPost, "/ai/stream/{streamId}/whip", strings.NewReader(sdpOffer)) + whipReq.SetPathValue("streamId", "teststream-streamid") + whipReq.Header.Set("Content-Type", "application/sdp") + + w := httptest.NewRecorder() + handler.ServeHTTP(w, whipReq) + assert.Equal(t, http.StatusCreated, w.Code) + + newParams, err := getStreamRequestParams(stream) + assert.NoError(t, err) + assert.NotNil(t, newParams.liveParams.kickInput) + + //stop the WHIP connection + time.Sleep(2 * time.Millisecond) //wait for setup + //add kickOrch because we are not calling runStream which would have added it + newParams.liveParams.kickOrch = func(error) {} + stream.UpdateStreamParams(newParams) + newParams.liveParams.kickInput(errors.New("test complete")) +} + +// Test GetStreamData handler +func TestGetStreamDataHandler(t *testing.T) { + + t.Run("StreamData_MissingStreamId", func(t *testing.T) { + // Test with missing stream ID - should return 400 + ls := &LivepeerServer{} + handler := ls.UpdateStream() + req := httptest.NewRequest(http.MethodPost, "/ai/stream/{streamId}/update", nil) + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + assert.Equal(t, http.StatusBadRequest, w.Code) + assert.Contains(t, w.Body.String(), "Missing stream name") + }) + + t.Run("StreamData_DataOutputWorking", func(t *testing.T) { + node := mockJobLivepeerNode() + node.WorkDir = t.TempDir() + server := httptest.NewServer(http.HandlerFunc(orchTokenHandler)) + defer server.Close() + node.OrchestratorPool = newStubOrchestratorPool(node, []string{server.URL}) + ls := &LivepeerServer{LivepeerNode: node} + drivers.NodeStorage = drivers.NewMemoryDriver(nil) + + // Prepare a valid gatewayJob + jobParams := JobParameters{EnableVideoIngress: true, EnableVideoEgress: true, EnableDataOutput: true} + paramsStr := marshalToString(t, jobParams) + jobReq := &JobRequest{ + Capability: "test-capability", + Parameters: paramsStr, + Timeout: 10, + } + orchJob := &orchJob{Req: jobReq, Params: &jobParams} + gatewayJob := &gatewayJob{Job: orchJob} + + // Prepare a valid StartRequest body for /ai/stream/start + startReq := StartRequest{ + Stream: "teststream", + RtmpOutput: "rtmp://output", + StreamId: "streamid", + Params: "{}", + } + body, _ := json.Marshal(startReq) + req := httptest.NewRequest(http.MethodPost, "/ai/stream/start", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + + urls, code, err := ls.setupStream(context.Background(), req, gatewayJob) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, code) + assert.NotNil(t, urls) + assert.Equal(t, "teststream-streamid", urls.StreamId) //combination of stream name (Stream) and id (StreamId) + + stream, ok := ls.LivepeerNode.LivePipelines[urls.StreamId] + assert.True(t, ok) + assert.NotNil(t, stream) + + params, err := getStreamRequestParams(stream) + assert.NoError(t, err) + assert.NotNil(t, params.liveParams) + + // Write some test data first + writer, err := params.liveParams.dataWriter.Next() + assert.NoError(t, err) + writer.Write([]byte("initial-data")) + writer.Close() + + handler := ls.GetStreamData() + dataReq := httptest.NewRequest(http.MethodGet, "/ai/stream/{streamId}/data", nil) + dataReq.SetPathValue("streamId", "teststream-streamid") + + // Create a context with timeout to prevent infinite blocking + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + dataReq = dataReq.WithContext(ctx) + + // Start writing more segments in a goroutine + go func() { + time.Sleep(10 * time.Millisecond) // Give handler time to start + + // Write additional segments + for i := 0; i < 2; i++ { + writer, err := params.liveParams.dataWriter.Next() + if err != nil { + break + } + writer.Write([]byte(fmt.Sprintf("test-data-%d", i))) + writer.Close() + time.Sleep(5 * time.Millisecond) + } + + // Close the writer to signal EOF + time.Sleep(10 * time.Millisecond) + params.liveParams.dataWriter.Close() + }() + + w := httptest.NewRecorder() + handler.ServeHTTP(w, dataReq) + + // Check response + responseBody := w.Body.String() + + // Verify we received some SSE data + assert.Contains(t, responseBody, "data: ", "Should have received SSE data") + + // Check for our test data + if strings.Contains(responseBody, "data: ") { + lines := strings.Split(responseBody, "\n") + dataFound := false + for _, line := range lines { + if strings.HasPrefix(line, "data: ") && strings.Contains(line, "data") { + dataFound = true + break + } + } + assert.True(t, dataFound, "Should have found data in SSE response") + } + }) +} + +// Test UpdateStream handler +func TestUpdateStreamHandler(t *testing.T) { + t.Run("UpdateStream_MissingStreamId", func(t *testing.T) { + // Test with missing stream ID - should return 400 + ls := &LivepeerServer{} + handler := ls.UpdateStream() + req := httptest.NewRequest(http.MethodPost, "/ai/stream/{streamId}/update", nil) + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + assert.Equal(t, http.StatusBadRequest, w.Code) + assert.Contains(t, w.Body.String(), "Missing stream name") + }) + + t.Run("Basic_StreamNotFound", func(t *testing.T) { + // Test with non-existent stream - should return 404 + node := mockJobLivepeerNode() + ls := &LivepeerServer{LivepeerNode: node} + + req := httptest.NewRequest(http.MethodPost, "/ai/stream/{streamId}/update", + strings.NewReader(`{"param1": "value1", "param2": "value2"}`)) + req.SetPathValue("streamId", "non-existent-stream") + req.Header.Set("Content-Type", "application/json") + + w := httptest.NewRecorder() + handler := ls.UpdateStream() + handler.ServeHTTP(w, req) + + assert.Equal(t, http.StatusNotFound, w.Code) + assert.Contains(t, w.Body.String(), "Stream not found") + }) + + t.Run("UpdateStream_WithOrchestratorControlChannel", func(t *testing.T) { + // Setup test infrastructure with mock orchestrator and trickle server + node := mockJobLivepeerNode() + + // Set up trickle-enabled orchestrator server + mux := http.NewServeMux() + mockOrch := &mockOrchestrator{} + + lp := &lphttp{orchestrator: nil, transRPC: mux, node: node} + lp.trickleSrv = trickle.ConfigureServer(trickle.TrickleServerConfig{ + Mux: mux, + BasePath: TrickleHTTPPath, + Autocreate: true, + }) + + // Register other required endpoints + mux.HandleFunc("/process/token", orchTokenHandler) + mux.HandleFunc("/ai/stream/start", orchAIStreamStartHandler) + + server := httptest.NewServer(lp) + defer server.Close() + + // Configure mock orchestrator + parsedURL, _ := url.Parse(server.URL) + capabilitySrv := httptest.NewServer(http.HandlerFunc(orchCapabilityUrlHandler)) + defer capabilitySrv.Close() + + lp.orchestrator = &testStreamOrch{mockOrchestrator: mockOrch, svc: parsedURL, capURL: capabilitySrv.URL} + + // Setup LivepeerServer + node.OrchestratorPool = newStubOrchestratorPool(node, []string{server.URL}) + ls := &LivepeerServer{LivepeerNode: node} + drivers.NodeStorage = drivers.NewMemoryDriver(nil) + + // Create a stream with control publisher + streamID := "test-stream" + controlURL := fmt.Sprintf("%s%stest-stream-control", server.URL, TrickleHTTPPath) + controlPub, err := trickle.NewTricklePublisher(controlURL) + assert.NoError(t, err) + + // Create minimal AI session + token := createMockJobToken(server.URL) + sess, err := tokenToAISession(*token) + assert.NoError(t, err) + + params := aiRequestParams{ + liveParams: &liveRequestParams{ + requestID: "req-1", + sess: &sess, + stream: streamID, + streamID: streamID, + sendErrorEvent: func(err error) {}, + segmentReader: media.NewSwitchableSegmentReader(), + }, + node: node, + } + + // Create and setup stream + stream := node.NewLivePipeline("req-1", streamID, "test-capability", params, nil) + stream.ControlPub = controlPub + + // Test update with valid stream and control publisher + updateData := `{"param1": "updated_value1", "param2": "updated_value2"}` + req := httptest.NewRequest(http.MethodPost, "/ai/stream/{streamId}/update", + strings.NewReader(updateData)) + req.SetPathValue("streamId", streamID) + req.Header.Set("Content-Type", "application/json") + + w := httptest.NewRecorder() + handler := ls.UpdateStream() + handler.ServeHTTP(w, req) + + // Should succeed + assert.Equal(t, http.StatusOK, w.Code) + + // Verify stream params were cached + assert.Equal(t, []byte(updateData), stream.Params) + + // Clean up + stream.StopStream(nil) + controlPub.Close() + }) + + t.Run("UpdateStream_WithoutControlChannel", func(t *testing.T) { + // Test stream update when no orchestrator control channel is available + node := mockJobLivepeerNode() + ls := &LivepeerServer{LivepeerNode: node} + + streamID := "test-stream-no-control" + + // Create minimal AI session + server := httptest.NewServer(http.HandlerFunc(orchTokenHandler)) + defer server.Close() + token := createMockJobToken(server.URL) + sess, err := tokenToAISession(*token) + assert.NoError(t, err) + + params := aiRequestParams{ + liveParams: &liveRequestParams{ + requestID: "req-1", + sess: &sess, + stream: streamID, + streamID: streamID, + sendErrorEvent: func(err error) {}, + segmentReader: media.NewSwitchableSegmentReader(), + }, + node: node, + } + + // Create stream WITHOUT control publisher + stream := node.NewLivePipeline("req-1", streamID, "test-capability", params, nil) + stream.ControlPub = nil // Explicitly set to nil + + updateData := `{"param1": "cached_value"}` + req := httptest.NewRequest(http.MethodPost, "/ai/stream/{streamId}/update", + strings.NewReader(updateData)) + req.SetPathValue("streamId", streamID) + req.Header.Set("Content-Type", "application/json") + + w := httptest.NewRecorder() + handler := ls.UpdateStream() + handler.ServeHTTP(w, req) + + // Should still succeed (params cached locally) + assert.Equal(t, http.StatusOK, w.Code) + + // Verify stream params were cached even without control channel + assert.Equal(t, []byte(updateData), stream.Params) + + // Clean up + stream.StopStream(nil) + }) + + t.Run("UpdateStream_ErrorHandling", func(t *testing.T) { + // Test various error conditions + node := mockJobLivepeerNode() + ls := &LivepeerServer{LivepeerNode: node} + + // Test 1: Wrong HTTP method + req := httptest.NewRequest(http.MethodGet, "/ai/stream/{streamId}/update", nil) + req.SetPathValue("streamId", "test-stream") + w := httptest.NewRecorder() + ls.UpdateStream().ServeHTTP(w, req) + assert.Equal(t, http.StatusMethodNotAllowed, w.Code) + + // Test 2: Request too large + streamID := "test-stream-large" + token := createMockJobToken("http://example.com") + sess, _ := tokenToAISession(*token) + params := aiRequestParams{ + liveParams: &liveRequestParams{ + requestID: "req-1", + sess: &sess, + stream: streamID, + streamID: streamID, + sendErrorEvent: func(err error) {}, + segmentReader: media.NewSwitchableSegmentReader(), + }, + node: node, + } + stream := node.NewLivePipeline("req-1", streamID, "test-capability", params, nil) + + // Create a body larger than 10MB + largeData := bytes.Repeat([]byte("a"), 10*1024*1024+1) + req = httptest.NewRequest(http.MethodPost, "/ai/stream/{streamId}/update", + bytes.NewReader(largeData)) + req.SetPathValue("streamId", streamID) + w = httptest.NewRecorder() + + ls.UpdateStream().ServeHTTP(w, req) + assert.Equal(t, http.StatusBadRequest, w.Code) + assert.Contains(t, w.Body.String(), "http: request body too large") + + stream.StopStream(nil) + + // Test 3: Control publisher write failure (simulate network error) + streamID2 := "test-stream-net-error" + params2 := aiRequestParams{ + liveParams: &liveRequestParams{ + requestID: "req-2", + sess: &sess, + stream: streamID2, + streamID: streamID2, + sendErrorEvent: func(err error) {}, + segmentReader: media.NewSwitchableSegmentReader(), + }, + node: node, + } + stream2 := node.NewLivePipeline("req-2", streamID2, "test-capability", params2, nil) + + // Use a control publisher pointing to non-existent URL + badControlPub, err := trickle.NewTricklePublisher("http://localhost:1/nonexistent") + if err == nil { + stream2.ControlPub = badControlPub + + req = httptest.NewRequest(http.MethodPost, "/ai/stream/{streamId}/update", + strings.NewReader(`{"param": "value"}`)) + req.SetPathValue("streamId", streamID2) + req.Header.Set("Content-Type", "application/json") + w = httptest.NewRecorder() + + ls.UpdateStream().ServeHTTP(w, req) + + // Should return 500 due to control publisher write failure + assert.Equal(t, http.StatusInternalServerError, w.Code) + + // But params should still be cached + assert.Equal(t, []byte(`{"param": "value"}`), stream2.Params) + + badControlPub.Close() + } + + stream2.StopStream(nil) + }) +} + +// Test GetStreamStatus handler +func TestGetStreamStatusHandler(t *testing.T) { + ls := &LivepeerServer{} + handler := ls.GetStreamStatus() + // stream does not exist + req := httptest.NewRequest(http.MethodGet, "/ai/stream/{streamId}/status", nil) + req.SetPathValue("streamId", "any-stream") + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + assert.Equal(t, http.StatusNotFound, w.Code) + + // stream exists + node := mockJobLivepeerNode() + ls.LivepeerNode = node + node.NewLivePipeline("req-1", "any-stream", "test-capability", aiRequestParams{}, nil) + GatewayStatus.StoreKey("any-stream", "test", "test") + req = httptest.NewRequest(http.MethodGet, "/ai/stream/{streamId}/status", nil) + req.SetPathValue("streamId", "any-stream") + w = httptest.NewRecorder() + handler.ServeHTTP(w, req) + assert.Equal(t, http.StatusOK, w.Code) +} + +// Test sendPaymentForStream +func TestSendPaymentForStream(t *testing.T) { + t.Run("Success_ValidPayment", func(t *testing.T) { + // Setup + node := mockJobLivepeerNode() + mockSender := pm.MockSender{} + mockSender.On("StartSession", mock.Anything).Return("foo").Times(2) + mockSender.On("CreateTicketBatch", "foo", 60).Return(mockTicketBatch(60), nil).Once() + node.Sender = &mockSender + node.Balances = core.NewAddressBalances(10) + defer node.Balances.StopCleanup() + + // Create mock orchestrator server that handles token requests and payments + paymentReceived := false + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/process/token": + orchTokenHandler(w, r) + case "/ai/stream/payment": + paymentReceived = true + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"status": "payment_processed"}`)) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + node.OrchestratorPool = newStubOrchestratorPool(node, []string{server.URL}) + ls := &LivepeerServer{LivepeerNode: node} + drivers.NodeStorage = drivers.NewMemoryDriver(nil) + + // Create a mock stream with AI session + streamID := "test-payment-stream" + token := createMockJobToken(server.URL) + sess, err := tokenToAISession(*token) + assert.NoError(t, err) + + params := aiRequestParams{ + liveParams: &liveRequestParams{ + requestID: "req-1", + sess: &sess, + stream: streamID, + streamID: streamID, + sendErrorEvent: func(err error) {}, + segmentReader: media.NewSwitchableSegmentReader(), + }, + node: node, + } + + stream := node.NewLivePipeline("req-1", streamID, "test-capability", params, nil) + + // Create a job sender + jobSender := &core.JobSender{ + Addr: "0x1111111111111111111111111111111111111111", + Sig: "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", + } + + // Test sendPaymentForStream + ctx := context.Background() + err = ls.sendPaymentForStream(ctx, stream, jobSender) + + // Should succeed + assert.NoError(t, err) + + // Verify payment was sent to orchestrator + assert.True(t, paymentReceived, "Payment should have been sent to orchestrator") + + // Clean up + stream.StopStream(nil) + }) + + t.Run("Error_GetTokenFailed", func(t *testing.T) { + // Setup node without orchestrator pool + node := mockJobLivepeerNode() + // Set up mock sender to prevent nil pointer dereference + mockSender := pm.MockSender{} + mockSender.On("StartSession", mock.Anything).Return("foo") + mockSender.On("CreateTicketBatch", mock.Anything, mock.Anything).Return(mockTicketBatch(10), nil) + node.Sender = &mockSender + node.Balances = core.NewAddressBalances(10) + defer node.Balances.StopCleanup() + + ls := &LivepeerServer{LivepeerNode: node} + + // Create a stream with invalid session + streamID := "test-invalid-token" + invalidToken := createMockJobToken("http://nonexistent-server.com") + sess, _ := tokenToAISession(*invalidToken) + params := aiRequestParams{ + liveParams: &liveRequestParams{ + requestID: "req-1", + sess: &sess, + stream: streamID, + streamID: streamID, + sendErrorEvent: func(err error) {}, + segmentReader: media.NewSwitchableSegmentReader(), + }, + node: node, + } + stream := node.NewLivePipeline("req-1", streamID, "test-capability", params, nil) + + jobSender := &core.JobSender{ + Addr: "0x1111111111111111111111111111111111111111", + Sig: "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", + } + + // Should fail to get new token + err := ls.sendPaymentForStream(context.Background(), stream, jobSender) + assert.Error(t, err) + assert.Contains(t, err.Error(), "nonexistent-server.com") + + stream.StopStream(nil) + }) + + t.Run("Error_PaymentCreationFailed", func(t *testing.T) { + // Test with node that has no sender (payment creation will fail) + node := mockJobLivepeerNode() + // node.Sender is nil by default + + server := httptest.NewServer(http.HandlerFunc(orchTokenHandler)) + defer server.Close() + node.OrchestratorPool = newStubOrchestratorPool(node, []string{server.URL}) + ls := &LivepeerServer{LivepeerNode: node} + + streamID := "test-payment-creation-fail" + token := createMockJobToken(server.URL) + sess, _ := tokenToAISession(*token) + params := aiRequestParams{ + liveParams: &liveRequestParams{ + requestID: "req-1", + sess: &sess, + stream: streamID, + streamID: streamID, + sendErrorEvent: func(err error) {}, + segmentReader: media.NewSwitchableSegmentReader(), + }, + node: node, + } + stream := node.NewLivePipeline("req-1", streamID, "test-capability", params, nil) + + jobSender := &core.JobSender{ + Addr: "0x1111111111111111111111111111111111111111", + Sig: "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", + } + + // Should continue even if payment creation fails (no payment required) + err := ls.sendPaymentForStream(context.Background(), stream, jobSender) + assert.NoError(t, err) // Should not error, just logs and continues + + stream.StopStream(nil) + }) + + t.Run("Error_OrchestratorPaymentFailed", func(t *testing.T) { + // Setup node with sender to create payments + node := mockJobLivepeerNode() + mockSender := pm.MockSender{} + mockSender.On("StartSession", mock.Anything).Return("foo").Times(2) + mockSender.On("CreateTicketBatch", "foo", 60).Return(mockTicketBatch(60), nil).Once() + node.Sender = &mockSender + node.Balances = core.NewAddressBalances(10) + defer node.Balances.StopCleanup() + + // Create mock orchestrator that returns error for payments + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/process/token": + orchTokenHandler(w, r) + case "/ai/stream/payment": + http.Error(w, "Payment processing failed", http.StatusInternalServerError) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + node.OrchestratorPool = newStubOrchestratorPool(node, []string{server.URL}) + ls := &LivepeerServer{LivepeerNode: node} + drivers.NodeStorage = drivers.NewMemoryDriver(nil) + + streamID := "test-payment-error" + token := createMockJobToken(server.URL) + sess, _ := tokenToAISession(*token) + params := aiRequestParams{ + liveParams: &liveRequestParams{ + requestID: "req-1", + sess: &sess, + stream: streamID, + streamID: streamID, + sendErrorEvent: func(err error) {}, + segmentReader: media.NewSwitchableSegmentReader(), + }, + node: node, + } + stream := node.NewLivePipeline("req-1", streamID, "test-capability", params, nil) + + jobSender := &core.JobSender{ + Addr: "0x1111111111111111111111111111111111111111", + Sig: "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", + } + + // Should fail with payment error + err := ls.sendPaymentForStream(context.Background(), stream, jobSender) + assert.Error(t, err) + assert.Contains(t, err.Error(), "unexpected status code") + + stream.StopStream(nil) + }) + + t.Run("Error_TokenToSessionConversionNoPrice", func(t *testing.T) { + // Test where tokenToAISession fails + node := mockJobLivepeerNode() + + // Set up mock sender to prevent nil pointer dereference + mockSender := pm.MockSender{} + mockSender.On("StartSession", mock.Anything).Return("foo") + mockSender.On("CreateTicketBatch", mock.Anything, mock.Anything).Return(mockTicketBatch(10), nil) + node.Sender = &mockSender + node.Balances = core.NewAddressBalances(10) + defer node.Balances.StopCleanup() + + // Create a server that returns invalid token response + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/process/token" { + // Return malformed token that will cause tokenToAISession to fail + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"invalid": "token_structure"}`)) + return + } + http.NotFound(w, r) + })) + defer server.Close() + + node.OrchestratorPool = newStubOrchestratorPool(node, []string{server.URL}) + ls := &LivepeerServer{LivepeerNode: node} + + // Create stream with valid initial session + streamID := "test-token-no-price" + token := createMockJobToken(server.URL) + sess, _ := tokenToAISession(*token) + params := aiRequestParams{ + liveParams: &liveRequestParams{ + requestID: "req-1", + sess: &sess, + stream: streamID, + streamID: streamID, + sendErrorEvent: func(err error) {}, + segmentReader: media.NewSwitchableSegmentReader(), + }, + node: node, + } + stream := node.NewLivePipeline("req-1", streamID, "test-capability", params, nil) + + jobSender := &core.JobSender{ + Addr: "0x1111111111111111111111111111111111111111", + Sig: "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", + } + + // Should fail during token to session conversion + err := ls.sendPaymentForStream(context.Background(), stream, jobSender) + assert.NoError(t, err) + + stream.StopStream(nil) + }) + + t.Run("Success_StreamParamsUpdated", func(t *testing.T) { + // Test that stream params are updated with new session after token refresh + node := mockJobLivepeerNode() + mockSender := pm.MockSender{} + mockSender.On("StartSession", mock.Anything).Return("foo").Times(2) + mockSender.On("CreateTicketBatch", "foo", 60).Return(mockTicketBatch(60), nil).Once() + node.Sender = &mockSender + node.Balances = core.NewAddressBalances(10) + defer node.Balances.StopCleanup() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/process/token": + orchTokenHandler(w, r) + case "/ai/stream/payment": + w.WriteHeader(http.StatusOK) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + node.OrchestratorPool = newStubOrchestratorPool(node, []string{server.URL}) + ls := &LivepeerServer{LivepeerNode: node} + drivers.NodeStorage = drivers.NewMemoryDriver(nil) + + streamID := "test-params-update" + originalToken := createMockJobToken(server.URL) + originalSess, _ := tokenToAISession(*originalToken) + originalSessionAddr := originalSess.Address() + + params := aiRequestParams{ + liveParams: &liveRequestParams{ + requestID: "req-1", + sess: &originalSess, + stream: streamID, + streamID: streamID, + sendErrorEvent: func(err error) {}, + segmentReader: media.NewSwitchableSegmentReader(), + }, + node: node, + } + stream := node.NewLivePipeline("req-1", streamID, "test-capability", params, nil) + + jobSender := &core.JobSender{ + Addr: "0x1111111111111111111111111111111111111111", + Sig: "0x1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef", + } + + // Send payment + err := ls.sendPaymentForStream(context.Background(), stream, jobSender) + assert.NoError(t, err) + + // Verify that stream params were updated with new session + updatedParams, err := getStreamRequestParams(stream) + assert.NoError(t, err) + + // The session should be updated (new token fetched) + updatedSessionAddr := updatedParams.liveParams.sess.Address() + // In a real scenario, this might be different, but our mock returns the same token + // The important thing is that UpdateStreamParams was called + assert.NotNil(t, updatedParams.liveParams.sess) + assert.Equal(t, originalSessionAddr, updatedSessionAddr) // Same because mock returns same token + + stream.StopStream(nil) + }) +} + +func TestTokenSessionConversion(t *testing.T) { + token := createMockJobToken("http://example.com") + sess, err := tokenToAISession(*token) + assert.True(t, err != nil || sess != (AISession{})) + assert.NotNil(t, sess.OrchestratorInfo) + assert.NotNil(t, sess.OrchestratorInfo.TicketParams) + + assert.NotEmpty(t, sess.Address()) + assert.NotEmpty(t, sess.Transcoder()) + + _, err = sessionToToken(&sess) + assert.True(t, err != nil || true) +} + +func TestGetStreamRequestParams(t *testing.T) { + _, err := getStreamRequestParams(nil) + assert.Error(t, err) +} + +// createMockMediaMTXServer creates a simple mock MediaMTX server that returns 200 OK to all requests +func createMockMediaMTXServer(t *testing.T) *httptest.Server { + mux := http.NewServeMux() + + // Simple handler that returns 200 OK to any request + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + t.Logf("Mock MediaMTX: %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusOK) + w.Write([]byte("OK")) + }) + + // Create a listener on port 9997 specifically + listener, err := net.Listen("tcp", ":9997") + if err != nil { + t.Fatalf("Failed to listen on port 9997: %v", err) + } + + server := &httptest.Server{ + Listener: listener, + Config: &http.Server{Handler: mux}, + } + server.Start() + + t.Cleanup(func() { + server.Close() + }) + + return server +} From a6c8d886b594a4b8296d8590cb7576437644e56f Mon Sep 17 00:00:00 2001 From: Brad P Date: Wed, 24 Sep 2025 07:03:40 -0500 Subject: [PATCH 55/57] make stream payment processing more lenient --- server/job_stream.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/server/job_stream.go b/server/job_stream.go index 6a7f1918cc..f46a6d3a59 100644 --- a/server/job_stream.go +++ b/server/job_stream.go @@ -344,7 +344,7 @@ func (ls *LivepeerServer) sendPaymentForStream(ctx context.Context, stream *core } req := &JobRequest{Request: string(jobDetailsStr), Parameters: "{}", Capability: stream.Pipeline, Sender: jobSender.Addr, - Timeout: 60, + Timeout: 70, } //sign the request job := gatewayJob{Job: &orchJob{Req: req}, node: ls.LivepeerNode} @@ -1163,6 +1163,7 @@ func (h *lphttp) StartStream(w http.ResponseWriter, r *http.Request) { pmtCheckDur := 23 * time.Second //run slightly faster than gateway so can return updated balance pmtTicker := time.NewTicker(pmtCheckDur) defer pmtTicker.Stop() + shouldStopStreamNextRound := false for { select { case <-stream.StreamCtx.Done(): @@ -1178,6 +1179,13 @@ func (h *lphttp) StartStream(w http.ResponseWriter, r *http.Request) { senderBalance := getPaymentBalance(orch, orchJob.Sender, orchJob.Req.Capability) if senderBalance != nil { if senderBalance.Cmp(big.NewRat(0, 1)) < 0 { + if !shouldStopStreamNextRound { + //warn once + clog.Warningf(ctx, "Insufficient balance for stream capability, will stop stream next round if not replenished sender=%s capability=%s balance=%s", orchJob.Sender, orchJob.Req.Capability, senderBalance.FloatString(0)) + shouldStopStreamNextRound = true + continue + } + clog.Infof(ctx, "Insufficient balance, stopping stream %s for sender %s", orchJob.Req.ID, orchJob.Sender) _, exists := h.node.ExternalCapabilities.Streams[orchJob.Req.ID] if exists { From b1771b720c7681daa294e7a7ca3f38918e96f8de Mon Sep 17 00:00:00 2001 From: Brad P Date: Wed, 24 Sep 2025 15:33:40 -0500 Subject: [PATCH 56/57] add startTime to liveParams for First Segment Delay calcs --- server/job_stream.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/job_stream.go b/server/job_stream.go index f46a6d3a59..1f953193a6 100644 --- a/server/job_stream.go +++ b/server/job_stream.go @@ -594,7 +594,6 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, job // kickInput will close the whip connection // localRTMPPrefix set by ENV variable LIVE_AI_PLAYBACK_HOST ssr := media.NewSwitchableSegmentReader() //this converts ingest to segments to send to Orchestrator - params := aiRequestParams{ node: ls.LivepeerNode, os: drivers.NodeStorage.NewSession(requestID), @@ -602,6 +601,7 @@ func (ls *LivepeerServer) setupStream(ctx context.Context, r *http.Request, job liveParams: &liveRequestParams{ segmentReader: ssr, + startTime: time.Now(), rtmpOutputs: rtmpOutputs, stream: streamID, //live video to video uses stream name, byoc combines to one id paymentProcessInterval: ls.livePaymentInterval, From 9c94845bf3b1421ae0ebc5687b36d64c0561d0f7 Mon Sep 17 00:00:00 2001 From: Brad P Date: Wed, 1 Oct 2025 13:53:38 -0500 Subject: [PATCH 57/57] fix spacing --- common/testutil.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/testutil.go b/common/testutil.go index 1687c9ad63..1b4155efba 100644 --- a/common/testutil.go +++ b/common/testutil.go @@ -93,7 +93,7 @@ func IgnoreRoutines() []goleak.Option { "github.com/livepeer/go-livepeer/core.(*Balances).StartCleanup", "internal/synctest.Run", "testing/synctest.testingSynctestTest", - "github.com/livepeer/go-livepeer/core.(*Balances).StartCleanup", "github.com/livepeer/go-livepeer/server.startTrickleSubscribe.func2", "github.com/livepeer/go-livepeer/server.startTrickleSubscribe", + "github.com/livepeer/go-livepeer/core.(*Balances).StartCleanup", "github.com/livepeer/go-livepeer/server.startTrickleSubscribe.func2", "github.com/livepeer/go-livepeer/server.startTrickleSubscribe", "net/http.(*persistConn).writeLoop", "net/http.(*persistConn).readLoop", "io.(*pipe).read", "github.com/livepeer/go-livepeer/media.gatherIncomingTracks", }