Skip to content
This repository was archived by the owner on Oct 6, 2025. It is now read-only.

Commit f064505

Browse files
doringemandoringeman
authored
Merge pull request #151 from doringeman/requests-sse
Add requests monitoring
2 parents fd5ec6e + 2a43e46 commit f064505

File tree

89 files changed

+3543
-214
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

89 files changed

+3543
-214
lines changed

‎commands/requests.go‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
package commands
2+
3+
import (
4+
"bufio"
5+
"fmt"
6+
"io"
7+
"strings"
8+
9+
"github.com/docker/model-cli/commands/completion"
10+
"github.com/spf13/cobra"
11+
)
12+
13+
func newRequestsCmd() *cobra.Command {
14+
var model string
15+
var follow bool
16+
var includeExisting bool
17+
c := &cobra.Command{
18+
Use: "requests [OPTIONS]",
19+
Short: "Fetch requests+responses from Docker Model Runner",
20+
PreRunE: func(cmd *cobra.Command, args []string) error {
21+
// Make --include-existing only available when --follow is set.
22+
if includeExisting && !follow {
23+
return fmt.Errorf("--include-existing can only be used with --follow")
24+
}
25+
return nil
26+
},
27+
RunE: func(cmd *cobra.Command, args []string) error {
28+
if _, err := ensureStandaloneRunnerAvailable(cmd.Context(), cmd); err != nil {
29+
return fmt.Errorf("unable to initialize standalone model runner: %w", err)
30+
}
31+
32+
responseBody, cancel, err := desktopClient.Requests(model, follow, includeExisting)
33+
if err != nil {
34+
errMsg := "Failed to get requests"
35+
if model != "" {
36+
errMsg = errMsg + " for " + model
37+
}
38+
err = handleClientError(err, errMsg)
39+
return handleNotRunningError(err)
40+
}
41+
defer cancel()
42+
43+
if follow {
44+
scanner := bufio.NewScanner(responseBody)
45+
cmd.Println("Connected to request stream. Press Ctrl+C to stop.")
46+
var currentEvent string
47+
for scanner.Scan() {
48+
select {
49+
case <-cmd.Context().Done():
50+
return nil
51+
default:
52+
}
53+
line := scanner.Text()
54+
if strings.HasPrefix(line, "event: ") {
55+
currentEvent = strings.TrimPrefix(line, "event: ")
56+
} else if strings.HasPrefix(line, "data: ") &&
57+
(currentEvent == "new_request" || currentEvent == "existing_request") {
58+
data := strings.TrimPrefix(line, "data: ")
59+
cmd.Println(data)
60+
}
61+
}
62+
cmd.Println("Stream closed by server.")
63+
} else {
64+
body, err := io.ReadAll(responseBody)
65+
if err != nil {
66+
return fmt.Errorf("failed to read response body: %w", err)
67+
}
68+
cmd.Print(string(body))
69+
}
70+
71+
return nil
72+
},
73+
ValidArgsFunction: completion.NoComplete,
74+
}
75+
c.Flags().BoolVarP(&follow, "follow", "f", false, "Follow requests stream")
76+
c.Flags().BoolVar(&includeExisting, "include-existing", false,
77+
"Include existing requests when starting to follow (only available with --follow)")
78+
c.Flags().StringVar(&model, "model", "", "Specify the model to filter requests")
79+
// Enable completion for the --model flag.
80+
_ = c.RegisterFlagCompletionFunc("model", completion.ModelNames(getDesktopClient, 1))
81+
return c
82+
}

‎commands/root.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,7 @@ func NewRootCmd(cli *command.DockerCli) *cobra.Command {
110110
newPSCmd(),
111111
newDFCmd(),
112112
newUnloadCmd(),
113+
newRequestsCmd(),
113114
)
114115
return rootCmd
115116
}

‎desktop/desktop.go‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"html"
1010
"io"
1111
"net/http"
12+
"net/url"
1213
"strconv"
1314
"strings"
1415
"time"
@@ -657,6 +658,63 @@ func (c *Client) ConfigureBackend(request scheduling.ConfigureRequest) error {
657658
return nil
658659
}
659660

661+
// Requests returns a response body and a cancel function to ensure proper cleanup.
662+
func (c *Client) Requests(modelFilter string, streaming bool, includeExisting bool) (io.ReadCloser, func(), error) {
663+
path := c.modelRunner.URL(inference.InferencePrefix + "/requests")
664+
var queryParams []string
665+
if modelFilter != "" {
666+
queryParams = append(queryParams, "model="+url.QueryEscape(modelFilter))
667+
}
668+
if includeExisting && streaming {
669+
queryParams = append(queryParams, "include_existing=true")
670+
}
671+
if len(queryParams) > 0 {
672+
path += "?" + strings.Join(queryParams, "&")
673+
}
674+
675+
req, err := http.NewRequest(http.MethodGet, path, nil)
676+
if err != nil {
677+
return nil, nil, fmt.Errorf("failed to create request: %w", err)
678+
}
679+
680+
if streaming {
681+
req.Header.Set("Accept", "text/event-stream")
682+
req.Header.Set("Cache-Control", "no-cache")
683+
} else {
684+
req.Header.Set("Accept", "application/json")
685+
}
686+
req.Header.Set("User-Agent", "docker-model-cli/"+Version)
687+
688+
resp, err := c.modelRunner.Client().Do(req)
689+
if err != nil {
690+
if streaming {
691+
return nil, nil, c.handleQueryError(fmt.Errorf("failed to connect to stream: %w", err), path)
692+
}
693+
return nil, nil, c.handleQueryError(err, path)
694+
}
695+
696+
if resp.StatusCode != http.StatusOK {
697+
if resp.StatusCode == http.StatusNotFound {
698+
body, _ := io.ReadAll(resp.Body)
699+
resp.Body.Close()
700+
return nil, nil, fmt.Errorf("%s", strings.TrimSpace(string(body)))
701+
}
702+
703+
resp.Body.Close()
704+
if streaming {
705+
return nil, nil, fmt.Errorf("stream request failed with status: %d", resp.StatusCode)
706+
}
707+
return nil, nil, fmt.Errorf("failed to list requests: %s", resp.Status)
708+
}
709+
710+
// Return the response body and a cancel function that closes it.
711+
cancel := func() {
712+
resp.Body.Close()
713+
}
714+
715+
return resp.Body, cancel, nil
716+
}
717+
660718
// doRequest is a helper function that performs HTTP requests and handles 503 responses
661719
func (c *Client) doRequest(method, path string, body io.Reader) (*http.Response, error) {
662720
return c.doRequestWithAuth(method, path, body, "", "")

‎docs/reference/docker_model.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ cname:
1515
- docker model ps
1616
- docker model pull
1717
- docker model push
18+
- docker model requests
1819
- docker model rm
1920
- docker model run
2021
- docker model status
@@ -32,6 +33,7 @@ clink:
3233
- docker_model_ps.yaml
3334
- docker_model_pull.yaml
3435
- docker_model_push.yaml
36+
- docker_model_requests.yaml
3537
- docker_model_rm.yaml
3638
- docker_model_run.yaml
3739
- docker_model_status.yaml

‎docs/reference/docker_model_requests.yaml‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
command: docker model requests
2+
short: Fetch requests+responses from Docker Model Runner
3+
long: Fetch requests+responses from Docker Model Runner
4+
usage: docker model requests [OPTIONS]
5+
pname: docker model
6+
plink: docker_model.yaml
7+
options:
8+
- option: follow
9+
shorthand: f
10+
value_type: bool
11+
default_value: "false"
12+
description: Follow requests stream
13+
deprecated: false
14+
hidden: false
15+
experimental: false
16+
experimentalcli: false
17+
kubernetes: false
18+
swarm: false
19+
- option: include-existing
20+
value_type: bool
21+
default_value: "false"
22+
description: |
23+
Include existing requests when starting to follow (only available with --follow)
24+
deprecated: false
25+
hidden: false
26+
experimental: false
27+
experimentalcli: false
28+
kubernetes: false
29+
swarm: false
30+
- option: model
31+
value_type: string
32+
description: Specify the model to filter requests
33+
deprecated: false
34+
hidden: false
35+
experimental: false
36+
experimentalcli: false
37+
kubernetes: false
38+
swarm: false
39+
deprecated: false
40+
hidden: false
41+
experimental: false
42+
experimentalcli: false
43+
kubernetes: false
44+
swarm: false
45+

‎docs/reference/model.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ Docker Model Runner
1616
| [`ps`](model_ps.md) | List running models |
1717
| [`pull`](model_pull.md) | Pull a model from Docker Hub or HuggingFace to your local environment |
1818
| [`push`](model_push.md) | Push a model to Docker Hub |
19+
| [`requests`](model_requests.md) | Fetch requests+responses from Docker Model Runner |
1920
| [`rm`](model_rm.md) | Remove local models downloaded from Docker Hub |
2021
| [`run`](model_run.md) | Run a model and interact with it using a submitted prompt or chat mode |
2122
| [`status`](model_status.md) | Check if the Docker Model Runner is running |

‎docs/reference/model_requests.md‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# docker model requests
2+
3+
<!---MARKER_GEN_START-->
4+
Fetch requests+responses from Docker Model Runner
5+
6+
### Options
7+
8+
| Name | Type | Default | Description |
9+
|:---------------------|:---------|:--------|:---------------------------------------------------------------------------------|
10+
| `-f`, `--follow` | `bool` | | Follow requests stream |
11+
| `--include-existing` | `bool` | | Include existing requests when starting to follow (only available with --follow) |
12+
| `--model` | `string` | | Specify the model to filter requests |
13+
14+
15+
<!---MARKER_GEN_END-->
16+

‎go.mod‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,8 @@ require (
1111
github.com/docker/docker v28.2.2+incompatible
1212
github.com/docker/go-connections v0.5.0
1313
github.com/docker/go-units v0.5.0
14-
github.com/docker/model-distribution v0.0.0-20250822172258-8fe9daa4a4da
15-
github.com/docker/model-runner v0.0.0-20250822173738-5341c9fc2974
14+
github.com/docker/model-distribution v0.0.0-20250905083217-3f098b3d8058
15+
github.com/docker/model-runner v0.0.0-20250911130340-38bb0171c947
1616
github.com/fatih/color v1.15.0
1717
github.com/google/go-containerregistry v0.20.6
1818
github.com/mattn/go-isatty v0.0.20
@@ -63,6 +63,7 @@ require (
6363
github.com/jaypipes/pcidb v1.0.1 // indirect
6464
github.com/json-iterator/go v1.1.12 // indirect
6565
github.com/klauspost/compress v1.18.0 // indirect
66+
github.com/kolesnikovae/go-winjob v1.0.0 // indirect
6667
github.com/mattn/go-colorable v0.1.13 // indirect
6768
github.com/mattn/go-runewidth v0.0.16 // indirect
6869
github.com/mattn/go-shellwords v1.0.12 // indirect
@@ -103,7 +104,7 @@ require (
103104
golang.org/x/exp v0.0.0-20250106191152-7588d65b2ba8 // indirect
104105
golang.org/x/mod v0.25.0 // indirect
105106
golang.org/x/net v0.41.0 // indirect
106-
golang.org/x/sys v0.33.0 // indirect
107+
golang.org/x/sys v0.35.0 // indirect
107108
golang.org/x/text v0.26.0 // indirect
108109
golang.org/x/time v0.9.0 // indirect
109110
golang.org/x/tools v0.34.0 // indirect
@@ -117,3 +118,5 @@ require (
117118
gotest.tools/v3 v3.5.2 // indirect
118119
howett.net/plist v1.0.1 // indirect
119120
)
121+
122+
replace github.com/kolesnikovae/go-winjob => github.com/docker/go-winjob v0.0.0-20250829235554-57b487ebcbc5

‎go.sum‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -77,11 +77,13 @@ github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQ
7777
github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw=
7878
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
7979
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
80+
github.com/docker/go-winjob v0.0.0-20250829235554-57b487ebcbc5 h1:dxSFEb0EEmvceIawSFNDMrvKakRz2t+2WYpY3dFAT04=
81+
github.com/docker/go-winjob v0.0.0-20250829235554-57b487ebcbc5/go.mod h1:ICOGmIXdwhfid7rQP+tLvDJqVg0lHdEk3pI5nsapTtg=
8082
github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
81-
github.com/docker/model-distribution v0.0.0-20250822172258-8fe9daa4a4da h1:ml99WBfcLnsy1frXQR4X+5WAC0DoGtwZyGoU/xBsDQM=
82-
github.com/docker/model-distribution v0.0.0-20250822172258-8fe9daa4a4da/go.mod h1:dThpO9JoG5Px3i+rTluAeZcqLGw8C0qepuEL4gL2o/c=
83-
github.com/docker/model-runner v0.0.0-20250822173738-5341c9fc2974 h1:/uF17tBEtsE6T2Xgg4cgrrqNcQ02gY5Lp98je+2K0nQ=
84-
github.com/docker/model-runner v0.0.0-20250822173738-5341c9fc2974/go.mod h1:1Q2QRB5vob542x6P5pQXlGTYs5bYPxNG6ePcjTndA0A=
83+
github.com/docker/model-distribution v0.0.0-20250905083217-3f098b3d8058 h1:whffgQ1pmiMFVrxRhJKA9yyCJXvmVX6iiohU9ezKCx0=
84+
github.com/docker/model-distribution v0.0.0-20250905083217-3f098b3d8058/go.mod h1:dThpO9JoG5Px3i+rTluAeZcqLGw8C0qepuEL4gL2o/c=
85+
github.com/docker/model-runner v0.0.0-20250911130340-38bb0171c947 h1:6Dz1SFZONEd8tlKetn2Gu6v5HDJI/YtUFwkqHGwrsV0=
86+
github.com/docker/model-runner v0.0.0-20250911130340-38bb0171c947/go.mod h1:cl7panafjkSHllYCCGYAzty2aUvbwk55Gi35v06XL80=
8587
github.com/dvsekhvalnov/jose2go v0.0.0-20170216131308-f21a8cedbbae/go.mod h1:7BvyPhdbLxMXIYTFPLsyJRFMsKmOZnQmzh6Gb+uquuM=
8688
github.com/elastic/go-sysinfo v1.15.3 h1:W+RnmhKFkqPTCRoFq2VCTmsT4p/fwpo+3gKNQsn1XU0=
8789
github.com/elastic/go-sysinfo v1.15.3/go.mod h1:K/cNrqYTDrSoMh2oDkYEMS2+a72GRxMvNP+GC+vRIlo=
@@ -345,8 +347,8 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
345347
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
346348
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
347349
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
348-
golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
349-
golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
350+
golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=
351+
golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
350352
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
351353
golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg=
352354
golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ=

‎vendor/github.com/docker/model-runner/pkg/inference/backend.go‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)