Merge branch 'main' into inspect-remote-model

# Conflicts:
#	commands/inspect.go

Author: ilopezluna

commands/compose.go

@@ -5,11 +5,11 @@ import (
 	"errors"
 	"fmt"
 	"slices"
-	"strconv"
 	"strings"
 
 	"github.com/docker/model-cli/desktop"
-	"github.com/docker/model-cli/pkg/standalone"
+	"github.com/docker/model-runner/pkg/inference/backends/llamacpp"
+	"github.com/docker/model-runner/pkg/inference/scheduling"
 	"github.com/spf13/cobra"
 )
 
@@ -28,6 +28,9 @@ func newComposeCmd() *cobra.Command {
 
 func newUpCommand() *cobra.Command {
 	var models []string
+	var ctxSize int64
+	var rawRuntimeFlags string
+	var backend string
 	c := &cobra.Command{
 		Use: "up",
 		RunE: func(cmd *cobra.Command, args []string) error {
@@ -37,31 +40,61 @@ func newUpCommand() *cobra.Command {
 				return err
 			}
 
-			if err := ensureStandaloneRunnerAvailable(cmd.Context(), nil); err != nil {
+			sendInfo("Initializing model runner...")
+			if ctxSize != 4096 {
+				sendInfo(fmt.Sprintf("Setting context size to %d", ctxSize))
+			}
+			if rawRuntimeFlags != "" {
+				sendInfo("Setting raw runtime flags to " + rawRuntimeFlags)
+			}
+
+			kind := modelRunner.EngineKind()
+			standalone, err := ensureStandaloneRunnerAvailable(cmd.Context(), nil)
+			if err != nil {
 				_ = sendErrorf("Failed to initialize standalone model runner: %v", err)
 				return fmt.Errorf("Failed to initialize standalone model runner: %w", err)
+			} else if ((kind == desktop.ModelRunnerEngineKindMoby || kind == desktop.ModelRunnerEngineKindCloud) &&
+				standalone == nil) ||
+				(standalone != nil && (standalone.gatewayIP == "" || standalone.gatewayPort == 0)) {
+				return errors.New("unable to determine standalone runner endpoint")
 			}
 
 			if err := downloadModelsOnlyIfNotFound(desktopClient, models); err != nil {
 				return err
 			}
 
-			if kind := modelRunner.EngineKind(); kind == desktop.ModelRunnerEngineKindDesktop {
-				// TODO: Get the actual URL from Docker Desktop via some API.
+			for _, model := range models {
+				if err := desktopClient.ConfigureBackend(scheduling.ConfigureRequest{
+					Model:           model,
+					ContextSize:     ctxSize,
+					RawRuntimeFlags: rawRuntimeFlags,
+				}); err != nil {
+					configErrFmtString := "failed to configure backend for model %s with context-size %d and runtime-flags %s"
+					_ = sendErrorf(configErrFmtString+": %v", model, ctxSize, rawRuntimeFlags, err)
+					return fmt.Errorf(configErrFmtString+": %w", model, ctxSize, rawRuntimeFlags, err)
+				}
+				sendInfo("Successfully configured backend for model " + model)
+			}
+
+			switch kind {
+			case desktop.ModelRunnerEngineKindDesktop:
 				_ = setenv("URL", "http://model-runner.docker.internal/engines/v1/")
-			} else if kind == desktop.ModelRunnerEngineKindMobyManual {
+			case desktop.ModelRunnerEngineKindMobyManual:
 				_ = setenv("URL", modelRunner.URL("/engines/v1/"))
-			} else if kind == desktop.ModelRunnerEngineKindMoby {
-				// TODO: Use more robust detection in Moby-like environments.
-				_ = setenv("URL", "http://host.docker.internal:"+strconv.Itoa(standalone.DefaultControllerPortMoby)+"/engines/v1/")
-			} else if kind == desktop.ModelRunnerEngineKindCloud {
-				// TODO: Use more robust detection in Cloud environments.
-				_ = setenv("URL", "http://host.docker.internal:"+strconv.Itoa(standalone.DefaultControllerPortCloud)+"/engines/v1/")
+			case desktop.ModelRunnerEngineKindCloud:
+				fallthrough
+			case desktop.ModelRunnerEngineKindMoby:
+				_ = setenv("URL", fmt.Sprintf("http://%s:%d/engines/v1", standalone.gatewayIP, standalone.gatewayPort))
+			default:
+				return fmt.Errorf("unhandled engine kind: %v", kind)
 			}
 			return nil
 		},
 	}
 	c.Flags().StringArrayVar(&models, "model", nil, "model to use")
+	c.Flags().Int64Var(&ctxSize, "context-size", -1, "context size for the model")
+	c.Flags().StringVar(&rawRuntimeFlags, "runtime-flags", "", "raw runtime flags to pass to the inference engine")
+	c.Flags().StringVar(&backend, "backend", llamacpp.Name, "inference backend to use")
 	return c
 }
 

commands/inspect.go

@@ -26,12 +26,12 @@ func newInspectCmd() *cobra.Command {
 			return nil
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
+			if _, err := ensureStandaloneRunnerAvailable(cmd.Context(), cmd); err != nil {
+				return fmt.Errorf("unable to initialize standalone model runner: %w", err)
+			}
 			if openai && remote {
 				return fmt.Errorf("--openai and --remote flags are not compatible")
 			}
-			if err := ensureStandaloneRunnerAvailable(cmd.Context(), cmd); err != nil {
-				return fmt.Errorf("unable to initialize standalone model runner: %w", err)
-			}
 			inspectedModel, err := inspectModel(args, openai, remote, desktopClient)
 			if err != nil {
 				return err

commands/install-runner.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"time"
 
+	"github.com/docker/docker/api/types/container"
 	"github.com/docker/model-cli/commands/completion"
 	"github.com/docker/model-cli/desktop"
 	gpupkg "github.com/docker/model-cli/pkg/gpu"
@@ -42,21 +43,49 @@ func waitForStandaloneRunnerAfterInstall(ctx context.Context) error {
 	return errors.New("standalone model runner took too long to initialize")
 }
 
+// standaloneRunner encodes the standalone runner configuration, if one exists.
+type standaloneRunner struct {
+	// hostPort is the port that the runner is listening to on the host.
+	hostPort uint16
+	// gatewayIP is the gateway IP address that the runner is listening on.
+	gatewayIP string
+	// gatewayPort is the gateway port that the runner is listening on.
+	gatewayPort uint16
+}
+
+// inspectStandaloneRunner inspects a standalone runner container and extracts
+// its configuration.
+func inspectStandaloneRunner(container container.Summary) *standaloneRunner {
+	result := &standaloneRunner{}
+	for _, port := range container.Ports {
+		if port.IP == "127.0.0.1" {
+			result.hostPort = port.PublicPort
+		} else {
+			// We don't really have a good way of knowing what the gateway IP
+			// address is, but in the standard standalone configuration we only
+			// bind to two interfaces: 127.0.0.1 and the gateway interface.
+			result.gatewayIP = port.IP
+			result.gatewayPort = port.PublicPort
+		}
+	}
+	return result
+}
+
 // ensureStandaloneRunnerAvailable is a utility function that other commands can
 // use to initialize a default standalone model runner. It is a no-op in
 // unsupported contexts or if automatic installs have been disabled.
-func ensureStandaloneRunnerAvailable(ctx context.Context, printer standalone.StatusPrinter) error {
+func ensureStandaloneRunnerAvailable(ctx context.Context, printer standalone.StatusPrinter) (*standaloneRunner, error) {
 	// If we're not in a supported model runner context, then don't do anything.
 	engineKind := modelRunner.EngineKind()
 	standaloneSupported := engineKind == desktop.ModelRunnerEngineKindMoby ||
 		engineKind == desktop.ModelRunnerEngineKindCloud
 	if !standaloneSupported {
-		return nil
+		return nil, nil
 	}
 
 	// If automatic installation has been disabled, then don't do anything.
 	if os.Getenv("MODEL_RUNNER_NO_AUTO_INSTALL") != "" {
-		return nil
+		return nil, nil
 	}
 
 	// Ensure that the output printer is non-nil.
@@ -67,32 +96,32 @@ func ensureStandaloneRunnerAvailable(ctx context.Context, printer standalone.Sta
 	// Create a Docker client for the active context.
 	dockerClient, err := desktop.DockerClientForContext(dockerCLI, dockerCLI.CurrentContext())
 	if err != nil {
-		return fmt.Errorf("failed to create Docker client: %w", err)
+		return nil, fmt.Errorf("failed to create Docker client: %w", err)
 	}
 
 	// Check if a model runner container exists.
-	container, _, err := standalone.FindControllerContainer(ctx, dockerClient)
+	containerID, _, container, err := standalone.FindControllerContainer(ctx, dockerClient)
 	if err != nil {
-		return fmt.Errorf("unable to identify existing standalone model runner: %w", err)
-	} else if container != "" {
-		return nil
+		return nil, fmt.Errorf("unable to identify existing standalone model runner: %w", err)
+	} else if containerID != "" {
+		return inspectStandaloneRunner(container), nil
 	}
 
 	// Automatically determine GPU support.
 	gpu, err := gpupkg.ProbeGPUSupport(ctx, dockerClient)
 	if err != nil {
-		return fmt.Errorf("unable to probe GPU support: %w", err)
+		return nil, fmt.Errorf("unable to probe GPU support: %w", err)
 	}
 
 	// Ensure that we have an up-to-date copy of the image.
 	if err := standalone.EnsureControllerImage(ctx, dockerClient, gpu, printer); err != nil {
-		return fmt.Errorf("unable to pull latest standalone model runner image: %w", err)
+		return nil, fmt.Errorf("unable to pull latest standalone model runner image: %w", err)
 	}
 
 	// Ensure that we have a model storage volume.
 	modelStorageVolume, err := standalone.EnsureModelStorageVolume(ctx, dockerClient, printer)
 	if err != nil {
-		return fmt.Errorf("unable to initialize standalone model storage: %w", err)
+		return nil, fmt.Errorf("unable to initialize standalone model storage: %w", err)
 	}
 
 	// Create the model runner container.
@@ -101,11 +130,28 @@ func ensureStandaloneRunnerAvailable(ctx context.Context, printer standalone.Sta
 		port = standalone.DefaultControllerPortCloud
 	}
 	if err := standalone.CreateControllerContainer(ctx, dockerClient, port, false, gpu, modelStorageVolume, printer); err != nil {
-		return fmt.Errorf("unable to initialize standalone model runner container: %w", err)
+		return nil, fmt.Errorf("unable to initialize standalone model runner container: %w", err)
 	}
 
 	// Poll until we get a response from the model runner.
-	return waitForStandaloneRunnerAfterInstall(ctx)
+	if err := waitForStandaloneRunnerAfterInstall(ctx); err != nil {
+		return nil, err
+	}
+
+	// Find the runner container.
+	//
+	// TODO: We should actually find this before calling
+	// waitForStandaloneRunnerAfterInstall (or have CreateControllerContainer
+	// return the container information), and probably pass the target
+	// information info waitForStandaloneRunnerAfterInstall, but let's wait
+	// until we do listener port customization / detection in the next PR.
+	containerID, _, container, err = standalone.FindControllerContainer(ctx, dockerClient)
+	if err != nil {
+		return nil, fmt.Errorf("unable to identify existing standalone model runner: %w", err)
+	} else if containerID == "" {
+		return nil, errors.New("standalone model runner not found after installation")
+	}
+	return inspectStandaloneRunner(container), nil
 }
 
 func newInstallRunner() *cobra.Command {
@@ -149,7 +195,7 @@ func newInstallRunner() *cobra.Command {
 			}
 
 			// Check if an active model runner container already exists.
-			if ctrID, ctrName, err := standalone.FindControllerContainer(cmd.Context(), dockerClient); err != nil {
+			if ctrID, ctrName, _, err := standalone.FindControllerContainer(cmd.Context(), dockerClient); err != nil {
 				return err
 			} else if ctrID != "" {
 				if ctrName != "" {

commands/list.go

@@ -30,7 +30,7 @@ func newListCmd() *cobra.Command {
 			if !jsonFormat && !openai && !quiet {
 				standaloneInstallPrinter = cmd
 			}
-			if err := ensureStandaloneRunnerAvailable(cmd.Context(), standaloneInstallPrinter); err != nil {
+			if _, err := ensureStandaloneRunnerAvailable(cmd.Context(), standaloneInstallPrinter); err != nil {
 				return fmt.Errorf("unable to initialize standalone model runner: %w", err)
 			}
 			models, err := listModels(openai, desktopClient, quiet, jsonFormat)

commands/logs.go

@@ -44,7 +44,7 @@ func newLogsCmd() *cobra.Command {
 				if err != nil {
 					return fmt.Errorf("failed to create Docker client: %w", err)
 				}
-				ctrID, _, err := standalone.FindControllerContainer(cmd.Context(), dockerClient)
+				ctrID, _, _, err := standalone.FindControllerContainer(cmd.Context(), dockerClient)
 				if err != nil {
 					return fmt.Errorf("unable to identify Model Runner container: %w", err)
 				} else if ctrID == "" {

commands/pull.go

@@ -23,7 +23,7 @@ func newPullCmd() *cobra.Command {
 			return nil
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
-			if err := ensureStandaloneRunnerAvailable(cmd.Context(), cmd); err != nil {
+			if _, err := ensureStandaloneRunnerAvailable(cmd.Context(), cmd); err != nil {
 				return fmt.Errorf("unable to initialize standalone model runner: %w", err)
 			}
 			return pullModel(cmd, desktopClient, args[0])

commands/push.go

@@ -23,7 +23,7 @@ func newPushCmd() *cobra.Command {
 			return nil
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
-			if err := ensureStandaloneRunnerAvailable(cmd.Context(), cmd); err != nil {
+			if _, err := ensureStandaloneRunnerAvailable(cmd.Context(), cmd); err != nil {
 				return fmt.Errorf("unable to initialize standalone model runner: %w", err)
 			}
 			return pushModel(cmd, desktopClient, args[0])

commands/rm.go

@@ -24,7 +24,7 @@ func newRemoveCmd() *cobra.Command {
 			return nil
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
-			if err := ensureStandaloneRunnerAvailable(cmd.Context(), cmd); err != nil {
+			if _, err := ensureStandaloneRunnerAvailable(cmd.Context(), cmd); err != nil {
 				return fmt.Errorf("unable to initialize standalone model runner: %w", err)
 			}
 			response, err := desktopClient.Remove(args, force)

commands/run.go

@@ -33,7 +33,7 @@ func newRunCmd() *cobra.Command {
 				}
 			}
 
-			if err := ensureStandaloneRunnerAvailable(cmd.Context(), cmd); err != nil {
+			if _, err := ensureStandaloneRunnerAvailable(cmd.Context(), cmd); err != nil {
 				return fmt.Errorf("unable to initialize standalone model runner: %w", err)
 			}
 

commands/status.go

@@ -15,7 +15,7 @@ func newStatusCmd() *cobra.Command {
 		Use:   "status",
 		Short: "Check if the Docker Model Runner is running",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			if err := ensureStandaloneRunnerAvailable(cmd.Context(), cmd); err != nil {
+			if _, err := ensureStandaloneRunnerAvailable(cmd.Context(), cmd); err != nil {
 				return fmt.Errorf("unable to initialize standalone model runner: %w", err)
 			}
 			status := desktopClient.Status()