Stacks onboarding with support for "remote" stacks

README.md

@@ -172,7 +172,7 @@ The Defang CLI recognizes the following environment variables:
 - `DEFANG_NO_CACHE` - If set to `true`, disables pull-through caching of container images; defaults to `false`
 - `DEFANG_ORG` - The name of the organization to use; defaults to the user's GitHub name
 - `DEFANG_PREFIX` - The prefix to use for all BYOC resources; defaults to `Defang`
-- `DEFANG_PROVIDER` - The name of the cloud provider to use, `auto` (default), `aws`, `digitalocean`, `gcp`, or `defang`
+- `DEFANG_PROVIDER` - The name of the cloud provider to use, `aws`, `digitalocean`, `gcp`, or `defang`
 - `DEFANG_PULUMI_BACKEND` - The Pulumi backend URL or `"pulumi-cloud"`; defaults to a self-hosted backend
 - `DEFANG_PULUMI_DEBUG` - If set to `true`, enables debug logging for Pulumi operations; defaults to `false`
 - `DEFANG_PULUMI_DIFF` - If set to `true`, shows the Pulumi diff during deployments; defaults to `false`

pkgs/npm/README.md

@@ -42,7 +42,7 @@ The Defang CLI recognizes the following environment variables:
 - `DEFANG_NO_CACHE` - If set to `true`, disables pull-through caching of container images; defaults to `false`
 - `DEFANG_ORG` - The name of the organization to use; defaults to the user's GitHub name
 - `DEFANG_PREFIX` - The prefix to use for all BYOC resources; defaults to `Defang`
-- `DEFANG_PROVIDER` - The name of the cloud provider to use, `auto` (default), `aws`, `digitalocean`, `gcp`, or `defang`
+- `DEFANG_PROVIDER` - The name of the cloud provider to use, `aws`, `digitalocean`, `gcp`, or `defang`
 - `DEFANG_PULUMI_BACKEND` - The Pulumi backend URL or `"pulumi-cloud"`; defaults to a self-hosted backend
 - `DEFANG_PULUMI_DEBUG` - If set to `true`, enables debug logging for Pulumi operations; defaults to `false`
 - `DEFANG_PULUMI_DIFF` - If set to `true`, shows the Pulumi diff during deployments; defaults to `false`

src/.goreleaser.yml

@@ -88,7 +88,7 @@ release:
     2. Extract the archive. This should reveal the binary file for Defang.
     3. Manually place the binary file in a directory that's included in your system's `PATH` environment variable.
     ### Additional Step for MacOS Users
-    If you're having trouble running the binary on MacOS, please check our [FAQs](https://docs.defang.io/docs/faq#im-having-trouble-running-the-binary-on-my-mac-what-should-i-do).
+    If you're having trouble running the binary on MacOS, please check our [FAQs](https://docs.defang.io/docs/intro/faq/questions#im-having-trouble-running-the-binary-on-my-mac-what-should-i-do).
 
     Please remember this software is in beta, so please report any issues or feedback through our GitHub page. Your help in improving Defang is greatly appreciated!
   # mode: keep-existing

src/README.md

@@ -42,7 +42,7 @@ The Defang CLI recognizes the following environment variables:
 - `DEFANG_NO_CACHE` - If set to `true`, disables pull-through caching of container images; defaults to `false`
 - `DEFANG_ORG` - The name of the organization to use; defaults to the user's GitHub name
 - `DEFANG_PREFIX` - The prefix to use for all BYOC resources; defaults to `Defang`
-- `DEFANG_PROVIDER` - The name of the cloud provider to use, `auto` (default), `aws`, `digitalocean`, `gcp`, or `defang`
+- `DEFANG_PROVIDER` - The name of the cloud provider to use, `aws`, `digitalocean`, `gcp`, or `defang`
 - `DEFANG_PULUMI_BACKEND` - The Pulumi backend URL or `"pulumi-cloud"`; defaults to a self-hosted backend
 - `DEFANG_PULUMI_DEBUG` - If set to `true`, enables debug logging for Pulumi operations; defaults to `false`
 - `DEFANG_PULUMI_DIFF` - If set to `true`, shows the Pulumi diff during deployments; defaults to `false`

src/cmd/cli/command/cd.go

@@ -5,7 +5,6 @@ import (
 	"os"
 
 	"github.com/DefangLabs/defang/src/pkg/cli"
-	cliClient "github.com/DefangLabs/defang/src/pkg/cli/client"
 	"github.com/DefangLabs/defang/src/pkg/cli/client/byoc/aws"
 	"github.com/DefangLabs/defang/src/pkg/cli/compose"
 	"github.com/DefangLabs/defang/src/pkg/term"
@@ -36,19 +35,19 @@ var cdCmd = &cobra.Command{
 func bootstrapCommand(cmd *cobra.Command, args []string, command string) error {
 	ctx := cmd.Context()
 	loader := configureLoader(cmd)
-	provider, err := newProviderChecked(ctx, loader)
-	if err != nil {
-		return err
-	}
 
 	if len(args) == 0 {
-		projectName, err := cliClient.LoadProjectNameWithFallback(ctx, loader, provider)
+		projectName, err := loader.LoadProjectName(ctx)
 		if err != nil {
 			return err
 		}
 		args = []string{projectName}
 	}
 
+	provider, err := newProviderChecked(ctx, args[0], false)
+	if err != nil {
+		return err
+	}
 	var errs []error
 	for _, projectName := range args {
 		err := canIUseProvider(ctx, provider, projectName, 0)
@@ -103,8 +102,7 @@ var cdTearDownCmd = &cobra.Command{
 	RunE: func(cmd *cobra.Command, args []string) error {
 		force, _ := cmd.Flags().GetBool("force")
 
-		loader := configureLoader(cmd)
-		provider, err := newProviderChecked(cmd.Context(), loader)
+		provider, err := newProviderChecked(cmd.Context(), "", false)
 		if err != nil {
 			return err
 		}
@@ -122,7 +120,7 @@ var cdListCmd = &cobra.Command{
 		remote, _ := cmd.Flags().GetBool("remote")
 		all, _ := cmd.Flags().GetBool("all")
 
-		provider, err := newProviderChecked(cmd.Context(), nil)
+		provider, err := newProviderChecked(cmd.Context(), "", false)
 		if err != nil {
 			return err
 		}
@@ -156,7 +154,9 @@ var cdPreviewCmd = &cobra.Command{
 			return err
 		}
 
-		provider, err := newProviderChecked(cmd.Context(), loader)
+		projectNameFlag, _ := cmd.Flags().GetString("project-name")
+		saveStacksToWkDir := projectNameFlag == ""
+		provider, err := newProviderChecked(cmd.Context(), project.Name, saveStacksToWkDir)
 		if err != nil {
 			return err
 		}
@@ -183,8 +183,7 @@ var cdInstallCmd = &cobra.Command{
 	Short:       "Install the CD resources into the cluster",
 	Hidden:      true, // users shouldn't have to run this manually, because it's done on deploy
 	RunE: func(cmd *cobra.Command, args []string) error {
-		loader := configureLoader(cmd)
-		provider, err := newProviderChecked(cmd.Context(), loader)
+		provider, err := newProviderChecked(cmd.Context(), "", false)
 		if err != nil {
 			return err
 		}

src/cmd/cli/command/commands.go

@@ -16,6 +16,7 @@ import (
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/DefangLabs/defang/src/pkg"
 	"github.com/DefangLabs/defang/src/pkg/agent"
+	agentTools "github.com/DefangLabs/defang/src/pkg/agent/tools"
 	"github.com/DefangLabs/defang/src/pkg/cli"
 	cliClient "github.com/DefangLabs/defang/src/pkg/cli/client"
 	"github.com/DefangLabs/defang/src/pkg/cli/client/byoc"
@@ -24,6 +25,7 @@ import (
 	"github.com/DefangLabs/defang/src/pkg/clouds/aws"
 	"github.com/DefangLabs/defang/src/pkg/debug"
 	"github.com/DefangLabs/defang/src/pkg/dryrun"
+	"github.com/DefangLabs/defang/src/pkg/elicitations"
 	"github.com/DefangLabs/defang/src/pkg/login"
 	"github.com/DefangLabs/defang/src/pkg/logs"
 	"github.com/DefangLabs/defang/src/pkg/mcp"
@@ -86,7 +88,7 @@ func Execute(ctx context.Context) error {
 
 		if strings.Contains(err.Error(), "maximum number of projects") {
 			projectName := "<name>"
-			provider, err := newProviderChecked(ctx, nil)
+			provider, err := newProviderChecked(ctx, projectName, false)
 			if err != nil {
 				return err
 			}
@@ -433,10 +435,15 @@ var RootCmd = &cobra.Command{
 			}
 		}
 
+		// TODO: consider connecting to fabric before loading the stack
+		// so we can support loading stacks from remote
 		// Read the global flags again from any .defang files in the cwd
 		err = global.loadDotDefang(global.getStackName(cmd.Flags()))
 		if err != nil {
-			return err
+			// if the stack file does not exist, continue without error, we will load it from the remote later.
+			if !errors.Is(err, os.ErrNotExist) {
+				return err
+			}
 		}
 
 		err = global.syncFlagsWithEnv(cmd.Flags())
@@ -524,9 +531,8 @@ var whoamiCmd = &cobra.Command{
 	Args:  cobra.NoArgs,
 	Short: "Show the current user",
 	RunE: func(cmd *cobra.Command, args []string) error {
-		loader := configureLoader(cmd)
 		global.NonInteractive = true // don't show provider prompt
-		provider, err := newProvider(cmd.Context(), loader)
+		provider, err := newProviderChecked(cmd.Context(), "", false)
 		if err != nil {
 			term.Debug("unable to get provider:", err)
 		}
@@ -576,7 +582,9 @@ var certGenerateCmd = &cobra.Command{
 			return err
 		}
 
-		provider, err := newProviderChecked(cmd.Context(), loader)
+		projectNameFlag, _ := cmd.Flags().GetString("project-name")
+		saveStacksToWkDir := projectNameFlag == ""
+		provider, err := newProviderChecked(cmd.Context(), project.Name, saveStacksToWkDir)
 		if err != nil {
 			return err
 		}
@@ -748,12 +756,13 @@ var configSetCmd = &cobra.Command{
 
 		// Make sure we have a project to set config for before asking for a value
 		loader := configureLoader(cmd)
-		provider, err := newProviderChecked(cmd.Context(), loader)
+		projectName, err := loader.LoadProjectName(cmd.Context())
 		if err != nil {
 			return err
 		}
-
-		projectName, err := cliClient.LoadProjectNameWithFallback(cmd.Context(), loader, provider)
+		projectNameFlag, _ := cmd.Flags().GetString("project-name")
+		saveStacksToWkDir := projectNameFlag == ""
+		provider, err := newProviderChecked(cmd.Context(), projectName, saveStacksToWkDir)
 		if err != nil {
 			return err
 		}
@@ -879,12 +888,13 @@ var configDeleteCmd = &cobra.Command{
 	Short:       "Removes one or more config values",
 	RunE: func(cmd *cobra.Command, names []string) error {
 		loader := configureLoader(cmd)
-		provider, err := newProviderChecked(cmd.Context(), loader)
+		projectName, err := loader.LoadProjectName(cmd.Context())
 		if err != nil {
 			return err
 		}
-
-		projectName, err := cliClient.LoadProjectNameWithFallback(cmd.Context(), loader, provider)
+		projectNameFlag, _ := cmd.Flags().GetString("project-name")
+		saveStacksToWkDir := projectNameFlag == ""
+		provider, err := newProviderChecked(cmd.Context(), projectName, saveStacksToWkDir)
 		if err != nil {
 			return err
 		}
@@ -912,12 +922,14 @@ var configListCmd = &cobra.Command{
 	Short:       "List configs",
 	RunE: func(cmd *cobra.Command, args []string) error {
 		loader := configureLoader(cmd)
-		provider, err := newProviderChecked(cmd.Context(), loader)
+
+		projectName, err := loader.LoadProjectName(cmd.Context())
 		if err != nil {
 			return err
 		}
-
-		projectName, err := cliClient.LoadProjectNameWithFallback(cmd.Context(), loader, provider)
+		projectNameFlag, _ := cmd.Flags().GetString("project-name")
+		saveStacksToWkDir := projectNameFlag == ""
+		provider, err := newProviderChecked(cmd.Context(), projectName, saveStacksToWkDir)
 		if err != nil {
 			return err
 		}
@@ -943,12 +955,13 @@ var debugCmd = &cobra.Command{
 		}
 
 		loader := configureLoader(cmd)
-		_, err := newProviderChecked(ctx, loader)
+		project, err := loader.LoadProject(ctx)
 		if err != nil {
 			return err
 		}
-
-		project, err := loader.LoadProject(ctx)
+		projectNameFlag, _ := cmd.Flags().GetString("project-name")
+		saveStacksToWkDir := projectNameFlag == ""
+		_, err = newProviderChecked(ctx, project.Name, saveStacksToWkDir)
 		if err != nil {
 			return err
 		}
@@ -992,12 +1005,13 @@ var deleteCmd = &cobra.Command{
 		var tail, _ = cmd.Flags().GetBool("tail")
 
 		loader := configureLoader(cmd)
-		provider, err := newProviderChecked(cmd.Context(), loader)
+		projectName, err := loader.LoadProjectName(cmd.Context())
 		if err != nil {
 			return err
 		}
-
-		projectName, err := cliClient.LoadProjectNameWithFallback(cmd.Context(), loader, provider)
+		projectNameFlag, _ := cmd.Flags().GetString("project-name")
+		saveStacksToWkDir := projectNameFlag == ""
+		provider, err := newProviderChecked(cmd.Context(), projectName, saveStacksToWkDir)
 		if err != nil {
 			return err
 		}
@@ -1238,7 +1252,7 @@ var providerDescription = map[cliClient.ProviderID]string{
 	cliClient.ProviderGCP:    "Deploy to Google Cloud Platform using gcloud Application Default Credentials.",
 }
 
-func updateProviderID(ctx context.Context, loader cliClient.Loader) error {
+func updateProviderID(ctx context.Context) error {
 	extraMsg := ""
 	whence := "default project"
 
@@ -1255,24 +1269,16 @@ func updateProviderID(ctx context.Context, loader cliClient.Loader) error {
 
 	switch global.ProviderID {
 	case cliClient.ProviderAuto:
-		if global.NonInteractive {
-			// Defaults to defang provider in non-interactive mode
-			if awsInEnv() {
-				term.Warn("Using Defang playground, but AWS environment variables were detected; did you forget --provider=aws or DEFANG_PROVIDER=aws?")
-			}
-			if doInEnv() {
-				term.Warn("Using Defang playground, but DIGITALOCEAN_TOKEN environment variable was detected; did you forget --provider=digitalocean or DEFANG_PROVIDER=digitalocean?")
-			}
-			if gcpInEnv() {
-				term.Warn("Using Defang playground, but GCP_PROJECT_ID/CLOUDSDK_CORE_PROJECT environment variable was detected; did you forget --provider=gcp or DEFANG_PROVIDER=gcp?")
-			}
-			global.ProviderID = cliClient.ProviderDefang
-		} else {
-			var err error
-			if whence, err = determineProviderID(ctx, loader); err != nil {
-				return err
-			}
+		if awsInEnv() {
+			term.Warn("Using Defang playground, but AWS environment variables were detected; did you forget --provider=aws or DEFANG_PROVIDER=aws?")
 		}
+		if doInEnv() {
+			term.Warn("Using Defang playground, but DIGITALOCEAN_TOKEN environment variable was detected; did you forget --provider=digitalocean or DEFANG_PROVIDER=digitalocean?")
+		}
+		if gcpInEnv() {
+			term.Warn("Using Defang playground, but GCP_PROJECT_ID/CLOUDSDK_CORE_PROJECT environment variable was detected; did you forget --provider=gcp or DEFANG_PROVIDER=gcp?")
+		}
+		global.ProviderID = cliClient.ProviderDefang
 	case cliClient.ProviderAWS:
 		if !awsInConfig(ctx) {
 			term.Warn("AWS provider was selected, but AWS environment is not set")
@@ -1294,19 +1300,32 @@ func updateProviderID(ctx context.Context, loader cliClient.Loader) error {
 	return nil
 }
 
-func newProvider(ctx context.Context, loader cliClient.Loader) (cliClient.Provider, error) {
-	if err := updateProviderID(ctx, loader); err != nil {
+func newProvider(ctx context.Context) (cliClient.Provider, error) {
+	if err := updateProviderID(ctx); err != nil {
 		return nil, err
 	}
 
 	provider := cli.NewProvider(ctx, global.ProviderID, global.Client, global.Stack)
 	return provider, nil
 }
 
-func newProviderChecked(ctx context.Context, loader cliClient.Loader) (cliClient.Provider, error) {
-	provider, err := newProvider(ctx, loader)
+type providerCreator struct{}
+
+func (pc *providerCreator) NewProvider(ctx context.Context, providerId cliClient.ProviderID, client cliClient.FabricClient, stack string) cliClient.Provider {
+	return cli.NewProvider(ctx, providerId, client, stack)
+}
+
+func newProviderChecked(ctx context.Context, projectName string, useWkDir bool) (cliClient.Provider, error) {
+	if global.NonInteractive {
+		return newProvider(ctx)
+	}
+	pc := &providerCreator{}
+	elicitationsClient := elicitations.NewSurveyClient(os.Stdin, os.Stdout, os.Stderr)
+	ec := elicitations.NewController(elicitationsClient)
+	pp := agentTools.NewProviderPreparer(pc, ec, global.Client)
+	_, provider, err := pp.SetupProvider(ctx, projectName, &global.Stack, useWkDir)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to setup provider: %w", err)
 	}
 	_, err = provider.AccountInfo(ctx)
 	return provider, err
@@ -1316,40 +1335,6 @@ func canIUseProvider(ctx context.Context, provider cliClient.Provider, projectNa
 	return cliClient.CanIUseProvider(ctx, global.Client, provider, projectName, global.Stack, serviceCount)
 }
 
-func determineProviderID(ctx context.Context, loader cliClient.Loader) (string, error) {
-	var projectName string
-	if loader != nil {
-		var err error
-		projectName, err = loader.LoadProjectName(ctx)
-		if err != nil {
-			term.Warnf("Unable to load project: %v", err)
-		}
-
-		if projectName != "" && !RootCmd.PersistentFlags().Changed("provider") { // If user manually selected auto provider, do not load from remote
-			resp, err := global.Client.GetSelectedProvider(ctx, &defangv1.GetSelectedProviderRequest{Project: projectName})
-			if err != nil {
-				term.Debugf("Unable to get selected provider: %v", err)
-			} else if resp.Provider != defangv1.Provider_PROVIDER_UNSPECIFIED {
-				global.ProviderID.SetValue(resp.Provider)
-				return "stored preference", nil
-			}
-		}
-	}
-
-	whence, err := interactiveSelectProvider(cliClient.AllProviders())
-
-	// Save the selected provider to the fabric
-	if projectName != "" {
-		if err := global.Client.SetSelectedProvider(ctx, &defangv1.SetSelectedProviderRequest{Project: projectName, Provider: global.ProviderID.Value()}); err != nil {
-			term.Debugf("Unable to save selected provider to defang server: %v", err)
-		} else {
-			term.Printf("%v is now the default provider for project %v and will auto-select next time if no other provider is specified. Use --provider=auto to reselect.", global.ProviderID, projectName)
-		}
-	}
-
-	return whence, err
-}
-
 func interactiveSelectProvider(providers []cliClient.ProviderID) (string, error) {
 	if len(providers) < 2 {
 		panic("interactiveSelectProvider called with less than 2 providers")