From 2cb0d099eca123bfdcbb5b621ac5c78c8ec74785 Mon Sep 17 00:00:00 2001 From: Nathanael DEMACON Date: Wed, 14 Jun 2023 16:08:46 +0200 Subject: [PATCH] feat(terraform): fetch parents and children --- internal/core/bootstrap.go | 8 +- internal/core/printer.go | 39 +++++--- internal/core/shell.go | 4 +- internal/core/shell_disabled.go | 2 +- internal/core/testing.go | 4 +- internal/terraform/association.go | 97 ++++++++++++++++--- internal/terraform/hcl.go | 151 +++++++++++++++++++++++++++--- 7 files changed, 260 insertions(+), 45 deletions(-) diff --git a/internal/core/bootstrap.go b/internal/core/bootstrap.go index 3f7b55160a..a4cf8a06ca 100644 --- a/internal/core/bootstrap.go +++ b/internal/core/bootstrap.go @@ -150,7 +150,7 @@ func Bootstrap(config *BootstrapConfig) (exitCode int, result interface{}, err e isClientFromBootstrapConfig = false client, err = createAnonymousClient(httpClient, config.BuildInfo) if err != nil { - printErr := printer.Print(err, nil) + printErr := printer.Print(client, err, nil) if printErr != nil { _, _ = fmt.Fprintln(config.Stderr, printErr) } @@ -201,7 +201,7 @@ func Bootstrap(config *BootstrapConfig) (exitCode int, result interface{}, err e // Load CLI config cliCfg, err := cliConfig.LoadConfig(ExtractCliConfigPath(ctx)) if err != nil { - printErr := printer.Print(err, nil) + printErr := printer.Print(meta.Client, err, nil) if printErr != nil { _, _ = fmt.Fprintln(config.Stderr, printErr) } @@ -270,7 +270,7 @@ func Bootstrap(config *BootstrapConfig) (exitCode int, result interface{}, err e if cliErr, ok := err.(*CliError); ok && cliErr.Code != 0 { errorCode = cliErr.Code } - printErr := printer.Print(err, nil) + printErr := printer.Print(meta.Client, err, nil) if printErr != nil { _, _ = fmt.Fprintln(os.Stderr, err) } @@ -278,7 +278,7 @@ func Bootstrap(config *BootstrapConfig) (exitCode int, result interface{}, err e } if meta.command != nil { - printErr := printer.Print(meta.result, meta.command.getHumanMarshalerOpt()) + printErr := printer.Print(meta.Client, meta.result, meta.command.getHumanMarshalerOpt()) if printErr != nil { _, _ = fmt.Fprintln(config.Stderr, printErr) } diff --git a/internal/core/printer.go b/internal/core/printer.go index 2c333c6e6f..b90fba28e0 100644 --- a/internal/core/printer.go +++ b/internal/core/printer.go @@ -13,6 +13,7 @@ import ( "github.com/scaleway/scaleway-cli/v2/internal/gofields" "github.com/scaleway/scaleway-cli/v2/internal/human" "github.com/scaleway/scaleway-cli/v2/internal/terraform" + "github.com/scaleway/scaleway-sdk-go/scw" ) // Type defines an formatter format. @@ -44,8 +45,12 @@ const ( // Option to enable pretty output on json printer. PrinterOptJSONPretty = "pretty" - // Option to enable pretty output on json printer. - PrinterOptTerraformWithChildren = "with-children" + // Option to disable parents output on terraform printer. + PrinterOptTerraformSkipParents = "skip-parents" + // Option to disable children output on terraform printer. + PrinterOptTerraformSkipChildren = "skip-children" + // Option to disable parents and children output on terraform printer. + PrinterOptTerraformSkipParentsAndChildren = "skip-parents-and-children" ) type PrinterConfig struct { @@ -115,11 +120,16 @@ func setupJSONPrinter(printer *Printer, opts string) error { func setupTerraformPrinter(printer *Printer, opts string) error { printer.printerType = PrinterTypeTerraform switch opts { - case PrinterOptTerraformWithChildren: - printer.terraformWithChildren = true + case PrinterOptTerraformSkipParents: + printer.terraformSkipParents = true + case PrinterOptTerraformSkipChildren: + printer.terraformSkipChildren = true + case PrinterOptTerraformSkipParentsAndChildren: + printer.terraformSkipParents = true + printer.terraformSkipChildren = true case "": default: - return fmt.Errorf("invalid option %s for terraform outout. Valid options are: %s", opts, PrinterOptTerraformWithChildren) + return fmt.Errorf("invalid option %s for terraform outout. Valid options are: %s and %s", opts, PrinterOptTerraformSkipParents, PrinterOptTerraformSkipChildren) } terraformVersion, err := terraform.GetLocalClientVersion() @@ -173,8 +183,10 @@ type Printer struct { // Enable pretty print on json output jsonPretty bool - // Enable children fetching on terraform output - terraformWithChildren bool + // Disable children fetching on terraform output + terraformSkipParents bool + // Disable children fetching on terraform output + terraformSkipChildren bool // go template to use on template output template *template.Template @@ -183,7 +195,7 @@ type Printer struct { humanFields []string } -func (p *Printer) Print(data interface{}, opt *human.MarshalOpt) error { +func (p *Printer) Print(client *scw.Client, data interface{}, opt *human.MarshalOpt) error { // No matter the printer type if data is a RawResult we should print it as is. if rawResult, isRawResult := data.(RawResult); isRawResult { _, err := p.stdout.Write(rawResult) @@ -201,7 +213,7 @@ func (p *Printer) Print(data interface{}, opt *human.MarshalOpt) error { case PrinterTypeYAML: err = p.printYAML(data) case PrinterTypeTerraform: - err = p.printTerraform(data) + err = p.printTerraform(client, data) case PrinterTypeTemplate: err = p.printTemplate(data) default: @@ -322,13 +334,18 @@ func (p *Printer) printYAML(data interface{}) error { return encoder.Encode(data) } -func (p *Printer) printTerraform(data interface{}) error { +func (p *Printer) printTerraform(client *scw.Client, data interface{}) error { writer := p.stdout if _, isError := data.(error); isError { return p.printHuman(data, nil) } - hcl, err := terraform.GetHCL(data) + hcl, err := terraform.GetHCL(&terraform.GetHCLConfig{ + Client: client, + Data: data, + SkipParents: p.terraformSkipParents, + SkipChildren: p.terraformSkipChildren, + }) if err != nil { return err } diff --git a/internal/core/shell.go b/internal/core/shell.go index c7e3cd55c0..f4b2684b18 100644 --- a/internal/core/shell.go +++ b/internal/core/shell.go @@ -267,7 +267,7 @@ func shellExecutor(rootCmd *cobra.Command, printer *Printer, meta *meta) func(s return } - printErr := printer.Print(err, nil) + printErr := printer.Print(meta.Client, err, nil) if printErr != nil { _, _ = fmt.Fprintln(os.Stderr, err) } @@ -283,7 +283,7 @@ func shellExecutor(rootCmd *cobra.Command, printer *Printer, meta *meta) func(s autoCompleteCache.Update(meta.command.Namespace) - printErr := printer.Print(meta.result, meta.command.getHumanMarshalerOpt()) + printErr := printer.Print(meta.Client, meta.result, meta.command.getHumanMarshalerOpt()) if printErr != nil { _, _ = fmt.Fprintln(os.Stderr, printErr) } diff --git a/internal/core/shell_disabled.go b/internal/core/shell_disabled.go index 659c6c8250..0b19943a17 100644 --- a/internal/core/shell_disabled.go +++ b/internal/core/shell_disabled.go @@ -11,7 +11,7 @@ import ( ) func RunShell(ctx context.Context, printer *Printer, meta *meta, rootCmd *cobra.Command, args []string) { - err := printer.Print(fmt.Errorf("shell is currently disabled on freebsd"), nil) + err := printer.Print(meta.Client, fmt.Errorf("shell is currently disabled on freebsd"), nil) if err != nil { _, _ = fmt.Fprintln(os.Stderr, err) } diff --git a/internal/core/testing.go b/internal/core/testing.go index fedf126dfb..576ee63d5d 100644 --- a/internal/core/testing.go +++ b/internal/core/testing.go @@ -717,11 +717,11 @@ func marshalGolden(t *testing.T, ctx *CheckFuncCtx) string { require.NoError(t, err) if ctx.Err != nil { - err = jsonPrinter.Print(ctx.Err, nil) + err = jsonPrinter.Print(nil, ctx.Err, nil) require.NoError(t, err) } if ctx.Result != nil { - err = jsonPrinter.Print(ctx.Result, nil) + err = jsonPrinter.Print(nil, ctx.Result, nil) require.NoError(t, err) } diff --git a/internal/terraform/association.go b/internal/terraform/association.go index 8754829fc0..d384598314 100644 --- a/internal/terraform/association.go +++ b/internal/terraform/association.go @@ -3,21 +3,32 @@ package terraform import ( "reflect" + "github.com/scaleway/scaleway-sdk-go/api/account/v2" "github.com/scaleway/scaleway-sdk-go/api/baremetal/v1" container "github.com/scaleway/scaleway-sdk-go/api/container/v1beta1" "github.com/scaleway/scaleway-sdk-go/api/instance/v1" + "github.com/scaleway/scaleway-sdk-go/scw" ) -type associationSubResource struct { - TerraformAttributeName string - Command string - AsDataSource bool +type associationParent struct { + Fetcher func(client *scw.Client, data interface{}) (interface{}, error) + AsDataSource bool +} + +type associationChild struct { + // { + // []: + // } + ParentFieldMap map[string]string + + Fetcher func(client *scw.Client, data interface{}) (interface{}, error) } type association struct { ResourceName string ImportFormat string - SubResources map[string]*associationSubResource + Parents map[string]*associationParent + Children []*associationChild } // const importFormatID = "{{ .Region }}/{{ .ID }}" @@ -36,21 +47,79 @@ var associations = map[interface{}]*association{ &container.Container{}: { ResourceName: "scaleway_container", ImportFormat: importFormatRegionID, - SubResources: map[string]*associationSubResource{ - "NamespaceID": { - TerraformAttributeName: "namespace_id", - Command: "container namespace get {{ .NamespaceID }}", + Parents: map[string]*associationParent{ + "namespace_id": { + Fetcher: func(client *scw.Client, raw interface{}) (interface{}, error) { + api := container.NewAPI(client) + data := raw.(*container.Container) + + return api.GetNamespace(&container.GetNamespaceRequest{ + NamespaceID: data.NamespaceID, + Region: data.Region, + }) + }, }, }, }, &container.Namespace{}: { ResourceName: "scaleway_container_namespace", ImportFormat: importFormatRegionID, - SubResources: map[string]*associationSubResource{ - "ProjectID": { - TerraformAttributeName: "project_id", - Command: "container project get project-id={{ .ProjectID }}", - AsDataSource: true, + Parents: map[string]*associationParent{ + "project_id": { + AsDataSource: true, + Fetcher: func(client *scw.Client, raw interface{}) (interface{}, error) { + api := account.NewAPI(client) + data := raw.(*container.Namespace) + + return api.GetProject(&account.GetProjectRequest{ + ProjectID: data.ProjectID, + }) + }, + }, + }, + Children: []*associationChild{ + { + ParentFieldMap: map[string]string{ + "namespace_id": "id", + }, + Fetcher: func(client *scw.Client, raw interface{}) (interface{}, error) { + api := container.NewAPI(client) + data := raw.(*container.Namespace) + + res, err := api.ListContainers(&container.ListContainersRequest{ + NamespaceID: data.ID, + Region: data.Region, + }) + if err != nil { + return nil, err + } + + return res.Containers, nil + }, + }, + }, + }, + &account.Project{}: { + ResourceName: "scaleway_account_project", + ImportFormat: "{{ .ID }}", + Children: []*associationChild{ + { + ParentFieldMap: map[string]string{ + "project_id": "id", + }, + Fetcher: func(client *scw.Client, raw interface{}) (interface{}, error) { + api := container.NewAPI(client) + data := raw.(*account.Project) + + res, err := api.ListNamespaces(&container.ListNamespacesRequest{ + ProjectID: &data.ID, + }) + if err != nil { + return nil, err + } + + return res.Namespaces, nil + }, }, }, }, diff --git a/internal/terraform/hcl.go b/internal/terraform/hcl.go index 6f92a68855..4627284f1d 100644 --- a/internal/terraform/hcl.go +++ b/internal/terraform/hcl.go @@ -7,6 +7,10 @@ import ( "os" "path/filepath" "reflect" + "regexp" + "strings" + + "github.com/scaleway/scaleway-sdk-go/scw" ) func getResourceID(format string, data interface{}) (string, error) { @@ -32,21 +36,21 @@ type hclImportTemplateData struct { const hclImportTemplate = ` terraform { required_providers { - scaleway = { + scaleway = { source = "scaleway/scaleway" - } + } } required_version = ">= 0.13" } - + import { # ID of the cloud resource # Check provider documentation for importable resources and format id = "{{ .ResourceID }}" - + # Resource address to = {{ .ResourceName }}.main -} +} ` func createImportFile(directory string, association *association, data interface{}) error { @@ -77,10 +81,53 @@ func createImportFile(directory string, association *association, data interface return nil } -func GetHCL(data interface{}) (string, error) { - association, ok := getAssociation(data) +var ( + resourceReferenceRe = regexp.MustCompile(`(?P(data)|(resource)) "(?P[a-z_]+)" "(?P[a-z_]+)"`) + resourceReferenceResourceTypeIndex = resourceReferenceRe.SubexpIndex("type") + resourceReferenceResourceModuleIndex = resourceReferenceRe.SubexpIndex("module") + resourceReferenceResourceNameIndex = resourceReferenceRe.SubexpIndex("name") +) + +func getResourceReferenceFromOutput(output string) (resourceModule string, resourceName string) { + matches := resourceReferenceRe.FindAllStringSubmatch(output, -1) + if matches == nil { + return "", "" + } + + match := matches[len(matches)-1] + + resourceType := match[resourceReferenceResourceTypeIndex] + resourceModule = match[resourceReferenceResourceModuleIndex] + resourceName = match[resourceReferenceResourceNameIndex] + + if resourceType == "data" { + resourceModule = fmt.Sprintf("data.%s", resourceModule) + } + + return +} + +type GetHCLConfig struct { + Client *scw.Client + Data interface{} + + SkipParents bool + SkipChildren bool +} + +func GetHCL(config *GetHCLConfig) (string, error) { + association, ok := getAssociation(config.Data) if !ok { - return "", fmt.Errorf("no terraform association found for this resource type (%s)", reflect.TypeOf(data).Name()) + resourceType := "nil" + if typeOf := reflect.TypeOf(config.Data); typeOf != nil { + resourceType = typeOf.Name() + + if resourceType == "" { + resourceType = typeOf.String() + } + } + + return "", fmt.Errorf("no terraform association found for this resource type (%s)", resourceType) } // Create temporary directory @@ -90,7 +137,7 @@ func GetHCL(data interface{}) (string, error) { } defer os.RemoveAll(tmpDir) - err = createImportFile(tmpDir, association, data) + err = createImportFile(tmpDir, association, config.Data) if err != nil { return "", err } @@ -112,10 +159,92 @@ func GetHCL(data interface{}) (string, error) { } // Read the generated output - output, err := os.ReadFile(filepath.Join(tmpDir, "output.tf")) + outputRaw, err := os.ReadFile(filepath.Join(tmpDir, "output.tf")) if err != nil { return "", err } - return string(output), nil + output := string(outputRaw) + // Remove first 4 lines (terraform header) + lines := strings.Split(output, "\n") + output = strings.Join(lines[4:], "\n") + + if config.Client == nil { + return output, nil + } + + parents := make([]string, 0, len(association.Parents)) + children := make([]string, 0, len(association.Children)) + + if !config.SkipParents { + for attributeName, resource := range association.Parents { + resourceData, err := resource.Fetcher(config.Client, config.Data) + if err != nil { + return "", err + } + + resourceOutput, err := GetHCL(&GetHCLConfig{ + Client: config.Client, + Data: resourceData, + SkipChildren: true, + }) + if err != nil { + return "", err + } + + resourceModule, resourceName := getResourceReferenceFromOutput(resourceOutput) + + parents = append(parents, resourceOutput) + + re := regexp.MustCompile(fmt.Sprintf(`%s([ \t]+)= .*`, attributeName)) + matches := re.FindAllStringSubmatch(output, -1) + spaces := matches[len(matches)-1][1] + + output = re.ReplaceAllString(output, fmt.Sprintf("%s%s= %s.%s", attributeName, spaces, resourceModule, resourceName)) + } + } + + if !config.SkipChildren { + parentResourceModule, parentResourceName := getResourceReferenceFromOutput(output) + + for _, child := range association.Children { + resourceData, err := child.Fetcher(config.Client, config.Data) + if err != nil { + return "", err + } + + // resourceData SHOULD be a slice + slice := reflect.ValueOf(resourceData) + for i := 0; i < slice.Len(); i++ { + resourceOutput, err := GetHCL(&GetHCLConfig{ + Client: config.Client, + Data: slice.Index(i).Interface(), + SkipParents: true, + }) + if err != nil { + return "", err + } + + for childField, parentField := range child.ParentFieldMap { + re := regexp.MustCompile(fmt.Sprintf(`%s([ \t]+)= .*`, childField)) + matches := re.FindAllStringSubmatch(resourceOutput, -1) + spaces := matches[len(matches)-1][1] + + resourceOutput = re.ReplaceAllString(resourceOutput, fmt.Sprintf("%s%s= %s.%s.%s", childField, spaces, parentResourceModule, parentResourceName, parentField)) + } + + children = append(children, resourceOutput) + } + } + } + + for _, parent := range parents { + output = parent + "\n" + output + } + + for _, child := range children { + output = output + "\n" + child + } + + return output, nil }