diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000000..9e1e322cea --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,43 @@ +name: Test +on: + push: + branches: [main] + pull_request: + branches: [main] +jobs: + test: + name: unit test and integration test + runs-on: ubuntu-latest + # Add "id-token" with the intended permissions for gcp auth. + permissions: + contents: "read" + id-token: "write" + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v3 + - name: Set up Go 1.x + uses: actions/setup-go@v3 + with: + go-version-file: "go.mod" + - name: Get dependencies + run: | + go mod download + - name: Test + run: | + make test + - name: Install Terraform + uses: hashicorp/setup-terraform@v2 + with: + terraform_version: 1.3.1 + # disable terraform_wrapper to avoid running node. + terraform_wrapper: false + - name: Terraform version + run: terraform --version + - name: Authenticate to Google Cloud + uses: "google-github-actions/auth@v0.4.0" + with: + workload_identity_provider: "${{ secrets.WIF_PROVIDER }}" # e.g. - projects/123456789/locations/global/workloadIdentityPools/my-pool/providers/my-provider + service_account: "${{ secrets.WIF_SERVICE_ACCOUNT }}" # e.g. - my-service-account@my-project.iam.gserviceaccount.com + - name: Integration test + run: | + TEST_PROJECT=${{ secrets.TEST_PROJECT}} TEST_ORG_ID=${{ secrets.TEST_ORG_ID}} TEST_ANCESTRY=${{ secrets.TEST_ANCESTRY}} TEST_FOLDER_ID=${{ secrets.TEST_FOLDER_ID}} make test-integration diff --git a/Makefile b/Makefile index 1878e0ba6a..ed9901c0ec 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,9 @@ test: - GO111MODULE=on go test ./... + GO111MODULE=on go test -short ./... + +test-integration: + go version + terraform --version + go test ./e2etest .PHONY: test diff --git a/cai2hcl/cai2hcl.go b/cai2hcl/cai2hcl.go index ca0715bfb0..ecaa4583f6 100644 --- a/cai2hcl/cai2hcl.go +++ b/cai2hcl/cai2hcl.go @@ -21,7 +21,11 @@ type Options struct { // Convert converts Asset into HCL. func Convert(assets []*caiasset.Asset, options *Options) ([]byte, error) { if options == nil || options.ErrorLogger == nil { - return nil, fmt.Errorf("logger is not initialized") + logger, err := zap.NewDevelopment() + if err != nil { + return nil, fmt.Errorf("error initiating logger: %w", err) + } + options.ErrorLogger = logger } // Group resources from the same tf resource type for convert. diff --git a/e2etest/compute_instance_test.go b/e2etest/compute_instance_test.go new file mode 100644 index 0000000000..57ddc4e350 --- /dev/null +++ b/e2etest/compute_instance_test.go @@ -0,0 +1,25 @@ +package e2etest + +import ( + "os" + "testing" +) + +func TestComputeInstance(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode.") + return + } + + tfFiles := []string{ + "full_compute_instance", + } + tmpDir := os.TempDir() + data := initTestData() + + for _, name := range tfFiles { + t.Run(name, func(t *testing.T) { + roundtripTest(t, name, tmpDir, data) + }) + } +} diff --git a/e2etest/e2e.go b/e2etest/e2e.go new file mode 100644 index 0000000000..62401d06fb --- /dev/null +++ b/e2etest/e2e.go @@ -0,0 +1,269 @@ +package e2etest + +import ( + "bytes" + "context" + "io/ioutil" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "text/template" + "time" + + "github.com/GoogleCloudPlatform/terraform-google-conversion/v2/cai2hcl" + "github.com/GoogleCloudPlatform/terraform-google-conversion/v2/tfplan2cai" + "github.com/GoogleCloudPlatform/terraform-validator/converters/google" + "github.com/google/go-cmp/cmp" + "github.com/hashicorp/hcl/hcl/printer" +) + +type testData struct { + TFVersion string + Provider map[string]string + Project map[string]string + Time map[string]string + OrgID string + FolderID string + Ancestry string +} + +// initTestData initializes the variables used for testing. As tests rely on +// environment variables, the parsing of those are only done once. +func initTestData() *testData { + credentials := getTestCredsFromEnv() + project := getTestProjectFromEnv() + org := getTestOrgFromEnv(nil) + billingAccount := getTestBillingAccountFromEnv(nil) + folder, ok := os.LookupEnv("TEST_FOLDER_ID") + if !ok { + log.Printf("Missing required env var TEST_FOLDER_ID. Default (%s) will be used.", defaultFolder) + folder = defaultFolder + } + ancestry, ok := os.LookupEnv("TEST_ANCESTRY") + if !ok { + log.Printf("Missing required env var TEST_ANCESTRY. Default (%s) will be used.", defaultAncestry) + ancestry = defaultAncestry + } + providerVersion := defaultProviderVersion + //As time is not information in terraform resource data, time is fixed for testing purposes + fixedTime := time.Date(2021, time.April, 14, 15, 16, 17, 0, time.UTC) + return &testData{ + TFVersion: "0.12", + Provider: map[string]string{ + "version": providerVersion, + "project": project, + "credentials": credentials, + }, + Time: map[string]string{ + "RFC3339Nano": fixedTime.Format(time.RFC3339Nano), + }, + Project: map[string]string{ + "Name": "My Project Name", + "ProjectId": "my-project-id", + "BillingAccountName": billingAccount, + "Number": "1234567890", + }, + OrgID: org, + FolderID: folder, + Ancestry: ancestry, + } +} + +func roundtripTest(t *testing.T, name string, tmpDir string, data *testData) { + dir, err := ioutil.TempDir(tmpDir, "terraform") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + // tf file for credential and versions + generateHeaderFile(t, filepath.Join(dir, "header.tf"), data) + + // fill template to generate the tf file + generateTestFiles(t, "../testdata", dir, name+".tf", data) + + // terraform init + terraform plan + terraformWorkflow(t, dir, name) + + // convert from tf json plan to assets + tfJSONPlanPath := filepath.Join(dir, name+".tfplan.json") + jsonPlan, err := ioutil.ReadFile(tfJSONPlanPath) + if err != nil { + t.Fatalf("cannot read %q, got: %s", tfJSONPlanPath, err) + } + gotAssets, err := tfplan2cai.Convert(context.Background(), + jsonPlan, + &tfplan2cai.Options{ + DefaultProject: data.Provider["project"], + }, + ) + if err != nil { + t.Fatalf("tfplan2cai.Convert() = %s, want = nil", err) + } + + // convert from assets to hcl + var assetsInput []*google.Asset + for ix := range gotAssets { + assetsInput = append(assetsInput, &gotAssets[ix]) + } + gotTFPlanBytes, err := cai2hcl.Convert(assetsInput, &cai2hcl.Options{}) + if err != nil { + t.Fatalf("cai2hcl.Convert() = %s, want = nil", err) + } + + // compare results + tfFilePath := filepath.Join(dir, name+".tf") + tfBytes, err := ioutil.ReadFile(tfFilePath) + if err != nil { + t.Fatalf("Error parsing %s: %s", tfFilePath, err) + } + wantTFPlanBytes, err := printer.Format(tfBytes) + if err != nil { + t.Fatalf("Error format %s: %s", tfFilePath, err) + } + if diff := cmp.Diff(string(wantTFPlanBytes), string(gotTFPlanBytes)); diff != "" { + t.Fatalf("want = %v, got = %v, diff = %s", string(wantTFPlanBytes), string(gotTFPlanBytes), diff) + } +} + +func terraformWorkflow(t *testing.T, dir, name string) { + terraformInit(t, "terraform", dir) + terraformPlan(t, "terraform", dir, name+".tfplan") + payload := terraformShow(t, "terraform", dir, name+".tfplan") + saveFile(t, dir, name+".tfplan.json", payload) +} + +func terraformInit(t *testing.T, executable, dir string) { + terraformExec(t, executable, dir, "init", "-input=false") +} + +func terraformPlan(t *testing.T, executable, dir, tfplan string) { + terraformExec(t, executable, dir, "plan", "-input=false", "-refresh=false", "-out", tfplan) +} + +func terraformShow(t *testing.T, executable, dir, tfplan string) []byte { + return terraformExec(t, executable, dir, "show", "--json", tfplan) +} + +func terraformExec(t *testing.T, executable, dir string, args ...string) []byte { + cmd := exec.Command(executable, args...) + cmd.Env = []string{"HOME=" + filepath.Join(dir, "fakehome")} + cmd.Dir = dir + wantError := false + payload, _ := run(t, cmd, wantError) + return payload +} + +func saveFile(t *testing.T, dir, filename string, payload []byte) { + fullpath := filepath.Join(dir, filename) + f, err := os.Create(fullpath) + if err != nil { + t.Fatalf("error while creating file %s, error %v", fullpath, err) + } + _, err = f.Write(payload) + if err != nil { + t.Fatalf("error while writing to file %s, error %v", fullpath, err) + } +} + +// run a command and call t.Fatal on non-zero exit. +func run(t *testing.T, cmd *exec.Cmd, wantError bool) ([]byte, []byte) { + var stderr, stdout bytes.Buffer + cmd.Stderr, cmd.Stdout = &stderr, &stdout + err := cmd.Run() + if gotError := (err != nil); gotError != wantError { + t.Fatalf("running %s: \nerror=%v \nstderr=%s \nstdout=%s", cmdToString(cmd), err, stderr.String(), stdout.String()) + } + // Print env, stdout and stderr if verbose flag is used. + if len(cmd.Env) != 0 { + t.Logf("=== Environment Variable of %s ===", cmdToString(cmd)) + t.Log(strings.Join(cmd.Env, "\n")) + } + if stdout.String() != "" { + t.Logf("=== STDOUT of %s ===", cmdToString(cmd)) + t.Log(stdout.String()) + } + if stderr.String() != "" { + t.Logf("=== STDERR of %s ===", cmdToString(cmd)) + t.Log(stderr.String()) + } + return stdout.Bytes(), stderr.Bytes() +} + +// cmdToString clones the logic of https://golang.org/pkg/os/exec/#Cmd.String. +func cmdToString(c *exec.Cmd) string { + // report the exact executable path (plus args) + b := new(strings.Builder) + b.WriteString(c.Path) + for _, a := range c.Args[1:] { + b.WriteByte(' ') + b.WriteString(a) + } + return b.String() +} + +func generateTestFiles(t *testing.T, sourceDir string, targetDir string, fileName string, data interface{}) { + funcMap := template.FuncMap{ + "pastLastSlash": func(s string) string { + split := strings.Split(s, "/") + return split[len(split)-1] + }, + } + tmpls, err := template.New("").Funcs(funcMap). + ParseGlob(filepath.Join(sourceDir, fileName)) + if err != nil { + t.Fatalf("generateTestFiles: %v", err) + } + for _, tmpl := range tmpls.Templates() { + if tmpl.Name() == "" { + continue // Skip base template. + } + path := filepath.Join(targetDir, tmpl.Name()) + f, err := os.Create(path) + if err != nil { + t.Fatalf("creating terraform file %v: %v", path, err) + } + if err := tmpl.Execute(f, data); err != nil { + t.Fatalf("templating terraform file %v: %v", path, err) + } + if err := f.Close(); err != nil { + t.Fatalf("closing file %v: %v", path, err) + } + t.Logf("Successfully created file %v", path) + } +} + +func generateHeaderFile(t *testing.T, path string, data interface{}) { + t.Helper() + headerTemplate := ` + terraform { + required_providers { + google = { + source = "hashicorp/google" + version = "~> {{.Provider.version}}" + } + } + } + + provider "google" { + {{if .Provider.credentials }}credentials = "{{.Provider.credentials}}"{{end}} + } + ` + tmpl, err := template.New("header").Parse(headerTemplate) + if err != nil { + t.Fatal(err) + } + f, err := os.Create(path) + if err != nil { + t.Fatalf("creating terraform file %v: %v", path, err) + } + if err := tmpl.Execute(f, data); err != nil { + t.Fatalf("templating terraform file %v: %v", path, err) + } + if err := f.Close(); err != nil { + t.Fatalf("closing file %v: %v", path, err) + } +} diff --git a/e2etest/environment.go b/e2etest/environment.go new file mode 100644 index 0000000000..f9377b33da --- /dev/null +++ b/e2etest/environment.go @@ -0,0 +1,122 @@ +package e2etest + +import ( + "log" + "os" + "path/filepath" + "testing" + + google "github.com/GoogleCloudPlatform/terraform-validator/converters/google/resources" +) + +const ( + samplePolicyPath = "../testdata/sample_policies" + defaultAncestry = "organization/12345/folder/67890" + defaultBillingAccount = "000AA0-A0B00A-AA00AA" + defaultCustId = "A00ccc00a" + defaultFirestoreProject = "firebar" + defaultFolder = "67890" + defaultIdentityUser = "foo" + defaultOrganization = "12345" + defaultOrganizationDomain = "meep.test.com" + defaultOrganizationTarget = "13579" + defaultProject = "foobar" + defaultProviderVersion = "4.28.0" + defaultRegion = "us-central1" + defaultServiceAccount = "meep@foobar.iam.gserviceaccount.com" +) + +func Nprintf(format string, params map[string]interface{}) string { + return google.Nprintf(format, params) +} + +// testAccPreCheck ensures at least one of the project env variables is set. +func getTestProjectFromEnv() string { + project := multiEnvSearch([]string{"TEST_PROJECT", "GOOGLE_PROJECT"}) + if project == "" { + log.Printf("Missing required env var TEST_PROJECT. Default (%s) will be used.", defaultProject) + project = defaultProject + } + + return project +} + +// testAccPreCheck ensures at least one of the credentials env variables is set. +func getTestCredsFromEnv() string { + cwd, err := os.Getwd() + if err != nil { + log.Fatalf("cannot get current directory: %v", err) + } + + credentials := multiEnvSearch([]string{"TEST_CREDENTIALS", "GOOGLE_APPLICATION_CREDENTIALS"}) + if credentials != "" { + // Make credentials path relative to repo root rather than + // test/ dir if it is a relative path. + if !filepath.IsAbs(credentials) { + credentials = filepath.Join(cwd, "..", credentials) + } + } else { + log.Printf("missing env var TEST_CREDENTIALS, will try to use Application Default Credentials") + } + + return credentials +} + +// testAccPreCheck ensures at least one of the region env variables is set. +func getTestRegionFromEnv() string { + return defaultRegion +} + +func getTestCustIdFromEnv(t *testing.T) string { + return defaultCustId +} + +func getTestIdentityUserFromEnv(t *testing.T) string { + return defaultIdentityUser +} + +// Firestore can't be enabled at the same time as Datastore, so we need a new +// project to manage it until we can enable Firestore programmatically. +func getTestFirestoreProjectFromEnv(t *testing.T) string { + return defaultFirestoreProject +} + +func getTestOrgFromEnv(t *testing.T) string { + org, ok := os.LookupEnv("TEST_ORG_ID") + if !ok { + log.Printf("Missing required env var TEST_ORG_ID. Default (%s) will be used.", defaultOrganization) + org = defaultOrganization + } + + return org +} + +func getTestOrgDomainFromEnv(t *testing.T) string { + return defaultOrganizationDomain +} + +func getTestOrgTargetFromEnv(t *testing.T) string { + return defaultOrganizationTarget +} + +func getTestBillingAccountFromEnv(t *testing.T) string { + return defaultBillingAccount +} + +func getTestServiceAccountFromEnv(t *testing.T) string { + return defaultServiceAccount +} + +func multiEnvSearch(ks []string) string { + for _, k := range ks { + if v := os.Getenv(k); v != "" { + return v + } + } + return "" +} + +func shouldOutputGeneratedFiles() bool { + _, ok := os.LookupEnv("TFV_CREATE_GENERATED_FILES") + return ok +} diff --git a/go.mod b/go.mod index 6f224d2cd8..ced14827e4 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/GoogleCloudPlatform/terraform-google-conversion/v2 -go 1.17 +go 1.18 require ( github.com/GoogleCloudPlatform/terraform-validator v0.16.1 diff --git a/tfplan2cai/tfplan_to_cai_test.go b/tfplan2cai/tfplan_to_cai_test.go index e18700758f..384f5c446e 100644 --- a/tfplan2cai/tfplan_to_cai_test.go +++ b/tfplan2cai/tfplan_to_cai_test.go @@ -25,7 +25,7 @@ func TestConvert_noResourceChanges(t *testing.T) { if err != nil { t.Fatalf("Error parsing %s: %s", f, err) } - options := &Options{} + options := &Options{Offline: true} assets, err := Convert(ctx, jsonPlan, options) assert.Empty(t, assets) assert.Empty(t, err)