Sync main jan 7 feature branch resource identity

.ci/gcb-pr-downstream-generation-and-test.yml

@@ -265,6 +265,7 @@ steps:
         - $BUILD_ID
         - $PROJECT_ID
         - "23"  # Build step
+        - "true"
 
     - name: 'gcr.io/graphite-docker-images/go-plus'
       entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'

.ci/magician/cmd/check_cassettes.go

@@ -77,7 +77,7 @@ var checkCassettesCmd = &cobra.Command{
 
 		ctlr := source.NewController(env["GOPATH"], "modular-magician", githubToken, rnr)
 
-		vt, err := vcr.NewTester(env, "ci-vcr-cassettes", "vcr-check-cassettes", rnr)
+		vt, err := vcr.NewTester(env, "ci-vcr-cassettes", "vcr-check-cassettes", rnr, false)
 		if err != nil {
 			return fmt.Errorf("error creating VCR tester: %w", err)
 		}

.ci/magician/cmd/collect_nightly_test_status.go

@@ -43,6 +43,7 @@ type TestInfo struct {
 	Status          string    `json:"status"`
 	Service         string    `json:"service"`
 	Resource        string    `json:"resource"`
+	CommitSha       string    `json:"commit_sha"`
 	ErrorMessage    string    `json:"error_message"`
 	LogLink         string    `json:"log_link"`
 	ProviderVersion string    `json:"provider_version"`
@@ -189,6 +190,7 @@ func createTestReport(pVersion provider.Version, tc TeamcityClient, gcs Cloudsto
 				Status:          testResult.Status,
 				Service:         serviceName,
 				Resource:        convertTestNameToResource(testResult.Name),
+				CommitSha:       build.Number,
 				ErrorMessage:    errorMessage,
 				LogLink:         logLink,
 				ProviderVersion: strings.ToUpper(pVersion.String()),

.ci/magician/cmd/generate_comment.go

@@ -89,7 +89,7 @@ type diffCommentData struct {
 	MissingServiceLabels []string
 	MissingTests         map[string]*MissingTestInfo
 	MissingDocs          *MissingDocsSummary
-	AddedResources       []string
+	MultipleResources    []string
 	Errors               []Errors
 }
 
@@ -377,7 +377,8 @@ func execGenerateComment(prNumber int, ghTokenMagicModules, buildId, buildStep,
 
 	// Check if multiple resources were added.
 	multipleResourcesState := "success"
-	if len(uniqueAddedResources) > 1 {
+	data.MultipleResources = multipleResources(maps.Keys(uniqueAddedResources))
+	if len(data.MultipleResources) > 1 {
 		multipleResourcesState = "failure"
 		for _, label := range pullRequest.Labels {
 			if label.Name == allowMultipleResourcesLabel {
@@ -391,8 +392,6 @@ func execGenerateComment(prNumber int, ghTokenMagicModules, buildId, buildStep,
 		fmt.Printf("Error posting terraform-provider-multiple-resources build status for pr %d commit %s: %v\n", prNumber, commitSha, err)
 		errors["Other"] = append(errors["Other"], "Failed to update missing-service-labels status check with state: "+multipleResourcesState)
 	}
-	data.AddedResources = maps.Keys(uniqueAddedResources)
-	slices.Sort(data.AddedResources)
 
 	// Compute affected resources based on changed files
 	changedFilesAffectedResources := map[string]struct{}{}
@@ -642,6 +641,38 @@ func formatDiffComment(data diffCommentData) (string, error) {
 	return sb.String(), nil
 }
 
+// addedMultipleResources returns a sorted slice of resource names that are considered "separate" resources.
+// In particular, IAM resources are merged with the parent resource as part of this check.
+func multipleResources(resources []string) []string {
+	if len(resources) == 0 {
+		return nil
+	}
+	iam := map[string]struct{}{}
+	final := map[string]struct{}{}
+
+	for _, r := range resources {
+		if k, found := strings.CutSuffix(r, "_iam_member"); found {
+			iam[k] = struct{}{}
+		} else if k, found := strings.CutSuffix(r, "_iam_binding"); found {
+			iam[k] = struct{}{}
+		} else if k, found := strings.CutSuffix(r, "_iam_policy"); found {
+			iam[k] = struct{}{}
+		} else {
+			final[k] = struct{}{}
+		}
+	}
+
+	for r, _ := range iam {
+		if _, ok := final[r]; !ok {
+			final[r+"_iam_*"] = struct{}{}
+		}
+	}
+
+	ret := maps.Keys(final)
+	slices.Sort(ret)
+	return ret
+}
+
 var resourceFileRegexp = regexp.MustCompile(`^.*/services/[^/]+/(?:data_source_|resource_|iam_)(.*?)(?:_test|_sweeper|_iam_test|_generated_test|_internal_test)?.go`)
 var resourceDocsRegexp = regexp.MustCompile(`^.*website/docs/(?:r|d)/(.*).html.markdown`)
 

.ci/magician/cmd/generate_comment_test.go

@@ -246,7 +246,7 @@ func TestFormatDiffComment(t *testing.T) {
 		},
 		"multiple resources are displayed": {
 			data: diffCommentData{
-				AddedResources: []string{"google_redis_instance", "google_alloydb_cluster"},
+				MultipleResources: []string{"google_redis_instance", "google_alloydb_cluster"},
 			},
 			expectedStrings: []string{
 				"## Diff report",
@@ -352,6 +352,72 @@ func TestFormatDiffComment(t *testing.T) {
 	}
 }
 
+func TestMultipleResources(t *testing.T) {
+	cases := []struct {
+		name      string
+		resources []string
+		want      []string
+	}{
+		{
+			name: "no resources",
+		},
+		{
+			name:      "single non-iam",
+			resources: []string{"google_redis_instance"},
+			want:      []string{"google_redis_instance"},
+		},
+		{
+			name:      "multiple non-iam",
+			resources: []string{"google_redis_instance", "google_alloydb_cluster"},
+			want:      []string{"google_alloydb_cluster", "google_redis_instance"},
+		},
+		{
+			name:      "single iam only",
+			resources: []string{"google_redis_instance_iam_member", "google_redis_instance_iam_policy", "google_redis_instance_iam_binding"},
+			want:      []string{"google_redis_instance_iam_*"},
+		},
+		{
+			name:      "single iam with parent",
+			resources: []string{"google_redis_instance_iam_member", "google_redis_instance_iam_policy", "google_redis_instance_iam_binding", "google_redis_instance"},
+			want:      []string{"google_redis_instance"},
+		},
+		{
+			name: "multiple iam",
+			resources: []string{
+				"google_redis_instance_iam_member",
+				"google_redis_instance_iam_policy",
+				"google_redis_instance_iam_binding",
+				"google_alloydb_cluster_iam_member",
+				"google_alloydb_cluster_iam_policy",
+				"google_alloydb_cluster_iam_binding",
+			},
+			want: []string{"google_alloydb_cluster_iam_*", "google_redis_instance_iam_*"},
+		},
+		{
+			name: "multiple iam with parent",
+			resources: []string{
+				"google_redis_instance_iam_member",
+				"google_redis_instance_iam_policy",
+				"google_redis_instance_iam_binding",
+				"google_alloydb_cluster_iam_member",
+				"google_alloydb_cluster_iam_policy",
+				"google_alloydb_cluster_iam_binding",
+				"google_redis_instance",
+			},
+			want: []string{"google_alloydb_cluster_iam_*", "google_redis_instance"},
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			got := multipleResources(tc.resources)
+			assert.Equal(t, tc.want, got)
+		})
+	}
+}
+
 func TestFileToResource(t *testing.T) {
 	cases := map[string]struct {
 		path string

.ci/magician/cmd/templates/DIFF_COMMENT.md.tmpl

@@ -51,10 +51,10 @@ If you believe this detection to be incorrect please raise the concern with your
 An `override-missing-service-label` label can be added to allow merging.
 {{end}}
 
-{{- if gt (len .AddedResources) 1 }}
+{{- if gt (len .MultipleResources) 1 }}
 ## Multiple resources added
 
-This PR adds multiple new resources: {{range $i, $resource := .AddedResources}}{{ if gt $i 0}}, {{end}}`{{$resource}}`{{end}}. This makes review significantly more difficult. Please split it into multiple PRs, one per resource.
+This PR adds multiple new resources: {{range $i, $resource := .MultipleResources}}{{ if gt $i 0}}, {{end}}`{{$resource}}`{{end}}. This makes review significantly more difficult. Please split it into multiple PRs, one per resource.
 An `override-multiple-resources` label can be added to allow merging.
 {{end}}
 

.ci/magician/cmd/test_eap_vcr.go

@@ -120,7 +120,7 @@ The following environment variables are required:
 		if err != nil {
 			return err
 		}
-		vt, err := vcr.NewTester(env, "ci-vcr-cassettes", "ci-vcr-logs", rnr)
+		vt, err := vcr.NewTester(env, "ci-vcr-cassettes", "ci-vcr-logs", rnr, false)
 		if err != nil {
 			return err
 		}

.ci/magician/cmd/test_terraform_vcr.go

@@ -93,7 +93,8 @@ It expects the following arguments:
 	3. Build ID
 	4. Project ID where Cloud Builds are located
 	5. Build step number
-
+	6. Enable async upload cassettes
+
 The following environment variables are required:
 ` + listTTVRequiredEnvironmentVariables(),
 	RunE: func(cmd *cobra.Command, args []string) error {
@@ -134,13 +135,17 @@ The following environment variables are required:
 		}
 		ctlr := source.NewController(env["GOPATH"], "modular-magician", env["GITHUB_TOKEN_DOWNSTREAMS"], rnr)
 
-		vt, err := vcr.NewTester(env, "ci-vcr-cassettes", "ci-vcr-logs", rnr)
-		if err != nil {
-			return fmt.Errorf("error creating VCR tester: %w", err)
+		if len(args) < 5 {
+			return fmt.Errorf("wrong number of arguments %d, expected >=5", len(args))
+		}
+		enableAsyncUploadCassettes := false
+		if len(args) > 5 {
+			enableAsyncUploadCassettes = strings.ToLower(args[5]) == "true"
 		}
 
-		if len(args) != 5 {
-			return fmt.Errorf("wrong number of arguments %d, expected 5", len(args))
+		vt, err := vcr.NewTester(env, "ci-vcr-cassettes", "ci-vcr-logs", rnr, enableAsyncUploadCassettes)
+		if err != nil {
+			return fmt.Errorf("error creating VCR tester: %w", err)
 		}
 
 		return execTestTerraformVCR(args[0], args[1], args[2], args[3], args[4], baseBranch, gh, rnr, ctlr, vt)
@@ -260,10 +265,11 @@ func execTestTerraformVCR(prNumber, mmCommitSha, buildID, projectID, buildStep,
 	}
 	if len(replayingResult.FailedTests) > 0 {
 		recordingResult, recordingErr := vt.RunParallel(vcr.RunOptions{
-			Mode:     vcr.Recording,
-			Version:  provider.Beta,
-			TestDirs: testDirs,
-			Tests:    replayingResult.FailedTests,
+			Mode:             vcr.Recording,
+			Version:          provider.Beta,
+			TestDirs:         testDirs,
+			Tests:            replayingResult.FailedTests,
+			UploadBranchName: newBranch,
 		})
 		if recordingErr != nil {
 			testState = "failure"

.ci/magician/cmd/vcr_cassette_update.go

@@ -103,7 +103,7 @@ var vcrCassetteUpdateCmd = &cobra.Command{
 		}
 		ctlr := source.NewController(env["GOPATH"], "hashicorp", env["GITHUB_TOKEN_CLASSIC"], rnr)
 
-		vt, err := vcr.NewTester(env, "ci-vcr-cassettes", "", rnr)
+		vt, err := vcr.NewTester(env, "ci-vcr-cassettes", "", rnr, true)
 		if err != nil {
 			return fmt.Errorf("error creating VCR tester: %w", err)
 		}

.ci/magician/cmd/vcr_cassette_update_test.go

@@ -407,7 +407,7 @@ func TestExecVCRCassetteUpdate(t *testing.T) {
 			ctlr := source.NewController("gopath", "hashicorp", "token", rnr)
 			vt, err := vcr.NewTester(map[string]string{
 				"SA_KEY": "sa_key",
-			}, "ci-vcr-cassettes", "", rnr)
+			}, "ci-vcr-cassettes", "", rnr, false)
 			if err != nil {
 				t.Fatalf("Failed to create new tester: %v", err)
 			}

.ci/magician/github/membership_data.go

@@ -86,8 +86,8 @@ var (
 		"melinath": {
 			vacations: []Vacation{
 				{
-					startDate: newDate(2025, 9, 17),
-					endDate:   newDate(2025, 9, 22),
+					startDate: newDate(2025, 12, 18),
+					endDate:   newDate(2026, 1, 6),
 				},
 			},
 		},

.ci/magician/go.mod

@@ -15,6 +15,7 @@ require (
 
 require (
 	cloud.google.com/go/storage v1.50.0
+	github.com/fsnotify/fsnotify v1.9.0
 	github.com/google/go-cmp v0.6.0
 	github.com/google/go-github/v68 v68.0.0
 	github.com/otiai10/copy v1.12.0

.ci/magician/go.sum

@@ -57,6 +57,8 @@ github.com/envoyproxy/protoc-gen-validate v1.1.0 h1:tntQDh69XqOCOZsDz0lVJQez/2L6
 github.com/envoyproxy/protoc-gen-validate v1.1.0/go.mod h1:sXRDRVmzEbkM7CVcM06s9shE/m23dg3wzjl0UWqJ2q4=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
+github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
 github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=