Skip to content

[Do Not Merge] Test pr 16040 #16073

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
shuyama1 wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

[Do Not Merge] Test pr 16040 #16073

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
08d52f7
Migrate gsutil usage to gcloud storage
bhandarivijay-png Dec 29, 2025
0178e63
fix
shuyama1 Jan 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions .ci/magician/cmd/vcr_cassette_update.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,8 +123,8 @@ func execVCRCassetteUpdate(buildID, today string, rnr ExecRunner, ctlr *source.C
// main cassettes backup
// incase nightly run goes wrong. this will be used to restore the cassettes
cassettePath := vt.CassettePath(provider.Beta)
args := []string{"-m", "-q", "cp", filepath.Join(cassettePath, "*"), bucketPrefix + "/main_cassettes_backup/fixtures/"}
if _, err := rnr.Run("gsutil", args, nil); err != nil {
args := []string{"storage", "cp", filepath.Join(cassettePath, "*"), bucketPrefix + "/main_cassettes_backup/fixtures/"}
if _, err := rnr.Run("gcloud", args, nil); err != nil {
return fmt.Errorf("error backup cassettes: %w", err)
}

Expand Down Expand Up @@ -222,18 +222,18 @@ func execVCRCassetteUpdate(buildID, today string, rnr ExecRunner, ctlr *source.C
}

func uploadLogsToGCS(src, dest string, rnr ExecRunner) (string, error) {
return uploadToGCS(src, dest, []string{"-h", "Content-Type:text/plain", "-q", "cp", "-r"}, rnr)
return uploadToGCS(src, dest, []string{"storage", "cp", "--content-type=text/plain", "--recursive"}, rnr)
}

func uploadCassettesToGCS(src, dest string, rnr ExecRunner) (string, error) {
return uploadToGCS(src, dest, []string{"-m", "-q", "cp"}, rnr)
return uploadToGCS(src, dest, []string{"storage", "cp"}, rnr)
}

func uploadToGCS(src, dest string, opts []string, rnr ExecRunner) (string, error) {
fmt.Printf("uploading from %s to %s\n", src, dest)
args := append(opts, src, dest)
fmt.Println("gsutil", args)
return rnr.Run("gsutil", args, nil)
fmt.Println("gcloud", args)
return rnr.Run("gcloud", args, nil)
}

func formatVCRCassettesUpdateReplaying(data vcrCassetteUpdateReplayingResult) (string, error) {
Expand Down
21 changes: 11 additions & 10 deletions .ci/magician/cmd/vcr_merge.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ var vcrMergeCmd = &cobra.Command{

It then performs the following operations:
1. Get the latest closed PR matching the reference commit SHA.
2. Run gsutil to list, copy, and remove the vcr cassettes fixtures.
2. Run gcloud storage to list, copy, and remove the vcr cassettes fixtures.
`,
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
Expand Down Expand Up @@ -88,11 +88,12 @@ func mergeCassettes(basePath, baseBranch, prPath string, runner source.Runner) {

func listCassettes(path string, runner source.Runner) error {
lsArgs := []string{
"storage",
"ls",
path,
}
fmt.Println("Running command: ", "gsutil", lsArgs)
ret, err := runner.Run("gsutil", lsArgs, nil)
fmt.Println("Running command: ", "gcloud", lsArgs)
ret, err := runner.Run("gcloud", lsArgs, nil)
if err != nil {
return err
}
Expand All @@ -102,26 +103,26 @@ func listCassettes(path string, runner source.Runner) error {

func cpCassettes(src, dest string, runner source.Runner) {
cpArgs := []string{
"-m",
"storage",
"cp",
src,
dest,
}
fmt.Println("Running command: ", "gsutil", cpArgs)
if _, err := runner.Run("gsutil", cpArgs, nil); err != nil {
fmt.Println("Running command: ", "gcloud", cpArgs)
if _, err := runner.Run("gcloud", cpArgs, nil); err != nil {
fmt.Println("Error in copy: ", err)
}
}

func rmCassettes(dest string, runner source.Runner) {
rmArgs := []string{
"-m",
"storage",
"rm",
"-r",
"--recursive",
dest,
}
fmt.Println("Running command: ", "gsutil", rmArgs)
if _, err := runner.Run("gsutil", rmArgs, nil); err != nil {
fmt.Println("Running command: ", "gcloud", rmArgs)
if _, err := runner.Run("gcloud", rmArgs, nil); err != nil {
fmt.Println("Error in remove: ", err)
}
}
Expand Down
2 changes: 1 addition & 1 deletion .ci/magician/cmd/vcr_merge_eap.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ var vcrMergeEapCmd = &cobra.Command{
1. CL number

It then performs the following operations:
1. Run gsutil to list, copy, and remove the vcr cassettes fixtures.
1. Run gcloud storage to list, copy, and remove the vcr cassettes fixtures.
`,
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
Expand Down
28 changes: 14 additions & 14 deletions .ci/magician/cmd/vcr_merge_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,25 +22,25 @@ func TestExecVCRMerge(t *testing.T) {
baseBranch: "main",
commitSha: "sha",
calledMethods: []string{
"gsutil ls gs://ci-vcr-cassettes/refs/heads/auto-pr-123/fixtures/",
"gsutil -m cp gs://ci-vcr-cassettes/refs/heads/auto-pr-123/fixtures/* gs://ci-vcr-cassettes/fixtures/",
"gsutil -m rm -r gs://ci-vcr-cassettes/refs/heads/auto-pr-123/",
"gsutil ls gs://ci-vcr-cassettes/beta/refs/heads/auto-pr-123/fixtures/",
"gsutil -m cp gs://ci-vcr-cassettes/beta/refs/heads/auto-pr-123/fixtures/* gs://ci-vcr-cassettes/beta/fixtures/",
"gsutil -m rm -r gs://ci-vcr-cassettes/beta/refs/heads/auto-pr-123/",
"gcloud storage ls gs://ci-vcr-cassettes/refs/heads/auto-pr-123/fixtures/",
"gcloud storage cp gs://ci-vcr-cassettes/refs/heads/auto-pr-123/fixtures/* gs://ci-vcr-cassettes/fixtures/",
"gcloud storage rm --recursive gs://ci-vcr-cassettes/refs/heads/auto-pr-123/",
"gcloud storage ls gs://ci-vcr-cassettes/beta/refs/heads/auto-pr-123/fixtures/",
"gcloud storage cp gs://ci-vcr-cassettes/beta/refs/heads/auto-pr-123/fixtures/* gs://ci-vcr-cassettes/beta/fixtures/",
"gcloud storage rm --recursive gs://ci-vcr-cassettes/beta/refs/heads/auto-pr-123/",
},
},
{
name: "base branch is not main",
baseBranch: "test-branch",
commitSha: "sha",
calledMethods: []string{
"gsutil ls gs://ci-vcr-cassettes/refs/heads/auto-pr-123/fixtures/",
"gsutil -m cp gs://ci-vcr-cassettes/refs/heads/auto-pr-123/fixtures/* gs://ci-vcr-cassettes/refs/branches/test-branch/fixtures/",
"gsutil -m rm -r gs://ci-vcr-cassettes/refs/heads/auto-pr-123/",
"gsutil ls gs://ci-vcr-cassettes/beta/refs/heads/auto-pr-123/fixtures/",
"gsutil -m cp gs://ci-vcr-cassettes/beta/refs/heads/auto-pr-123/fixtures/* gs://ci-vcr-cassettes/beta/refs/branches/test-branch/fixtures/",
"gsutil -m rm -r gs://ci-vcr-cassettes/beta/refs/heads/auto-pr-123/",
"gcloud storage ls gs://ci-vcr-cassettes/refs/heads/auto-pr-123/fixtures/",
"gcloud storage cp gs://ci-vcr-cassettes/refs/heads/auto-pr-123/fixtures/* gs://ci-vcr-cassettes/refs/branches/test-branch/fixtures/",
"gcloud storage rm --recursive gs://ci-vcr-cassettes/refs/heads/auto-pr-123/",
"gcloud storage ls gs://ci-vcr-cassettes/beta/refs/heads/auto-pr-123/fixtures/",
"gcloud storage cp gs://ci-vcr-cassettes/beta/refs/heads/auto-pr-123/fixtures/* gs://ci-vcr-cassettes/beta/refs/branches/test-branch/fixtures/",
"gcloud storage rm --recursive gs://ci-vcr-cassettes/beta/refs/heads/auto-pr-123/",
},
},
{
Expand All @@ -54,8 +54,8 @@ func TestExecVCRMerge(t *testing.T) {
lsReturnedError: true,
baseBranch: "main",
calledMethods: []string{
"gsutil ls gs://ci-vcr-cassettes/refs/heads/auto-pr-123/fixtures/",
"gsutil ls gs://ci-vcr-cassettes/beta/refs/heads/auto-pr-123/fixtures/",
"gcloud storage ls gs://ci-vcr-cassettes/refs/heads/auto-pr-123/fixtures/",
"gcloud storage ls gs://ci-vcr-cassettes/beta/refs/heads/auto-pr-123/fixtures/",
},
},
}
Expand Down
2 changes: 1 addition & 1 deletion mmv1/third_party/terraform/.teamcity/components/builds/build_steps.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ fun BuildSteps.saveArtifactsToGCS() {
fi

# Copy logs to GCS
gsutil -m cp %teamcity.build.checkoutDir%/debug* gs://teamcity-logs/${'$'}{FOLDER}/%env.BUILD_NUMBER%/
gcloud storage cp %teamcity.build.checkoutDir%/debug* gs://teamcity-logs/${'$'}{FOLDER}/%env.BUILD_NUMBER%/

# Cleanup
rm google-account.json
Expand Down
2 changes: 1 addition & 1 deletion mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2202,7 +2202,7 @@ resource "google_storage_bucket_object" "init_script" {
content = <<EOL
#!/bin/bash
echo "init action success" >> /tmp/%s
gsutil cp /tmp/%s ${google_storage_bucket.init_bucket.url}
gcloud storage cp /tmp/%s ${google_storage_bucket.init_bucket.url}
EOL

}
Expand Down
4 changes: 2 additions & 2 deletions ...d_party/terraform/services/storage/data_source_storage_object_signed_url_internal_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,9 @@ const fakeCredentials = `{
}
`

// The following values are derived from the output of the `gsutil signurl` command.
// The following values are derived from the output of the `gcloud storage sign-url` command.
// i.e.
// gsutil signurl fake_creds.json gs://tf-test-bucket-6159205297736845881/path/to/file
// gcloud storage sign-url --private-key-file=fake_creds.json gs://tf-test-bucket-6159205297736845881/path/to/file
// URL HTTP Method Expiration Signed URL
// gs://tf-test-bucket-6159205297736845881/path/to/file GET 2016-08-12 14:03:30 https://storage.googleapis.com/tf-test-bucket-6159205297736845881/path/to/[email protected]&Expires=1470967410&Signature=JJvE2Jc%2BeoagyS1qRACKBGUkgLkKjw7cGymHhtB4IzzN3nbXDqr0acRWGy0%2BEpZ3HYNDalEYsK0lR9Q0WCgty5I0JKmPIuo9hOYa1xTNH%2B22xiWsekxGV%2FcA9FXgWpi%2BFt7fBmMk4dhDe%2BuuYc7N79hd0FYuSBNW1Wp32Bluoe4SNkNAB%2BuIDd9KqPzqs09UAbBoz2y4WxXOQnRyR8GAfb8B%2FDtv62gYjtmp%2F6%2Fyr6xj7byWKZdQt8kEftQLTQmP%2F17Efjp6p%2BXo71Q0F9IhAFiqWfp3Ij8hHDSebLcVb2ULXyHNNQpHBOhFgALrFW3I6Uc3WciLEOsBS9Ej3EGdTg%3D%3D

Expand Down
Loading