Skip to content

Commit c207d8f

Browse files
drichmdbdrichmdb
committed
testing
This commit changes dbtools to dual write all of its uploads to a new bucket, cdn-origin-db-tools. In the future we will remove puts to the old bucket and only serve artifacts from the new bucket using our CDN.
1 parent 4558399 commit c207d8f

File tree

3 files changed

+114
-18
lines changed

3 files changed

+114
-18
lines changed

‎common.yml

+21
Original file line numberDiff line numberDiff line change
@@ -343,17 +343,31 @@ functions:
343343
content_type: application/octet-stream
344344

345345
"upload release packages to s3":
346+
- command: ec2.assume_role
347+
params:
348+
role_arn: "arn:aws:iam::119629040606:role/s3-access.cdn-origin-db-tools"
346349
- command: shell.exec
347350
params:
348351
working_dir: src/github.com/mongodb/mongo-tools
352+
env:
353+
NEW_AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
354+
NEW_AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
355+
NEW_AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN}
349356
script: |
350357
${_set_shell_env}
351358
go run release/release.go upload-release
352359
353360
"upload release json feed to s3":
361+
- command: ec2.assume_role
362+
params:
363+
role_arn: "arn:aws:iam::119629040606:role/s3-access.cdn-origin-db-tools"
354364
- command: shell.exec
355365
params:
356366
working_dir: src/github.com/mongodb/mongo-tools
367+
env:
368+
NEW_AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
369+
NEW_AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
370+
NEW_AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN}
357371
script: |
358372
${_set_shell_env}
359373
go run release/release.go upload-json
@@ -369,9 +383,16 @@ functions:
369383
permissions: public-read
370384

371385
"generate full JSON feed":
386+
- command: ec2.assume_role
387+
params:
388+
role_arn: "arn:aws:iam::119629040606:role/s3-access.cdn-origin-db-tools"
372389
- command: shell.exec
373390
params:
374391
working_dir: src/github.com/mongodb/mongo-tools
392+
env:
393+
NEW_AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
394+
NEW_AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
395+
NEW_AWS_SESSION_TOKEN: ${AWS_SESSION_TOKEN}
375396
script: |
376397
${_set_shell_env}
377398
go run release/release.go generate-full-json

‎release/aws/aws.go

+21-1
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
"regexp"
1515

1616
"github.com/aws/aws-sdk-go/aws"
17+
"github.com/aws/aws-sdk-go/aws/credentials"
1718
"github.com/aws/aws-sdk-go/aws/session"
1819
"github.com/aws/aws-sdk-go/service/s3"
1920
"github.com/aws/aws-sdk-go/service/s3/s3manager"
@@ -24,6 +25,7 @@ var awsClient *AWS
2425

2526
type AWS struct {
2627
session *session.Session
28+
acl string
2729
}
2830

2931
func initializeClient() error {
@@ -40,10 +42,28 @@ func initializeClient() error {
4042

4143
awsClient = &AWS{
4244
session: s,
45+
acl: "public-read",
4346
}
4447
return nil
4548
}
4649

50+
func NewClientFromCredentials(id, secret, token, acl string) (*AWS, error) {
51+
s, err := session.NewSession(&aws.Config{
52+
Region: aws.String("us-east-1"),
53+
Credentials: credentials.NewStaticCredentials(id, secret, token),
54+
})
55+
if err != nil {
56+
return nil, fmt.Errorf("failed to create AWS session: %w", err)
57+
}
58+
59+
c := &AWS{
60+
session: s,
61+
acl: acl,
62+
}
63+
64+
return c, nil
65+
}
66+
4767
// GetClient returns the global AWS client.
4868
// It initializes the AWS client if it hasn't already been initialized.
4969
func GetClient() (*AWS, error) {
@@ -75,7 +95,7 @@ func (a *AWS) UploadBytes(bucket, objPath, filename string, reader io.Reader) er
7595
_, err := uploader.Upload(&s3manager.UploadInput{
7696
Bucket: aws.String(bucket),
7797
Key: aws.String(key),
78-
ACL: aws.String("public-read"),
98+
ACL: aws.String(a.acl),
7999
Body: reader,
80100
})
81101
if err != nil {

‎release/release.go

+72-17
Original file line numberDiff line numberDiff line change
@@ -953,13 +953,20 @@ func generateFullReleaseJSON(v version.Version) {
953953
return
954954
}
955955

956-
awsClient, err := aws.GetClient()
957-
check(err, "get aws client")
956+
awsClientOld, err := aws.GetClient()
957+
check(err, "get aws client old")
958958

959-
feed, err := awsClient.GenerateFullReleaseFeedFromObjects()
959+
newAccessKeyID := os.Getenv("NEW_AWS_ACCESS_KEY_ID")
960+
newSecretAccessKey := os.Getenv("NEW_AWS_SECRET_ACCESS_KEY")
961+
newSessionToken := os.Getenv("NEW_AWS_SESSION_TOKEN")
962+
963+
awsClientNew, err := aws.NewClientFromCredentials(newAccessKeyID, newSecretAccessKey, newSessionToken, "private")
964+
check(err, "new aws client from credentials")
965+
966+
feed, err := awsClientOld.GenerateFullReleaseFeedFromObjects()
960967
check(err, "generate full release feed from s3 objects")
961968

962-
uploadFeedFile("full.json", feed, awsClient)
969+
uploadFeedFile("full.json", feed, awsClientOld, awsClientNew)
963970
}
964971

965972
func uploadReleaseJSON(v version.Version) {
@@ -996,8 +1003,15 @@ func uploadReleaseJSON(v version.Version) {
9961003
log.Fatalf("found %d sign tasks, but expected %d", len(signTasks), pfCount)
9971004
}
9981005

999-
awsClient, err := aws.GetClient()
1000-
check(err, "get aws client")
1006+
awsClientOld, err := aws.GetClient()
1007+
check(err, "get aws client old")
1008+
1009+
newAccessKeyID := os.Getenv("NEW_AWS_ACCESS_KEY_ID")
1010+
newSecretAccessKey := os.Getenv("NEW_AWS_SECRET_ACCESS_KEY")
1011+
newSessionToken := os.Getenv("NEW_AWS_SESSION_TOKEN")
1012+
1013+
awsClientNew, err := aws.NewClientFromCredentials(newAccessKeyID, newSecretAccessKey, newSessionToken, "private")
1014+
check(err, "new aws client from credentials")
10011015

10021016
// Accumulate all downloaded artifacts from sign tasks for JSON feed.
10031017
var dls []*download.ToolsDownload
@@ -1061,8 +1075,18 @@ func uploadReleaseJSON(v version.Version) {
10611075
}
10621076

10631077
// Download the current full.json
1064-
buff, err := awsClient.DownloadFile("downloads.mongodb.org", "tools/db/full.json")
1065-
check(err, "download full.json")
1078+
const addr = "https://downloads.mongodb.org/tools/db/full.json"
1079+
res, err := http.Get(addr)
1080+
check(err, "http get full.json")
1081+
1082+
defer res.Body.Close()
1083+
1084+
buff, err := io.ReadAll(res.Body)
1085+
check(err, "read full.json body")
1086+
1087+
if res.StatusCode != http.StatusOK {
1088+
panic(fmt.Errorf("get full.json status %d: %s", res.StatusCode, string(buff)))
1089+
}
10661090

10671091
var fullFeed download.JSONFeed
10681092

@@ -1074,7 +1098,7 @@ func uploadReleaseJSON(v version.Version) {
10741098
fullFeed.Versions,
10751099
&download.ToolsVersion{Version: v.String(), Downloads: dls},
10761100
)
1077-
uploadFeedFile("full.json", &fullFeed, awsClient)
1101+
uploadFeedFile("full.json", &fullFeed, awsClientOld, awsClientNew)
10781102

10791103
// Upload only the most recent version to release.json
10801104
var feed download.JSONFeed
@@ -1083,23 +1107,32 @@ func uploadReleaseJSON(v version.Version) {
10831107
&download.ToolsVersion{Version: v.String(), Downloads: dls},
10841108
)
10851109

1086-
uploadFeedFile("release.json", &feed, awsClient)
1110+
uploadFeedFile("release.json", &feed, awsClientOld, awsClientNew)
10871111
}
10881112

1089-
func uploadFeedFile(filename string, feed *download.JSONFeed, awsClient *aws.AWS) {
1113+
func uploadFeedFile(filename string, feed *download.JSONFeed, awsClientOld, awsClientNew *aws.AWS) {
10901114
var feedBuffer bytes.Buffer
10911115

10921116
jsonEncoder := json.NewEncoder(&feedBuffer)
10931117
jsonEncoder.SetIndent("", " ")
10941118
err := jsonEncoder.Encode(*feed)
10951119
check(err, "encode json feed")
10961120

1121+
feedBytes := feedBuffer.Bytes()
1122+
10971123
log.Printf(
10981124
"uploading download feed to https://s3.amazonaws.com/downloads.mongodb.org/tools/db/%s\n",
10991125
filename,
11001126
)
1101-
err = awsClient.UploadBytes("downloads.mongodb.org", "/tools/db", filename, &feedBuffer)
1102-
check(err, "upload json feed")
1127+
err = awsClientOld.UploadBytes("downloads.mongodb.org", "/tools/db", filename, bytes.NewReader(feedBytes))
1128+
check(err, "upload json feed old")
1129+
1130+
log.Printf(
1131+
"uploading download feed to s3://cdn-origin-db-tools/tools/db/%s\n",
1132+
filename,
1133+
)
1134+
err = awsClientNew.UploadBytes("cdn-origin-db-tools", "/tools/db", filename, bytes.NewReader(feedBytes))
1135+
check(err, "upload json feed new")
11031136
}
11041137

11051138
func uploadRelease(v version.Version) {
@@ -1129,8 +1162,15 @@ func uploadRelease(v version.Version) {
11291162
log.Fatalf("found %d sign tasks, but expected one", len(signTasks))
11301163
}
11311164

1132-
awsClient, err := aws.GetClient()
1133-
check(err, "get aws client")
1165+
awsClientOld, err := aws.GetClient()
1166+
check(err, "get old aws client")
1167+
1168+
newAccessKeyID := os.Getenv("NEW_AWS_ACCESS_KEY_ID")
1169+
newSecretAccessKey := os.Getenv("NEW_AWS_SECRET_ACCESS_KEY")
1170+
newSessionToken := os.Getenv("NEW_AWS_SESSION_TOKEN")
1171+
1172+
awsClientNew, err := aws.NewClientFromCredentials(newAccessKeyID, newSecretAccessKey, newSessionToken, "private")
1173+
check(err, "new aws client from credentials")
11341174

11351175
for _, task := range signTasks {
11361176
log.Printf("\ngetting artifacts for %s\n", task.Variant)
@@ -1182,14 +1222,29 @@ func uploadRelease(v version.Version) {
11821222
" uploading to https://s3.amazonaws.com/downloads.mongodb.org/tools/db/%s\n",
11831223
stableFile,
11841224
)
1185-
err = awsClient.UploadFile("downloads.mongodb.org", "/tools/db", stableFile)
1225+
err = awsClientOld.UploadFile("downloads.mongodb.org", "/tools/db", stableFile)
11861226
check(err, "uploading %q file to S3", stableFile)
11871227
log.Printf(
11881228
" uploading to https://s3.amazonaws.com/downloads.mongodb.org/tools/db/%s\n",
11891229
latestStableFile,
11901230
)
1191-
err = awsClient.UploadFile("downloads.mongodb.org", "/tools/db", latestStableFile)
1231+
1232+
err = awsClientOld.UploadFile("downloads.mongodb.org", "/tools/db", latestStableFile)
11921233
check(err, "uploading %q file to S3", latestStableFile)
1234+
1235+
log.Printf(
1236+
" uploading to s3://cdn-origin-db-tools/tools/db/%s\n",
1237+
stableFile,
1238+
)
1239+
err = awsClientNew.UploadFile("cdn-origin-db-tools", "/tools/db", stableFile)
1240+
check(err, "uploading %q file to S3 new", stableFile)
1241+
log.Printf(
1242+
" uploading to s3://cdn-origin-db-tools/tools/db/%s\n",
1243+
latestStableFile,
1244+
)
1245+
1246+
err = awsClientNew.UploadFile("cdn-origin-db-tools", "/tools/db", latestStableFile)
1247+
check(err, "uploading %q file to S3 new", latestStableFile)
11931248
}
11941249
}
11951250
}

0 commit comments

Comments
 (0)