Skip to content

Commit a9f7261

Browse files
djosephsendjosephsen
authored
feat: s3 support for the blob sync (#1449)
## This PR Intent of this pr is to add S3 bucket support to the existing "blob" sync. ### Related Issues fixes #1376 ### Notes Marking as a draft for now until I can find my aws creds and live-test it. ### Follow-up Tasks integration testing is yet to be performed ### How to test unit tests provided Signed-off-by: Dave Josephsen <[email protected]>
1 parent 431fbb4 commit a9f7261

File tree

10 files changed

+138
-86
lines changed

10 files changed

+138
-86
lines changed

‎core/go.mod

+21
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,26 @@ require (
5656
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
5757
github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
5858
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
59+
github.com/aws/aws-sdk-go v1.55.5 // indirect
60+
github.com/aws/aws-sdk-go-v2 v1.30.3 // indirect
61+
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 // indirect
62+
github.com/aws/aws-sdk-go-v2/config v1.27.27 // indirect
63+
github.com/aws/aws-sdk-go-v2/credentials v1.17.27 // indirect
64+
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 // indirect
65+
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.10 // indirect
66+
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 // indirect
67+
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 // indirect
68+
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
69+
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.15 // indirect
70+
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect
71+
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.17 // indirect
72+
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 // indirect
73+
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15 // indirect
74+
github.com/aws/aws-sdk-go-v2/service/s3 v1.58.3 // indirect
75+
github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 // indirect
76+
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 // indirect
77+
github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 // indirect
78+
github.com/aws/smithy-go v1.20.3 // indirect
5979
github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df // indirect
6080
github.com/beorn7/perks v1.0.1 // indirect
6181
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
@@ -84,6 +104,7 @@ require (
84104
github.com/googleapis/gax-go/v2 v2.13.0 // indirect
85105
github.com/grpc-ecosystem/grpc-gateway/v2 v2.23.0 // indirect
86106
github.com/imdario/mergo v0.3.16 // indirect
107+
github.com/jmespath/go-jmespath v0.4.0 // indirect
87108
github.com/josharian/intern v1.0.0 // indirect
88109
github.com/json-iterator/go v1.1.12 // indirect
89110
github.com/klauspost/compress v1.17.9 // indirect

‎core/go.sum

+2-80
Large diffs are not rendered by default.

‎core/pkg/sync/blob/blob_sync.go

+1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"gocloud.dev/blob"
1313
_ "gocloud.dev/blob/azureblob" // needed to initialize Azure Blob Storage driver
1414
_ "gocloud.dev/blob/gcsblob" // needed to initialize GCS driver
15+
_ "gocloud.dev/blob/s3blob" // needed to initialize s3 driver
1516
)
1617

1718
type Sync struct {

‎core/pkg/sync/builder/syncbuilder.go

+36-2
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ const (
3232
syncProviderHTTP = "http"
3333
syncProviderGcs = "gcs"
3434
syncProviderAzblob = "azblob"
35+
syncProviderS3 = "s3"
3536
)
3637

3738
var (
@@ -43,6 +44,7 @@ var (
4344
regFile *regexp.Regexp
4445
regGcs *regexp.Regexp
4546
regAzblob *regexp.Regexp
47+
regS3 *regexp.Regexp
4648
)
4749

4850
func init() {
@@ -54,6 +56,7 @@ func init() {
5456
regFile = regexp.MustCompile("^file:")
5557
regGcs = regexp.MustCompile("^gs://.+?/")
5658
regAzblob = regexp.MustCompile("^azblob://.+?/")
59+
regS3 = regexp.MustCompile("^s3://.+?/")
5760
}
5861

5962
type ISyncBuilder interface {
@@ -119,12 +122,15 @@ func (sb *SyncBuilder) syncFromConfig(sourceConfig sync.SourceConfig, logger *lo
119122
case syncProviderAzblob:
120123
logger.Debug(fmt.Sprintf("using blob sync-provider with azblob driver for: %s", sourceConfig.URI))
121124
return sb.newAzblob(sourceConfig, logger)
125+
case syncProviderS3:
126+
logger.Debug(fmt.Sprintf("using blob sync-provider with s3 driver for: %s", sourceConfig.URI))
127+
return sb.newS3(sourceConfig, logger), nil
122128

123129
default:
124130
return nil, fmt.Errorf("invalid sync provider: %s, must be one of with "+
125-
"'%s', '%s', '%s', '%s', '%s', '%s', '%s' or '%s'",
131+
"'%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s' or '%s'",
126132
sourceConfig.Provider, syncProviderFile, syncProviderFsNotify, syncProviderFileInfo,
127-
syncProviderKubernetes, syncProviderHTTP, syncProviderGrpc, syncProviderGcs, syncProviderAzblob)
133+
syncProviderKubernetes, syncProviderHTTP, syncProviderGrpc, syncProviderGcs, syncProviderAzblob, syncProviderS3)
128134
}
129135
}
130136

@@ -284,6 +290,34 @@ func (sb *SyncBuilder) newAzblob(config sync.SourceConfig, logger *logger.Logger
284290
}, nil
285291
}
286292

293+
func (sb *SyncBuilder) newS3(config sync.SourceConfig, logger *logger.Logger) *blobSync.Sync {
294+
// Extract bucket uri and object name from the full URI:
295+
// gs://bucket/path/to/object results in gs://bucket/ as bucketUri and
296+
// path/to/object as an object name.
297+
bucketURI := regS3.FindString(config.URI)
298+
objectName := regS3.ReplaceAllString(config.URI, "")
299+
300+
// Defaults to 5 seconds if interval is not set.
301+
var interval uint32 = 5
302+
if config.Interval != 0 {
303+
interval = config.Interval
304+
}
305+
306+
return &blobSync.Sync{
307+
Bucket: bucketURI,
308+
Object: objectName,
309+
310+
BlobURLMux: blob.DefaultURLMux(),
311+
312+
Logger: logger.WithFields(
313+
zap.String("component", "sync"),
314+
zap.String("sync", "s3"),
315+
),
316+
Interval: interval,
317+
Cron: cron.New(),
318+
}
319+
}
320+
287321
type IK8sClientBuilder interface {
288322
GetK8sClient() (dynamic.Interface, error)
289323
}

‎core/pkg/sync/builder/syncbuilder_test.go

+59
Original file line numberDiff line numberDiff line change
@@ -242,6 +242,10 @@ func Test_SyncsFromFromConfig(t *testing.T) {
242242
URI: "azblob://bucket/path/to/file",
243243
Provider: syncProviderAzblob,
244244
},
245+
{
246+
URI: "s3://bucket/path/to/file",
247+
Provider: syncProviderS3,
248+
},
245249
},
246250
},
247251
wantSyncs: []sync.ISync{
@@ -252,6 +256,7 @@ func Test_SyncsFromFromConfig(t *testing.T) {
252256
&kubernetes.Sync{},
253257
&blob.Sync{},
254258
&blob.Sync{},
259+
&blob.Sync{},
255260
},
256261
wantErr: false,
257262
},
@@ -418,3 +423,57 @@ func Test_AzblobConfig(t *testing.T) {
418423
})
419424
}
420425
}
426+
427+
func Test_S3Config(t *testing.T) {
428+
lg := logger.NewLogger(nil, false)
429+
defaultInterval := uint32(5)
430+
tests := []struct {
431+
name string
432+
uri string
433+
interval uint32
434+
expectedBucket string
435+
expectedObject string
436+
expectedInterval uint32
437+
}{
438+
{
439+
name: "simple path",
440+
uri: "s3://bucket/path/to/object",
441+
interval: 10,
442+
expectedBucket: "s3://bucket/",
443+
expectedObject: "path/to/object",
444+
expectedInterval: 10,
445+
},
446+
{
447+
name: "default interval",
448+
uri: "s3://bucket/path/to/object",
449+
expectedBucket: "s3://bucket/",
450+
expectedObject: "path/to/object",
451+
expectedInterval: defaultInterval,
452+
},
453+
{
454+
name: "no object set", // Blob syncer will return error when fetching
455+
uri: "s3://bucket/",
456+
expectedBucket: "s3://bucket/",
457+
expectedObject: "",
458+
expectedInterval: defaultInterval,
459+
},
460+
{
461+
name: "malformed uri", // Blob syncer will return error when opening bucket
462+
uri: "malformed",
463+
expectedBucket: "",
464+
expectedObject: "malformed",
465+
expectedInterval: defaultInterval,
466+
},
467+
}
468+
for _, tt := range tests {
469+
t.Run(tt.name, func(t *testing.T) {
470+
s3Sync := NewSyncBuilder().newS3(sync.SourceConfig{
471+
URI: tt.uri,
472+
Interval: tt.interval,
473+
}, lg)
474+
require.Equal(t, tt.expectedBucket, s3Sync.Bucket)
475+
require.Equal(t, tt.expectedObject, s3Sync.Object)
476+
require.Equal(t, int(tt.expectedInterval), int(s3Sync.Interval))
477+
})
478+
}
479+
}

‎core/pkg/sync/builder/utils.go

+5
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,11 @@ func ParseSyncProviderURIs(uris []string) ([]sync.SourceConfig, error) {
7979
URI: uri,
8080
Provider: syncProviderAzblob,
8181
})
82+
case regS3.Match(uriB):
83+
syncProvidersParsed = append(syncProvidersParsed, sync.SourceConfig{
84+
URI: uri,
85+
Provider: syncProviderS3,
86+
})
8287
default:
8388
return syncProvidersParsed, fmt.Errorf("invalid sync uri argument: %s, must start with 'file:', "+
8489
"'http(s)://', 'grpc(s)://', 'gs://', 'azblob://' or 'core.openfeature.dev'", uri)

‎core/pkg/sync/builder/utils_test.go

+11-1
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,8 @@ func TestParseSource(t *testing.T) {
3030
{"uri":"host:port","provider":"grpc"},
3131
{"uri":"default/my-crd","provider":"kubernetes"},
3232
{"uri":"gs://bucket-name/path/to/file","provider":"gcs"},
33-
{"uri":"azblob://bucket-name/path/to/file","provider":"azblob"}
33+
{"uri":"azblob://bucket-name/path/to/file","provider":"azblob"},
34+
{"uri":"s3://bucket-name/path/to/file","provider":"s3"}
3435
]`,
3536
expectErr: false,
3637
out: []sync.SourceConfig{
@@ -59,6 +60,10 @@ func TestParseSource(t *testing.T) {
5960
URI: "azblob://bucket-name/path/to/file",
6061
Provider: syncProviderAzblob,
6162
},
63+
{
64+
URI: "s3://bucket-name/path/to/file",
65+
Provider: syncProviderS3,
66+
},
6267
},
6368
},
6469
"multiple-syncs-with-options": {
@@ -194,6 +199,7 @@ func TestParseSyncProviderURIs(t *testing.T) {
194199
"core.openfeature.dev/default/my-crd",
195200
"gs://bucket-name/path/to/file",
196201
"azblob://bucket-name/path/to/file",
202+
"s3://bucket-name/path/to/file",
197203
},
198204
expectErr: false,
199205
out: []sync.SourceConfig{
@@ -227,6 +233,10 @@ func TestParseSyncProviderURIs(t *testing.T) {
227233
URI: "azblob://bucket-name/path/to/file",
228234
Provider: syncProviderAzblob,
229235
},
236+
{
237+
URI: "s3://bucket-name/path/to/file",
238+
Provider: syncProviderS3,
239+
},
230240
},
231241
},
232242
"empty": {

‎schemas

‎spec

Submodule spec updated 31 files

‎test-harness

0 commit comments

Comments
 (0)