From 6c5ab38da1ac2f9056387da63362244a2fe0ec03 Mon Sep 17 00:00:00 2001 From: Josh Scott Date: Fri, 23 Jul 2021 12:42:38 -0500 Subject: [PATCH 1/2] Bugfix: get all s3 objects --- Makefile | 7 +------ go.mod | 4 +++- go.sum | 18 ++++++++++++++---- pkg/looking-glass/downloader/s3.go | 18 ++++++++++-------- 4 files changed, 28 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index ca6231c..989e7f0 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ TAG?="" # Run all tests .PHONY: test -test: fmt lint vet test-unit go-mod-tidy +test: fmt vet test-unit go-mod-tidy # Run unit tests .PHONY: test-unit @@ -22,11 +22,6 @@ go-mod-tidy: fmt: test -z "$(shell gofmt -l .)" -# Run linter -.PHONY: lint -lint: - golint -set_exit_status ./... - # Run vet .PHONY: vet vet: diff --git a/go.mod b/go.mod index 1eccfaf..4e63412 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,8 @@ require ( github.com/spf13/cobra v0.0.4 github.com/spf13/viper v1.4.0 github.com/stretchr/testify v1.3.0 - golang.org/x/net v0.0.0-20190522155817-f3200d17e092 + golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 // indirect + golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be + golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect ) diff --git a/go.sum b/go.sum index ff1974f..7b9e8a9 100644 --- a/go.sum +++ b/go.sum @@ -180,8 +180,9 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181001203147-e3636079e1a4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -189,8 +190,10 @@ golang.org/x/net v0.0.0-20180926154720-4dfa2610cdf3/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -203,12 +206,19 @@ golang.org/x/sys v0.0.0-20180906133057-8cf3aee42992/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs= diff --git a/pkg/looking-glass/downloader/s3.go b/pkg/looking-glass/downloader/s3.go index ce1ad61..5575690 100644 --- a/pkg/looking-glass/downloader/s3.go +++ b/pkg/looking-glass/downloader/s3.go @@ -92,21 +92,23 @@ func createAwsSession(awsKey string, awsSecret string, awsRegion string) (*sessi // ListObjects lists the objects available in the S3 bucket func (s3s *s3) ListObjects() ([]string, error) { var objects []string + path := &sss.ListObjectsV2Input{ + Bucket: aws.String(s3s.awsBucket), + Prefix: aws.String(s3s.awsPrefix), + } - resp, err := sss.New(&s3s.awsSession). - ListObjects(&sss.ListObjectsInput{ - Bucket: aws.String(s3s.awsBucket), - Prefix: aws.String(s3s.awsPrefix), + err := sss.New(&s3s.awsSession). + ListObjectsV2Pages(path, func(page *sss.ListObjectsV2Output, lastPage bool) bool { + for _, obj := range page.Contents { + objects = append(objects, *obj.Key) + } + return lastPage }) if err != nil { return nil, err } - for _, obj := range resp.Contents { - objects = append(objects, *obj.Key) - } - return objects, nil } From 2ab559975c8148eca0d2ee4ef74e888ccdf32236 Mon Sep 17 00:00:00 2001 From: Josh Scott Date: Fri, 23 Jul 2021 13:31:35 -0500 Subject: [PATCH 2/2] Fix error in logic We should return !lastPage because returning true continues iterating. --- pkg/looking-glass/downloader/github.go | 5 +++++ pkg/looking-glass/downloader/s3.go | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/looking-glass/downloader/github.go b/pkg/looking-glass/downloader/github.go index d514019..17737d5 100644 --- a/pkg/looking-glass/downloader/github.go +++ b/pkg/looking-glass/downloader/github.go @@ -186,6 +186,11 @@ func (ghd *githubDownloader) GetObject(sourceObj string, targetPath string) erro // Download the asset rc, _, err := ghd.client.Repositories.DownloadReleaseAsset(ctx, ghd.repoOwner, ghd.repoName, assetID, http.DefaultClient) + + if err != nil { + return err + } + _, err = io.Copy(f, rc) if err != nil { diff --git a/pkg/looking-glass/downloader/s3.go b/pkg/looking-glass/downloader/s3.go index 5575690..e3f588c 100644 --- a/pkg/looking-glass/downloader/s3.go +++ b/pkg/looking-glass/downloader/s3.go @@ -102,7 +102,7 @@ func (s3s *s3) ListObjects() ([]string, error) { for _, obj := range page.Contents { objects = append(objects, *obj.Key) } - return lastPage + return !lastPage }) if err != nil {