From 9e5778c77f45fbf5d7524ba802727874bbd0859b Mon Sep 17 00:00:00 2001 From: andre-urbani Date: Thu, 27 Nov 2025 10:25:23 +0000 Subject: [PATCH 01/10] add cloudflare cache purge --- api/versions.go | 18 ++++++++++ config/config.go | 9 ++++- config/config_test.go | 2 ++ go.mod | 6 ++-- go.sum | 4 +-- utils/utils.go | 66 ++++++++++++++++++++++++++++++++++ utils/utils_test.go | 84 +++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 184 insertions(+), 5 deletions(-) diff --git a/api/versions.go b/api/versions.go index 5d28e354..155566dd 100644 --- a/api/versions.go +++ b/api/versions.go @@ -8,9 +8,11 @@ import ( "net/http" "strconv" "strings" + "time" "github.com/ONSdigital/dp-api-clients-go/v2/headers" errs "github.com/ONSdigital/dp-dataset-api/apierrors" + "github.com/ONSdigital/dp-dataset-api/config" "github.com/ONSdigital/dp-dataset-api/models" "github.com/ONSdigital/dp-dataset-api/utils" dpresponse "github.com/ONSdigital/dp-net/v3/handlers/response" @@ -789,6 +791,22 @@ func (api *DatasetAPI) putState(w http.ResponseWriter, r *http.Request) { handleVersionAPIErr(ctx, err, w, logData) return } + cfg, err := config.Get() + if err != nil { + log.Error(ctx, "failed to get config for cache purge", err, logData) + } else { + httpClient := &http.Client{Timeout: 10 * time.Second} + baseURL := cfg.CloudflareAPIURL + if baseURL == "" { + baseURL = "https://api.cloudflare.com/client/v4" + } + err = utils.PurgeCache(ctx, datasetID, edition, baseURL, cfg.CloudflareZoneID, cfg.CloudflareAPIToken, httpClient) + if err != nil { + log.Error(ctx, "failed to purge cloudflare cache", err, logData) + } else { + log.Info(ctx, "successfully triggered cloudflare cache purge", logData) + } + } } setJSONContentType(w) diff --git a/config/config.go b/config/config.go index 9eeac249..1cd0e31e 100644 --- a/config/config.go +++ b/config/config.go @@ -59,6 +59,9 @@ type Configuration struct { OTServiceName string `envconfig:"OTEL_SERVICE_NAME"` OTBatchTimeout time.Duration `envconfig:"OTEL_BATCH_TIMEOUT"` OtelEnabled bool `envconfig:"OTEL_ENABLED"` + CloudflareZoneID string `envconfig:"CLOUDFLARE_ZONE_ID"` + CloudflareAPIToken string `envconfig:"CLOUDFLARE_API_TOKEN" json:"-"` + CloudflareAPIURL string `envconfig:"CLOUDFLARE_API_URL"` MongoConfig } @@ -114,7 +117,11 @@ func Get() (*Configuration, error) { DefaultMaxLimit: 1000, DefaultLimit: 20, DefaultOffset: 0, - MaxRequestOptions: 100, // Maximum number of options acceptable in an incoming Patch request. Compromise between one option per call (inefficient) and an order of 100k options per call, for census data (memory and computationally expensive) + MaxRequestOptions: 100, // Maximum number of options acceptable in an incoming Patch request. Compromise between one option per call (inefficient) and an order of 100k options per call, for census data (memory and computationally expensive) + CloudflareZoneID: "9f1eec58caedd8e902395a065c120073", // this is not a real zone id, purely for local testing purposes as any 32 character length alphanumeric string will work + CloudflareAPIToken: "test-token", + CloudflareAPIURL: "http://cloud-flare-stub:22500", + MongoConfig: MongoConfig{ MongoDriverConfig: mongodriver.MongoDriverConfig{ ClusterEndpoint: "localhost:27017", diff --git a/config/config_test.go b/config/config_test.go index f7f02e78..40afab50 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -61,6 +61,8 @@ func TestSpec(t *testing.T) { So(cfg.IsWriteConcernMajorityEnabled, ShouldEqual, true) So(cfg.DatasetAPIURL, ShouldEqual, "http://localhost:22000") So(cfg.CodeListAPIURL, ShouldEqual, "http://localhost:22400") + So(cfg.CloudflareAPIToken, ShouldEqual, "test-token") + So(cfg.CloudflareZoneID, ShouldEqual, "9f1eec58caedd8e902395a065c120073") }) }) }) diff --git a/go.mod b/go.mod index e9e2c13a..382e37b2 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/ONSdigital/dp-dataset-api -go 1.24 +go 1.24.0 + +toolchain go1.24.1 require ( github.com/ONSdigital/dp-api-clients-go v1.43.0 @@ -16,7 +18,7 @@ require ( github.com/ONSdigital/dp-mongodb/v3 v3.8.0 github.com/ONSdigital/dp-net/v3 v3.5.0 github.com/ONSdigital/dp-otel-go v0.0.8 - github.com/ONSdigital/log.go/v2 v2.5.0 + github.com/ONSdigital/log.go/v2 v2.5.1 github.com/cucumber/godog v0.15.1 github.com/golang/glog v1.2.5 github.com/google/go-cmp v0.7.0 diff --git a/go.sum b/go.sum index 82a09914..0f413693 100644 --- a/go.sum +++ b/go.sum @@ -65,8 +65,8 @@ github.com/ONSdigital/log.go v1.1.0/go.mod h1:0hOVuYR3bDUI30VRo48d5KHfJIoe+spuPX github.com/ONSdigital/log.go/v2 v2.0.0/go.mod h1:PR7vXrv9dZKUc7SI/0toxBbStk84snmybBnWpe+xY2o= github.com/ONSdigital/log.go/v2 v2.0.5/go.mod h1:PR7vXrv9dZKUc7SI/0toxBbStk84snmybBnWpe+xY2o= github.com/ONSdigital/log.go/v2 v2.0.9/go.mod h1:VyTDkL82FtiAkaNFaT+bURBhLbP7NsIx4rkVbdpiuEg= -github.com/ONSdigital/log.go/v2 v2.5.0 h1:gFHAn6tLOzkhC9hiAFgFxzNBh5Uz06KyULQ9aQyM9tE= -github.com/ONSdigital/log.go/v2 v2.5.0/go.mod h1:0ilpZzc5lVoBlXC/s5m8EaQETbe0yT8Z+p4QhKy0fpY= +github.com/ONSdigital/log.go/v2 v2.5.1 h1:GCM270UHSP5+mv4OaQ2oHiWp0FiSgfnM6imW2zpAslw= +github.com/ONSdigital/log.go/v2 v2.5.1/go.mod h1:KZNEweCUHD8dKwhlvoRvgd2Y2aUIuU3H9/MmbFyVzW8= github.com/Shopify/sarama v1.38.1 h1:lqqPUPQZ7zPqYlWpTh+LQ9bhYNu2xJL6k1SJN4WVe2A= github.com/Shopify/sarama v1.38.1/go.mod h1:iwv9a67Ha8VNa+TifujYoWGxWnu2kNVAQdSdZ4X2o5g= github.com/Shopify/toxiproxy/v2 v2.5.0 h1:i4LPT+qrSlKNtQf5QliVjdP08GyAH8+BUIc9gT0eahc= diff --git a/utils/utils.go b/utils/utils.go index 1b99010e..e9263566 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -1,7 +1,12 @@ package utils import ( + "bytes" "context" + "encoding/json" + "fmt" + "io" + "net/http" "net/url" "strconv" "strings" @@ -791,3 +796,64 @@ func RewriteDistributions(ctx context.Context, results *[]models.Distribution, d } return items, nil } + +type Prefixes struct { + Prefixes []string `json:"prefixes,omitempty"` +} + +type HTTPClient interface { + Do(req *http.Request) (*http.Response, error) +} + +func PurgeCache(ctx context.Context, datasetID, editionID, baseURL, zoneID, apiToken string, client HTTPClient) error { + url := fmt.Sprintf("%s/zones/%s/purge_cache", baseURL, zoneID) + webDatasetURL := fmt.Sprintf("www.ons.gov.uk/datasets/%s", datasetID) + webEditionsURL := fmt.Sprintf("www.ons.gov.uk/datasets/%s/editions", datasetID) + webVersionsURL := fmt.Sprintf("www.ons.gov.uk/datasets/%s/editions/%s/versions", datasetID, editionID) + + apiDatasetURL := fmt.Sprintf("api.beta.ons.gov.uk/v1/datasets/%s", datasetID) + apiEditionsURL := fmt.Sprintf("api.beta.ons.gov.uk/v1/datasets/%s/editions", datasetID) + apiVersionsURL := fmt.Sprintf("api.beta.ons.gov.uk/v1/datasets/%s/editions/%s/versions", datasetID, editionID) + + prefixes := Prefixes{ + Prefixes: []string{ + webDatasetURL, + webEditionsURL, + webVersionsURL, + apiDatasetURL, + apiEditionsURL, + apiVersionsURL, + }, + } + + payload, err := json.Marshal(prefixes) + if err != nil { + return fmt.Errorf("failed to marshal purge prefixes: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(payload)) + if err != nil { + return fmt.Errorf("failed to create purge request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", apiToken)) + + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("failed to send purge request to cloudflare: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("cloudflare API returned status %d: %s", resp.StatusCode, string(body)) + } + + log.Info(ctx, "successfully purged cloudflare cache", log.Data{ + "dataset_id": datasetID, + "edition": editionID, + }) + + return nil +} diff --git a/utils/utils_test.go b/utils/utils_test.go index 782f5448..bb9c1388 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -2,7 +2,10 @@ package utils import ( "context" + "encoding/json" + "io" "net/http" + "net/http/httptest" neturl "net/url" "testing" "time" @@ -6255,3 +6258,84 @@ func TestRewriteDistributions_Error(t *testing.T) { }) }) } + +func TestPurgeCache(t *testing.T) { + Convey("Given a dataset and edition", t, func() { + ctx := context.Background() + datasetID := "test-dataset" + editionID := "test-edition" + zoneID := "test-zone" + apiToken := "test-token" + baseURL := "http://base-url" + + Convey("When cache purge succeeds", func() { + var capturedRequest *http.Request + var capturedBody []byte + + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + capturedRequest = r + capturedBody, _ = io.ReadAll(r.Body) + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"success": true}`)) + })) + defer mockServer.Close() + + mockClient := &http.Client{ + Transport: &mockTransport{server: mockServer}, + } + + err := PurgeCache(ctx, datasetID, editionID, baseURL, zoneID, apiToken, mockClient) + + Convey("Then no error is returned", func() { + So(err, ShouldBeNil) + }) + + Convey("And the request has correct headers", func() { + So(capturedRequest.Header.Get("Content-Type"), ShouldEqual, "application/json") + So(capturedRequest.Header.Get("Authorization"), ShouldEqual, "Bearer test-token") + }) + + Convey("And the request contains all 6 URL prefixes", func() { + var prefixes Prefixes + json.Unmarshal(capturedBody, &prefixes) + + So(len(prefixes.Prefixes), ShouldEqual, 6) + So(prefixes.Prefixes[0], ShouldEqual, "www.ons.gov.uk/datasets/test-dataset") + So(prefixes.Prefixes[1], ShouldEqual, "www.ons.gov.uk/datasets/test-dataset/editions") + So(prefixes.Prefixes[2], ShouldEqual, "www.ons.gov.uk/datasets/test-dataset/editions/test-edition/versions") + So(prefixes.Prefixes[3], ShouldEqual, "api.beta.ons.gov.uk/v1/datasets/test-dataset") + So(prefixes.Prefixes[4], ShouldEqual, "api.beta.ons.gov.uk/v1/datasets/test-dataset/editions") + So(prefixes.Prefixes[5], ShouldEqual, "api.beta.ons.gov.uk/v1/datasets/test-dataset/editions/test-edition/versions") + }) + }) + + Convey("When Cloudflare returns an error", func() { + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(`{"error": "something went wrong"}`)) + })) + defer mockServer.Close() + + mockClient := &http.Client{ + Transport: &mockTransport{server: mockServer}, + } + + err := PurgeCache(ctx, datasetID, editionID, baseURL, zoneID, apiToken, mockClient) + + Convey("Then an error is returned", func() { + So(err, ShouldNotBeNil) + So(err.Error(), ShouldContainSubstring, "cloudflare API returned status 500") + }) + }) + }) +} + +type mockTransport struct { + server *httptest.Server +} + +func (m *mockTransport) RoundTrip(req *http.Request) (*http.Response, error) { + req.URL.Scheme = "http" + req.URL.Host = m.server.URL[7:] + return http.DefaultTransport.RoundTrip(req) +} From 69ef42fd4d6339225bba2be43d504037b7d5cf05 Mon Sep 17 00:00:00 2001 From: andre-urbani Date: Thu, 27 Nov 2025 10:49:13 +0000 Subject: [PATCH 02/10] update config test --- config/config_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/config/config_test.go b/config/config_test.go index 40afab50..d78e93aa 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -63,6 +63,7 @@ func TestSpec(t *testing.T) { So(cfg.CodeListAPIURL, ShouldEqual, "http://localhost:22400") So(cfg.CloudflareAPIToken, ShouldEqual, "test-token") So(cfg.CloudflareZoneID, ShouldEqual, "9f1eec58caedd8e902395a065c120073") + So(cfg.CloudflareAPIURL, ShouldEqual, "http://cloud-flare-stub:22500") }) }) }) From 75dec890d84c96d937b22efdd9c2c0f44d135e2c Mon Sep 17 00:00:00 2001 From: andre-urbani Date: Thu, 27 Nov 2025 12:15:26 +0000 Subject: [PATCH 03/10] fix lint --- utils/utils.go | 5 ++--- utils/utils_test.go | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/utils/utils.go b/utils/utils.go index 909d160a..4753504b 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -807,7 +807,7 @@ type HTTPClient interface { } func PurgeCache(ctx context.Context, datasetID, editionID, baseURL, zoneID, apiToken string, client HTTPClient) error { - url := fmt.Sprintf("%s/zones/%s/purge_cache", baseURL, zoneID) + cloudflareURL := fmt.Sprintf("%s/zones/%s/purge_cache", baseURL, zoneID) webDatasetURL := fmt.Sprintf("www.ons.gov.uk/datasets/%s", datasetID) webEditionsURL := fmt.Sprintf("www.ons.gov.uk/datasets/%s/editions", datasetID) webVersionsURL := fmt.Sprintf("www.ons.gov.uk/datasets/%s/editions/%s/versions", datasetID, editionID) @@ -832,7 +832,7 @@ func PurgeCache(ctx context.Context, datasetID, editionID, baseURL, zoneID, apiT return fmt.Errorf("failed to marshal purge prefixes: %w", err) } - req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(payload)) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, cloudflareURL, bytes.NewReader(payload)) if err != nil { return fmt.Errorf("failed to create purge request: %w", err) } @@ -857,7 +857,6 @@ func PurgeCache(ctx context.Context, datasetID, editionID, baseURL, zoneID, apiT }) return nil - } func GenerateDistributionsDownloadURLs(datasetID, edition string, version int, distributions *[]models.Distribution) *[]models.Distribution { diff --git a/utils/utils_test.go b/utils/utils_test.go index 1021f198..8e8f3850 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -6309,7 +6309,6 @@ func TestPurgeCache(t *testing.T) { Convey("Then an error is returned", func() { So(err, ShouldNotBeNil) So(err.Error(), ShouldContainSubstring, "cloudflare API returned status 500") - }) }) }) From f130695c4632f370ade0356733db5b2629ba4186 Mon Sep 17 00:00:00 2001 From: andre-urbani Date: Wed, 3 Dec 2025 09:22:03 +0000 Subject: [PATCH 04/10] update to use cloudflare sdk --- api/api.go | 5 +- api/dataset_test.go | 4 +- api/versions.go | 22 +-- api/webendpoints_test.go | 2 +- cloudflare/client.go | 140 ++++++++++++++++++ cloudflare/client_test.go | 84 +++++++++++ config/config.go | 2 + dimension/dimension_test.go | 2 +- features/cloudflare_cache_purge.feature | 187 ++++++++++++++++++++++++ features/steps/dataset_component.go | 108 ++++++++++---- features/steps/steps.go | 44 +++++- go.mod | 4 + go.sum | 8 + instance/instance_external_test.go | 2 +- service/initialise.go | 49 +++++++ service/interfaces.go | 2 + service/mock/initialiser.go | 63 +++++++- service/service.go | 32 +++- service/service_test.go | 28 ++++ utils/utils.go | 65 -------- utils/utils_test.go | 84 ----------- 21 files changed, 728 insertions(+), 209 deletions(-) create mode 100644 cloudflare/client.go create mode 100644 cloudflare/client_test.go create mode 100644 features/cloudflare_cache_purge.feature diff --git a/api/api.go b/api/api.go index e9c2a628..b9e0ebbc 100644 --- a/api/api.go +++ b/api/api.go @@ -10,6 +10,7 @@ import ( "github.com/ONSdigital/dp-authorisation/auth" "github.com/ONSdigital/dp-dataset-api/application" + "github.com/ONSdigital/dp-dataset-api/cloudflare" "github.com/ONSdigital/dp-dataset-api/config" "github.com/ONSdigital/dp-dataset-api/dimension" "github.com/ONSdigital/dp-dataset-api/instance" @@ -77,10 +78,11 @@ type DatasetAPI struct { smDatasetAPI *application.StateMachineDatasetAPI filesAPIClient filesAPISDK.Clienter authToken string + cloudflareClient cloudflare.Clienter } // Setup creates a new Dataset API instance and register the API routes based on the application configuration. -func Setup(ctx context.Context, cfg *config.Configuration, router *mux.Router, dataStore store.DataStore, urlBuilder *url.Builder, downloadGenerators map[models.DatasetType]DownloadsGenerator, datasetPermissions, permissions AuthHandler, enableURLRewriting bool, smDatasetAPI *application.StateMachineDatasetAPI) *DatasetAPI { +func Setup(ctx context.Context, cfg *config.Configuration, router *mux.Router, dataStore store.DataStore, urlBuilder *url.Builder, downloadGenerators map[models.DatasetType]DownloadsGenerator, datasetPermissions, permissions AuthHandler, enableURLRewriting bool, smDatasetAPI *application.StateMachineDatasetAPI, cloudflareClient cloudflare.Clienter) *DatasetAPI { api := &DatasetAPI{ dataStore: dataStore, host: cfg.DatasetAPIURL, @@ -100,6 +102,7 @@ func Setup(ctx context.Context, cfg *config.Configuration, router *mux.Router, d MaxRequestOptions: cfg.MaxRequestOptions, defaultLimit: cfg.DefaultLimit, smDatasetAPI: smDatasetAPI, + cloudflareClient: cloudflareClient, } paginator := pagination.NewPaginator(cfg.DefaultLimit, cfg.DefaultOffset, cfg.DefaultMaxLimit) diff --git a/api/dataset_test.go b/api/dataset_test.go index 3f55a02d..331311fc 100644 --- a/api/dataset_test.go +++ b/api/dataset_test.go @@ -161,7 +161,7 @@ func GetAPIWithCMDMocks(mockedDataStore store.Storer, mockedGeneratedDownloads D StateMachine: application.NewStateMachine(testContext, states, transitions, store.DataStore{Backend: mockedDataStore}), } - return Setup(testContext, cfg, mux.NewRouter(), store.DataStore{Backend: mockedDataStore}, urlBuilder, mockedMapGeneratedDownloads, datasetPermissions, permissions, enableURLRewriting, &mockStatemachineDatasetAPI) + return Setup(testContext, cfg, mux.NewRouter(), store.DataStore{Backend: mockedDataStore}, urlBuilder, mockedMapGeneratedDownloads, datasetPermissions, permissions, enableURLRewriting, &mockStatemachineDatasetAPI, nil) } // GetAPIWithCMDMocks also used in other tests, so exported @@ -217,7 +217,7 @@ func GetAPIWithCantabularMocks(mockedDataStore store.Storer, mockedGeneratedDown StateMachine: application.NewStateMachine(testContext, states, transitions, store.DataStore{Backend: mockedDataStore}), } - return Setup(testContext, cfg, mux.NewRouter(), store.DataStore{Backend: mockedDataStore}, urlBuilder, mockedMapGeneratedDownloads, datasetPermissions, permissions, enableURLRewriting, &mockStatemachineDatasetAPI) + return Setup(testContext, cfg, mux.NewRouter(), store.DataStore{Backend: mockedDataStore}, urlBuilder, mockedMapGeneratedDownloads, datasetPermissions, permissions, enableURLRewriting, &mockStatemachineDatasetAPI, nil) } func createRequestWithAuth(method, target string, body io.Reader) *http.Request { diff --git a/api/versions.go b/api/versions.go index babfc2b4..f454513b 100644 --- a/api/versions.go +++ b/api/versions.go @@ -8,11 +8,9 @@ import ( "net/http" "strconv" "strings" - "time" "github.com/ONSdigital/dp-api-clients-go/v2/headers" errs "github.com/ONSdigital/dp-dataset-api/apierrors" - "github.com/ONSdigital/dp-dataset-api/config" "github.com/ONSdigital/dp-dataset-api/models" "github.com/ONSdigital/dp-dataset-api/utils" dpresponse "github.com/ONSdigital/dp-net/v3/handlers/response" @@ -804,30 +802,22 @@ func (api *DatasetAPI) putState(w http.ResponseWriter, r *http.Request) { Type: models.Static.String(), } - updatedVersion, err := api.smDatasetAPI.AmendVersion(r.Context(), vars, versionUpdate) + _, err = api.smDatasetAPI.AmendVersion(r.Context(), vars, versionUpdate) if err != nil { handleVersionAPIErr(ctx, err, w, logData) return } - if stateUpdate.State == models.PublishedState && updatedVersion.Distributions != nil && len(*updatedVersion.Distributions) > 0 { - err = api.publishDistributionFiles(ctx, updatedVersion, logData) + if stateUpdate.State == models.PublishedState && currentVersion.Distributions != nil && len(*currentVersion.Distributions) > 0 { + err = api.publishDistributionFiles(ctx, currentVersion, logData) if err != nil { log.Error(ctx, "putState endpoint: failed to publish distribution files", err, logData) handleVersionAPIErr(ctx, err, w, logData) return } - cfg, err := config.Get() - if err != nil { - log.Error(ctx, "failed to get config for cache purge", err, logData) - } else { - httpClient := &http.Client{Timeout: 10 * time.Second} - baseURL := cfg.CloudflareAPIURL - if baseURL == "" { - baseURL = "https://api.cloudflare.com/client/v4" - } - err = utils.PurgeCache(ctx, datasetID, edition, baseURL, cfg.CloudflareZoneID, cfg.CloudflareAPIToken, httpClient) - if err != nil { + + if api.cloudflareClient != nil { + if err := api.cloudflareClient.PurgeCacheByPrefix(ctx, datasetID, edition); err != nil { log.Error(ctx, "failed to purge cloudflare cache", err, logData) } else { log.Info(ctx, "successfully triggered cloudflare cache purge", logData) diff --git a/api/webendpoints_test.go b/api/webendpoints_test.go index 85c5af0c..ff024a04 100644 --- a/api/webendpoints_test.go +++ b/api/webendpoints_test.go @@ -368,5 +368,5 @@ func GetWebAPIWithMocks(ctx context.Context, mockedDataStore store.Storer, mocke cfg.DatasetAPIURL = host cfg.EnablePrivateEndpoints = false - return Setup(ctx, cfg, mux.NewRouter(), store.DataStore{Backend: mockedDataStore}, urlBuilder, mockedMapDownloadGenerators, datasetPermissions, permissions, enableURLRewriting, &mockStatemachineDatasetAPI) + return Setup(ctx, cfg, mux.NewRouter(), store.DataStore{Backend: mockedDataStore}, urlBuilder, mockedMapDownloadGenerators, datasetPermissions, permissions, enableURLRewriting, &mockStatemachineDatasetAPI, nil) } diff --git a/cloudflare/client.go b/cloudflare/client.go new file mode 100644 index 00000000..a233d7a4 --- /dev/null +++ b/cloudflare/client.go @@ -0,0 +1,140 @@ +package cloudflare + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "github.com/ONSdigital/log.go/v2/log" + "github.com/cloudflare/cloudflare-go" +) + +// Clienter defines the interface for Cloudflare cache operations +type Clienter interface { + PurgeCacheByPrefix(ctx context.Context, datasetID, editionID string) error +} + +// Client wraps the Cloudflare API client +type Client struct { + api *cloudflare.API + zoneID string + baseURL string + httpClient *http.Client + apiToken string +} + +// New creates a new Cloudflare client +func New(apiToken, zoneID string, useSDK bool, baseURL ...string) (*Client, error) { + if apiToken == "" { + return nil, fmt.Errorf("cloudflare API token is required") + } + if zoneID == "" { + return nil, fmt.Errorf("cloudflare zone ID is required") + } + + // if SDK disabled for local testing with the cloudflare stub, use the HTTP client + if !useSDK && len(baseURL) > 0 && baseURL[0] != "" { + return &Client{ + api: nil, + zoneID: zoneID, + baseURL: baseURL[0], + apiToken: apiToken, + httpClient: &http.Client{Timeout: 10 * time.Second}, + }, nil + } + + // use real Cloudflare SDK + api, err := cloudflare.NewWithAPIToken(apiToken) + if err != nil { + return nil, fmt.Errorf("failed to create cloudflare client: %w", err) + } + + return &Client{ + api: api, + zoneID: zoneID, + }, nil +} + +// PurgeCacheByPrefix purges the Cloudflare cache for dataset-related URLs +func (c *Client) PurgeCacheByPrefix(ctx context.Context, datasetID, editionID string) error { + prefixes := buildPrefixes(datasetID, editionID) + + logData := log.Data{ + "dataset_id": datasetID, + "edition": editionID, + "prefixes": prefixes, + } + log.Info(ctx, "purging cloudflare cache", logData) + + // local mock + if c.baseURL != "" { + return c.purgeCacheViaHTTP(ctx, prefixes) + } + + // for sdk + return c.purgeCacheViaSDK(ctx, prefixes) +} + +// purgeCacheViaSDK uses the cloudflare sdk +func (c *Client) purgeCacheViaSDK(ctx context.Context, prefixes []string) error { + params := cloudflare.PurgeCacheRequest{ + Prefixes: prefixes, + } + + _, err := c.api.PurgeCache(ctx, c.zoneID, params) + if err != nil { + return fmt.Errorf("failed to purge cache: %w", err) + } + + return nil +} + +// purgeCacheViaHTTP makes direct HTTP call for local cloudflare stub +func (c *Client) purgeCacheViaHTTP(ctx context.Context, prefixes []string) error { + url := fmt.Sprintf("%s/zones/%s/purge_cache", c.baseURL, c.zoneID) + + payload := map[string]interface{}{ + "prefixes": prefixes, + } + + body, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("failed to marshal request: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(body)) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", c.apiToken)) + + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("failed to make request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + bodyBytes, _ := io.ReadAll(resp.Body) + return fmt.Errorf("purge failed with status %d: %s", resp.StatusCode, string(bodyBytes)) + } + + return nil +} + +func buildPrefixes(datasetID, editionID string) []string { + return []string{ + fmt.Sprintf("www.ons.gov.uk/datasets/%s", datasetID), + fmt.Sprintf("www.ons.gov.uk/datasets/%s/editions", datasetID), + fmt.Sprintf("www.ons.gov.uk/datasets/%s/editions/%s/versions", datasetID, editionID), + fmt.Sprintf("api.beta.ons.gov.uk/v1/datasets/%s", datasetID), + fmt.Sprintf("api.beta.ons.gov.uk/v1/datasets/%s/editions", datasetID), + fmt.Sprintf("api.beta.ons.gov.uk/v1/datasets/%s/editions/%s/versions", datasetID, editionID), + } +} diff --git a/cloudflare/client_test.go b/cloudflare/client_test.go new file mode 100644 index 00000000..bc74c49c --- /dev/null +++ b/cloudflare/client_test.go @@ -0,0 +1,84 @@ +package cloudflare_test + +import ( + "testing" + + "github.com/ONSdigital/dp-dataset-api/cloudflare" + . "github.com/smartystreets/goconvey/convey" +) + +func TestNew(t *testing.T) { + Convey("Given valid API token and zone ID", t, func() { + apiToken := "test-token" + zoneID := "test-zone-id" + useSDK := false + + Convey("When creating a new Cloudflare client", func() { + client, err := cloudflare.New(apiToken, zoneID, useSDK) + + Convey("Then no error is returned", func() { + So(err, ShouldBeNil) + }) + + Convey("And the client is not nil", func() { + So(client, ShouldNotBeNil) + }) + }) + }) + + Convey("Given an empty API token", t, func() { + apiToken := "" + zoneID := "test-zone-id" + useSDK := false + + Convey("When creating a new Cloudflare client", func() { + client, err := cloudflare.New(apiToken, zoneID, useSDK) + + Convey("Then an error is returned", func() { + So(err, ShouldNotBeNil) + So(err.Error(), ShouldContainSubstring, "API token is required") + }) + + Convey("And the client is nil", func() { + So(client, ShouldBeNil) + }) + }) + }) + + Convey("Given an empty zone ID", t, func() { + apiToken := "test-token" + zoneID := "" + useSDK := false + + Convey("When creating a new Cloudflare client", func() { + client, err := cloudflare.New(apiToken, zoneID, useSDK) + + Convey("Then an error is returned", func() { + So(err, ShouldNotBeNil) + So(err.Error(), ShouldContainSubstring, "zone ID is required") + }) + + Convey("And the client is nil", func() { + So(client, ShouldBeNil) + }) + }) + }) + + Convey("Given both empty API token and zone ID", t, func() { + apiToken := "" + zoneID := "" + useSDK := false + + Convey("When creating a new Cloudflare client", func() { + client, err := cloudflare.New(apiToken, zoneID, useSDK) + + Convey("Then an error is returned", func() { + So(err, ShouldNotBeNil) + }) + + Convey("And the client is nil", func() { + So(client, ShouldBeNil) + }) + }) + }) +} diff --git a/config/config.go b/config/config.go index 1cd0e31e..dd44f8c6 100644 --- a/config/config.go +++ b/config/config.go @@ -47,6 +47,7 @@ type Configuration struct { EnablePermissionsAuth bool `envconfig:"ENABLE_PERMISSIONS_AUTH"` EnableObservationEndpoint bool `envconfig:"ENABLE_OBSERVATION_ENDPOINT"` EnableURLRewriting bool `envconfig:"ENABLE_URL_REWRITING"` + EnableCloudflareSDK bool `envconfig:"ENABLE_CLOUDFLARE_SDK"` DisableGraphDBDependency bool `envconfig:"DISABLE_GRAPH_DB_DEPENDENCY"` KafkaVersion string `envconfig:"KAFKA_VERSION"` DefaultMaxLimit int `envconfig:"DEFAULT_MAXIMUM_LIMIT"` @@ -112,6 +113,7 @@ func Get() (*Configuration, error) { EnablePermissionsAuth: false, EnableObservationEndpoint: true, EnableURLRewriting: false, + EnableCloudflareSDK: false, DisableGraphDBDependency: false, KafkaVersion: "1.0.2", DefaultMaxLimit: 1000, diff --git a/dimension/dimension_test.go b/dimension/dimension_test.go index d6ad9ace..73ad577e 100644 --- a/dimension/dimension_test.go +++ b/dimension/dimension_test.go @@ -1544,7 +1544,7 @@ func getAPIWithCMDMocks(ctx context.Context, mockedDataStore store.Storer, mocke datasetPermissions := getAuthorisationHandlerMock() permissions := getAuthorisationHandlerMock() - return api.Setup(ctx, cfg, mux.NewRouter(), store.DataStore{Backend: mockedDataStore}, urlBuilder, downloadGenerators, datasetPermissions, permissions, enableURLRewriting, &mockStatemachineDatasetAPI) + return api.Setup(ctx, cfg, mux.NewRouter(), store.DataStore{Backend: mockedDataStore}, urlBuilder, downloadGenerators, datasetPermissions, permissions, enableURLRewriting, &mockStatemachineDatasetAPI, nil) } func getAuthorisationHandlerMock() *mocks.AuthHandlerMock { diff --git a/features/cloudflare_cache_purge.feature b/features/cloudflare_cache_purge.feature new file mode 100644 index 00000000..80159e09 --- /dev/null +++ b/features/cloudflare_cache_purge.feature @@ -0,0 +1,187 @@ +Feature: Cloudflare Cache Purging on Version Publication + + Scenario: Cloudflare cache is purged when version is published + Given I have a static dataset with version: + """ + { + "dataset": { + "id": "cache-test-dataset", + "title": "Cache Test Dataset", + "state": "associated", + "type": "static", + "links": { + "editions": { + "href": "/datasets/cache-test-dataset/editions" + }, + "self": { + "href": "/datasets/cache-test-dataset" + } + } + }, + "version": { + "id": "cache-test-version", + "edition": "2025", + "edition_title": "2025 Edition", + "links": { + "dataset": { + "id": "cache-test-dataset" + }, + "edition": { + "href": "/datasets/cache-test-dataset/editions/2025", + "id": "2025" + }, + "self": { + "href": "/datasets/cache-test-dataset/editions/2025/versions/1" + }, + "version": { + "href": "/datasets/cache-test-dataset/editions/2025/versions/1", + "id": "1" + } + }, + "version": 1, + "release_date": "2025-01-01T09:00:00.000Z", + "state": "approved", + "type": "static", + "distributions": [ + { + "title": "csv", + "format": "csv", + "media_type": "text/csv", + "download_url": "/downloads/datasets/cache-test-dataset/editions/2025/versions/1.csv", + "byte_size": 125000 + } + ] + } + } + """ + And private endpoints are enabled + And I am identified as "user@ons.gov.uk" + And I am authorised + When I PUT "/datasets/cache-test-dataset/editions/2025/versions/1/state" + """ + { + "state": "published" + } + """ + Then the HTTP status code should be "200" + And cloudflare cache purge should have been called for dataset "cache-test-dataset" and edition "2025" + + Scenario: Cloudflare cache is not purged when version transitions to non-published state + Given I have a static dataset with version: + """ + { + "dataset": { + "id": "cache-test-non-publish", + "title": "Cache Test Non-Publish", + "state": "created", + "type": "static" + }, + "version": { + "id": "cache-test-version-2", + "edition": "2025", + "edition_title": "2025 Edition", + "links": { + "dataset": { + "id": "cache-test-non-publish" + }, + "edition": { + "href": "/datasets/cache-test-non-publish/editions/2025", + "id": "2025" + }, + "self": { + "href": "/datasets/cache-test-non-publish/editions/2025/versions/1" + } + }, + "version": 1, + "release_date": "2025-01-01T09:00:00.000Z", + "state": "created", + "type": "static", + "distributions": [ + { + "title": "csv", + "format": "csv", + "media_type": "text/csv", + "download_url": "/downloads/datasets/cache-test-non-publish/editions/2025/versions/1.csv", + "byte_size": 125000 + } + ] + } + } + """ + And private endpoints are enabled + And I am identified as "user@ons.gov.uk" + And I am authorised + When I PUT "/datasets/cache-test-non-publish/editions/2025/versions/1/state" + """ + { + "state": "associated" + } + """ + Then the HTTP status code should be "200" + And cloudflare cache purge should not have been called + + Scenario: Publication succeeds even if Cloudflare cache purge fails + Given I have a static dataset with version: + """ + { + "dataset": { + "id": "cache-test-fail", + "title": "Cache Test Fail", + "state": "associated", + "type": "static", + "links": { + "editions": { + "href": "/datasets/cache-test-fail/editions" + }, + "self": { + "href": "/datasets/cache-test-fail" + } + } + }, + "version": { + "id": "cache-test-version-3", + "edition": "2025", + "edition_title": "2025 Edition", + "links": { + "dataset": { + "id": "cache-test-fail" + }, + "edition": { + "href": "/datasets/cache-test-fail/editions/2025", + "id": "2025" + }, + "self": { + "href": "/datasets/cache-test-fail/editions/2025/versions/1" + }, + "version": { + "href": "/datasets/cache-test-fail/editions/2025/versions/1", + "id": "1" + } + }, + "version": 1, + "release_date": "2025-01-01T09:00:00.000Z", + "state": "approved", + "type": "static", + "distributions": [ + { + "title": "csv", + "format": "csv", + "media_type": "text/csv", + "download_url": "/downloads/datasets/cache-test-fail/editions/2025/versions/1.csv", + "byte_size": 125000 + } + ] + } + } + """ + And private endpoints are enabled + And I am identified as "user@ons.gov.uk" + And I am authorised + And cloudflare cache purge is configured to fail + When I PUT "/datasets/cache-test-fail/editions/2025/versions/1/state" + """ + { + "state": "published" + } + """ + Then the HTTP status code should be "200" diff --git a/features/steps/dataset_component.go b/features/steps/dataset_component.go index ca62a795..4599ff84 100644 --- a/features/steps/dataset_component.go +++ b/features/steps/dataset_component.go @@ -8,12 +8,14 @@ import ( componenttest "github.com/ONSdigital/dp-component-test" "github.com/ONSdigital/dp-component-test/utils" + "github.com/ONSdigital/dp-dataset-api/cloudflare" "github.com/ONSdigital/dp-dataset-api/config" "github.com/ONSdigital/dp-dataset-api/mongo" "github.com/ONSdigital/dp-dataset-api/service" serviceMock "github.com/ONSdigital/dp-dataset-api/service/mock" "github.com/ONSdigital/dp-dataset-api/store" storeMock "github.com/ONSdigital/dp-dataset-api/store/datastoretest" + "github.com/ONSdigital/dp-files-api/files" filesAPISDK "github.com/ONSdigital/dp-files-api/sdk" filesAPISDKMocks "github.com/ONSdigital/dp-files-api/sdk/mocks" "github.com/ONSdigital/dp-healthcheck/healthcheck" @@ -24,17 +26,19 @@ import ( ) type DatasetComponent struct { - ErrorFeature componenttest.ErrorFeature - apiFeature *componenttest.APIFeature - svc *service.Service - errorChan chan error - MongoClient *mongo.Mongo - Config *config.Configuration - HTTPServer *http.Server - ServiceRunning bool - consumer kafka.IConsumerGroup - producer kafka.IProducer - initialiser service.Initialiser + ErrorFeature componenttest.ErrorFeature + apiFeature *componenttest.APIFeature + svc *service.Service + errorChan chan error + MongoClient *mongo.Mongo + Config *config.Configuration + HTTPServer *http.Server + ServiceRunning bool + consumer kafka.IConsumerGroup + producer kafka.IProducer + initialiser service.Initialiser + MockCloudflare *MockCloudflareClient + CloudflarePurgeCalls []CloudflarePurgeCall } func NewDatasetComponent(mongoURI, zebedeeURL string) (*DatasetComponent, error) { @@ -42,8 +46,14 @@ func NewDatasetComponent(mongoURI, zebedeeURL string) (*DatasetComponent, error) HTTPServer: &http.Server{ ReadHeaderTimeout: 60 * time.Second, }, - errorChan: make(chan error), - ServiceRunning: false, + errorChan: make(chan error), + ServiceRunning: false, + CloudflarePurgeCalls: []CloudflarePurgeCall{}, + } + + c.MockCloudflare = &MockCloudflareClient{ + PurgeCalls: &c.CloudflarePurgeCalls, + ShouldFail: false, } var err error @@ -92,6 +102,10 @@ func (c *DatasetComponent) Reset() error { c.Config.EnablePrivateEndpoints = false c.Config.EnableURLRewriting = false + + c.CloudflarePurgeCalls = []CloudflarePurgeCall{} + c.MockCloudflare.ShouldFail = false + // Resets back to Mocked Kafka c.setInitialiserMock() @@ -237,6 +251,17 @@ func (c *DatasetComponent) DoGetGraphDBOk(context.Context) (store.GraphDB, servi func (c *DatasetComponent) DoGetFilesAPIClientOk(ctx context.Context, cfg *config.Configuration) (filesAPISDK.Clienter, error) { return &filesAPISDKMocks.ClienterMock{ + GetFileFunc: func(ctx context.Context, filePath string) (*files.StoredRegisteredMetaData, error) { + return &files.StoredRegisteredMetaData{ + Path: filePath, + IsPublishable: true, + State: "UPLOADED", + SizeInBytes: 1000, + }, nil + }, + MarkFilePublishedFunc: func(ctx context.Context, filePath string) error { + return nil + }, DeleteFileFunc: func(ctx context.Context, filePath string) error { if filePath == "/fail/to/delete.csv" { return fmt.Errorf("failed to delete file at path: %s", filePath) @@ -246,23 +271,56 @@ func (c *DatasetComponent) DoGetFilesAPIClientOk(ctx context.Context, cfg *confi }, nil } +func (c *DatasetComponent) DoGetCloudflareClientOk(ctx context.Context, cfg *config.Configuration) (cloudflare.Clienter, error) { + return c.MockCloudflare, nil +} + func (c *DatasetComponent) setInitialiserMock() { c.initialiser = &serviceMock.InitialiserMock{ - DoGetMongoDBFunc: c.DoGetMongoDB, - DoGetGraphDBFunc: c.DoGetGraphDBOk, - DoGetFilesAPIClientFunc: c.DoGetFilesAPIClientOk, - DoGetKafkaProducerFunc: c.DoGetMockedKafkaProducerOk, - DoGetHealthCheckFunc: c.DoGetHealthcheckOk, - DoGetHTTPServerFunc: c.DoGetHTTPServer, + DoGetMongoDBFunc: c.DoGetMongoDB, + DoGetGraphDBFunc: c.DoGetGraphDBOk, + DoGetFilesAPIClientFunc: c.DoGetFilesAPIClientOk, + DoGetKafkaProducerFunc: c.DoGetMockedKafkaProducerOk, + DoGetHealthCheckFunc: c.DoGetHealthcheckOk, + DoGetHTTPServerFunc: c.DoGetHTTPServer, + DoGetCloudflareClientFunc: c.DoGetCloudflareClientOk, } } func (c *DatasetComponent) setInitialiserRealKafka() { c.initialiser = &serviceMock.InitialiserMock{ - DoGetMongoDBFunc: c.DoGetMongoDB, - DoGetGraphDBFunc: c.DoGetGraphDBOk, - DoGetFilesAPIClientFunc: c.DoGetFilesAPIClientOk, - DoGetKafkaProducerFunc: c.DoGetKafkaProducer, - DoGetHealthCheckFunc: c.DoGetHealthcheckOk, - DoGetHTTPServerFunc: c.DoGetHTTPServer, + DoGetMongoDBFunc: c.DoGetMongoDB, + DoGetGraphDBFunc: c.DoGetGraphDBOk, + DoGetFilesAPIClientFunc: c.DoGetFilesAPIClientOk, + DoGetKafkaProducerFunc: c.DoGetKafkaProducer, + DoGetHealthCheckFunc: c.DoGetHealthcheckOk, + DoGetHTTPServerFunc: c.DoGetHTTPServer, + DoGetCloudflareClientFunc: c.DoGetCloudflareClientOk, } } + +type CloudflarePurgeCall struct { + DatasetID string + EditionID string +} + +type MockCloudflareClient struct { + PurgeCalls *[]CloudflarePurgeCall + ShouldFail bool +} + +func (m *MockCloudflareClient) PurgeCacheByPrefix(ctx context.Context, datasetID, editionID string) error { + if m.ShouldFail { + return fmt.Errorf("mock cloudflare purge failed") + } + + *m.PurgeCalls = append(*m.PurgeCalls, CloudflarePurgeCall{ + DatasetID: datasetID, + EditionID: editionID, + }) + + log.Info(ctx, "mock cloudflare cache purge called", log.Data{ + "dataset_id": datasetID, + "edition_id": editionID, + }) + return nil +} diff --git a/features/steps/steps.go b/features/steps/steps.go index 98774b40..6c1a088a 100644 --- a/features/steps/steps.go +++ b/features/steps/steps.go @@ -55,6 +55,9 @@ func (c *DatasetComponent) RegisterSteps(ctx *godog.ScenarioContext) { ctx.Step(`^the response header "([^"]*)" should not be empty$`, c.theResponseHeaderShouldNotBeEmpty) ctx.Step(`^the dataset "([^"]*)" should have next equal to current$`, c.theDatasetShouldHaveNextEqualToCurrent) ctx.Step(`^the "([^"]*)" feature flag is "([^"]*)"$`, c.theFeatureFlagIs) + ctx.Step(`^cloudflare cache purge should have been called for dataset "([^"]*)" and edition "([^"]*)"$`, c.cloudflareCachePurgeShouldHaveBeenCalled) + ctx.Step(`^cloudflare cache purge should not have been called$`, c.cloudflareCachePurgeShouldNotHaveBeenCalled) + ctx.Step(`^cloudflare cache purge is configured to fail$`, c.cloudflareCachePurgeIsConfiguredToFail) } func (c *DatasetComponent) theFeatureFlagIs(flagName, status string) error { @@ -514,11 +517,9 @@ func (c *DatasetComponent) iHaveStaticDatasetWithVersion(jsonData *godog.DocStri Dataset models.Dataset `json:"dataset"` Version models.Version `json:"version"` } - if err := json.Unmarshal([]byte(jsonData.Content), &data); err != nil { return fmt.Errorf("failed to unmarshal static dataset data: %w", err) } - datasetID := data.Dataset.ID data.Dataset.Type = "static" datasetUp := models.DatasetUpdate{ @@ -530,23 +531,30 @@ func (c *DatasetComponent) iHaveStaticDatasetWithVersion(jsonData *godog.DocStri if err := c.putDocumentInDatabase(datasetUp, datasetID, datasetsCollection, 0); err != nil { return fmt.Errorf("failed to insert static dataset: %w", err) } - versionID := data.Version.ID + if data.Version.Links == nil { + data.Version.Links = &models.VersionLinks{} + } + + if data.Version.Links.Self == nil { + data.Version.Links.Self = &models.LinkObject{ + HRef: fmt.Sprintf("/datasets/%s/editions/%s/versions/%d", datasetID, data.Version.Edition, data.Version.Version), + } + } + if data.Version.Links.Version == nil { data.Version.Links.Version = &models.LinkObject{ HRef: data.Version.Links.Self.HRef, - ID: "1", + ID: fmt.Sprintf("%d", data.Version.Version), } } data.Version.ETag = "etag-" + versionID - versionsCollection := c.MongoClient.ActualCollectionName(config.VersionsCollection) if err := c.putDocumentInDatabase(data.Version, versionID, versionsCollection, 0); err != nil { return fmt.Errorf("failed to insert static version: %w", err) } - return nil } @@ -626,3 +634,27 @@ func (c *DatasetComponent) theResponseHeaderShouldNotBeEmpty(header string) erro } return nil } + +func (c *DatasetComponent) cloudflareCachePurgeShouldHaveBeenCalled(datasetID, editionID string) error { + for _, call := range c.CloudflarePurgeCalls { + if call.DatasetID == datasetID && call.EditionID == editionID { + return nil + } + } + + return fmt.Errorf("expected cloudflare cache purge to be called for dataset '%s' and edition '%s', but it was not. Calls made: %+v", + datasetID, editionID, c.CloudflarePurgeCalls) +} + +func (c *DatasetComponent) cloudflareCachePurgeShouldNotHaveBeenCalled() error { + if len(c.CloudflarePurgeCalls) > 0 { + return fmt.Errorf("expected no cloudflare cache purge calls, but got %d calls: %+v", + len(c.CloudflarePurgeCalls), c.CloudflarePurgeCalls) + } + return nil +} + +func (c *DatasetComponent) cloudflareCachePurgeIsConfiguredToFail() error { + c.MockCloudflare.ShouldFail = true + return nil +} diff --git a/go.mod b/go.mod index a79bb42f..2198fe4d 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/ONSdigital/dp-net/v3 v3.9.0 github.com/ONSdigital/dp-otel-go v0.0.8 github.com/ONSdigital/log.go/v2 v2.5.1 + github.com/cloudflare/cloudflare-go v0.116.0 github.com/cucumber/godog v0.15.1 github.com/golang/glog v1.2.5 github.com/google/go-cmp v0.7.0 @@ -81,9 +82,11 @@ require ( github.com/gobwas/httphead v0.1.0 // indirect github.com/gobwas/pool v0.2.1 // indirect github.com/gobwas/ws v1.4.0 // indirect + github.com/goccy/go-json v0.10.5 // indirect github.com/gofrs/uuid v4.4.0+incompatible // indirect github.com/golang-jwt/jwt/v4 v4.5.2 // indirect github.com/golang/snappy v1.0.0 // indirect + github.com/google/go-querystring v1.1.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gopherjs/gopherjs v1.17.2 // indirect github.com/gorilla/websocket v1.5.3 // indirect @@ -140,6 +143,7 @@ require ( golang.org/x/sync v0.16.0 // indirect golang.org/x/sys v0.34.0 // indirect golang.org/x/text v0.27.0 // indirect + golang.org/x/time v0.9.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect google.golang.org/grpc v1.73.0 // indirect diff --git a/go.sum b/go.sum index cec5d6fe..e7e58681 100644 --- a/go.sum +++ b/go.sum @@ -125,6 +125,8 @@ github.com/chromedp/chromedp v0.14.0 h1:/xE5m6wEBwivhalHwlCOyYfBcAJNwg4nLw96QiCf github.com/chromedp/chromedp v0.14.0/go.mod h1:rHzAv60xDE7VNy/MYtTUrYreSc0ujt2O1/C3bzctYBo= github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM= github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8= +github.com/cloudflare/cloudflare-go v0.116.0 h1:iRPMnTtnswRpELO65NTwMX4+RTdxZl+Xf/zi+HPE95s= +github.com/cloudflare/cloudflare-go v0.116.0/go.mod h1:Ds6urDwn/TF2uIU24mu7H91xkKP8gSAHxQ44DSZgVmU= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cucumber/gherkin/go/v26 v26.2.0 h1:EgIjePLWiPeslwIWmNQ3XHcypPsWAHoMCz/YEBKP4GI= github.com/cucumber/gherkin/go/v26 v26.2.0/go.mod h1:t2GAPnB8maCT4lkHL99BDCVNzCh1d7dBhCLt150Nr/0= @@ -181,6 +183,8 @@ github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs= github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v4.3.1+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA= @@ -202,6 +206,8 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -458,6 +464,8 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= diff --git a/instance/instance_external_test.go b/instance/instance_external_test.go index 8cc0db9e..1d7b0bb8 100644 --- a/instance/instance_external_test.go +++ b/instance/instance_external_test.go @@ -779,5 +779,5 @@ func getAPIWithCantabularMocks(ctx context.Context, mockedDataStore store.Storer cfg.DatasetAPIURL = "http://localhost:22000" cfg.EnablePrivateEndpoints = true - return api.Setup(ctx, cfg, mux.NewRouter(), store.DataStore{Backend: mockedDataStore}, urlBuilder, mockedMapDownloadGenerators, datasetPermissions, permissions, enableURLRewriting, &mockStatemachineDatasetAPI) + return api.Setup(ctx, cfg, mux.NewRouter(), store.DataStore{Backend: mockedDataStore}, urlBuilder, mockedMapDownloadGenerators, datasetPermissions, permissions, enableURLRewriting, &mockStatemachineDatasetAPI, nil) } diff --git a/service/initialise.go b/service/initialise.go index 2339f9e4..23ef49dd 100644 --- a/service/initialise.go +++ b/service/initialise.go @@ -4,6 +4,7 @@ import ( "context" "net/http" + "github.com/ONSdigital/dp-dataset-api/cloudflare" "github.com/ONSdigital/dp-dataset-api/config" "github.com/ONSdigital/dp-dataset-api/mongo" "github.com/ONSdigital/dp-dataset-api/store" @@ -22,6 +23,7 @@ type ExternalServiceList struct { HealthCheck bool MongoDB bool FilesAPIClient bool + CloudflareClient bool Init Initialiser } @@ -97,6 +99,20 @@ func (e *ExternalServiceList) GetFilesAPIClient(ctx context.Context, cfg *config return nil, nil } +// GetCloudflareClient returns a Cloudflare client +func (e *ExternalServiceList) GetCloudflareClient(ctx context.Context, cfg *config.Configuration) (cloudflare.Clienter, error) { + cloudflareClient, err := e.Init.DoGetCloudflareClient(ctx, cfg) + if err != nil { + log.Error(ctx, "failed to initialise cloudflare client", err) + return nil, err + } + if cloudflareClient != nil { + e.CloudflareClient = true + log.Info(ctx, "cloudflare client created successfully") + } + return cloudflareClient, nil +} + // DoGetHTTPServer creates an HTTP Server with the provided bind address and router func (e *Init) DoGetHTTPServer(bindAddr string, router http.Handler) HTTPServer { s := dphttp.NewServer(bindAddr, router) @@ -163,3 +179,36 @@ func (e *Init) DoGetMongoDB(ctx context.Context, cfg config.MongoConfig) (store. func (e *Init) DoGetFilesAPIClient(ctx context.Context, cfg *config.Configuration) (filesAPISDK.Clienter, error) { return filesAPISDK.New(cfg.FilesAPIURL, cfg.ServiceAuthToken), nil } + +// DoGetCloudflareClient returns a Cloudflare client +func (e *Init) DoGetCloudflareClient(ctx context.Context, cfg *config.Configuration) (cloudflare.Clienter, error) { + if cfg.CloudflareAPIToken == "" || cfg.CloudflareZoneID == "" { + log.Info(ctx, "cloudflare integration disabled: missing API token or zone ID") + return nil, nil + } + + var client cloudflare.Clienter + var err error + + // for local mock server when SDK is disabled + if !cfg.EnableCloudflareSDK && cfg.CloudflareAPIURL != "" { + client, err = cloudflare.New( + cfg.CloudflareAPIToken, + cfg.CloudflareZoneID, + cfg.EnableCloudflareSDK, + cfg.CloudflareAPIURL, + ) + } else { + client, err = cloudflare.New( + cfg.CloudflareAPIToken, + cfg.CloudflareZoneID, + cfg.EnableCloudflareSDK, + ) + } + + if err != nil { + return nil, err + } + + return client, nil +} diff --git a/service/interfaces.go b/service/interfaces.go index aba440af..1c438eed 100644 --- a/service/interfaces.go +++ b/service/interfaces.go @@ -4,6 +4,7 @@ import ( "context" "net/http" + "github.com/ONSdigital/dp-dataset-api/cloudflare" "github.com/ONSdigital/dp-dataset-api/config" "github.com/ONSdigital/dp-dataset-api/store" filesAPISDK "github.com/ONSdigital/dp-files-api/sdk" @@ -24,6 +25,7 @@ type Initialiser interface { DoGetGraphDB(ctx context.Context) (store.GraphDB, Closer, error) DoGetMongoDB(ctx context.Context, cfg config.MongoConfig) (store.MongoDB, error) DoGetFilesAPIClient(ctx context.Context, cfg *config.Configuration) (filesAPISDK.Clienter, error) + DoGetCloudflareClient(ctx context.Context, cfg *config.Configuration) (cloudflare.Clienter, error) } // HTTPServer defines the required methods from the HTTP server diff --git a/service/mock/initialiser.go b/service/mock/initialiser.go index 26691ca3..64e5d65c 100644 --- a/service/mock/initialiser.go +++ b/service/mock/initialiser.go @@ -5,6 +5,7 @@ package mock import ( "context" + "github.com/ONSdigital/dp-dataset-api/cloudflare" "github.com/ONSdigital/dp-dataset-api/config" "github.com/ONSdigital/dp-dataset-api/service" "github.com/ONSdigital/dp-dataset-api/store" @@ -24,6 +25,9 @@ var _ service.Initialiser = &InitialiserMock{} // // // make and configure a mocked service.Initialiser // mockedInitialiser := &InitialiserMock{ +// DoGetCloudflareClientFunc: func(ctx context.Context, cfg *config.Configuration) (cloudflare.Clienter, error) { +// panic("mock out the DoGetCloudflareClient method") +// }, // DoGetFilesAPIClientFunc: func(ctx context.Context, cfg *config.Configuration) (filesAPISDK.Clienter, error) { // panic("mock out the DoGetFilesAPIClient method") // }, @@ -49,6 +53,9 @@ var _ service.Initialiser = &InitialiserMock{} // // } type InitialiserMock struct { + // DoGetCloudflareClientFunc mocks the DoGetCloudflareClient method. + DoGetCloudflareClientFunc func(ctx context.Context, cfg *config.Configuration) (cloudflare.Clienter, error) + // DoGetFilesAPIClientFunc mocks the DoGetFilesAPIClient method. DoGetFilesAPIClientFunc func(ctx context.Context, cfg *config.Configuration) (filesAPISDK.Clienter, error) @@ -69,6 +76,13 @@ type InitialiserMock struct { // calls tracks calls to the methods. calls struct { + // DoGetCloudflareClient holds details about calls to the DoGetCloudflareClient method. + DoGetCloudflareClient []struct { + // Ctx is the ctx argument value. + Ctx context.Context + // Cfg is the cfg argument value. + Cfg *config.Configuration + } // DoGetFilesAPIClient holds details about calls to the DoGetFilesAPIClient method. DoGetFilesAPIClient []struct { // Ctx is the ctx argument value. @@ -116,12 +130,49 @@ type InitialiserMock struct { Cfg config.MongoConfig } } - lockDoGetFilesAPIClient sync.RWMutex - lockDoGetGraphDB sync.RWMutex - lockDoGetHTTPServer sync.RWMutex - lockDoGetHealthCheck sync.RWMutex - lockDoGetKafkaProducer sync.RWMutex - lockDoGetMongoDB sync.RWMutex + lockDoGetCloudflareClient sync.RWMutex + lockDoGetFilesAPIClient sync.RWMutex + lockDoGetGraphDB sync.RWMutex + lockDoGetHTTPServer sync.RWMutex + lockDoGetHealthCheck sync.RWMutex + lockDoGetKafkaProducer sync.RWMutex + lockDoGetMongoDB sync.RWMutex +} + +// DoGetCloudflareClient calls DoGetCloudflareClientFunc. +func (mock *InitialiserMock) DoGetCloudflareClient(ctx context.Context, cfg *config.Configuration) (cloudflare.Clienter, error) { + if mock.DoGetCloudflareClientFunc == nil { + panic("InitialiserMock.DoGetCloudflareClientFunc: method is nil but Initialiser.DoGetCloudflareClient was just called") + } + callInfo := struct { + Ctx context.Context + Cfg *config.Configuration + }{ + Ctx: ctx, + Cfg: cfg, + } + mock.lockDoGetCloudflareClient.Lock() + mock.calls.DoGetCloudflareClient = append(mock.calls.DoGetCloudflareClient, callInfo) + mock.lockDoGetCloudflareClient.Unlock() + return mock.DoGetCloudflareClientFunc(ctx, cfg) +} + +// DoGetCloudflareClientCalls gets all the calls that were made to DoGetCloudflareClient. +// Check the length with: +// +// len(mockedInitialiser.DoGetCloudflareClientCalls()) +func (mock *InitialiserMock) DoGetCloudflareClientCalls() []struct { + Ctx context.Context + Cfg *config.Configuration +} { + var calls []struct { + Ctx context.Context + Cfg *config.Configuration + } + mock.lockDoGetCloudflareClient.RLock() + calls = mock.calls.DoGetCloudflareClient + mock.lockDoGetCloudflareClient.RUnlock() + return calls } // DoGetFilesAPIClient calls DoGetFilesAPIClientFunc. diff --git a/service/service.go b/service/service.go index 348015b5..28aca1f1 100644 --- a/service/service.go +++ b/service/service.go @@ -11,6 +11,7 @@ import ( "github.com/ONSdigital/dp-authorisation/auth" "github.com/ONSdigital/dp-dataset-api/api" "github.com/ONSdigital/dp-dataset-api/application" + "github.com/ONSdigital/dp-dataset-api/cloudflare" "github.com/ONSdigital/dp-dataset-api/config" "github.com/ONSdigital/dp-dataset-api/download" adapter "github.com/ONSdigital/dp-dataset-api/kafka" @@ -51,6 +52,7 @@ type Service struct { generateCantabularDownloadsProducer kafka.IProducer identityClient *clientsidentity.Client filesAPIClient filesAPISDK.Clienter + cloudflareClient cloudflare.Clienter server HTTPServer healthCheck HealthChecker api *api.DatasetAPI @@ -232,6 +234,10 @@ func (svc *Service) Run(ctx context.Context, buildTime, gitCommit, version strin return err } + if err := svc.initCloudflareClient(ctx); err != nil { + log.Error(ctx, "failed to initialise cloudflare client, continuing without cache purging", err) + } + ds := store.DataStore{Backend: DatsetAPIStore{svc.mongoDB, svc.graphDB}} // Get GenerateDownloads Kafka Producer @@ -319,7 +325,7 @@ func (svc *Service) Run(ctx context.Context, buildTime, gitCommit, version strin datasetPermissions, permissions := getAuthorisationHandlers(ctx, svc.config) sm := GetStateMachine(ctx, ds) svc.smDS = application.Setup(ds, smDownloadGenerators, sm) - svc.api = api.Setup(ctx, svc.config, r, ds, urlBuilder, downloadGenerators, datasetPermissions, permissions, enableURLRewriting, svc.smDS) + svc.api = api.Setup(ctx, svc.config, r, ds, urlBuilder, downloadGenerators, datasetPermissions, permissions, enableURLRewriting, svc.smDS, svc.cloudflareClient) // Set the files API client on the DatasetAPI after initialisation if svc.config.EnablePrivateEndpoints && svc.filesAPIClient != nil { @@ -380,6 +386,30 @@ func (svc *Service) initFilesAPIClient(ctx context.Context) error { return err } +func (svc *Service) initCloudflareClient(ctx context.Context) error { + if svc.config.CloudflareAPIToken == "" || svc.config.CloudflareZoneID == "" { + log.Info(ctx, "cloudflare integration disabled: missing API token or zone ID") + svc.cloudflareClient = nil + return nil + } + + log.Info(ctx, "initialising cloudflare client", log.Data{ + "api_url": svc.config.CloudflareAPIURL, + "zone_id": svc.config.CloudflareZoneID, + }) + + var err error + svc.cloudflareClient, err = svc.serviceList.GetCloudflareClient(ctx, svc.config) + + if err != nil { + log.Error(ctx, "failed to create cloudflare client", err) + return err + } + + log.Info(ctx, "cloudflare client initialised successfully") + return nil +} + func createURLBuilder(cfg *config.Configuration) (*url.Builder, error) { websiteURL, err := neturl.Parse(cfg.WebsiteURL) if err != nil { diff --git a/service/service_test.go b/service/service_test.go index 9a7aca6c..dd6afb16 100644 --- a/service/service_test.go +++ b/service/service_test.go @@ -7,6 +7,7 @@ import ( "sync" "testing" + "github.com/ONSdigital/dp-dataset-api/cloudflare" "github.com/ONSdigital/dp-dataset-api/config" "github.com/ONSdigital/dp-dataset-api/service" serviceMock "github.com/ONSdigital/dp-dataset-api/service/mock" @@ -124,6 +125,9 @@ func TestRun(t *testing.T) { Convey("Given that initialising MongoDB returns an error", func() { initMock := &serviceMock.InitialiserMock{ DoGetMongoDBFunc: funcDoGetMongoDBErr, + DoGetCloudflareClientFunc: func(ctx context.Context, cfg *config.Configuration) (cloudflare.Clienter, error) { + return nil, nil + }, } svcErrors := make(chan error, 1) svcList := service.NewServiceList(initMock) @@ -144,6 +148,9 @@ func TestRun(t *testing.T) { initMock := &serviceMock.InitialiserMock{ DoGetMongoDBFunc: funcDoGetMongoDBOk, DoGetGraphDBFunc: funcDoGetGraphDBErr, + DoGetCloudflareClientFunc: func(ctx context.Context, cfg *config.Configuration) (cloudflare.Clienter, error) { + return nil, nil + }, } svcErrors := make(chan error, 1) svcList := service.NewServiceList(initMock) @@ -165,6 +172,9 @@ func TestRun(t *testing.T) { DoGetMongoDBFunc: funcDoGetMongoDBOk, DoGetGraphDBFunc: funcDoGetGraphDBOk, DoGetFilesAPIClientFunc: funcDoGetFilesAPIClientErr, + DoGetCloudflareClientFunc: func(ctx context.Context, cfg *config.Configuration) (cloudflare.Clienter, error) { + return nil, nil + }, } svcErrors := make(chan error, 1) svcList := service.NewServiceList(initMock) @@ -187,6 +197,9 @@ func TestRun(t *testing.T) { DoGetGraphDBFunc: funcDoGetGraphDBOk, DoGetFilesAPIClientFunc: funcDoGetFilesAPIClientOk, DoGetKafkaProducerFunc: funcDoGetKafkaProducerErr, + DoGetCloudflareClientFunc: func(ctx context.Context, cfg *config.Configuration) (cloudflare.Clienter, error) { + return nil, nil + }, } svcErrors := make(chan error, 1) svcList := service.NewServiceList(initMock) @@ -210,6 +223,9 @@ func TestRun(t *testing.T) { DoGetFilesAPIClientFunc: funcDoGetFilesAPIClientOk, DoGetKafkaProducerFunc: funcDoGetKafkaProducerOk, DoGetHealthCheckFunc: funcDoGetHealthcheckErr, + DoGetCloudflareClientFunc: func(ctx context.Context, cfg *config.Configuration) (cloudflare.Clienter, error) { + return nil, nil + }, } svcErrors := make(chan error, 1) svcList := service.NewServiceList(initMock) @@ -241,6 +257,9 @@ func TestRun(t *testing.T) { DoGetHealthCheckFunc: func(*config.Configuration, string, string, string) (service.HealthChecker, error) { return hcMockAddFail, nil }, + DoGetCloudflareClientFunc: func(ctx context.Context, cfg *config.Configuration) (cloudflare.Clienter, error) { + return nil, nil + }, } svcErrors := make(chan error, 1) svcList := service.NewServiceList(initMock) @@ -273,6 +292,9 @@ func TestRun(t *testing.T) { DoGetKafkaProducerFunc: funcDoGetKafkaProducerOk, DoGetHealthCheckFunc: funcDoGetHealthcheckOk, DoGetHTTPServerFunc: funcDoGetHTTPServer, + DoGetCloudflareClientFunc: func(ctx context.Context, cfg *config.Configuration) (cloudflare.Clienter, error) { + return nil, nil + }, } svcErrors := make(chan error, 1) svcList := service.NewServiceList(initMock) @@ -312,6 +334,9 @@ func TestRun(t *testing.T) { DoGetKafkaProducerFunc: funcDoGetKafkaProducerOk, DoGetHealthCheckFunc: funcDoGetHealthcheckOk, DoGetHTTPServerFunc: funcDoGetHTTPServer, + DoGetCloudflareClientFunc: func(ctx context.Context, cfg *config.Configuration) (cloudflare.Clienter, error) { + return nil, nil + }, } svcErrors := make(chan error, 1) svcList := service.NewServiceList(initMock) @@ -347,6 +372,9 @@ func TestRun(t *testing.T) { DoGetKafkaProducerFunc: funcDoGetKafkaProducerOk, DoGetHealthCheckFunc: funcDoGetHealthcheckOk, DoGetHTTPServerFunc: funcDoGetFailingHTTPServer, + DoGetCloudflareClientFunc: func(ctx context.Context, cfg *config.Configuration) (cloudflare.Clienter, error) { + return nil, nil + }, } svcErrors := make(chan error, 1) svcList := service.NewServiceList(initMock) diff --git a/utils/utils.go b/utils/utils.go index 4753504b..91ba31d1 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -1,12 +1,8 @@ package utils import ( - "bytes" "context" - "encoding/json" "fmt" - "io" - "net/http" "net/url" "path" "strconv" @@ -798,67 +794,6 @@ func RewriteDistributions(ctx context.Context, results *[]models.Distribution, d return items, nil } -type Prefixes struct { - Prefixes []string `json:"prefixes,omitempty"` -} - -type HTTPClient interface { - Do(req *http.Request) (*http.Response, error) -} - -func PurgeCache(ctx context.Context, datasetID, editionID, baseURL, zoneID, apiToken string, client HTTPClient) error { - cloudflareURL := fmt.Sprintf("%s/zones/%s/purge_cache", baseURL, zoneID) - webDatasetURL := fmt.Sprintf("www.ons.gov.uk/datasets/%s", datasetID) - webEditionsURL := fmt.Sprintf("www.ons.gov.uk/datasets/%s/editions", datasetID) - webVersionsURL := fmt.Sprintf("www.ons.gov.uk/datasets/%s/editions/%s/versions", datasetID, editionID) - - apiDatasetURL := fmt.Sprintf("api.beta.ons.gov.uk/v1/datasets/%s", datasetID) - apiEditionsURL := fmt.Sprintf("api.beta.ons.gov.uk/v1/datasets/%s/editions", datasetID) - apiVersionsURL := fmt.Sprintf("api.beta.ons.gov.uk/v1/datasets/%s/editions/%s/versions", datasetID, editionID) - - prefixes := Prefixes{ - Prefixes: []string{ - webDatasetURL, - webEditionsURL, - webVersionsURL, - apiDatasetURL, - apiEditionsURL, - apiVersionsURL, - }, - } - - payload, err := json.Marshal(prefixes) - if err != nil { - return fmt.Errorf("failed to marshal purge prefixes: %w", err) - } - - req, err := http.NewRequestWithContext(ctx, http.MethodPost, cloudflareURL, bytes.NewReader(payload)) - if err != nil { - return fmt.Errorf("failed to create purge request: %w", err) - } - - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", apiToken)) - - resp, err := client.Do(req) - if err != nil { - return fmt.Errorf("failed to send purge request to cloudflare: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return fmt.Errorf("cloudflare API returned status %d: %s", resp.StatusCode, string(body)) - } - - log.Info(ctx, "successfully purged cloudflare cache", log.Data{ - "dataset_id": datasetID, - "edition": editionID, - }) - - return nil -} - func GenerateDistributionsDownloadURLs(datasetID, edition string, version int, distributions *[]models.Distribution) *[]models.Distribution { if distributions == nil || len(*distributions) == 0 { return distributions diff --git a/utils/utils_test.go b/utils/utils_test.go index 8e8f3850..1c4bbf43 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -2,10 +2,7 @@ package utils import ( "context" - "encoding/json" - "io" "net/http" - "net/http/httptest" neturl "net/url" "testing" "time" @@ -6243,77 +6240,6 @@ func TestRewriteDistributions_Error(t *testing.T) { }) } -func TestPurgeCache(t *testing.T) { - Convey("Given a dataset and edition", t, func() { - ctx := context.Background() - datasetID := "test-dataset" - editionID := "test-edition" - zoneID := "test-zone" - apiToken := "test-token" - baseURL := "http://base-url" - - Convey("When cache purge succeeds", func() { - var capturedRequest *http.Request - var capturedBody []byte - - mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - capturedRequest = r - capturedBody, _ = io.ReadAll(r.Body) - w.WriteHeader(http.StatusOK) - w.Write([]byte(`{"success": true}`)) - })) - defer mockServer.Close() - - mockClient := &http.Client{ - Transport: &mockTransport{server: mockServer}, - } - - err := PurgeCache(ctx, datasetID, editionID, baseURL, zoneID, apiToken, mockClient) - - Convey("Then no error is returned", func() { - So(err, ShouldBeNil) - }) - - Convey("And the request has correct headers", func() { - So(capturedRequest.Header.Get("Content-Type"), ShouldEqual, "application/json") - So(capturedRequest.Header.Get("Authorization"), ShouldEqual, "Bearer test-token") - }) - - Convey("And the request contains all 6 URL prefixes", func() { - var prefixes Prefixes - json.Unmarshal(capturedBody, &prefixes) - - So(len(prefixes.Prefixes), ShouldEqual, 6) - So(prefixes.Prefixes[0], ShouldEqual, "www.ons.gov.uk/datasets/test-dataset") - So(prefixes.Prefixes[1], ShouldEqual, "www.ons.gov.uk/datasets/test-dataset/editions") - So(prefixes.Prefixes[2], ShouldEqual, "www.ons.gov.uk/datasets/test-dataset/editions/test-edition/versions") - So(prefixes.Prefixes[3], ShouldEqual, "api.beta.ons.gov.uk/v1/datasets/test-dataset") - So(prefixes.Prefixes[4], ShouldEqual, "api.beta.ons.gov.uk/v1/datasets/test-dataset/editions") - So(prefixes.Prefixes[5], ShouldEqual, "api.beta.ons.gov.uk/v1/datasets/test-dataset/editions/test-edition/versions") - }) - }) - - Convey("When Cloudflare returns an error", func() { - mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(`{"error": "something went wrong"}`)) - })) - defer mockServer.Close() - - mockClient := &http.Client{ - Transport: &mockTransport{server: mockServer}, - } - - err := PurgeCache(ctx, datasetID, editionID, baseURL, zoneID, apiToken, mockClient) - - Convey("Then an error is returned", func() { - So(err, ShouldNotBeNil) - So(err.Error(), ShouldContainSubstring, "cloudflare API returned status 500") - }) - }) - }) -} - func TestGenerateDistributionsDownloadURLs(t *testing.T) { Convey("Given a set of distributions", t, func() { distributions := &[]models.Distribution{ @@ -6397,13 +6323,3 @@ func TestGenerateDistributionsDownloadURLs(t *testing.T) { }) }) } - -type mockTransport struct { - server *httptest.Server -} - -func (m *mockTransport) RoundTrip(req *http.Request) (*http.Response, error) { - req.URL.Scheme = "http" - req.URL.Host = m.server.URL[7:] - return http.DefaultTransport.RoundTrip(req) -} From d7e5db686d8c253bb86eca088bcfc759970a387a Mon Sep 17 00:00:00 2001 From: andre-urbani Date: Wed, 3 Dec 2025 10:04:46 +0000 Subject: [PATCH 05/10] fix audit failure --- ci/build.yml | 2 +- ci/lint.yml | 2 +- ci/unit.yml | 2 +- go.mod | 30 +++++++++++------------ go.sum | 68 ++++++++++++++++++++++++++-------------------------- 5 files changed, 52 insertions(+), 52 deletions(-) diff --git a/ci/build.yml b/ci/build.yml index 050d72bb..33ffac33 100644 --- a/ci/build.yml +++ b/ci/build.yml @@ -5,7 +5,7 @@ image_resource: type: docker-image source: repository: golang - tag: 1.24.6-bullseye + tag: 1.24.11-bullseye inputs: - name: dp-dataset-api diff --git a/ci/lint.yml b/ci/lint.yml index 070e5f9e..43345946 100644 --- a/ci/lint.yml +++ b/ci/lint.yml @@ -5,7 +5,7 @@ image_resource: type: docker-image source: repository: onsdigital/dp-concourse-tools-lint-go - tag: 1.24.6-bullseye-golangci-lint-2 + tag: 1.24.11-bullseye-golangci-lint-2 inputs: - name: dp-dataset-api diff --git a/ci/unit.yml b/ci/unit.yml index 8a839d5f..887888a8 100644 --- a/ci/unit.yml +++ b/ci/unit.yml @@ -5,7 +5,7 @@ image_resource: type: docker-image source: repository: golang - tag: 1.24.6-bullseye + tag: 1.24.11-bullseye inputs: - name: dp-dataset-api diff --git a/go.mod b/go.mod index 2198fe4d..8d7f8c1b 100644 --- a/go.mod +++ b/go.mod @@ -2,18 +2,18 @@ module github.com/ONSdigital/dp-dataset-api go 1.24.0 -toolchain go1.24.1 +toolchain go1.24.11 require ( github.com/ONSdigital/dp-api-clients-go v1.43.0 - github.com/ONSdigital/dp-api-clients-go/v2 v2.269.0 + github.com/ONSdigital/dp-api-clients-go/v2 v2.270.0 github.com/ONSdigital/dp-assistdog v0.0.1 github.com/ONSdigital/dp-authorisation v0.5.0 github.com/ONSdigital/dp-component-test v0.28.0 github.com/ONSdigital/dp-files-api v1.14.0 github.com/ONSdigital/dp-graph/v2 v2.18.0 github.com/ONSdigital/dp-healthcheck v1.6.4 - github.com/ONSdigital/dp-kafka/v4 v4.2.0 + github.com/ONSdigital/dp-kafka/v4 v4.3.0 github.com/ONSdigital/dp-mongodb-in-memory v1.8.1 github.com/ONSdigital/dp-mongodb/v3 v3.8.0 github.com/ONSdigital/dp-net/v3 v3.9.0 @@ -32,8 +32,8 @@ require ( github.com/smartystreets/goconvey v1.8.1 github.com/stretchr/testify v1.11.1 go.mongodb.org/mongo-driver v1.17.6 - go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.62.0 - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 + go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.63.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 ) require ( @@ -123,26 +123,26 @@ require ( github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect github.com/yuin/gopher-lua v1.1.1 // indirect - go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/github.com/Shopify/sarama/otelsarama v0.43.0 // indirect go.opentelemetry.io/contrib/propagators/autoprop v0.61.0 // indirect go.opentelemetry.io/contrib/propagators/aws v1.36.0 // indirect go.opentelemetry.io/contrib/propagators/b3 v1.36.0 // indirect go.opentelemetry.io/contrib/propagators/jaeger v1.36.0 // indirect go.opentelemetry.io/contrib/propagators/ot v1.36.0 // indirect - go.opentelemetry.io/otel v1.37.0 // indirect + go.opentelemetry.io/otel v1.38.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0 // indirect - go.opentelemetry.io/otel/metric v1.37.0 // indirect - go.opentelemetry.io/otel/sdk v1.37.0 // indirect - go.opentelemetry.io/otel/trace v1.37.0 // indirect + go.opentelemetry.io/otel/metric v1.38.0 // indirect + go.opentelemetry.io/otel/sdk v1.38.0 // indirect + go.opentelemetry.io/otel/trace v1.38.0 // indirect go.opentelemetry.io/proto/otlp v1.7.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.40.0 // indirect - golang.org/x/net v0.42.0 // indirect - golang.org/x/sync v0.16.0 // indirect - golang.org/x/sys v0.34.0 // indirect - golang.org/x/text v0.27.0 // indirect + golang.org/x/crypto v0.45.0 // indirect + golang.org/x/net v0.47.0 // indirect + golang.org/x/sync v0.18.0 // indirect + golang.org/x/sys v0.38.0 // indirect + golang.org/x/text v0.31.0 // indirect golang.org/x/time v0.9.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect diff --git a/go.sum b/go.sum index e7e58681..8d6b0554 100644 --- a/go.sum +++ b/go.sum @@ -5,8 +5,8 @@ github.com/ONSdigital/dp-api-clients-go v1.34.3/go.mod h1:kX+YKuoLYLfkeLHMvQKRRy github.com/ONSdigital/dp-api-clients-go v1.41.1/go.mod h1:Ga1+ANjviu21NFJI9wp5NctJIdB4TJLDGbpQFl2V8Wc= github.com/ONSdigital/dp-api-clients-go v1.43.0 h1:0982P/YxnYXvba1RhEcFmwF3xywC4eXokWQ8YH3Mm24= github.com/ONSdigital/dp-api-clients-go v1.43.0/go.mod h1:V5MfINik+o3OAF985UXUoMjXIfrZe3JKYa5AhZn5jts= -github.com/ONSdigital/dp-api-clients-go/v2 v2.269.0 h1:13QGPBu/NmIwmPhSP0yTlPEp/U1u22dEdyi8lN5guhA= -github.com/ONSdigital/dp-api-clients-go/v2 v2.269.0/go.mod h1:bLseTP21r8LCStUEeOdVPyqtrTomOFP/azPjKWW4deA= +github.com/ONSdigital/dp-api-clients-go/v2 v2.270.0 h1:kTSud/+crx9ijAHb5wOCgVx9Lg/tn7gG6pqhJGlOYPA= +github.com/ONSdigital/dp-api-clients-go/v2 v2.270.0/go.mod h1:bLseTP21r8LCStUEeOdVPyqtrTomOFP/azPjKWW4deA= github.com/ONSdigital/dp-assistdog v0.0.1 h1:5jzUWZ3EmweVjSx+5c40oLh/+WBJtD3VOErzURy+B8g= github.com/ONSdigital/dp-assistdog v0.0.1/go.mod h1:Agb8iTx42/V+eSffc/gD2oXEvOcerN3GER76FAcBr+I= github.com/ONSdigital/dp-authorisation v0.5.0 h1:k1ROJ+vgd1hDWyjj+ZdvSZGlZw73PN1k7CFVZhdyILA= @@ -25,8 +25,8 @@ github.com/ONSdigital/dp-healthcheck v1.6.4 h1:FhWOuVmob36dYq7AzCdbgyf0Vk58IFitS github.com/ONSdigital/dp-healthcheck v1.6.4/go.mod h1:j3UNbGT4ZJg1chrRkPLE6YUVYCg1su3AAQ8frcBrvgc= github.com/ONSdigital/dp-kafka/v3 v3.11.0 h1:A2rOt2c+7unDM7McFvZEl4EsS0FbT9GJhGHR40iOM1M= github.com/ONSdigital/dp-kafka/v3 v3.11.0/go.mod h1:46LmGU5t2MrVITblJHsQp3kj24FzJV6c2huzxRHMnI4= -github.com/ONSdigital/dp-kafka/v4 v4.2.0 h1:nkBJ1enfYAIOXfp59//vzwkZfnAcw77JPRAW75Fv2Zs= -github.com/ONSdigital/dp-kafka/v4 v4.2.0/go.mod h1:aLlTVci5Br9se+aLtIde+zq3rZbOUeQAZCTCm/TCmrI= +github.com/ONSdigital/dp-kafka/v4 v4.3.0 h1:QGSB3v+ySj1VzuwG1M/BZLoA3dA/nrzYRqbmEKkqrc4= +github.com/ONSdigital/dp-kafka/v4 v4.3.0/go.mod h1:XBdgWfGNQOXJCiRxWUTBiFXBsuBhwM5yQyvquHKePHY= github.com/ONSdigital/dp-mocking v0.0.0-20190905163309-fee2702ad1b9/go.mod h1:BcIRgitUju//qgNePRBmNjATarTtynAgc0yV29VpLEk= github.com/ONSdigital/dp-mocking v0.11.0 h1:laln6e2JD4vtsYbg0cTw9ur1Xf390AUYdd85cG2UNQw= github.com/ONSdigital/dp-mocking v0.11.0/go.mod h1:oHkuukWnURnK7epY5TD5oYVkOwldR2La1D5LQBTxY0A= @@ -314,8 +314,8 @@ github.com/rdumont/assistdog v0.0.0-20240711132531-b5b791dd7452 h1:ddg1HcWj+0zux github.com/rdumont/assistdog v0.0.0-20240711132531-b5b791dd7452/go.mod h1:ahNDhvMiyr24YnmrypHHoQWzK/an1pgIV/7u+0OAlug= github.com/redis/go-redis/v9 v9.11.0 h1:E3S08Gl/nJNn5vkxd2i78wZxWAPNZgUNTp8WIJUAiIs= github.com/redis/go-redis/v9 v9.11.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= -github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= -github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybLANtM3mBXNUtOfsCFXeTsnBqCsx1KM= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= @@ -363,14 +363,14 @@ github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7 go.mongodb.org/mongo-driver v1.9.1/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= go.mongodb.org/mongo-driver v1.17.6 h1:87JUG1wZfWsr6rIz3ZmpH90rL5tea7O3IHuSwHUpsss= go.mongodb.org/mongo-driver v1.17.6/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/github.com/Shopify/sarama/otelsarama v0.43.0 h1:/RxdhdIi0HrKSzdWHLjureinjnGL5YQEYevaC/EAg1k= go.opentelemetry.io/contrib/instrumentation/github.com/Shopify/sarama/otelsarama v0.43.0/go.mod h1:BKzh9a9EE+vHuq99EwD2cEa+T+Ts1fQ6W3ovO80mjkY= -go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.62.0 h1:wbJnIwX0KTq1cpPaxh5p/uPMbmWvQBYKrRd4SdI91nk= -go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.62.0/go.mod h1:PiB67AUY2rooZsFDWZ8TBmpST1KB9fyrAd1NXxANZsM= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 h1:Hf9xI/XLML9ElpiHVDNwvqI0hIFlzV8dgIr35kV1kRU= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0/go.mod h1:NfchwuyNoMcZ5MLHwPrODwUF1HWCXWrL31s8gSAdIKY= +go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.63.0 h1:rATLgFjv0P9qyXQR/aChJ6JVbMtXOQjt49GgT36cBbk= +go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.63.0/go.mod h1:34csimR1lUhdT5HH4Rii9aKPrvBcnFRwxLwcevsU+Kk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg= go.opentelemetry.io/contrib/propagators/autoprop v0.61.0 h1:cxOVDJ30qfzV27G5p9WMtJUB/3cXC0iL+u9EV1fSOws= go.opentelemetry.io/contrib/propagators/autoprop v0.61.0/go.mod h1:Y+xiUbWetg65vAroDZcIzJ5wyPNWRH32EoIV9rIaa0g= go.opentelemetry.io/contrib/propagators/aws v1.36.0 h1:Txhy/1LZIbbnutftc5pdU8Y9vOQuAkuIOFXuLsdDejs= @@ -381,22 +381,22 @@ go.opentelemetry.io/contrib/propagators/jaeger v1.36.0 h1:SoCgXYF4ISDtNyfLUzsGDa go.opentelemetry.io/contrib/propagators/jaeger v1.36.0/go.mod h1:VHu48l0YTRKSObdPQ+Sb8xMZvdnJlN7yhHuHoPgNqHM= go.opentelemetry.io/contrib/propagators/ot v1.36.0 h1:UBoZjbx483GslNKYK2YpfvePTJV4BHGeFd8+b7dexiM= go.opentelemetry.io/contrib/propagators/ot v1.36.0/go.mod h1:adDDRry19/n9WoA7mSCMjoVJcmzK/bZYzX9SR+g2+W4= -go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= -go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 h1:dNzwXjZKpMpE2JhmO+9HsPl42NIXFIFSUSSs0fiqra0= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0/go.mod h1:90PoxvaEB5n6AOdZvi+yWJQoE95U8Dhhw2bSyRqnTD0= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0 h1:JgtbA0xkWHnTmYk7YusopJFX6uleBmAuZ8n05NEh8nQ= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0/go.mod h1:179AK5aar5R3eS9FucPy6rggvU0g52cvKId8pv4+v0c= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.37.0 h1:SNhVp/9q4Go/XHBkQ1/d5u9P/U+L1yaGPoi0x+mStaI= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.37.0/go.mod h1:tx8OOlGH6R4kLV67YaYO44GFXloEjGPZuMjEkaaqIp4= -go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= -go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= -go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= -go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= -go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= -go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= -go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= -go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.38.0 h1:kJxSDN4SgWWTjG/hPp3O7LCGLcHXFlvS2/FFOrwL+SE= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.38.0/go.mod h1:mgIOzS7iZeKJdeB8/NYHrJ48fdGc71Llo5bJ1J4DWUE= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os= go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -409,8 +409,8 @@ golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= -golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= +golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= +golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -424,14 +424,14 @@ golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= -golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201008141435-b3e1573b7520/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= -golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -449,8 +449,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= -golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -462,8 +462,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= -golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From 9d92ddc4c4cf37107b912821feb1ae7e4b39628a Mon Sep 17 00:00:00 2001 From: andre-urbani Date: Wed, 3 Dec 2025 10:43:58 +0000 Subject: [PATCH 06/10] fix audit failure --- ci/build.yml | 2 +- ci/lint.yml | 2 +- ci/unit.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/build.yml b/ci/build.yml index 33ffac33..4d3001a1 100644 --- a/ci/build.yml +++ b/ci/build.yml @@ -5,7 +5,7 @@ image_resource: type: docker-image source: repository: golang - tag: 1.24.11-bullseye + tag: 1.24.11-bookworm inputs: - name: dp-dataset-api diff --git a/ci/lint.yml b/ci/lint.yml index 43345946..ef4ff5b1 100644 --- a/ci/lint.yml +++ b/ci/lint.yml @@ -5,7 +5,7 @@ image_resource: type: docker-image source: repository: onsdigital/dp-concourse-tools-lint-go - tag: 1.24.11-bullseye-golangci-lint-2 + tag: 1.24.11-bookworm-golangci-lint-2 inputs: - name: dp-dataset-api diff --git a/ci/unit.yml b/ci/unit.yml index 887888a8..438d49ff 100644 --- a/ci/unit.yml +++ b/ci/unit.yml @@ -5,7 +5,7 @@ image_resource: type: docker-image source: repository: golang - tag: 1.24.11-bullseye + tag: 1.24.11-bookworm inputs: - name: dp-dataset-api From 2fa320bd6f0d3234f84090875378e146025c2fea Mon Sep 17 00:00:00 2001 From: andre-urbani Date: Wed, 3 Dec 2025 11:08:02 +0000 Subject: [PATCH 07/10] update yml file to relative path --- ci/lint.yml | 2 +- ci/unit.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/lint.yml b/ci/lint.yml index ef4ff5b1..880d44cf 100644 --- a/ci/lint.yml +++ b/ci/lint.yml @@ -14,4 +14,4 @@ run: path: dp-dataset-api/ci/scripts/lint.sh caches: - - path: /go + - path: go/ diff --git a/ci/unit.yml b/ci/unit.yml index 438d49ff..3bac97bc 100644 --- a/ci/unit.yml +++ b/ci/unit.yml @@ -14,4 +14,4 @@ run: path: dp-dataset-api/ci/scripts/unit.sh caches: - - path: /go + - path: go/ From b8881a182d0ab53286baebc09a986ab3c3afe2d9 Mon Sep 17 00:00:00 2001 From: andre-urbani Date: Wed, 3 Dec 2025 11:18:07 +0000 Subject: [PATCH 08/10] update go.mod to fix mongo unit test issues --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 8d7f8c1b..88199d95 100644 --- a/go.mod +++ b/go.mod @@ -104,7 +104,7 @@ require ( github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect - github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/compress v1.18.2 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect @@ -115,11 +115,11 @@ require ( github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 // indirect github.com/redis/go-redis/v9 v9.11.0 // indirect github.com/smarty/assertions v1.16.0 // indirect - github.com/spf13/afero v1.14.0 // indirect + github.com/spf13/afero v1.15.0 // indirect github.com/spf13/pflag v1.0.7 // indirect github.com/square/mongo-lock v0.0.0-20230808145049-cfcf499f6bf0 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect - github.com/xdg-go/scram v1.1.2 // indirect + github.com/xdg-go/scram v1.2.0 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect github.com/yuin/gopher-lua v1.1.1 // indirect diff --git a/go.sum b/go.sum index 8d6b0554..e23094e2 100644 --- a/go.sum +++ b/go.sum @@ -268,8 +268,8 @@ github.com/justinas/alice v1.2.0/go.mod h1:fN5HRH/reO/zrUflLfTN43t3vXvKzvZIENsNE github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -326,8 +326,8 @@ github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYl github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= -github.com/spf13/afero v1.14.0 h1:9tH6MapGnn/j0eb0yIXiLjERO8RB6xIVZRDCX7PtqWA= -github.com/spf13/afero v1.14.0/go.mod h1:acJQ8t0ohCGuMN3O+Pv0V0hgMxNYDlvdk+VTfyZmbYo= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.7 h1:vN6T9TfwStFPFM5XzjsvmzZkLuaLX+HS+0SeFLRgU6M= @@ -349,8 +349,8 @@ github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhV github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= -github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= -github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/scram v1.2.0 h1:bYKF2AEwG5rqd1BumT4gAnvwU/M9nBp2pTSxeZw7Wvs= +github.com/xdg-go/scram v1.2.0/go.mod h1:3dlrS0iBaWKYVt2ZfA4cj48umJZ+cAEbR6/SjLA88I8= github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= From 7e884ed5c12f644066081fae069fb3690191ce71 Mon Sep 17 00:00:00 2001 From: andre-urbani Date: Wed, 3 Dec 2025 12:57:40 +0000 Subject: [PATCH 09/10] update to use mongo testcontainers --- go.mod | 75 ++++++++++----- go.sum | 178 +++++++++++++++++++++++++++--------- mongo/dataset_test.go | 2 +- mongo/mongo_test_helpers.go | 34 ++++--- mongo/version_store_test.go | 6 +- 5 files changed, 217 insertions(+), 78 deletions(-) diff --git a/go.mod b/go.mod index 88199d95..8d579374 100644 --- a/go.mod +++ b/go.mod @@ -2,23 +2,20 @@ module github.com/ONSdigital/dp-dataset-api go 1.24.0 -toolchain go1.24.11 - require ( github.com/ONSdigital/dp-api-clients-go v1.43.0 - github.com/ONSdigital/dp-api-clients-go/v2 v2.270.0 + github.com/ONSdigital/dp-api-clients-go/v2 v2.269.0 github.com/ONSdigital/dp-assistdog v0.0.1 github.com/ONSdigital/dp-authorisation v0.5.0 github.com/ONSdigital/dp-component-test v0.28.0 github.com/ONSdigital/dp-files-api v1.14.0 github.com/ONSdigital/dp-graph/v2 v2.18.0 github.com/ONSdigital/dp-healthcheck v1.6.4 - github.com/ONSdigital/dp-kafka/v4 v4.3.0 - github.com/ONSdigital/dp-mongodb-in-memory v1.8.1 + github.com/ONSdigital/dp-kafka/v4 v4.2.0 github.com/ONSdigital/dp-mongodb/v3 v3.8.0 github.com/ONSdigital/dp-net/v3 v3.9.0 github.com/ONSdigital/dp-otel-go v0.0.8 - github.com/ONSdigital/log.go/v2 v2.5.1 + github.com/ONSdigital/log.go/v2 v2.5.0 github.com/cloudflare/cloudflare-go v0.116.0 github.com/cucumber/godog v0.15.1 github.com/golang/glog v1.2.5 @@ -31,15 +28,20 @@ require ( github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b github.com/smartystreets/goconvey v1.8.1 github.com/stretchr/testify v1.11.1 + github.com/testcontainers/testcontainers-go/modules/mongodb v0.40.0 go.mongodb.org/mongo-driver v1.17.6 - go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.63.0 - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 + go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.62.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 ) require ( + dario.cat/mergo v1.0.2 // indirect + github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect github.com/ONSdigital/dis-redis v0.3.0 // indirect github.com/ONSdigital/dp-authorisation/v2 v2.32.3 // indirect github.com/ONSdigital/dp-kafka/v3 v3.11.0 // indirect + github.com/ONSdigital/dp-mongodb-in-memory v1.8.1 // indirect github.com/ONSdigital/dp-net/v2 v2.22.0 // indirect github.com/ONSdigital/dp-permissions-api v1.3.0 // indirect github.com/ONSdigital/golang-neo4j-bolt-driver v0.0.0-20241121114036-9f4b82bb9d37 // indirect @@ -58,24 +60,36 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.11 // indirect github.com/aws/aws-sdk-go-v2/service/s3 v1.89.0 // indirect github.com/aws/smithy-go v1.23.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cenkalti/backoff/v5 v5.0.2 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chromedp/cdproto v0.0.0-20250724212937-08a3db8b4327 // indirect github.com/chromedp/chromedp v0.14.0 // indirect github.com/chromedp/sysutil v1.1.0 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/platforms v0.2.1 // indirect + github.com/cpuguy83/dockercfg v0.3.2 // indirect github.com/cucumber/gherkin/go/v26 v26.2.0 // indirect github.com/cucumber/messages/go/v21 v21.0.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/docker v28.5.1+incompatible // indirect + github.com/docker/go-connections v0.6.0 // indirect + github.com/docker/go-units v0.5.0 // indirect github.com/eapache/go-resiliency v1.7.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect + github.com/ebitengine/purego v0.8.4 // indirect github.com/fatih/color v1.18.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-avro/avro v0.0.0-20171219232920-444163702c11 // indirect github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator v9.31.0+incompatible // indirect @@ -104,45 +118,64 @@ require ( github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect - github.com/klauspost/compress v1.18.2 // indirect + github.com/klauspost/compress v1.18.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/magiconair/properties v1.8.10 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/maxcnunes/httpfake v1.2.4 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/go-archive v0.1.0 // indirect + github.com/moby/patternmatcher v0.6.0 // indirect + github.com/moby/sys/sequential v0.6.0 // indirect + github.com/moby/sys/user v0.4.0 // indirect + github.com/moby/sys/userns v0.1.0 // indirect + github.com/moby/term v0.5.0 // indirect github.com/montanaflynn/stats v0.7.1 // indirect + github.com/morikuni/aec v1.0.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 // indirect github.com/redis/go-redis/v9 v9.11.0 // indirect + github.com/shirou/gopsutil/v4 v4.25.6 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect github.com/smarty/assertions v1.16.0 // indirect - github.com/spf13/afero v1.15.0 // indirect + github.com/spf13/afero v1.14.0 // indirect github.com/spf13/pflag v1.0.7 // indirect github.com/square/mongo-lock v0.0.0-20230808145049-cfcf499f6bf0 // indirect + github.com/testcontainers/testcontainers-go v0.40.0 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect - github.com/xdg-go/scram v1.2.0 // indirect + github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect github.com/yuin/gopher-lua v1.1.1 // indirect - go.opentelemetry.io/auto/sdk v1.2.1 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/contrib/instrumentation/github.com/Shopify/sarama/otelsarama v0.43.0 // indirect go.opentelemetry.io/contrib/propagators/autoprop v0.61.0 // indirect go.opentelemetry.io/contrib/propagators/aws v1.36.0 // indirect go.opentelemetry.io/contrib/propagators/b3 v1.36.0 // indirect go.opentelemetry.io/contrib/propagators/jaeger v1.36.0 // indirect go.opentelemetry.io/contrib/propagators/ot v1.36.0 // indirect - go.opentelemetry.io/otel v1.38.0 // indirect + go.opentelemetry.io/otel v1.37.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0 // indirect - go.opentelemetry.io/otel/metric v1.38.0 // indirect - go.opentelemetry.io/otel/sdk v1.38.0 // indirect - go.opentelemetry.io/otel/trace v1.38.0 // indirect + go.opentelemetry.io/otel/metric v1.37.0 // indirect + go.opentelemetry.io/otel/sdk v1.37.0 // indirect + go.opentelemetry.io/otel/trace v1.37.0 // indirect go.opentelemetry.io/proto/otlp v1.7.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.45.0 // indirect - golang.org/x/net v0.47.0 // indirect - golang.org/x/sync v0.18.0 // indirect - golang.org/x/sys v0.38.0 // indirect - golang.org/x/text v0.31.0 // indirect + golang.org/x/crypto v0.43.0 // indirect + golang.org/x/net v0.45.0 // indirect + golang.org/x/sync v0.17.0 // indirect + golang.org/x/sys v0.37.0 // indirect + golang.org/x/text v0.30.0 // indirect golang.org/x/time v0.9.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect diff --git a/go.sum b/go.sum index e23094e2..58266bcb 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,11 @@ +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/ONSdigital/dis-redis v0.3.0 h1:aI0d3MsXPmRbe+okYAbrwEGBibhoCw+gfbgtJ3mA11c= github.com/ONSdigital/dis-redis v0.3.0/go.mod h1:CLbCwaEfJhifBM7PufwNi0mymys+xM6xNgwhihhSIHQ= github.com/ONSdigital/dp-api-clients-go v1.28.0/go.mod h1:iyJy6uRL4B6OYOJA0XMr5UHt6+Q8XmN9uwmURO+9Oj4= @@ -5,8 +13,8 @@ github.com/ONSdigital/dp-api-clients-go v1.34.3/go.mod h1:kX+YKuoLYLfkeLHMvQKRRy github.com/ONSdigital/dp-api-clients-go v1.41.1/go.mod h1:Ga1+ANjviu21NFJI9wp5NctJIdB4TJLDGbpQFl2V8Wc= github.com/ONSdigital/dp-api-clients-go v1.43.0 h1:0982P/YxnYXvba1RhEcFmwF3xywC4eXokWQ8YH3Mm24= github.com/ONSdigital/dp-api-clients-go v1.43.0/go.mod h1:V5MfINik+o3OAF985UXUoMjXIfrZe3JKYa5AhZn5jts= -github.com/ONSdigital/dp-api-clients-go/v2 v2.270.0 h1:kTSud/+crx9ijAHb5wOCgVx9Lg/tn7gG6pqhJGlOYPA= -github.com/ONSdigital/dp-api-clients-go/v2 v2.270.0/go.mod h1:bLseTP21r8LCStUEeOdVPyqtrTomOFP/azPjKWW4deA= +github.com/ONSdigital/dp-api-clients-go/v2 v2.269.0 h1:13QGPBu/NmIwmPhSP0yTlPEp/U1u22dEdyi8lN5guhA= +github.com/ONSdigital/dp-api-clients-go/v2 v2.269.0/go.mod h1:bLseTP21r8LCStUEeOdVPyqtrTomOFP/azPjKWW4deA= github.com/ONSdigital/dp-assistdog v0.0.1 h1:5jzUWZ3EmweVjSx+5c40oLh/+WBJtD3VOErzURy+B8g= github.com/ONSdigital/dp-assistdog v0.0.1/go.mod h1:Agb8iTx42/V+eSffc/gD2oXEvOcerN3GER76FAcBr+I= github.com/ONSdigital/dp-authorisation v0.5.0 h1:k1ROJ+vgd1hDWyjj+ZdvSZGlZw73PN1k7CFVZhdyILA= @@ -25,8 +33,8 @@ github.com/ONSdigital/dp-healthcheck v1.6.4 h1:FhWOuVmob36dYq7AzCdbgyf0Vk58IFitS github.com/ONSdigital/dp-healthcheck v1.6.4/go.mod h1:j3UNbGT4ZJg1chrRkPLE6YUVYCg1su3AAQ8frcBrvgc= github.com/ONSdigital/dp-kafka/v3 v3.11.0 h1:A2rOt2c+7unDM7McFvZEl4EsS0FbT9GJhGHR40iOM1M= github.com/ONSdigital/dp-kafka/v3 v3.11.0/go.mod h1:46LmGU5t2MrVITblJHsQp3kj24FzJV6c2huzxRHMnI4= -github.com/ONSdigital/dp-kafka/v4 v4.3.0 h1:QGSB3v+ySj1VzuwG1M/BZLoA3dA/nrzYRqbmEKkqrc4= -github.com/ONSdigital/dp-kafka/v4 v4.3.0/go.mod h1:XBdgWfGNQOXJCiRxWUTBiFXBsuBhwM5yQyvquHKePHY= +github.com/ONSdigital/dp-kafka/v4 v4.2.0 h1:nkBJ1enfYAIOXfp59//vzwkZfnAcw77JPRAW75Fv2Zs= +github.com/ONSdigital/dp-kafka/v4 v4.2.0/go.mod h1:aLlTVci5Br9se+aLtIde+zq3rZbOUeQAZCTCm/TCmrI= github.com/ONSdigital/dp-mocking v0.0.0-20190905163309-fee2702ad1b9/go.mod h1:BcIRgitUju//qgNePRBmNjATarTtynAgc0yV29VpLEk= github.com/ONSdigital/dp-mocking v0.11.0 h1:laln6e2JD4vtsYbg0cTw9ur1Xf390AUYdd85cG2UNQw= github.com/ONSdigital/dp-mocking v0.11.0/go.mod h1:oHkuukWnURnK7epY5TD5oYVkOwldR2La1D5LQBTxY0A= @@ -65,8 +73,8 @@ github.com/ONSdigital/log.go v1.1.0/go.mod h1:0hOVuYR3bDUI30VRo48d5KHfJIoe+spuPX github.com/ONSdigital/log.go/v2 v2.0.0/go.mod h1:PR7vXrv9dZKUc7SI/0toxBbStk84snmybBnWpe+xY2o= github.com/ONSdigital/log.go/v2 v2.0.5/go.mod h1:PR7vXrv9dZKUc7SI/0toxBbStk84snmybBnWpe+xY2o= github.com/ONSdigital/log.go/v2 v2.0.9/go.mod h1:VyTDkL82FtiAkaNFaT+bURBhLbP7NsIx4rkVbdpiuEg= -github.com/ONSdigital/log.go/v2 v2.5.1 h1:GCM270UHSP5+mv4OaQ2oHiWp0FiSgfnM6imW2zpAslw= -github.com/ONSdigital/log.go/v2 v2.5.1/go.mod h1:KZNEweCUHD8dKwhlvoRvgd2Y2aUIuU3H9/MmbFyVzW8= +github.com/ONSdigital/log.go/v2 v2.5.0 h1:gFHAn6tLOzkhC9hiAFgFxzNBh5Uz06KyULQ9aQyM9tE= +github.com/ONSdigital/log.go/v2 v2.5.0/go.mod h1:0ilpZzc5lVoBlXC/s5m8EaQETbe0yT8Z+p4QhKy0fpY= github.com/Shopify/sarama v1.38.1 h1:lqqPUPQZ7zPqYlWpTh+LQ9bhYNu2xJL6k1SJN4WVe2A= github.com/Shopify/sarama v1.38.1/go.mod h1:iwv9a67Ha8VNa+TifujYoWGxWnu2kNVAQdSdZ4X2o5g= github.com/Shopify/toxiproxy/v2 v2.5.0 h1:i4LPT+qrSlKNtQf5QliVjdP08GyAH8+BUIc9gT0eahc= @@ -115,6 +123,8 @@ github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cenkalti/backoff/v5 v5.0.2 h1:rIfFVxEf1QsI7E1ZHfp/B4DF/6QBAUhmgkxc0H7Zss8= github.com/cenkalti/backoff/v5 v5.0.2/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -127,7 +137,19 @@ github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipw github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8= github.com/cloudflare/cloudflare-go v0.116.0 h1:iRPMnTtnswRpELO65NTwMX4+RTdxZl+Xf/zi+HPE95s= github.com/cloudflare/cloudflare-go v0.116.0/go.mod h1:Ds6urDwn/TF2uIU24mu7H91xkKP8gSAHxQ44DSZgVmU= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= +github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/cucumber/gherkin/go/v26 v26.2.0 h1:EgIjePLWiPeslwIWmNQ3XHcypPsWAHoMCz/YEBKP4GI= github.com/cucumber/gherkin/go/v26 v26.2.0/go.mod h1:t2GAPnB8maCT4lkHL99BDCVNzCh1d7dBhCLt150Nr/0= github.com/cucumber/godog v0.15.1 h1:rb/6oHDdvVZKS66hrhpjFQFHjthFSrQBCOI1LwshNTI= @@ -143,12 +165,22 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v28.5.1+incompatible h1:Bm8DchhSD2J6PsFzxC35TZo4TLGR2PdW/E69rU45NhM= +github.com/docker/docker v28.5.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= +github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/eapache/go-resiliency v1.7.0 h1:n3NRTnBn5N0Cbi/IeOHuQn9s2UwVUH7Ga0ZWcP+9JTA= github.com/eapache/go-resiliency v1.7.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4ALJ04o5Qqpdz8XLIpNA3WM/iSIXqxtqo7UGVws= github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/ebitengine/purego v0.8.4 h1:CF7LEKg5FFOsASUj0+QwaXf8Ht6TlFxg09+S9wz0omw= +github.com/ebitengine/purego v0.8.4/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9/go.mod h1:uPmAp6Sws4L7+Q/OokbWDAK1ibXYhB3PXFP1kol5hPg= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= @@ -168,6 +200,8 @@ github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= @@ -204,6 +238,7 @@ github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= @@ -268,8 +303,8 @@ github.com/justinas/alice v1.2.0/go.mod h1:fN5HRH/reO/zrUflLfTN43t3vXvKzvZIENsNE github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= -github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -282,6 +317,10 @@ github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kUL github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= +github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= @@ -296,9 +335,31 @@ github.com/maxcnunes/httpfake v1.2.4 h1:l7s/N7zuG6XpzG+5dUolg5SSoR3hANQxqzAkv+lR github.com/maxcnunes/httpfake v1.2.4/go.mod h1:rWVxb0bLKtOUM/5hN3UO1VEdEitz1hfcTXs7UyiK6r0= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/go-archive v0.1.0 h1:Kk/5rdW/g+H8NHdJW2gsXyZ7UnzvJNOy6VKJqueWdcQ= +github.com/moby/go-archive v0.1.0/go.mod h1:G9B+YoujNohJmrIYFBpSd54GTUB4lt9S+xVQvsJyFuo= +github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= +github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= +github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= +github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= +github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= +github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= +github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw= github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= @@ -308,17 +369,23 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 h1:bsUq1dX0N8AOIL7EB/X911+m4EHsnWEHeJ0c+3TTBrg= github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rdumont/assistdog v0.0.0-20240711132531-b5b791dd7452 h1:ddg1HcWj+0zuxGEUuOoZlJu6iTjqhwYkCPVAjBPSrS8= github.com/rdumont/assistdog v0.0.0-20240711132531-b5b791dd7452/go.mod h1:ahNDhvMiyr24YnmrypHHoQWzK/an1pgIV/7u+0OAlug= github.com/redis/go-redis/v9 v9.11.0 h1:E3S08Gl/nJNn5vkxd2i78wZxWAPNZgUNTp8WIJUAiIs= github.com/redis/go-redis/v9 v9.11.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= -github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= -github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybLANtM3mBXNUtOfsCFXeTsnBqCsx1KM= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/shirou/gopsutil/v4 v4.25.6 h1:kLysI2JsKorfaFPcYmcJqbzROzsBWEOAtw6A7dIfqXs= +github.com/shirou/gopsutil/v4 v4.25.6/go.mod h1:PfybzyydfZcN+JMMjkF6Zb8Mq1A/VcogFFg7hj50W9c= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smarty/assertions v1.16.0 h1:EvHNkdRA4QHMrn75NZSoUQ/mAUXAYWfatfB01yTCzfY= github.com/smarty/assertions v1.16.0/go.mod h1:duaaFdCS0K9dnoM50iyek/eYINOZ64gbh1Xlf6LG7AI= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -326,8 +393,8 @@ github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYl github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= -github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= -github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= +github.com/spf13/afero v1.14.0 h1:9tH6MapGnn/j0eb0yIXiLjERO8RB6xIVZRDCX7PtqWA= +github.com/spf13/afero v1.14.0/go.mod h1:acJQ8t0ohCGuMN3O+Pv0V0hgMxNYDlvdk+VTfyZmbYo= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.7 h1:vN6T9TfwStFPFM5XzjsvmzZkLuaLX+HS+0SeFLRgU6M= @@ -337,20 +404,31 @@ github.com/square/mongo-lock v0.0.0-20230808145049-cfcf499f6bf0/go.mod h1:bLPJcG github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/testcontainers/testcontainers-go v0.40.0 h1:pSdJYLOVgLE8YdUY2FHQ1Fxu+aMnb6JfVz1mxk7OeMU= +github.com/testcontainers/testcontainers-go v0.40.0/go.mod h1:FSXV5KQtX2HAMlm7U3APNyLkkap35zNLxukw9oBi/MY= +github.com/testcontainers/testcontainers-go/modules/mongodb v0.40.0 h1:z/1qHeliTLDKNaJ7uOHOx1FjwghbcbYfga4dTFkF0hU= +github.com/testcontainers/testcontainers-go/modules/mongodb v0.40.0/go.mod h1:GaunAWwMXLtsMKG3xn2HYIBDbKddGArfcGsF2Aog81E= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= -github.com/xdg-go/scram v1.2.0 h1:bYKF2AEwG5rqd1BumT4gAnvwU/M9nBp2pTSxeZw7Wvs= -github.com/xdg-go/scram v1.2.0/go.mod h1:3dlrS0iBaWKYVt2ZfA4cj48umJZ+cAEbR6/SjLA88I8= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= @@ -360,17 +438,21 @@ github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfS github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.mongodb.org/mongo-driver v1.9.1/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= go.mongodb.org/mongo-driver v1.17.6 h1:87JUG1wZfWsr6rIz3ZmpH90rL5tea7O3IHuSwHUpsss= go.mongodb.org/mongo-driver v1.17.6/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= -go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= -go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.mongodb.org/mongo-driver/v2 v2.3.0 h1:sh55yOXA2vUjW1QYw/2tRlHSQViwDyPnW61AwpZ4rtU= +go.mongodb.org/mongo-driver/v2 v2.3.0/go.mod h1:jHeEDJHJq7tm6ZF45Issun9dbogjfnPySb1vXA7EeAI= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= go.opentelemetry.io/contrib/instrumentation/github.com/Shopify/sarama/otelsarama v0.43.0 h1:/RxdhdIi0HrKSzdWHLjureinjnGL5YQEYevaC/EAg1k= go.opentelemetry.io/contrib/instrumentation/github.com/Shopify/sarama/otelsarama v0.43.0/go.mod h1:BKzh9a9EE+vHuq99EwD2cEa+T+Ts1fQ6W3ovO80mjkY= -go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.63.0 h1:rATLgFjv0P9qyXQR/aChJ6JVbMtXOQjt49GgT36cBbk= -go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.63.0/go.mod h1:34csimR1lUhdT5HH4Rii9aKPrvBcnFRwxLwcevsU+Kk= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG0FI8OiXhBfcRtqqHcZcka+gU3cskNuf05R18= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg= +go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.62.0 h1:wbJnIwX0KTq1cpPaxh5p/uPMbmWvQBYKrRd4SdI91nk= +go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.62.0/go.mod h1:PiB67AUY2rooZsFDWZ8TBmpST1KB9fyrAd1NXxANZsM= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 h1:Hf9xI/XLML9ElpiHVDNwvqI0hIFlzV8dgIr35kV1kRU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0/go.mod h1:NfchwuyNoMcZ5MLHwPrODwUF1HWCXWrL31s8gSAdIKY= go.opentelemetry.io/contrib/propagators/autoprop v0.61.0 h1:cxOVDJ30qfzV27G5p9WMtJUB/3cXC0iL+u9EV1fSOws= go.opentelemetry.io/contrib/propagators/autoprop v0.61.0/go.mod h1:Y+xiUbWetg65vAroDZcIzJ5wyPNWRH32EoIV9rIaa0g= go.opentelemetry.io/contrib/propagators/aws v1.36.0 h1:Txhy/1LZIbbnutftc5pdU8Y9vOQuAkuIOFXuLsdDejs= @@ -381,22 +463,24 @@ go.opentelemetry.io/contrib/propagators/jaeger v1.36.0 h1:SoCgXYF4ISDtNyfLUzsGDa go.opentelemetry.io/contrib/propagators/jaeger v1.36.0/go.mod h1:VHu48l0YTRKSObdPQ+Sb8xMZvdnJlN7yhHuHoPgNqHM= go.opentelemetry.io/contrib/propagators/ot v1.36.0 h1:UBoZjbx483GslNKYK2YpfvePTJV4BHGeFd8+b7dexiM= go.opentelemetry.io/contrib/propagators/ot v1.36.0/go.mod h1:adDDRry19/n9WoA7mSCMjoVJcmzK/bZYzX9SR+g2+W4= -go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= -go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 h1:dNzwXjZKpMpE2JhmO+9HsPl42NIXFIFSUSSs0fiqra0= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0/go.mod h1:90PoxvaEB5n6AOdZvi+yWJQoE95U8Dhhw2bSyRqnTD0= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0 h1:JgtbA0xkWHnTmYk7YusopJFX6uleBmAuZ8n05NEh8nQ= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0/go.mod h1:179AK5aar5R3eS9FucPy6rggvU0g52cvKId8pv4+v0c= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.38.0 h1:kJxSDN4SgWWTjG/hPp3O7LCGLcHXFlvS2/FFOrwL+SE= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.38.0/go.mod h1:mgIOzS7iZeKJdeB8/NYHrJ48fdGc71Llo5bJ1J4DWUE= -go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= -go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= -go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= -go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= -go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= -go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= -go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= -go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.37.0 h1:SNhVp/9q4Go/XHBkQ1/d5u9P/U+L1yaGPoi0x+mStaI= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.37.0/go.mod h1:tx8OOlGH6R4kLV67YaYO44GFXloEjGPZuMjEkaaqIp4= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= +go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= +go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= +go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os= go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -409,8 +493,8 @@ golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= -golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= +golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -424,37 +508,45 @@ golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= -golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/net v0.45.0 h1:RLBg5JKixCy82FtLJpeNlVM0nrSqpCRYzVU1n8kj0tM= +golang.org/x/net v0.45.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201008141435-b3e1573b7520/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= -golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210414055047-fe65e336abe0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= +golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q= +golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -462,8 +554,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= +golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -495,3 +587,5 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= diff --git a/mongo/dataset_test.go b/mongo/dataset_test.go index 57020b87..5b897f5c 100644 --- a/mongo/dataset_test.go +++ b/mongo/dataset_test.go @@ -500,7 +500,7 @@ func TestIsStaticDataset(t *testing.T) { mongo, server, err := getTestMongoDB(ctx) So(err, ShouldBeNil) defer func() { - server.Stop(ctx) + server.Terminate(ctx) }() Convey("When IsStaticDataset is called with a static dataset ID", func() { diff --git a/mongo/mongo_test_helpers.go b/mongo/mongo_test_helpers.go index 92f5ae38..72c587e0 100644 --- a/mongo/mongo_test_helpers.go +++ b/mongo/mongo_test_helpers.go @@ -3,12 +3,13 @@ package mongo import ( "context" "fmt" + "net/url" "time" "github.com/ONSdigital/dp-dataset-api/config" "github.com/ONSdigital/dp-dataset-api/models" - mim "github.com/ONSdigital/dp-mongodb-in-memory" mongoDriver "github.com/ONSdigital/dp-mongodb/v3/mongodb" + testMongoContainer "github.com/testcontainers/testcontainers-go/modules/mongodb" ) var ( @@ -19,8 +20,8 @@ var ( unpublishedStaticID = "unpublished-static-id" ) -// getTestMongoDB initializes a MongoDB connection for use in tests -func getTestMongoDB(ctx context.Context) (*Mongo, *mim.Server, error) { +// getTestMongoDB initializes a MongoDB connection for use in tests using testcontainers +func getTestMongoDB(ctx context.Context) (*Mongo, *testMongoContainer.MongoDBContainer, error) { mongoVersion := "4.4.8" cfg, err := config.Get() @@ -28,28 +29,39 @@ func getTestMongoDB(ctx context.Context) (*Mongo, *mim.Server, error) { return nil, nil, err } - mongoServer, err := mim.Start(ctx, mongoVersion) + mongoContainer, err := testMongoContainer.Run(ctx, fmt.Sprintf("mongo:%s", mongoVersion)) if err != nil { - return nil, nil, err + return nil, nil, fmt.Errorf("failed to start mongo container: %w", err) } - mongoConfig := getTestMongoDriverConfig(mongoServer, cfg.Database, cfg.Collections) + + mongoConfig := getTestMongoDriverConfig(mongoContainer, cfg.Database, cfg.Collections) conn, err := mongoDriver.Open(mongoConfig) if err != nil { - return nil, nil, err + return nil, nil, fmt.Errorf("failed to open mongo connection: %w", err) } return &Mongo{ MongoConfig: cfg.MongoConfig, Connection: conn, - }, mongoServer, nil + }, mongoContainer, nil } -// Custom config to work with mongo in memory -func getTestMongoDriverConfig(mongoServer *mim.Server, database string, collections map[string]string) *mongoDriver.MongoDriverConfig { +// Custom config to work with testcontainers +func getTestMongoDriverConfig(mongoContainer *testMongoContainer.MongoDBContainer, database string, collections map[string]string) *mongoDriver.MongoDriverConfig { + connectionString, err := mongoContainer.ConnectionString(context.Background()) + if err != nil { + panic(fmt.Sprintf("failed to get connection string: %v", err)) + } + + connStringURL, err := url.Parse(connectionString) + if err != nil { + panic(fmt.Sprintf("failed to parse connection string: %v", err)) + } + return &mongoDriver.MongoDriverConfig{ ConnectTimeout: 5 * time.Second, QueryTimeout: 5 * time.Second, - ClusterEndpoint: mongoServer.URI(), + ClusterEndpoint: connStringURL.Host, Database: database, Collections: collections, } diff --git a/mongo/version_store_test.go b/mongo/version_store_test.go index 5a041ac7..9f386b3b 100644 --- a/mongo/version_store_test.go +++ b/mongo/version_store_test.go @@ -113,7 +113,7 @@ func TestGetAllStaticVersions(t *testing.T) { So(err, ShouldBeNil) defer func() { - server.Stop(ctx) + server.Terminate(ctx) }() versions, err := setupVersionsTestData(ctx, mongoStore) @@ -205,7 +205,7 @@ func TestDeleteStaticDatasetVersion(t *testing.T) { mongoStore, server, err := getTestMongoDB(ctx) So(err, ShouldBeNil) defer func() { - server.Stop(ctx) + server.Terminate(ctx) }() versions, err := setupVersionsTestData(ctx, mongoStore) @@ -233,7 +233,7 @@ func TestCheckEditionTitleExistsStatic(t *testing.T) { So(err, ShouldBeNil) defer func() { - server.Stop(ctx) + server.Terminate(ctx) }() versions, err := setupVersionsTestData(ctx, mongo) From bc7951f89b3414be485c596ecd57c89060fc8276 Mon Sep 17 00:00:00 2001 From: andre-urbani Date: Wed, 3 Dec 2025 13:12:46 +0000 Subject: [PATCH 10/10] update unit.yml file --- ci/unit.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/ci/unit.yml b/ci/unit.yml index 3bac97bc..0e0e147b 100644 --- a/ci/unit.yml +++ b/ci/unit.yml @@ -4,14 +4,19 @@ platform: linux image_resource: type: docker-image source: - repository: golang - tag: 1.24.11-bookworm + repository: onsdigital/dp-concourse-tools-docker-go + tag: 1.24.11-dind-24 inputs: - name: dp-dataset-api run: - path: dp-dataset-api/ci/scripts/unit.sh + path: bash + args: + - -exc + - | + /start_docker.sh + dp-dataset-api/ci/scripts/unit.sh caches: - - path: go/ + - path: go/ \ No newline at end of file