diff --git a/api/api.go b/api/api.go index e9c2a628..b9e0ebbc 100644 --- a/api/api.go +++ b/api/api.go @@ -10,6 +10,7 @@ import ( "github.com/ONSdigital/dp-authorisation/auth" "github.com/ONSdigital/dp-dataset-api/application" + "github.com/ONSdigital/dp-dataset-api/cloudflare" "github.com/ONSdigital/dp-dataset-api/config" "github.com/ONSdigital/dp-dataset-api/dimension" "github.com/ONSdigital/dp-dataset-api/instance" @@ -77,10 +78,11 @@ type DatasetAPI struct { smDatasetAPI *application.StateMachineDatasetAPI filesAPIClient filesAPISDK.Clienter authToken string + cloudflareClient cloudflare.Clienter } // Setup creates a new Dataset API instance and register the API routes based on the application configuration. -func Setup(ctx context.Context, cfg *config.Configuration, router *mux.Router, dataStore store.DataStore, urlBuilder *url.Builder, downloadGenerators map[models.DatasetType]DownloadsGenerator, datasetPermissions, permissions AuthHandler, enableURLRewriting bool, smDatasetAPI *application.StateMachineDatasetAPI) *DatasetAPI { +func Setup(ctx context.Context, cfg *config.Configuration, router *mux.Router, dataStore store.DataStore, urlBuilder *url.Builder, downloadGenerators map[models.DatasetType]DownloadsGenerator, datasetPermissions, permissions AuthHandler, enableURLRewriting bool, smDatasetAPI *application.StateMachineDatasetAPI, cloudflareClient cloudflare.Clienter) *DatasetAPI { api := &DatasetAPI{ dataStore: dataStore, host: cfg.DatasetAPIURL, @@ -100,6 +102,7 @@ func Setup(ctx context.Context, cfg *config.Configuration, router *mux.Router, d MaxRequestOptions: cfg.MaxRequestOptions, defaultLimit: cfg.DefaultLimit, smDatasetAPI: smDatasetAPI, + cloudflareClient: cloudflareClient, } paginator := pagination.NewPaginator(cfg.DefaultLimit, cfg.DefaultOffset, cfg.DefaultMaxLimit) diff --git a/api/dataset_test.go b/api/dataset_test.go index 3f55a02d..331311fc 100644 --- a/api/dataset_test.go +++ b/api/dataset_test.go @@ -161,7 +161,7 @@ func GetAPIWithCMDMocks(mockedDataStore store.Storer, mockedGeneratedDownloads D StateMachine: application.NewStateMachine(testContext, states, transitions, store.DataStore{Backend: mockedDataStore}), } - return Setup(testContext, cfg, mux.NewRouter(), store.DataStore{Backend: mockedDataStore}, urlBuilder, mockedMapGeneratedDownloads, datasetPermissions, permissions, enableURLRewriting, &mockStatemachineDatasetAPI) + return Setup(testContext, cfg, mux.NewRouter(), store.DataStore{Backend: mockedDataStore}, urlBuilder, mockedMapGeneratedDownloads, datasetPermissions, permissions, enableURLRewriting, &mockStatemachineDatasetAPI, nil) } // GetAPIWithCMDMocks also used in other tests, so exported @@ -217,7 +217,7 @@ func GetAPIWithCantabularMocks(mockedDataStore store.Storer, mockedGeneratedDown StateMachine: application.NewStateMachine(testContext, states, transitions, store.DataStore{Backend: mockedDataStore}), } - return Setup(testContext, cfg, mux.NewRouter(), store.DataStore{Backend: mockedDataStore}, urlBuilder, mockedMapGeneratedDownloads, datasetPermissions, permissions, enableURLRewriting, &mockStatemachineDatasetAPI) + return Setup(testContext, cfg, mux.NewRouter(), store.DataStore{Backend: mockedDataStore}, urlBuilder, mockedMapGeneratedDownloads, datasetPermissions, permissions, enableURLRewriting, &mockStatemachineDatasetAPI, nil) } func createRequestWithAuth(method, target string, body io.Reader) *http.Request { diff --git a/api/versions.go b/api/versions.go index f9f11e0f..4a340ec0 100644 --- a/api/versions.go +++ b/api/versions.go @@ -830,19 +830,27 @@ func (api *DatasetAPI) putState(w http.ResponseWriter, r *http.Request) { Type: models.Static.String(), } - updatedVersion, err := api.smDatasetAPI.AmendVersion(r.Context(), vars, versionUpdate) + _, err = api.smDatasetAPI.AmendVersion(r.Context(), vars, versionUpdate) if err != nil { handleVersionAPIErr(ctx, err, w, logData) return } - if stateUpdate.State == models.PublishedState && updatedVersion.Distributions != nil && len(*updatedVersion.Distributions) > 0 { - err = api.publishDistributionFiles(ctx, updatedVersion, logData) + if stateUpdate.State == models.PublishedState && currentVersion.Distributions != nil && len(*currentVersion.Distributions) > 0 { + err = api.publishDistributionFiles(ctx, currentVersion, logData) if err != nil { log.Error(ctx, "putState endpoint: failed to publish distribution files", err, logData) handleVersionAPIErr(ctx, err, w, logData) return } + + if api.cloudflareClient != nil { + if err := api.cloudflareClient.PurgeCacheByPrefix(ctx, datasetID, edition); err != nil { + log.Error(ctx, "failed to purge cloudflare cache", err, logData) + } else { + log.Info(ctx, "successfully triggered cloudflare cache purge", logData) + } + } } setJSONContentType(w) diff --git a/api/webendpoints_test.go b/api/webendpoints_test.go index 85c5af0c..ff024a04 100644 --- a/api/webendpoints_test.go +++ b/api/webendpoints_test.go @@ -368,5 +368,5 @@ func GetWebAPIWithMocks(ctx context.Context, mockedDataStore store.Storer, mocke cfg.DatasetAPIURL = host cfg.EnablePrivateEndpoints = false - return Setup(ctx, cfg, mux.NewRouter(), store.DataStore{Backend: mockedDataStore}, urlBuilder, mockedMapDownloadGenerators, datasetPermissions, permissions, enableURLRewriting, &mockStatemachineDatasetAPI) + return Setup(ctx, cfg, mux.NewRouter(), store.DataStore{Backend: mockedDataStore}, urlBuilder, mockedMapDownloadGenerators, datasetPermissions, permissions, enableURLRewriting, &mockStatemachineDatasetAPI, nil) } diff --git a/ci/build.yml b/ci/build.yml index 050d72bb..4d3001a1 100644 --- a/ci/build.yml +++ b/ci/build.yml @@ -5,7 +5,7 @@ image_resource: type: docker-image source: repository: golang - tag: 1.24.6-bullseye + tag: 1.24.11-bookworm inputs: - name: dp-dataset-api diff --git a/ci/lint.yml b/ci/lint.yml index 070e5f9e..880d44cf 100644 --- a/ci/lint.yml +++ b/ci/lint.yml @@ -5,7 +5,7 @@ image_resource: type: docker-image source: repository: onsdigital/dp-concourse-tools-lint-go - tag: 1.24.6-bullseye-golangci-lint-2 + tag: 1.24.11-bookworm-golangci-lint-2 inputs: - name: dp-dataset-api @@ -14,4 +14,4 @@ run: path: dp-dataset-api/ci/scripts/lint.sh caches: - - path: /go + - path: go/ diff --git a/ci/unit.yml b/ci/unit.yml index 8a839d5f..0e0e147b 100644 --- a/ci/unit.yml +++ b/ci/unit.yml @@ -4,14 +4,19 @@ platform: linux image_resource: type: docker-image source: - repository: golang - tag: 1.24.6-bullseye + repository: onsdigital/dp-concourse-tools-docker-go + tag: 1.24.11-dind-24 inputs: - name: dp-dataset-api run: - path: dp-dataset-api/ci/scripts/unit.sh + path: bash + args: + - -exc + - | + /start_docker.sh + dp-dataset-api/ci/scripts/unit.sh caches: - - path: /go + - path: go/ \ No newline at end of file diff --git a/cloudflare/client.go b/cloudflare/client.go new file mode 100644 index 00000000..a233d7a4 --- /dev/null +++ b/cloudflare/client.go @@ -0,0 +1,140 @@ +package cloudflare + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "github.com/ONSdigital/log.go/v2/log" + "github.com/cloudflare/cloudflare-go" +) + +// Clienter defines the interface for Cloudflare cache operations +type Clienter interface { + PurgeCacheByPrefix(ctx context.Context, datasetID, editionID string) error +} + +// Client wraps the Cloudflare API client +type Client struct { + api *cloudflare.API + zoneID string + baseURL string + httpClient *http.Client + apiToken string +} + +// New creates a new Cloudflare client +func New(apiToken, zoneID string, useSDK bool, baseURL ...string) (*Client, error) { + if apiToken == "" { + return nil, fmt.Errorf("cloudflare API token is required") + } + if zoneID == "" { + return nil, fmt.Errorf("cloudflare zone ID is required") + } + + // if SDK disabled for local testing with the cloudflare stub, use the HTTP client + if !useSDK && len(baseURL) > 0 && baseURL[0] != "" { + return &Client{ + api: nil, + zoneID: zoneID, + baseURL: baseURL[0], + apiToken: apiToken, + httpClient: &http.Client{Timeout: 10 * time.Second}, + }, nil + } + + // use real Cloudflare SDK + api, err := cloudflare.NewWithAPIToken(apiToken) + if err != nil { + return nil, fmt.Errorf("failed to create cloudflare client: %w", err) + } + + return &Client{ + api: api, + zoneID: zoneID, + }, nil +} + +// PurgeCacheByPrefix purges the Cloudflare cache for dataset-related URLs +func (c *Client) PurgeCacheByPrefix(ctx context.Context, datasetID, editionID string) error { + prefixes := buildPrefixes(datasetID, editionID) + + logData := log.Data{ + "dataset_id": datasetID, + "edition": editionID, + "prefixes": prefixes, + } + log.Info(ctx, "purging cloudflare cache", logData) + + // local mock + if c.baseURL != "" { + return c.purgeCacheViaHTTP(ctx, prefixes) + } + + // for sdk + return c.purgeCacheViaSDK(ctx, prefixes) +} + +// purgeCacheViaSDK uses the cloudflare sdk +func (c *Client) purgeCacheViaSDK(ctx context.Context, prefixes []string) error { + params := cloudflare.PurgeCacheRequest{ + Prefixes: prefixes, + } + + _, err := c.api.PurgeCache(ctx, c.zoneID, params) + if err != nil { + return fmt.Errorf("failed to purge cache: %w", err) + } + + return nil +} + +// purgeCacheViaHTTP makes direct HTTP call for local cloudflare stub +func (c *Client) purgeCacheViaHTTP(ctx context.Context, prefixes []string) error { + url := fmt.Sprintf("%s/zones/%s/purge_cache", c.baseURL, c.zoneID) + + payload := map[string]interface{}{ + "prefixes": prefixes, + } + + body, err := json.Marshal(payload) + if err != nil { + return fmt.Errorf("failed to marshal request: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(body)) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", c.apiToken)) + + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("failed to make request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + bodyBytes, _ := io.ReadAll(resp.Body) + return fmt.Errorf("purge failed with status %d: %s", resp.StatusCode, string(bodyBytes)) + } + + return nil +} + +func buildPrefixes(datasetID, editionID string) []string { + return []string{ + fmt.Sprintf("www.ons.gov.uk/datasets/%s", datasetID), + fmt.Sprintf("www.ons.gov.uk/datasets/%s/editions", datasetID), + fmt.Sprintf("www.ons.gov.uk/datasets/%s/editions/%s/versions", datasetID, editionID), + fmt.Sprintf("api.beta.ons.gov.uk/v1/datasets/%s", datasetID), + fmt.Sprintf("api.beta.ons.gov.uk/v1/datasets/%s/editions", datasetID), + fmt.Sprintf("api.beta.ons.gov.uk/v1/datasets/%s/editions/%s/versions", datasetID, editionID), + } +} diff --git a/cloudflare/client_test.go b/cloudflare/client_test.go new file mode 100644 index 00000000..bc74c49c --- /dev/null +++ b/cloudflare/client_test.go @@ -0,0 +1,84 @@ +package cloudflare_test + +import ( + "testing" + + "github.com/ONSdigital/dp-dataset-api/cloudflare" + . "github.com/smartystreets/goconvey/convey" +) + +func TestNew(t *testing.T) { + Convey("Given valid API token and zone ID", t, func() { + apiToken := "test-token" + zoneID := "test-zone-id" + useSDK := false + + Convey("When creating a new Cloudflare client", func() { + client, err := cloudflare.New(apiToken, zoneID, useSDK) + + Convey("Then no error is returned", func() { + So(err, ShouldBeNil) + }) + + Convey("And the client is not nil", func() { + So(client, ShouldNotBeNil) + }) + }) + }) + + Convey("Given an empty API token", t, func() { + apiToken := "" + zoneID := "test-zone-id" + useSDK := false + + Convey("When creating a new Cloudflare client", func() { + client, err := cloudflare.New(apiToken, zoneID, useSDK) + + Convey("Then an error is returned", func() { + So(err, ShouldNotBeNil) + So(err.Error(), ShouldContainSubstring, "API token is required") + }) + + Convey("And the client is nil", func() { + So(client, ShouldBeNil) + }) + }) + }) + + Convey("Given an empty zone ID", t, func() { + apiToken := "test-token" + zoneID := "" + useSDK := false + + Convey("When creating a new Cloudflare client", func() { + client, err := cloudflare.New(apiToken, zoneID, useSDK) + + Convey("Then an error is returned", func() { + So(err, ShouldNotBeNil) + So(err.Error(), ShouldContainSubstring, "zone ID is required") + }) + + Convey("And the client is nil", func() { + So(client, ShouldBeNil) + }) + }) + }) + + Convey("Given both empty API token and zone ID", t, func() { + apiToken := "" + zoneID := "" + useSDK := false + + Convey("When creating a new Cloudflare client", func() { + client, err := cloudflare.New(apiToken, zoneID, useSDK) + + Convey("Then an error is returned", func() { + So(err, ShouldNotBeNil) + }) + + Convey("And the client is nil", func() { + So(client, ShouldBeNil) + }) + }) + }) +} diff --git a/config/config.go b/config/config.go index 9eeac249..dd44f8c6 100644 --- a/config/config.go +++ b/config/config.go @@ -47,6 +47,7 @@ type Configuration struct { EnablePermissionsAuth bool `envconfig:"ENABLE_PERMISSIONS_AUTH"` EnableObservationEndpoint bool `envconfig:"ENABLE_OBSERVATION_ENDPOINT"` EnableURLRewriting bool `envconfig:"ENABLE_URL_REWRITING"` + EnableCloudflareSDK bool `envconfig:"ENABLE_CLOUDFLARE_SDK"` DisableGraphDBDependency bool `envconfig:"DISABLE_GRAPH_DB_DEPENDENCY"` KafkaVersion string `envconfig:"KAFKA_VERSION"` DefaultMaxLimit int `envconfig:"DEFAULT_MAXIMUM_LIMIT"` @@ -59,6 +60,9 @@ type Configuration struct { OTServiceName string `envconfig:"OTEL_SERVICE_NAME"` OTBatchTimeout time.Duration `envconfig:"OTEL_BATCH_TIMEOUT"` OtelEnabled bool `envconfig:"OTEL_ENABLED"` + CloudflareZoneID string `envconfig:"CLOUDFLARE_ZONE_ID"` + CloudflareAPIToken string `envconfig:"CLOUDFLARE_API_TOKEN" json:"-"` + CloudflareAPIURL string `envconfig:"CLOUDFLARE_API_URL"` MongoConfig } @@ -109,12 +113,17 @@ func Get() (*Configuration, error) { EnablePermissionsAuth: false, EnableObservationEndpoint: true, EnableURLRewriting: false, + EnableCloudflareSDK: false, DisableGraphDBDependency: false, KafkaVersion: "1.0.2", DefaultMaxLimit: 1000, DefaultLimit: 20, DefaultOffset: 0, - MaxRequestOptions: 100, // Maximum number of options acceptable in an incoming Patch request. Compromise between one option per call (inefficient) and an order of 100k options per call, for census data (memory and computationally expensive) + MaxRequestOptions: 100, // Maximum number of options acceptable in an incoming Patch request. Compromise between one option per call (inefficient) and an order of 100k options per call, for census data (memory and computationally expensive) + CloudflareZoneID: "9f1eec58caedd8e902395a065c120073", // this is not a real zone id, purely for local testing purposes as any 32 character length alphanumeric string will work + CloudflareAPIToken: "test-token", + CloudflareAPIURL: "http://cloud-flare-stub:22500", + MongoConfig: MongoConfig{ MongoDriverConfig: mongodriver.MongoDriverConfig{ ClusterEndpoint: "localhost:27017", diff --git a/config/config_test.go b/config/config_test.go index f7f02e78..d78e93aa 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -61,6 +61,9 @@ func TestSpec(t *testing.T) { So(cfg.IsWriteConcernMajorityEnabled, ShouldEqual, true) So(cfg.DatasetAPIURL, ShouldEqual, "http://localhost:22000") So(cfg.CodeListAPIURL, ShouldEqual, "http://localhost:22400") + So(cfg.CloudflareAPIToken, ShouldEqual, "test-token") + So(cfg.CloudflareZoneID, ShouldEqual, "9f1eec58caedd8e902395a065c120073") + So(cfg.CloudflareAPIURL, ShouldEqual, "http://cloud-flare-stub:22500") }) }) }) diff --git a/dimension/dimension_test.go b/dimension/dimension_test.go index d6ad9ace..73ad577e 100644 --- a/dimension/dimension_test.go +++ b/dimension/dimension_test.go @@ -1544,7 +1544,7 @@ func getAPIWithCMDMocks(ctx context.Context, mockedDataStore store.Storer, mocke datasetPermissions := getAuthorisationHandlerMock() permissions := getAuthorisationHandlerMock() - return api.Setup(ctx, cfg, mux.NewRouter(), store.DataStore{Backend: mockedDataStore}, urlBuilder, downloadGenerators, datasetPermissions, permissions, enableURLRewriting, &mockStatemachineDatasetAPI) + return api.Setup(ctx, cfg, mux.NewRouter(), store.DataStore{Backend: mockedDataStore}, urlBuilder, downloadGenerators, datasetPermissions, permissions, enableURLRewriting, &mockStatemachineDatasetAPI, nil) } func getAuthorisationHandlerMock() *mocks.AuthHandlerMock { diff --git a/features/cloudflare_cache_purge.feature b/features/cloudflare_cache_purge.feature new file mode 100644 index 00000000..80159e09 --- /dev/null +++ b/features/cloudflare_cache_purge.feature @@ -0,0 +1,187 @@ +Feature: Cloudflare Cache Purging on Version Publication + + Scenario: Cloudflare cache is purged when version is published + Given I have a static dataset with version: + """ + { + "dataset": { + "id": "cache-test-dataset", + "title": "Cache Test Dataset", + "state": "associated", + "type": "static", + "links": { + "editions": { + "href": "/datasets/cache-test-dataset/editions" + }, + "self": { + "href": "/datasets/cache-test-dataset" + } + } + }, + "version": { + "id": "cache-test-version", + "edition": "2025", + "edition_title": "2025 Edition", + "links": { + "dataset": { + "id": "cache-test-dataset" + }, + "edition": { + "href": "/datasets/cache-test-dataset/editions/2025", + "id": "2025" + }, + "self": { + "href": "/datasets/cache-test-dataset/editions/2025/versions/1" + }, + "version": { + "href": "/datasets/cache-test-dataset/editions/2025/versions/1", + "id": "1" + } + }, + "version": 1, + "release_date": "2025-01-01T09:00:00.000Z", + "state": "approved", + "type": "static", + "distributions": [ + { + "title": "csv", + "format": "csv", + "media_type": "text/csv", + "download_url": "/downloads/datasets/cache-test-dataset/editions/2025/versions/1.csv", + "byte_size": 125000 + } + ] + } + } + """ + And private endpoints are enabled + And I am identified as "user@ons.gov.uk" + And I am authorised + When I PUT "/datasets/cache-test-dataset/editions/2025/versions/1/state" + """ + { + "state": "published" + } + """ + Then the HTTP status code should be "200" + And cloudflare cache purge should have been called for dataset "cache-test-dataset" and edition "2025" + + Scenario: Cloudflare cache is not purged when version transitions to non-published state + Given I have a static dataset with version: + """ + { + "dataset": { + "id": "cache-test-non-publish", + "title": "Cache Test Non-Publish", + "state": "created", + "type": "static" + }, + "version": { + "id": "cache-test-version-2", + "edition": "2025", + "edition_title": "2025 Edition", + "links": { + "dataset": { + "id": "cache-test-non-publish" + }, + "edition": { + "href": "/datasets/cache-test-non-publish/editions/2025", + "id": "2025" + }, + "self": { + "href": "/datasets/cache-test-non-publish/editions/2025/versions/1" + } + }, + "version": 1, + "release_date": "2025-01-01T09:00:00.000Z", + "state": "created", + "type": "static", + "distributions": [ + { + "title": "csv", + "format": "csv", + "media_type": "text/csv", + "download_url": "/downloads/datasets/cache-test-non-publish/editions/2025/versions/1.csv", + "byte_size": 125000 + } + ] + } + } + """ + And private endpoints are enabled + And I am identified as "user@ons.gov.uk" + And I am authorised + When I PUT "/datasets/cache-test-non-publish/editions/2025/versions/1/state" + """ + { + "state": "associated" + } + """ + Then the HTTP status code should be "200" + And cloudflare cache purge should not have been called + + Scenario: Publication succeeds even if Cloudflare cache purge fails + Given I have a static dataset with version: + """ + { + "dataset": { + "id": "cache-test-fail", + "title": "Cache Test Fail", + "state": "associated", + "type": "static", + "links": { + "editions": { + "href": "/datasets/cache-test-fail/editions" + }, + "self": { + "href": "/datasets/cache-test-fail" + } + } + }, + "version": { + "id": "cache-test-version-3", + "edition": "2025", + "edition_title": "2025 Edition", + "links": { + "dataset": { + "id": "cache-test-fail" + }, + "edition": { + "href": "/datasets/cache-test-fail/editions/2025", + "id": "2025" + }, + "self": { + "href": "/datasets/cache-test-fail/editions/2025/versions/1" + }, + "version": { + "href": "/datasets/cache-test-fail/editions/2025/versions/1", + "id": "1" + } + }, + "version": 1, + "release_date": "2025-01-01T09:00:00.000Z", + "state": "approved", + "type": "static", + "distributions": [ + { + "title": "csv", + "format": "csv", + "media_type": "text/csv", + "download_url": "/downloads/datasets/cache-test-fail/editions/2025/versions/1.csv", + "byte_size": 125000 + } + ] + } + } + """ + And private endpoints are enabled + And I am identified as "user@ons.gov.uk" + And I am authorised + And cloudflare cache purge is configured to fail + When I PUT "/datasets/cache-test-fail/editions/2025/versions/1/state" + """ + { + "state": "published" + } + """ + Then the HTTP status code should be "200" diff --git a/features/steps/dataset_component.go b/features/steps/dataset_component.go index ca62a795..4599ff84 100644 --- a/features/steps/dataset_component.go +++ b/features/steps/dataset_component.go @@ -8,12 +8,14 @@ import ( componenttest "github.com/ONSdigital/dp-component-test" "github.com/ONSdigital/dp-component-test/utils" + "github.com/ONSdigital/dp-dataset-api/cloudflare" "github.com/ONSdigital/dp-dataset-api/config" "github.com/ONSdigital/dp-dataset-api/mongo" "github.com/ONSdigital/dp-dataset-api/service" serviceMock "github.com/ONSdigital/dp-dataset-api/service/mock" "github.com/ONSdigital/dp-dataset-api/store" storeMock "github.com/ONSdigital/dp-dataset-api/store/datastoretest" + "github.com/ONSdigital/dp-files-api/files" filesAPISDK "github.com/ONSdigital/dp-files-api/sdk" filesAPISDKMocks "github.com/ONSdigital/dp-files-api/sdk/mocks" "github.com/ONSdigital/dp-healthcheck/healthcheck" @@ -24,17 +26,19 @@ import ( ) type DatasetComponent struct { - ErrorFeature componenttest.ErrorFeature - apiFeature *componenttest.APIFeature - svc *service.Service - errorChan chan error - MongoClient *mongo.Mongo - Config *config.Configuration - HTTPServer *http.Server - ServiceRunning bool - consumer kafka.IConsumerGroup - producer kafka.IProducer - initialiser service.Initialiser + ErrorFeature componenttest.ErrorFeature + apiFeature *componenttest.APIFeature + svc *service.Service + errorChan chan error + MongoClient *mongo.Mongo + Config *config.Configuration + HTTPServer *http.Server + ServiceRunning bool + consumer kafka.IConsumerGroup + producer kafka.IProducer + initialiser service.Initialiser + MockCloudflare *MockCloudflareClient + CloudflarePurgeCalls []CloudflarePurgeCall } func NewDatasetComponent(mongoURI, zebedeeURL string) (*DatasetComponent, error) { @@ -42,8 +46,14 @@ func NewDatasetComponent(mongoURI, zebedeeURL string) (*DatasetComponent, error) HTTPServer: &http.Server{ ReadHeaderTimeout: 60 * time.Second, }, - errorChan: make(chan error), - ServiceRunning: false, + errorChan: make(chan error), + ServiceRunning: false, + CloudflarePurgeCalls: []CloudflarePurgeCall{}, + } + + c.MockCloudflare = &MockCloudflareClient{ + PurgeCalls: &c.CloudflarePurgeCalls, + ShouldFail: false, } var err error @@ -92,6 +102,10 @@ func (c *DatasetComponent) Reset() error { c.Config.EnablePrivateEndpoints = false c.Config.EnableURLRewriting = false + + c.CloudflarePurgeCalls = []CloudflarePurgeCall{} + c.MockCloudflare.ShouldFail = false + // Resets back to Mocked Kafka c.setInitialiserMock() @@ -237,6 +251,17 @@ func (c *DatasetComponent) DoGetGraphDBOk(context.Context) (store.GraphDB, servi func (c *DatasetComponent) DoGetFilesAPIClientOk(ctx context.Context, cfg *config.Configuration) (filesAPISDK.Clienter, error) { return &filesAPISDKMocks.ClienterMock{ + GetFileFunc: func(ctx context.Context, filePath string) (*files.StoredRegisteredMetaData, error) { + return &files.StoredRegisteredMetaData{ + Path: filePath, + IsPublishable: true, + State: "UPLOADED", + SizeInBytes: 1000, + }, nil + }, + MarkFilePublishedFunc: func(ctx context.Context, filePath string) error { + return nil + }, DeleteFileFunc: func(ctx context.Context, filePath string) error { if filePath == "/fail/to/delete.csv" { return fmt.Errorf("failed to delete file at path: %s", filePath) @@ -246,23 +271,56 @@ func (c *DatasetComponent) DoGetFilesAPIClientOk(ctx context.Context, cfg *confi }, nil } +func (c *DatasetComponent) DoGetCloudflareClientOk(ctx context.Context, cfg *config.Configuration) (cloudflare.Clienter, error) { + return c.MockCloudflare, nil +} + func (c *DatasetComponent) setInitialiserMock() { c.initialiser = &serviceMock.InitialiserMock{ - DoGetMongoDBFunc: c.DoGetMongoDB, - DoGetGraphDBFunc: c.DoGetGraphDBOk, - DoGetFilesAPIClientFunc: c.DoGetFilesAPIClientOk, - DoGetKafkaProducerFunc: c.DoGetMockedKafkaProducerOk, - DoGetHealthCheckFunc: c.DoGetHealthcheckOk, - DoGetHTTPServerFunc: c.DoGetHTTPServer, + DoGetMongoDBFunc: c.DoGetMongoDB, + DoGetGraphDBFunc: c.DoGetGraphDBOk, + DoGetFilesAPIClientFunc: c.DoGetFilesAPIClientOk, + DoGetKafkaProducerFunc: c.DoGetMockedKafkaProducerOk, + DoGetHealthCheckFunc: c.DoGetHealthcheckOk, + DoGetHTTPServerFunc: c.DoGetHTTPServer, + DoGetCloudflareClientFunc: c.DoGetCloudflareClientOk, } } func (c *DatasetComponent) setInitialiserRealKafka() { c.initialiser = &serviceMock.InitialiserMock{ - DoGetMongoDBFunc: c.DoGetMongoDB, - DoGetGraphDBFunc: c.DoGetGraphDBOk, - DoGetFilesAPIClientFunc: c.DoGetFilesAPIClientOk, - DoGetKafkaProducerFunc: c.DoGetKafkaProducer, - DoGetHealthCheckFunc: c.DoGetHealthcheckOk, - DoGetHTTPServerFunc: c.DoGetHTTPServer, + DoGetMongoDBFunc: c.DoGetMongoDB, + DoGetGraphDBFunc: c.DoGetGraphDBOk, + DoGetFilesAPIClientFunc: c.DoGetFilesAPIClientOk, + DoGetKafkaProducerFunc: c.DoGetKafkaProducer, + DoGetHealthCheckFunc: c.DoGetHealthcheckOk, + DoGetHTTPServerFunc: c.DoGetHTTPServer, + DoGetCloudflareClientFunc: c.DoGetCloudflareClientOk, } } + +type CloudflarePurgeCall struct { + DatasetID string + EditionID string +} + +type MockCloudflareClient struct { + PurgeCalls *[]CloudflarePurgeCall + ShouldFail bool +} + +func (m *MockCloudflareClient) PurgeCacheByPrefix(ctx context.Context, datasetID, editionID string) error { + if m.ShouldFail { + return fmt.Errorf("mock cloudflare purge failed") + } + + *m.PurgeCalls = append(*m.PurgeCalls, CloudflarePurgeCall{ + DatasetID: datasetID, + EditionID: editionID, + }) + + log.Info(ctx, "mock cloudflare cache purge called", log.Data{ + "dataset_id": datasetID, + "edition_id": editionID, + }) + return nil +} diff --git a/features/steps/steps.go b/features/steps/steps.go index 98774b40..6c1a088a 100644 --- a/features/steps/steps.go +++ b/features/steps/steps.go @@ -55,6 +55,9 @@ func (c *DatasetComponent) RegisterSteps(ctx *godog.ScenarioContext) { ctx.Step(`^the response header "([^"]*)" should not be empty$`, c.theResponseHeaderShouldNotBeEmpty) ctx.Step(`^the dataset "([^"]*)" should have next equal to current$`, c.theDatasetShouldHaveNextEqualToCurrent) ctx.Step(`^the "([^"]*)" feature flag is "([^"]*)"$`, c.theFeatureFlagIs) + ctx.Step(`^cloudflare cache purge should have been called for dataset "([^"]*)" and edition "([^"]*)"$`, c.cloudflareCachePurgeShouldHaveBeenCalled) + ctx.Step(`^cloudflare cache purge should not have been called$`, c.cloudflareCachePurgeShouldNotHaveBeenCalled) + ctx.Step(`^cloudflare cache purge is configured to fail$`, c.cloudflareCachePurgeIsConfiguredToFail) } func (c *DatasetComponent) theFeatureFlagIs(flagName, status string) error { @@ -514,11 +517,9 @@ func (c *DatasetComponent) iHaveStaticDatasetWithVersion(jsonData *godog.DocStri Dataset models.Dataset `json:"dataset"` Version models.Version `json:"version"` } - if err := json.Unmarshal([]byte(jsonData.Content), &data); err != nil { return fmt.Errorf("failed to unmarshal static dataset data: %w", err) } - datasetID := data.Dataset.ID data.Dataset.Type = "static" datasetUp := models.DatasetUpdate{ @@ -530,23 +531,30 @@ func (c *DatasetComponent) iHaveStaticDatasetWithVersion(jsonData *godog.DocStri if err := c.putDocumentInDatabase(datasetUp, datasetID, datasetsCollection, 0); err != nil { return fmt.Errorf("failed to insert static dataset: %w", err) } - versionID := data.Version.ID + if data.Version.Links == nil { + data.Version.Links = &models.VersionLinks{} + } + + if data.Version.Links.Self == nil { + data.Version.Links.Self = &models.LinkObject{ + HRef: fmt.Sprintf("/datasets/%s/editions/%s/versions/%d", datasetID, data.Version.Edition, data.Version.Version), + } + } + if data.Version.Links.Version == nil { data.Version.Links.Version = &models.LinkObject{ HRef: data.Version.Links.Self.HRef, - ID: "1", + ID: fmt.Sprintf("%d", data.Version.Version), } } data.Version.ETag = "etag-" + versionID - versionsCollection := c.MongoClient.ActualCollectionName(config.VersionsCollection) if err := c.putDocumentInDatabase(data.Version, versionID, versionsCollection, 0); err != nil { return fmt.Errorf("failed to insert static version: %w", err) } - return nil } @@ -626,3 +634,27 @@ func (c *DatasetComponent) theResponseHeaderShouldNotBeEmpty(header string) erro } return nil } + +func (c *DatasetComponent) cloudflareCachePurgeShouldHaveBeenCalled(datasetID, editionID string) error { + for _, call := range c.CloudflarePurgeCalls { + if call.DatasetID == datasetID && call.EditionID == editionID { + return nil + } + } + + return fmt.Errorf("expected cloudflare cache purge to be called for dataset '%s' and edition '%s', but it was not. Calls made: %+v", + datasetID, editionID, c.CloudflarePurgeCalls) +} + +func (c *DatasetComponent) cloudflareCachePurgeShouldNotHaveBeenCalled() error { + if len(c.CloudflarePurgeCalls) > 0 { + return fmt.Errorf("expected no cloudflare cache purge calls, but got %d calls: %+v", + len(c.CloudflarePurgeCalls), c.CloudflarePurgeCalls) + } + return nil +} + +func (c *DatasetComponent) cloudflareCachePurgeIsConfiguredToFail() error { + c.MockCloudflare.ShouldFail = true + return nil +} diff --git a/go.mod b/go.mod index b5e15f85..8d579374 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/ONSdigital/dp-dataset-api -go 1.24 +go 1.24.0 require ( github.com/ONSdigital/dp-api-clients-go v1.43.0 @@ -12,11 +12,11 @@ require ( github.com/ONSdigital/dp-graph/v2 v2.18.0 github.com/ONSdigital/dp-healthcheck v1.6.4 github.com/ONSdigital/dp-kafka/v4 v4.2.0 - github.com/ONSdigital/dp-mongodb-in-memory v1.8.1 github.com/ONSdigital/dp-mongodb/v3 v3.8.0 github.com/ONSdigital/dp-net/v3 v3.9.0 github.com/ONSdigital/dp-otel-go v0.0.8 github.com/ONSdigital/log.go/v2 v2.5.0 + github.com/cloudflare/cloudflare-go v0.116.0 github.com/cucumber/godog v0.15.1 github.com/golang/glog v1.2.5 github.com/google/go-cmp v0.7.0 @@ -28,15 +28,20 @@ require ( github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b github.com/smartystreets/goconvey v1.8.1 github.com/stretchr/testify v1.11.1 + github.com/testcontainers/testcontainers-go/modules/mongodb v0.40.0 go.mongodb.org/mongo-driver v1.17.6 go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.62.0 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 ) require ( + dario.cat/mergo v1.0.2 // indirect + github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect github.com/ONSdigital/dis-redis v0.3.0 // indirect github.com/ONSdigital/dp-authorisation/v2 v2.32.3 // indirect github.com/ONSdigital/dp-kafka/v3 v3.11.0 // indirect + github.com/ONSdigital/dp-mongodb-in-memory v1.8.1 // indirect github.com/ONSdigital/dp-net/v2 v2.22.0 // indirect github.com/ONSdigital/dp-permissions-api v1.3.0 // indirect github.com/ONSdigital/golang-neo4j-bolt-driver v0.0.0-20241121114036-9f4b82bb9d37 // indirect @@ -55,33 +60,47 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.11 // indirect github.com/aws/aws-sdk-go-v2/service/s3 v1.89.0 // indirect github.com/aws/smithy-go v1.23.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cenkalti/backoff/v5 v5.0.2 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chromedp/cdproto v0.0.0-20250724212937-08a3db8b4327 // indirect github.com/chromedp/chromedp v0.14.0 // indirect github.com/chromedp/sysutil v1.1.0 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/platforms v0.2.1 // indirect + github.com/cpuguy83/dockercfg v0.3.2 // indirect github.com/cucumber/gherkin/go/v26 v26.2.0 // indirect github.com/cucumber/messages/go/v21 v21.0.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/docker v28.5.1+incompatible // indirect + github.com/docker/go-connections v0.6.0 // indirect + github.com/docker/go-units v0.5.0 // indirect github.com/eapache/go-resiliency v1.7.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect + github.com/ebitengine/purego v0.8.4 // indirect github.com/fatih/color v1.18.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-avro/avro v0.0.0-20171219232920-444163702c11 // indirect github.com/go-json-experiment/json v0.0.0-20250725192818-e39067aee2d2 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator v9.31.0+incompatible // indirect github.com/gobwas/httphead v0.1.0 // indirect github.com/gobwas/pool v0.2.1 // indirect github.com/gobwas/ws v1.4.0 // indirect + github.com/goccy/go-json v0.10.5 // indirect github.com/gofrs/uuid v4.4.0+incompatible // indirect github.com/golang-jwt/jwt/v4 v4.5.2 // indirect github.com/golang/snappy v1.0.0 // indirect + github.com/google/go-querystring v1.1.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gopherjs/gopherjs v1.17.2 // indirect github.com/gorilla/websocket v1.5.3 // indirect @@ -101,23 +120,42 @@ require ( github.com/jtolds/gls v4.20.0+incompatible // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/magiconair/properties v1.8.10 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/maxcnunes/httpfake v1.2.4 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/go-archive v0.1.0 // indirect + github.com/moby/patternmatcher v0.6.0 // indirect + github.com/moby/sys/sequential v0.6.0 // indirect + github.com/moby/sys/user v0.4.0 // indirect + github.com/moby/sys/userns v0.1.0 // indirect + github.com/moby/term v0.5.0 // indirect github.com/montanaflynn/stats v0.7.1 // indirect + github.com/morikuni/aec v1.0.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 // indirect github.com/redis/go-redis/v9 v9.11.0 // indirect + github.com/shirou/gopsutil/v4 v4.25.6 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect github.com/smarty/assertions v1.16.0 // indirect github.com/spf13/afero v1.14.0 // indirect github.com/spf13/pflag v1.0.7 // indirect github.com/square/mongo-lock v0.0.0-20230808145049-cfcf499f6bf0 // indirect + github.com/testcontainers/testcontainers-go v0.40.0 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect github.com/yuin/gopher-lua v1.1.1 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/contrib/instrumentation/github.com/Shopify/sarama/otelsarama v0.43.0 // indirect go.opentelemetry.io/contrib/propagators/autoprop v0.61.0 // indirect @@ -133,11 +171,12 @@ require ( go.opentelemetry.io/otel/trace v1.37.0 // indirect go.opentelemetry.io/proto/otlp v1.7.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.40.0 // indirect - golang.org/x/net v0.42.0 // indirect - golang.org/x/sync v0.16.0 // indirect - golang.org/x/sys v0.34.0 // indirect - golang.org/x/text v0.27.0 // indirect + golang.org/x/crypto v0.43.0 // indirect + golang.org/x/net v0.45.0 // indirect + golang.org/x/sync v0.17.0 // indirect + golang.org/x/sys v0.37.0 // indirect + golang.org/x/text v0.30.0 // indirect + golang.org/x/time v0.9.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20250603155806-513f23925822 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250603155806-513f23925822 // indirect google.golang.org/grpc v1.73.0 // indirect diff --git a/go.sum b/go.sum index 3d314ba7..58266bcb 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,11 @@ +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/ONSdigital/dis-redis v0.3.0 h1:aI0d3MsXPmRbe+okYAbrwEGBibhoCw+gfbgtJ3mA11c= github.com/ONSdigital/dis-redis v0.3.0/go.mod h1:CLbCwaEfJhifBM7PufwNi0mymys+xM6xNgwhihhSIHQ= github.com/ONSdigital/dp-api-clients-go v1.28.0/go.mod h1:iyJy6uRL4B6OYOJA0XMr5UHt6+Q8XmN9uwmURO+9Oj4= @@ -115,6 +123,8 @@ github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cenkalti/backoff/v5 v5.0.2 h1:rIfFVxEf1QsI7E1ZHfp/B4DF/6QBAUhmgkxc0H7Zss8= github.com/cenkalti/backoff/v5 v5.0.2/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -125,7 +135,21 @@ github.com/chromedp/chromedp v0.14.0 h1:/xE5m6wEBwivhalHwlCOyYfBcAJNwg4nLw96QiCf github.com/chromedp/chromedp v0.14.0/go.mod h1:rHzAv60xDE7VNy/MYtTUrYreSc0ujt2O1/C3bzctYBo= github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM= github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8= +github.com/cloudflare/cloudflare-go v0.116.0 h1:iRPMnTtnswRpELO65NTwMX4+RTdxZl+Xf/zi+HPE95s= +github.com/cloudflare/cloudflare-go v0.116.0/go.mod h1:Ds6urDwn/TF2uIU24mu7H91xkKP8gSAHxQ44DSZgVmU= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= +github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= +github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/cucumber/gherkin/go/v26 v26.2.0 h1:EgIjePLWiPeslwIWmNQ3XHcypPsWAHoMCz/YEBKP4GI= github.com/cucumber/gherkin/go/v26 v26.2.0/go.mod h1:t2GAPnB8maCT4lkHL99BDCVNzCh1d7dBhCLt150Nr/0= github.com/cucumber/godog v0.15.1 h1:rb/6oHDdvVZKS66hrhpjFQFHjthFSrQBCOI1LwshNTI= @@ -141,12 +165,22 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/docker v28.5.1+incompatible h1:Bm8DchhSD2J6PsFzxC35TZo4TLGR2PdW/E69rU45NhM= +github.com/docker/docker v28.5.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= +github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/eapache/go-resiliency v1.7.0 h1:n3NRTnBn5N0Cbi/IeOHuQn9s2UwVUH7Ga0ZWcP+9JTA= github.com/eapache/go-resiliency v1.7.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4ALJ04o5Qqpdz8XLIpNA3WM/iSIXqxtqo7UGVws= github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/ebitengine/purego v0.8.4 h1:CF7LEKg5FFOsASUj0+QwaXf8Ht6TlFxg09+S9wz0omw= +github.com/ebitengine/purego v0.8.4/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9/go.mod h1:uPmAp6Sws4L7+Q/OokbWDAK1ibXYhB3PXFP1kol5hPg= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= @@ -166,6 +200,8 @@ github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= @@ -181,6 +217,8 @@ github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs= github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v4.3.1+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA= @@ -200,8 +238,11 @@ github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -276,6 +317,10 @@ github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kUL github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= +github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= @@ -290,9 +335,31 @@ github.com/maxcnunes/httpfake v1.2.4 h1:l7s/N7zuG6XpzG+5dUolg5SSoR3hANQxqzAkv+lR github.com/maxcnunes/httpfake v1.2.4/go.mod h1:rWVxb0bLKtOUM/5hN3UO1VEdEitz1hfcTXs7UyiK6r0= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/go-archive v0.1.0 h1:Kk/5rdW/g+H8NHdJW2gsXyZ7UnzvJNOy6VKJqueWdcQ= +github.com/moby/go-archive v0.1.0/go.mod h1:G9B+YoujNohJmrIYFBpSd54GTUB4lt9S+xVQvsJyFuo= +github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= +github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= +github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= +github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= +github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= +github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= +github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw= github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= @@ -302,6 +369,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 h1:bsUq1dX0N8AOIL7EB/X911+m4EHsnWEHeJ0c+3TTBrg= github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rdumont/assistdog v0.0.0-20240711132531-b5b791dd7452 h1:ddg1HcWj+0zuxGEUuOoZlJu6iTjqhwYkCPVAjBPSrS8= @@ -313,6 +382,10 @@ github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWN github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybLANtM3mBXNUtOfsCFXeTsnBqCsx1KM= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/shirou/gopsutil/v4 v4.25.6 h1:kLysI2JsKorfaFPcYmcJqbzROzsBWEOAtw6A7dIfqXs= +github.com/shirou/gopsutil/v4 v4.25.6/go.mod h1:PfybzyydfZcN+JMMjkF6Zb8Mq1A/VcogFFg7hj50W9c= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smarty/assertions v1.16.0 h1:EvHNkdRA4QHMrn75NZSoUQ/mAUXAYWfatfB01yTCzfY= github.com/smarty/assertions v1.16.0/go.mod h1:duaaFdCS0K9dnoM50iyek/eYINOZ64gbh1Xlf6LG7AI= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -331,15 +404,26 @@ github.com/square/mongo-lock v0.0.0-20230808145049-cfcf499f6bf0/go.mod h1:bLPJcG github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/testcontainers/testcontainers-go v0.40.0 h1:pSdJYLOVgLE8YdUY2FHQ1Fxu+aMnb6JfVz1mxk7OeMU= +github.com/testcontainers/testcontainers-go v0.40.0/go.mod h1:FSXV5KQtX2HAMlm7U3APNyLkkap35zNLxukw9oBi/MY= +github.com/testcontainers/testcontainers-go/modules/mongodb v0.40.0 h1:z/1qHeliTLDKNaJ7uOHOx1FjwghbcbYfga4dTFkF0hU= +github.com/testcontainers/testcontainers-go/modules/mongodb v0.40.0/go.mod h1:GaunAWwMXLtsMKG3xn2HYIBDbKddGArfcGsF2Aog81E= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= @@ -354,9 +438,13 @@ github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfS github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.mongodb.org/mongo-driver v1.9.1/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= go.mongodb.org/mongo-driver v1.17.6 h1:87JUG1wZfWsr6rIz3ZmpH90rL5tea7O3IHuSwHUpsss= go.mongodb.org/mongo-driver v1.17.6/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= +go.mongodb.org/mongo-driver/v2 v2.3.0 h1:sh55yOXA2vUjW1QYw/2tRlHSQViwDyPnW61AwpZ4rtU= +go.mongodb.org/mongo-driver/v2 v2.3.0/go.mod h1:jHeEDJHJq7tm6ZF45Issun9dbogjfnPySb1vXA7EeAI= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= go.opentelemetry.io/contrib/instrumentation/github.com/Shopify/sarama/otelsarama v0.43.0 h1:/RxdhdIi0HrKSzdWHLjureinjnGL5YQEYevaC/EAg1k= @@ -381,6 +469,8 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 h1:dNzwXjZKpMpE2JhmO+9 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0/go.mod h1:90PoxvaEB5n6AOdZvi+yWJQoE95U8Dhhw2bSyRqnTD0= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0 h1:JgtbA0xkWHnTmYk7YusopJFX6uleBmAuZ8n05NEh8nQ= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0/go.mod h1:179AK5aar5R3eS9FucPy6rggvU0g52cvKId8pv4+v0c= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.37.0 h1:SNhVp/9q4Go/XHBkQ1/d5u9P/U+L1yaGPoi0x+mStaI= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.37.0/go.mod h1:tx8OOlGH6R4kLV67YaYO44GFXloEjGPZuMjEkaaqIp4= go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= @@ -403,8 +493,8 @@ golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= -golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= +golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= +golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -418,37 +508,45 @@ golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= -golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= +golang.org/x/net v0.45.0 h1:RLBg5JKixCy82FtLJpeNlVM0nrSqpCRYzVU1n8kj0tM= +golang.org/x/net v0.45.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201008141435-b3e1573b7520/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= -golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210414055047-fe65e336abe0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= -golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= +golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q= +golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -456,8 +554,10 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= -golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= +golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= +golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -487,3 +587,5 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= diff --git a/instance/instance_external_test.go b/instance/instance_external_test.go index 8cc0db9e..1d7b0bb8 100644 --- a/instance/instance_external_test.go +++ b/instance/instance_external_test.go @@ -779,5 +779,5 @@ func getAPIWithCantabularMocks(ctx context.Context, mockedDataStore store.Storer cfg.DatasetAPIURL = "http://localhost:22000" cfg.EnablePrivateEndpoints = true - return api.Setup(ctx, cfg, mux.NewRouter(), store.DataStore{Backend: mockedDataStore}, urlBuilder, mockedMapDownloadGenerators, datasetPermissions, permissions, enableURLRewriting, &mockStatemachineDatasetAPI) + return api.Setup(ctx, cfg, mux.NewRouter(), store.DataStore{Backend: mockedDataStore}, urlBuilder, mockedMapDownloadGenerators, datasetPermissions, permissions, enableURLRewriting, &mockStatemachineDatasetAPI, nil) } diff --git a/mongo/dataset_test.go b/mongo/dataset_test.go index 57020b87..5b897f5c 100644 --- a/mongo/dataset_test.go +++ b/mongo/dataset_test.go @@ -500,7 +500,7 @@ func TestIsStaticDataset(t *testing.T) { mongo, server, err := getTestMongoDB(ctx) So(err, ShouldBeNil) defer func() { - server.Stop(ctx) + server.Terminate(ctx) }() Convey("When IsStaticDataset is called with a static dataset ID", func() { diff --git a/mongo/mongo_test_helpers.go b/mongo/mongo_test_helpers.go index 92f5ae38..72c587e0 100644 --- a/mongo/mongo_test_helpers.go +++ b/mongo/mongo_test_helpers.go @@ -3,12 +3,13 @@ package mongo import ( "context" "fmt" + "net/url" "time" "github.com/ONSdigital/dp-dataset-api/config" "github.com/ONSdigital/dp-dataset-api/models" - mim "github.com/ONSdigital/dp-mongodb-in-memory" mongoDriver "github.com/ONSdigital/dp-mongodb/v3/mongodb" + testMongoContainer "github.com/testcontainers/testcontainers-go/modules/mongodb" ) var ( @@ -19,8 +20,8 @@ var ( unpublishedStaticID = "unpublished-static-id" ) -// getTestMongoDB initializes a MongoDB connection for use in tests -func getTestMongoDB(ctx context.Context) (*Mongo, *mim.Server, error) { +// getTestMongoDB initializes a MongoDB connection for use in tests using testcontainers +func getTestMongoDB(ctx context.Context) (*Mongo, *testMongoContainer.MongoDBContainer, error) { mongoVersion := "4.4.8" cfg, err := config.Get() @@ -28,28 +29,39 @@ func getTestMongoDB(ctx context.Context) (*Mongo, *mim.Server, error) { return nil, nil, err } - mongoServer, err := mim.Start(ctx, mongoVersion) + mongoContainer, err := testMongoContainer.Run(ctx, fmt.Sprintf("mongo:%s", mongoVersion)) if err != nil { - return nil, nil, err + return nil, nil, fmt.Errorf("failed to start mongo container: %w", err) } - mongoConfig := getTestMongoDriverConfig(mongoServer, cfg.Database, cfg.Collections) + + mongoConfig := getTestMongoDriverConfig(mongoContainer, cfg.Database, cfg.Collections) conn, err := mongoDriver.Open(mongoConfig) if err != nil { - return nil, nil, err + return nil, nil, fmt.Errorf("failed to open mongo connection: %w", err) } return &Mongo{ MongoConfig: cfg.MongoConfig, Connection: conn, - }, mongoServer, nil + }, mongoContainer, nil } -// Custom config to work with mongo in memory -func getTestMongoDriverConfig(mongoServer *mim.Server, database string, collections map[string]string) *mongoDriver.MongoDriverConfig { +// Custom config to work with testcontainers +func getTestMongoDriverConfig(mongoContainer *testMongoContainer.MongoDBContainer, database string, collections map[string]string) *mongoDriver.MongoDriverConfig { + connectionString, err := mongoContainer.ConnectionString(context.Background()) + if err != nil { + panic(fmt.Sprintf("failed to get connection string: %v", err)) + } + + connStringURL, err := url.Parse(connectionString) + if err != nil { + panic(fmt.Sprintf("failed to parse connection string: %v", err)) + } + return &mongoDriver.MongoDriverConfig{ ConnectTimeout: 5 * time.Second, QueryTimeout: 5 * time.Second, - ClusterEndpoint: mongoServer.URI(), + ClusterEndpoint: connStringURL.Host, Database: database, Collections: collections, } diff --git a/mongo/version_store_test.go b/mongo/version_store_test.go index 5a041ac7..9f386b3b 100644 --- a/mongo/version_store_test.go +++ b/mongo/version_store_test.go @@ -113,7 +113,7 @@ func TestGetAllStaticVersions(t *testing.T) { So(err, ShouldBeNil) defer func() { - server.Stop(ctx) + server.Terminate(ctx) }() versions, err := setupVersionsTestData(ctx, mongoStore) @@ -205,7 +205,7 @@ func TestDeleteStaticDatasetVersion(t *testing.T) { mongoStore, server, err := getTestMongoDB(ctx) So(err, ShouldBeNil) defer func() { - server.Stop(ctx) + server.Terminate(ctx) }() versions, err := setupVersionsTestData(ctx, mongoStore) @@ -233,7 +233,7 @@ func TestCheckEditionTitleExistsStatic(t *testing.T) { So(err, ShouldBeNil) defer func() { - server.Stop(ctx) + server.Terminate(ctx) }() versions, err := setupVersionsTestData(ctx, mongo) diff --git a/service/initialise.go b/service/initialise.go index 2339f9e4..23ef49dd 100644 --- a/service/initialise.go +++ b/service/initialise.go @@ -4,6 +4,7 @@ import ( "context" "net/http" + "github.com/ONSdigital/dp-dataset-api/cloudflare" "github.com/ONSdigital/dp-dataset-api/config" "github.com/ONSdigital/dp-dataset-api/mongo" "github.com/ONSdigital/dp-dataset-api/store" @@ -22,6 +23,7 @@ type ExternalServiceList struct { HealthCheck bool MongoDB bool FilesAPIClient bool + CloudflareClient bool Init Initialiser } @@ -97,6 +99,20 @@ func (e *ExternalServiceList) GetFilesAPIClient(ctx context.Context, cfg *config return nil, nil } +// GetCloudflareClient returns a Cloudflare client +func (e *ExternalServiceList) GetCloudflareClient(ctx context.Context, cfg *config.Configuration) (cloudflare.Clienter, error) { + cloudflareClient, err := e.Init.DoGetCloudflareClient(ctx, cfg) + if err != nil { + log.Error(ctx, "failed to initialise cloudflare client", err) + return nil, err + } + if cloudflareClient != nil { + e.CloudflareClient = true + log.Info(ctx, "cloudflare client created successfully") + } + return cloudflareClient, nil +} + // DoGetHTTPServer creates an HTTP Server with the provided bind address and router func (e *Init) DoGetHTTPServer(bindAddr string, router http.Handler) HTTPServer { s := dphttp.NewServer(bindAddr, router) @@ -163,3 +179,36 @@ func (e *Init) DoGetMongoDB(ctx context.Context, cfg config.MongoConfig) (store. func (e *Init) DoGetFilesAPIClient(ctx context.Context, cfg *config.Configuration) (filesAPISDK.Clienter, error) { return filesAPISDK.New(cfg.FilesAPIURL, cfg.ServiceAuthToken), nil } + +// DoGetCloudflareClient returns a Cloudflare client +func (e *Init) DoGetCloudflareClient(ctx context.Context, cfg *config.Configuration) (cloudflare.Clienter, error) { + if cfg.CloudflareAPIToken == "" || cfg.CloudflareZoneID == "" { + log.Info(ctx, "cloudflare integration disabled: missing API token or zone ID") + return nil, nil + } + + var client cloudflare.Clienter + var err error + + // for local mock server when SDK is disabled + if !cfg.EnableCloudflareSDK && cfg.CloudflareAPIURL != "" { + client, err = cloudflare.New( + cfg.CloudflareAPIToken, + cfg.CloudflareZoneID, + cfg.EnableCloudflareSDK, + cfg.CloudflareAPIURL, + ) + } else { + client, err = cloudflare.New( + cfg.CloudflareAPIToken, + cfg.CloudflareZoneID, + cfg.EnableCloudflareSDK, + ) + } + + if err != nil { + return nil, err + } + + return client, nil +} diff --git a/service/interfaces.go b/service/interfaces.go index aba440af..1c438eed 100644 --- a/service/interfaces.go +++ b/service/interfaces.go @@ -4,6 +4,7 @@ import ( "context" "net/http" + "github.com/ONSdigital/dp-dataset-api/cloudflare" "github.com/ONSdigital/dp-dataset-api/config" "github.com/ONSdigital/dp-dataset-api/store" filesAPISDK "github.com/ONSdigital/dp-files-api/sdk" @@ -24,6 +25,7 @@ type Initialiser interface { DoGetGraphDB(ctx context.Context) (store.GraphDB, Closer, error) DoGetMongoDB(ctx context.Context, cfg config.MongoConfig) (store.MongoDB, error) DoGetFilesAPIClient(ctx context.Context, cfg *config.Configuration) (filesAPISDK.Clienter, error) + DoGetCloudflareClient(ctx context.Context, cfg *config.Configuration) (cloudflare.Clienter, error) } // HTTPServer defines the required methods from the HTTP server diff --git a/service/mock/initialiser.go b/service/mock/initialiser.go index 26691ca3..64e5d65c 100644 --- a/service/mock/initialiser.go +++ b/service/mock/initialiser.go @@ -5,6 +5,7 @@ package mock import ( "context" + "github.com/ONSdigital/dp-dataset-api/cloudflare" "github.com/ONSdigital/dp-dataset-api/config" "github.com/ONSdigital/dp-dataset-api/service" "github.com/ONSdigital/dp-dataset-api/store" @@ -24,6 +25,9 @@ var _ service.Initialiser = &InitialiserMock{} // // // make and configure a mocked service.Initialiser // mockedInitialiser := &InitialiserMock{ +// DoGetCloudflareClientFunc: func(ctx context.Context, cfg *config.Configuration) (cloudflare.Clienter, error) { +// panic("mock out the DoGetCloudflareClient method") +// }, // DoGetFilesAPIClientFunc: func(ctx context.Context, cfg *config.Configuration) (filesAPISDK.Clienter, error) { // panic("mock out the DoGetFilesAPIClient method") // }, @@ -49,6 +53,9 @@ var _ service.Initialiser = &InitialiserMock{} // // } type InitialiserMock struct { + // DoGetCloudflareClientFunc mocks the DoGetCloudflareClient method. + DoGetCloudflareClientFunc func(ctx context.Context, cfg *config.Configuration) (cloudflare.Clienter, error) + // DoGetFilesAPIClientFunc mocks the DoGetFilesAPIClient method. DoGetFilesAPIClientFunc func(ctx context.Context, cfg *config.Configuration) (filesAPISDK.Clienter, error) @@ -69,6 +76,13 @@ type InitialiserMock struct { // calls tracks calls to the methods. calls struct { + // DoGetCloudflareClient holds details about calls to the DoGetCloudflareClient method. + DoGetCloudflareClient []struct { + // Ctx is the ctx argument value. + Ctx context.Context + // Cfg is the cfg argument value. + Cfg *config.Configuration + } // DoGetFilesAPIClient holds details about calls to the DoGetFilesAPIClient method. DoGetFilesAPIClient []struct { // Ctx is the ctx argument value. @@ -116,12 +130,49 @@ type InitialiserMock struct { Cfg config.MongoConfig } } - lockDoGetFilesAPIClient sync.RWMutex - lockDoGetGraphDB sync.RWMutex - lockDoGetHTTPServer sync.RWMutex - lockDoGetHealthCheck sync.RWMutex - lockDoGetKafkaProducer sync.RWMutex - lockDoGetMongoDB sync.RWMutex + lockDoGetCloudflareClient sync.RWMutex + lockDoGetFilesAPIClient sync.RWMutex + lockDoGetGraphDB sync.RWMutex + lockDoGetHTTPServer sync.RWMutex + lockDoGetHealthCheck sync.RWMutex + lockDoGetKafkaProducer sync.RWMutex + lockDoGetMongoDB sync.RWMutex +} + +// DoGetCloudflareClient calls DoGetCloudflareClientFunc. +func (mock *InitialiserMock) DoGetCloudflareClient(ctx context.Context, cfg *config.Configuration) (cloudflare.Clienter, error) { + if mock.DoGetCloudflareClientFunc == nil { + panic("InitialiserMock.DoGetCloudflareClientFunc: method is nil but Initialiser.DoGetCloudflareClient was just called") + } + callInfo := struct { + Ctx context.Context + Cfg *config.Configuration + }{ + Ctx: ctx, + Cfg: cfg, + } + mock.lockDoGetCloudflareClient.Lock() + mock.calls.DoGetCloudflareClient = append(mock.calls.DoGetCloudflareClient, callInfo) + mock.lockDoGetCloudflareClient.Unlock() + return mock.DoGetCloudflareClientFunc(ctx, cfg) +} + +// DoGetCloudflareClientCalls gets all the calls that were made to DoGetCloudflareClient. +// Check the length with: +// +// len(mockedInitialiser.DoGetCloudflareClientCalls()) +func (mock *InitialiserMock) DoGetCloudflareClientCalls() []struct { + Ctx context.Context + Cfg *config.Configuration +} { + var calls []struct { + Ctx context.Context + Cfg *config.Configuration + } + mock.lockDoGetCloudflareClient.RLock() + calls = mock.calls.DoGetCloudflareClient + mock.lockDoGetCloudflareClient.RUnlock() + return calls } // DoGetFilesAPIClient calls DoGetFilesAPIClientFunc. diff --git a/service/service.go b/service/service.go index 348015b5..28aca1f1 100644 --- a/service/service.go +++ b/service/service.go @@ -11,6 +11,7 @@ import ( "github.com/ONSdigital/dp-authorisation/auth" "github.com/ONSdigital/dp-dataset-api/api" "github.com/ONSdigital/dp-dataset-api/application" + "github.com/ONSdigital/dp-dataset-api/cloudflare" "github.com/ONSdigital/dp-dataset-api/config" "github.com/ONSdigital/dp-dataset-api/download" adapter "github.com/ONSdigital/dp-dataset-api/kafka" @@ -51,6 +52,7 @@ type Service struct { generateCantabularDownloadsProducer kafka.IProducer identityClient *clientsidentity.Client filesAPIClient filesAPISDK.Clienter + cloudflareClient cloudflare.Clienter server HTTPServer healthCheck HealthChecker api *api.DatasetAPI @@ -232,6 +234,10 @@ func (svc *Service) Run(ctx context.Context, buildTime, gitCommit, version strin return err } + if err := svc.initCloudflareClient(ctx); err != nil { + log.Error(ctx, "failed to initialise cloudflare client, continuing without cache purging", err) + } + ds := store.DataStore{Backend: DatsetAPIStore{svc.mongoDB, svc.graphDB}} // Get GenerateDownloads Kafka Producer @@ -319,7 +325,7 @@ func (svc *Service) Run(ctx context.Context, buildTime, gitCommit, version strin datasetPermissions, permissions := getAuthorisationHandlers(ctx, svc.config) sm := GetStateMachine(ctx, ds) svc.smDS = application.Setup(ds, smDownloadGenerators, sm) - svc.api = api.Setup(ctx, svc.config, r, ds, urlBuilder, downloadGenerators, datasetPermissions, permissions, enableURLRewriting, svc.smDS) + svc.api = api.Setup(ctx, svc.config, r, ds, urlBuilder, downloadGenerators, datasetPermissions, permissions, enableURLRewriting, svc.smDS, svc.cloudflareClient) // Set the files API client on the DatasetAPI after initialisation if svc.config.EnablePrivateEndpoints && svc.filesAPIClient != nil { @@ -380,6 +386,30 @@ func (svc *Service) initFilesAPIClient(ctx context.Context) error { return err } +func (svc *Service) initCloudflareClient(ctx context.Context) error { + if svc.config.CloudflareAPIToken == "" || svc.config.CloudflareZoneID == "" { + log.Info(ctx, "cloudflare integration disabled: missing API token or zone ID") + svc.cloudflareClient = nil + return nil + } + + log.Info(ctx, "initialising cloudflare client", log.Data{ + "api_url": svc.config.CloudflareAPIURL, + "zone_id": svc.config.CloudflareZoneID, + }) + + var err error + svc.cloudflareClient, err = svc.serviceList.GetCloudflareClient(ctx, svc.config) + + if err != nil { + log.Error(ctx, "failed to create cloudflare client", err) + return err + } + + log.Info(ctx, "cloudflare client initialised successfully") + return nil +} + func createURLBuilder(cfg *config.Configuration) (*url.Builder, error) { websiteURL, err := neturl.Parse(cfg.WebsiteURL) if err != nil { diff --git a/service/service_test.go b/service/service_test.go index 9a7aca6c..dd6afb16 100644 --- a/service/service_test.go +++ b/service/service_test.go @@ -7,6 +7,7 @@ import ( "sync" "testing" + "github.com/ONSdigital/dp-dataset-api/cloudflare" "github.com/ONSdigital/dp-dataset-api/config" "github.com/ONSdigital/dp-dataset-api/service" serviceMock "github.com/ONSdigital/dp-dataset-api/service/mock" @@ -124,6 +125,9 @@ func TestRun(t *testing.T) { Convey("Given that initialising MongoDB returns an error", func() { initMock := &serviceMock.InitialiserMock{ DoGetMongoDBFunc: funcDoGetMongoDBErr, + DoGetCloudflareClientFunc: func(ctx context.Context, cfg *config.Configuration) (cloudflare.Clienter, error) { + return nil, nil + }, } svcErrors := make(chan error, 1) svcList := service.NewServiceList(initMock) @@ -144,6 +148,9 @@ func TestRun(t *testing.T) { initMock := &serviceMock.InitialiserMock{ DoGetMongoDBFunc: funcDoGetMongoDBOk, DoGetGraphDBFunc: funcDoGetGraphDBErr, + DoGetCloudflareClientFunc: func(ctx context.Context, cfg *config.Configuration) (cloudflare.Clienter, error) { + return nil, nil + }, } svcErrors := make(chan error, 1) svcList := service.NewServiceList(initMock) @@ -165,6 +172,9 @@ func TestRun(t *testing.T) { DoGetMongoDBFunc: funcDoGetMongoDBOk, DoGetGraphDBFunc: funcDoGetGraphDBOk, DoGetFilesAPIClientFunc: funcDoGetFilesAPIClientErr, + DoGetCloudflareClientFunc: func(ctx context.Context, cfg *config.Configuration) (cloudflare.Clienter, error) { + return nil, nil + }, } svcErrors := make(chan error, 1) svcList := service.NewServiceList(initMock) @@ -187,6 +197,9 @@ func TestRun(t *testing.T) { DoGetGraphDBFunc: funcDoGetGraphDBOk, DoGetFilesAPIClientFunc: funcDoGetFilesAPIClientOk, DoGetKafkaProducerFunc: funcDoGetKafkaProducerErr, + DoGetCloudflareClientFunc: func(ctx context.Context, cfg *config.Configuration) (cloudflare.Clienter, error) { + return nil, nil + }, } svcErrors := make(chan error, 1) svcList := service.NewServiceList(initMock) @@ -210,6 +223,9 @@ func TestRun(t *testing.T) { DoGetFilesAPIClientFunc: funcDoGetFilesAPIClientOk, DoGetKafkaProducerFunc: funcDoGetKafkaProducerOk, DoGetHealthCheckFunc: funcDoGetHealthcheckErr, + DoGetCloudflareClientFunc: func(ctx context.Context, cfg *config.Configuration) (cloudflare.Clienter, error) { + return nil, nil + }, } svcErrors := make(chan error, 1) svcList := service.NewServiceList(initMock) @@ -241,6 +257,9 @@ func TestRun(t *testing.T) { DoGetHealthCheckFunc: func(*config.Configuration, string, string, string) (service.HealthChecker, error) { return hcMockAddFail, nil }, + DoGetCloudflareClientFunc: func(ctx context.Context, cfg *config.Configuration) (cloudflare.Clienter, error) { + return nil, nil + }, } svcErrors := make(chan error, 1) svcList := service.NewServiceList(initMock) @@ -273,6 +292,9 @@ func TestRun(t *testing.T) { DoGetKafkaProducerFunc: funcDoGetKafkaProducerOk, DoGetHealthCheckFunc: funcDoGetHealthcheckOk, DoGetHTTPServerFunc: funcDoGetHTTPServer, + DoGetCloudflareClientFunc: func(ctx context.Context, cfg *config.Configuration) (cloudflare.Clienter, error) { + return nil, nil + }, } svcErrors := make(chan error, 1) svcList := service.NewServiceList(initMock) @@ -312,6 +334,9 @@ func TestRun(t *testing.T) { DoGetKafkaProducerFunc: funcDoGetKafkaProducerOk, DoGetHealthCheckFunc: funcDoGetHealthcheckOk, DoGetHTTPServerFunc: funcDoGetHTTPServer, + DoGetCloudflareClientFunc: func(ctx context.Context, cfg *config.Configuration) (cloudflare.Clienter, error) { + return nil, nil + }, } svcErrors := make(chan error, 1) svcList := service.NewServiceList(initMock) @@ -347,6 +372,9 @@ func TestRun(t *testing.T) { DoGetKafkaProducerFunc: funcDoGetKafkaProducerOk, DoGetHealthCheckFunc: funcDoGetHealthcheckOk, DoGetHTTPServerFunc: funcDoGetFailingHTTPServer, + DoGetCloudflareClientFunc: func(ctx context.Context, cfg *config.Configuration) (cloudflare.Clienter, error) { + return nil, nil + }, } svcErrors := make(chan error, 1) svcList := service.NewServiceList(initMock)