diff --git a/internal/command/init_test.go b/internal/command/init_test.go index 444a082bb410..6316dc3f8036 100644 --- a/internal/command/init_test.go +++ b/internal/command/init_test.go @@ -3897,6 +3897,59 @@ func TestInit_stateStore_newWorkingDir(t *testing.T) { } }) + t.Run("temporary: test showing use of network mirror in mock provider source", func(t *testing.T) { + // Create a temporary, uninitialized working directory with configuration including a state store + td := t.TempDir() + testCopyDir(t, testFixturePath("init-with-state-store"), td) + t.Chdir(td) + + // Mock provider still needs to be supplied via testingOverrides despite the mock network mirror + mockProvider := mockPluggableStateStorageProvider() + mockProviderAddress := addrs.NewDefaultProvider("test") + + // Set up mock provider source that mocks out downloading hashicorp/test v1.2.3 from a network mirror. + source := newHTTPMirrorProviderSourceUsingTestHttpServer(t, map[string][]string{ + "hashicorp/test": {"1.0.0", "1.2.3"}, + }, true) + + ui := new(cli.MockUi) + view, done := testView(t) + meta := Meta{ + Ui: ui, + View: view, + AllowExperimentalFeatures: true, + testingOverrides: &testingOverrides{ + Providers: map[addrs.Provider]providers.Factory{ + mockProviderAddress: providers.FactoryFixed(mockProvider), + }, + }, + ProviderSource: source, + } + c := &InitCommand{ + Meta: meta, + } + + args := []string{"-enable-pluggable-state-storage-experiment=true"} + code := c.Run(args) + testOutput := done(t) + if code != 0 { + t.Fatalf("expected code 0 exit code, got %d, output: \n%s", code, testOutput.All()) + } + + // Check output + output := testOutput.All() + expectedOutputs := []string{ + "Initializing the state store...", + " Installed hashicorp/test v1.2.3 (verified checksum)", // verified checksum message due to hashes matching those described by the network mirror. + "Terraform has been successfully initialized!", + } + for _, expected := range expectedOutputs { + if !strings.Contains(output, expected) { + t.Fatalf("expected output to include %q, but got':\n %s", expected, output) + } + } + }) + t.Run("the init command creates a backend state file, and the default workspace is not made by default", func(t *testing.T) { // Create a temporary, uninitialized working directory with configuration including a state store td := t.TempDir() @@ -5978,16 +6031,19 @@ func newMockProviderSource(t *testing.T, availableProviderVersions map[string][] return getproviders.NewMockSource(packages, nil) } -// newMockProviderSourceViaHTTP is similar to newMockProviderSource except that the metadata (PackageMeta) for each provider -// reports that the provider is going to be accessed via HTTP +// newMockProviderSourceViaHTTP returns a mock provider source that will return +// metadata for providers defined by the calling test code. That metadata will +// report that the provider package is available for download via HTTP from a given +// address. That address is built using the address parameter. // -// Provider binaries are not available via the mock HTTP provider source. This source is sufficient only to allow Terraform -// to complete the provider installation process while believing it's installing providers over HTTP. -// This method is not sufficient to enable Terraform to use providers with those names. +// The mock HTTP provider source returned by newMockProviderSourceViaHTTP is not +// sufficient for Terraform to complete a provider installation process successfully; +// the provider source will supply Terraform with metadata describing where packages +// can be downloaded from, only. Without an HTTP server that serves files matching the +// metadata returned from this source, Terraform will fail during provider download. // -// When using `newMockProviderSourceViaHTTP` to set a value for `(Meta).ProviderSource` in a test, also set up `testOverrides` -// in the same Meta. That way the provider source will allow the download process to complete, and when Terraform attempts to use -// those binaries it will instead use the testOverride providers. +// Use newMockProviderSourceUsingTestHttpServer, a helper that sets up a test HTTP server +// to use in combination with this source. func newMockProviderSourceViaHTTP(t *testing.T, availableProviderVersions map[string][]string, address string) (source *getproviders.MockSource) { t.Helper() var packages []getproviders.PackageMeta @@ -6015,13 +6071,15 @@ func newMockProviderSourceViaHTTP(t *testing.T, availableProviderVersions map[st return getproviders.NewMockSource(packages, nil) } -// newMockProviderSourceUsingTestHttpServer is a helper that makes it easier to use newMockProviderSourceViaHTTP. -// This helper sets up a test HTTP server for use with newMockProviderSourceViaHTTP, and configures a handler that will respond when -// Terraform attempts to download provider binaries during installation. The mock source is returned ready to use and all cleanup is -// handled internally to this helper. +// newMockProviderSourceUsingTestHttpServer is a helper that returns a mock provider +// source that is paired with a test HTTP server. The provider source will tell Terraform +// that a given provider can be downloaded via HTTP from a given URL, and the test HTTP +// server will enable Terraform to perform that download successfully. // -// This source is not sufficient for providers to be available to use during a test; when using this helper, also set up testOverrides in -// the same Meta to provide the actual provider implementations for use during the test. +// This source is not sufficient for providers to be available to _use_ during a test, +// it is only sufficient to enable a provider installation process to complete successfully. +// If your test expects a provider to be installed and then used, ensure that testOverrides +// provides the actual provider implementations for use during the test. func newMockProviderSourceUsingTestHttpServer(t *testing.T, availableProviderVersions map[string][]string) *getproviders.MockSource { t.Helper() @@ -6076,7 +6134,7 @@ func newMockProviderSourceUsingTestHttpServer(t *testing.T, availableProviderVer t.Fatalf("unexpected URL path, test doesn't define a matching provider version: %s", r.URL.Path) } - // This code returns data in the temporary file that's created by the mock provider source. + // This code returns data in the temporary file that's created by this test helper. // This 'download' is not used when Terraform uses the provider after the mock installation completes; // Terraform will look for will use testOverrides in the Meta set up for this test. // @@ -6101,6 +6159,239 @@ func newMockProviderSourceUsingTestHttpServer(t *testing.T, availableProviderVer return source } +// newHTTPMirrorProviderSourceUsingTestHttpServer returns an HTTPMirrorSource that is backed +// by a test HTTPS server that acts as a network mirror. The test HTTP server will serve the +// providers and versions defined in the input map using the Provider Network Mirror Protocol. +// +// Calling code has the option of allowing the mirror to report hashes for each provider version, +// or to force the mirror to not report hashes. +// +// All cleanup is handled internally using t.Cleanup. +// +// This source is not sufficient for providers to be available to _use_ during a test, +// it is only sufficient to enable a provider installation process to complete successfully. +// If your test expects a provider to be installed and then used, ensure that testOverrides +// provides the actual provider implementations for use during the test. +func newHTTPMirrorProviderSourceUsingTestHttpServer(t *testing.T, input map[string][]string, allowReturnHashes bool) *getproviders.HTTPMirrorSource { + t.Helper() + + // Get un-started server so we can obtain the port it'll run on. + server := httptest.NewUnstartedServer(nil) + address := server.Listener.Addr().String() + + // Parse input map for convenience + availableProviderVersions := make(map[addrs.Provider][]getproviders.Version) + for pSource, versions := range input { + addr := addrs.MustParseProviderSourceString(pSource) + var parsedVersions []getproviders.Version + for _, versionStr := range versions { + version := getproviders.MustParseVersion(versionStr) + parsedVersions = append(parsedVersions, version) + } + availableProviderVersions[addr] = parsedVersions + } + + // Create tmp files for each provider version defined in the availableProviderVersions map. + // These files are later served by the mock network mirror HTTP server. + tmpFileLocations := map[addrs.Provider]map[getproviders.Version]string{} + for addr, versions := range availableProviderVersions { + for _, version := range versions { + f, _, err := getproviders.CreateFakeFileWithChecksumForProvider(t, addr, version, getproviders.CurrentPlatform, "") // file cleanup already handled + if err != nil { + t.Fatalf("failed to create fake package for provider %s %s: %s", addr, version, err) + } + if _, ok := tmpFileLocations[addr]; !ok { + tmpFileLocations[addr] = make(map[getproviders.Version]string) + } + tmpFileLocations[addr][version] = f.Name() + } + } + + // Implement a network mirror + // + // First we define handlers for different endpoints, and then we set up the + // server to use those handlers. + + // Handler for endpoints that list available versions of a provider + // GET :hostname/:namespace/:type/index.json + handleListEndpoint := func(addr addrs.Provider, w http.ResponseWriter, r *http.Request) { + // Create response body with the versions available for this provider. + response := getproviders.ListVersionsResponseBody{ + Versions: make(map[string]struct{}), + } + versions := availableProviderVersions[addr] + for _, v := range versions { + response.Versions[v.String()] = struct{}{} + } + + b, err := json.Marshal(response) + if err != nil { + t.Fatalf("failed to marshal versions response for provider %s: %s", addr, err) + } + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + w.Write(b) + } + + // Handler for endpoints that list data about a specific versions of a provider + // GET :hostname/:namespace/:type/:version.json + handleVersionEndpoint := func(addr addrs.Provider, v getproviders.Version, w http.ResponseWriter, r *http.Request) { + // Create response body with metadata for single package that matches current platform + response := getproviders.ListInstallationPackagesResponseBody{ + Archives: make(map[string]*getproviders.ListInstallationPackagesArchiveMeta), + } + // E.g. terraform-provider-foobar_1.0.0_darwin_amd64.zip + path := fmt.Sprintf( + "terraform-provider-%s_%s_%s.zip", + addr.Type, + v.String(), + getproviders.CurrentPlatform.String(), + ) + + // Make hashes + if _, ok := tmpFileLocations[addr]; !ok { + t.Fatalf("no package metadata found in the mock network mirror for provider source %q", addr) + } + if _, ok := tmpFileLocations[addr][v]; !ok { + t.Fatalf("no package metadata found in the mock network mirror for provider source %q and version %q", addr, v) + } + fileLocation := tmpFileLocations[addr][v] + zHash, err := getproviders.PackageHashLegacyZipSHA(getproviders.PackageLocalArchive(fileLocation)) + if err != nil { + t.Fatalf("failed to compute hash for provider source %q and version %q: %s", addr, v, err) + } + h1Hash, err := getproviders.PackageHashV1(getproviders.PackageLocalArchive(fileLocation)) + if err != nil { + t.Fatalf("failed to compute hash for provider source %q and version %q: %s", addr, v, err) + } + + m := &getproviders.ListInstallationPackagesArchiveMeta{ + RelativeURL: path, + } + if allowReturnHashes { + // Test may want no hashes to be returned, to test the behaviour when no authentication data + // is available for a provider package. + m.Hashes = []string{ + zHash.String(), + h1Hash.String(), + } + } + response.Archives[getproviders.CurrentPlatform.String()] = m + + b, err := json.Marshal(response) + if err != nil { + t.Fatalf("failed to marshal versions response for provider %s: %s", addr, err) + } + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + w.Write(b) + } + + // Handler for endpoints that allow downloading a provider archive. + // GET hostname/:namespace/:type/:filename , where filename always ends in .zip + handleZipDownloadEndpoint := func(addr addrs.Provider, v getproviders.Version, w http.ResponseWriter, r *http.Request) { + // This code returns data in the temporary file that's created by this test helper. + // This 'download' is not used when Terraform uses the provider after the mock installation completes; + // Terraform will look for will use testOverrides in the Meta set up for this test. + // + // Although it's not used later we need to use this file (versus empty or made-up bytes) to enable installation + // logic to receive data with the correct checksum. + fileLocation, ok := tmpFileLocations[addr][v] + if !ok { + t.Fatalf("no package metadata found in the mock network mirror for provider source %q and version %q", addr, v) + } + + f, err := os.Open(fileLocation) + if err != nil { + t.Fatalf("failed to open mock source file: %s", err) + } + defer f.Close() + archiveBytes, err := io.ReadAll(f) + if err != nil { + t.Fatalf("failed to read mock source file: %s", err) + } + w.Header().Add("Content-Type", "application/zip") + w.WriteHeader(http.StatusOK) + w.Write(archiveBytes) + } + + // Set up how the server handles requests. + // This includes validating that the request matches a provider version that was specified in the input map + // from the calling test code. + server.Config = &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + parts := strings.Split(r.URL.Path, "/") + if len(parts) < 5 { + t.Fatalf("unexpected URL path in request to test network mirror: %s", r.URL.Path) + } + + // Parse provider source; avoid repeated logic in handlers below + providerSource := strings.Join(parts[1:len(parts)-1], "/") + providerAddr, diag := addrs.ParseProviderSourceString(providerSource) + if diag.HasErrors() { + t.Fatalf("failed to parse provider source from URL path %q: %s", r.URL.Path, diag.Err()) + } + + // Is the request for a provider source and version combo specified by the test? + if _, ok := availableProviderVersions[providerAddr]; !ok { + t.Fatalf("provider source %q is not available in the mock network mirror", providerAddr) + } + var v getproviders.Version + if !strings.HasSuffix(r.URL.Path, "/index.json") { + // Only parse and assert version for requests that include a version in the path + versionRegex := regexp.MustCompile(`([0-9]+\.[0-9]+\.[0-9]+)`) + versionStr := versionRegex.FindString(r.URL.Path) + if versionStr == "" { + t.Fatalf("expected to be able to parse version from URL path %q", r.URL.Path) + } + v = getproviders.MustParseVersion(versionStr) + + found := false + for _, pv := range availableProviderVersions[providerAddr] { + if pv.Same(v) { + found = true + break + } + } + if !found { + t.Fatalf("provider source %q and version %q is not available in the mock network mirror", providerAddr, v) + } + } + + switch { + case strings.HasSuffix(r.URL.Path, "/index.json"): + // List versions for a provider + // GET :hostname/:namespace/:type/index.json + handleListEndpoint(providerAddr, w, r) + return + case strings.HasSuffix(r.URL.Path, ".json"): + // Show archives for a specific version + // GET :hostname/:namespace/:type/:version.json + handleVersionEndpoint(providerAddr, v, w, r) + return + case strings.HasSuffix(r.URL.Path, ".zip"): + // Handle provider binary download requests. + // GET :hostname/:namespace/:type/:filename.zip + handleZipDownloadEndpoint(providerAddr, v, w, r) + return + default: + t.Fatalf("unhandled request to mock network mirror HTTP server:\npath: %s\n request: %#v", r.URL.Path, r) + } + })} + + server.Start() // Mock HTTP Mirror source doesn't enforce TLS + t.Cleanup(server.Close) + + // Set up mock provider source that mocks installation via HTTP. + url := &url.URL{ + Scheme: "http", // No TLS again + Host: address, + } + + source := getproviders.NewMockHTTPMirrorSource(t, url) + + return source +} + // installFakeProviderPackages installs a fake package for the given provider // names (interpreted as a "default" provider address) and versions into the // local plugin cache for the given "meta". diff --git a/internal/getproviders/http_mirror_source.go b/internal/getproviders/http_mirror_source.go index 4b8937986842..9a7c42a96d56 100644 --- a/internal/getproviders/http_mirror_source.go +++ b/internal/getproviders/http_mirror_source.go @@ -14,6 +14,7 @@ import ( "net/url" "path" "strings" + "testing" "github.com/hashicorp/go-retryablehttp" svchost "github.com/hashicorp/terraform-svchost" @@ -55,14 +56,35 @@ func NewHTTPMirrorSource(baseURL *url.URL, creds svcauth.CredentialsSource) *HTT } return nil } + // Enforce TLS + return newHTTPMirrorSourceWithHTTPClientTLS(baseURL, creds, httpClient) +} + +func NewMockHTTPMirrorSource(t *testing.T, baseURL *url.URL) *HTTPMirrorSource { + httpClient := httpclient.New() + httpClient.Timeout = requestTimeout + httpClient.CheckRedirect = func(req *http.Request, via []*http.Request) error { + // If we get redirected more than five times we'll assume we're + // in a redirect loop and bail out, rather than hanging forever. + if len(via) > 5 { + return fmt.Errorf("too many redirects") + } + return nil + } + + // In tests we don't enforce TLS and also there are no creds + var creds svcauth.CredentialsSource // nil return newHTTPMirrorSourceWithHTTPClient(baseURL, creds, httpClient) } -func newHTTPMirrorSourceWithHTTPClient(baseURL *url.URL, creds svcauth.CredentialsSource, httpClient *http.Client) *HTTPMirrorSource { +func newHTTPMirrorSourceWithHTTPClientTLS(baseURL *url.URL, creds svcauth.CredentialsSource, httpClient *http.Client) *HTTPMirrorSource { if baseURL.Scheme != "https" { panic("non-https URL for HTTP mirror") } + return newHTTPMirrorSourceWithHTTPClient(baseURL, creds, httpClient) +} +func newHTTPMirrorSourceWithHTTPClient(baseURL *url.URL, creds svcauth.CredentialsSource, httpClient *http.Client) *HTTPMirrorSource { // We borrow the retry settings and behaviors from the registry client, // because our needs here are very similar to those of the registry client. retryableClient := retryablehttp.NewClient() @@ -117,10 +139,7 @@ func (s *HTTPMirrorSource) AvailableVersions(ctx context.Context, provider addrs // If we got here then the response had status OK and so our body // will be non-nil and should contain some JSON for us to parse. - type ResponseBody struct { - Versions map[string]struct{} `json:"versions"` - } - var bodyContent ResponseBody + var bodyContent ListVersionsResponseBody dec := json.NewDecoder(body) if err := dec.Decode(&bodyContent); err != nil { @@ -181,14 +200,7 @@ func (s *HTTPMirrorSource) PackageMeta(ctx context.Context, provider addrs.Provi // If we got here then the response had status OK and so our body // will be non-nil and should contain some JSON for us to parse. - type ResponseArchiveMeta struct { - RelativeURL string `json:"url"` - Hashes []string - } - type ResponseBody struct { - Archives map[string]*ResponseArchiveMeta `json:"archives"` - } - var bodyContent ResponseBody + var bodyContent ListInstallationPackagesResponseBody dec := json.NewDecoder(body) if err := dec.Decode(&bodyContent); err != nil { @@ -247,6 +259,28 @@ func (s *HTTPMirrorSource) ForDisplay(provider addrs.Provider) string { return "provider mirror at " + s.baseURL.String() } +// ListVersionsResponseBody is the JSON structure of a response when a user queries the available versions +// for a provider in the network mirror, i.e. a GET to path :hostname/:namespace/:type/index.json +// See: https://developer.hashicorp.com/terraform/internals/provider-network-mirror-protocol#list-available-versions +type ListVersionsResponseBody struct { + Versions map[string]struct{} `json:"versions"` +} + +// ListInstallationPackagesResponseBody is the structure of the JSON response when a user queries the +// available packages for a given provider version in the network mirror. +// i.e. at the path :hostname/:namespace/:type/:version.json +// See: https://developer.hashicorp.com/terraform/internals/provider-network-mirror-protocol#list-available-installation-packages +type ListInstallationPackagesResponseBody struct { + Archives map[string]*ListInstallationPackagesArchiveMeta `json:"archives"` +} + +// ListInstallationPackagesArchiveMeta is the JSON structure of each archive entry for a specific provider +// version in the network mirror. See ListInstallationPackagesResponseBody. +type ListInstallationPackagesArchiveMeta struct { + RelativeURL string `json:"url"` + Hashes []string +} + // mirrorHost extracts the hostname portion of the configured base URL and // returns it as a svchost.Hostname, normalized in the usual ways. //