From 1b0953bb87972fb611c01ab73ce47fb4afa9c944 Mon Sep 17 00:00:00 2001 From: "Alan D. Salewski" Date: Mon, 31 Aug 2020 06:05:57 -0400 Subject: [PATCH] Incorporate go-getter support for 'git::' filepaths The go-getters library now has support for local file system paths to Git repositories, specified with the 'git::' forcing token. The feature works for both absolute and relative filepaths, and supports all the usual go-getter goodies including '//' delimited subdirs and URI-style query parameters.[0][1] We incorporate that capability into Terraform, which allows users to specify paths to locally present Git repositories from which to clone other Terrform modules on which they are dependent. When coupled with Git submodules, this creates a powerful way to manage Terraform modules at specific versions without requiring those modules to be available on the network (e.g., on GitHub): module "my_module" { source = "git::../git-submodules/tf-modules/some-tf-module?ref=v0.1.0" // ... } From the perspective of Terraform, such Git repositories are "remote" in the same way that repositories on GitHub are. Note that within a Terraform module "call" block, the filepaths specified are relative to the directory in which the *.tf file lives, not relative to the current working directory of the Terraform process. In order to support this feature, Terraform needs to supply that contextual information to go-getter to allow relative filepath resolution to work. In order to do so, we needed to switch over to using go-getter's new "Contextual Detector" API. It works in the same basic way as the traditional Detector API, but allows us to provide this additional information. In keeping with the "keep things simple" comment in the commit message of 2b2ac1f6deb9, we are here maintaining our custom go-getter detectors in two places. Only now each is called FooCtxDetector rather than FooDetector. Nevertheless, all except the GitCtxDetector do little more than "pass through" delegation to its analogous FooDetector counterpart. Fixes #25488 Fixes #21107 [0] https://github.com/hashicorp/go-getter/issues/268 [1] https://github.com/hashicorp/go-getter/pull/269 --- configs/configload/getter.go | 18 +++++----- internal/initwd/getter.go | 18 +++++----- internal/initwd/module_install.go | 59 +++++++++++++++++++++++++++++-- 3 files changed, 75 insertions(+), 20 deletions(-) diff --git a/configs/configload/getter.go b/configs/configload/getter.go index d0c4567b6dee..a37507a3bc70 100644 --- a/configs/configload/getter.go +++ b/configs/configload/getter.go @@ -17,13 +17,13 @@ import ( // any meddling that might be done by other go-getter callers linked into our // executable. -var goGetterDetectors = []getter.Detector{ - new(getter.GitHubDetector), - new(getter.GitDetector), - new(getter.BitBucketDetector), - new(getter.GCSDetector), - new(getter.S3Detector), - new(getter.FileDetector), +var goGetterCtxDetectors = []getter.CtxDetector{ + new(getter.GitHubCtxDetector), + new(getter.GitCtxDetector), + new(getter.BitBucketCtxDetector), + new(getter.GCSCtxDetector), + new(getter.S3CtxDetector), + new(getter.FileCtxDetector), } var goGetterNoDetectors = []getter.Detector{} @@ -80,12 +80,12 @@ type reusingGetter map[string]string // end-user-actionable error messages. At this time we do not have any // reasonable way to improve these error messages at this layer because // the underlying errors are not separatelyr recognizable. -func (g reusingGetter) getWithGoGetter(instPath, addr string) (string, error) { +func (g reusingGetter) getWithGoGetter(instPath, addr, srcAbs string) (string, error) { packageAddr, subDir := splitAddrSubdir(addr) log.Printf("[DEBUG] will download %q to %s", packageAddr, instPath) - realAddr, err := getter.Detect(packageAddr, instPath, goGetterDetectors) + realAddr, err := getter.CtxDetect(packageAddr, instPath, srcAbs, goGetterCtxDetectors) if err != nil { return "", err } diff --git a/internal/initwd/getter.go b/internal/initwd/getter.go index 79bb7d7b625a..54e557943086 100644 --- a/internal/initwd/getter.go +++ b/internal/initwd/getter.go @@ -20,13 +20,13 @@ import ( // any meddling that might be done by other go-getter callers linked into our // executable. -var goGetterDetectors = []getter.Detector{ - new(getter.GitHubDetector), - new(getter.GitDetector), - new(getter.BitBucketDetector), - new(getter.GCSDetector), - new(getter.S3Detector), - new(getter.FileDetector), +var goGetterCtxDetectors = []getter.CtxDetector{ + new(getter.GitHubCtxDetector), + new(getter.GitCtxDetector), + new(getter.BitBucketCtxDetector), + new(getter.GCSCtxDetector), + new(getter.S3CtxDetector), + new(getter.FileCtxDetector), } var goGetterNoDetectors = []getter.Detector{} @@ -83,12 +83,12 @@ type reusingGetter map[string]string // end-user-actionable error messages. At this time we do not have any // reasonable way to improve these error messages at this layer because // the underlying errors are not separately recognizable. -func (g reusingGetter) getWithGoGetter(instPath, addr string) (string, error) { +func (g reusingGetter) getWithGoGetter(instPath, addr, srcAbs string) (string, error) { packageAddr, subDir := splitAddrSubdir(addr) log.Printf("[DEBUG] will download %q to %s", packageAddr, instPath) - realAddr, err := getter.Detect(packageAddr, instPath, goGetterDetectors) + realAddr, err := getter.CtxDetect(packageAddr, instPath, srcAbs, goGetterCtxDetectors) if err != nil { return "", err } diff --git a/internal/initwd/module_install.go b/internal/initwd/module_install.go index 38a44c262666..b61976637633 100644 --- a/internal/initwd/module_install.go +++ b/internal/initwd/module_install.go @@ -450,7 +450,13 @@ func (i *ModuleInstaller) installRegistryModule(req *earlyconfig.ModuleRequest, log.Printf("[TRACE] ModuleInstaller: %s %s %s is available at %q", key, addr, latestMatch, dlAddr) - modDir, err := getter.getWithGoGetter(instPath, dlAddr) + srcDirAbs, sdDiags := i.calledFromSourceDirAbs(req) + if srcDirAbs == "" { + diags = append(diags, sdDiags...) + return nil, nil, diags + } + + modDir, err := getter.getWithGoGetter(instPath, dlAddr, srcDirAbs) if err != nil { // Errors returned by go-getter have very inconsistent quality as // end-user error messages, but for now we're accepting that because @@ -521,7 +527,13 @@ func (i *ModuleInstaller) installGoGetterModule(req *earlyconfig.ModuleRequest, return nil, diags } - modDir, err := getter.getWithGoGetter(instPath, req.SourceAddr) + srcDirAbs, sdDiags := i.calledFromSourceDirAbs(req) + if srcDirAbs == "" { + diags = append(diags, sdDiags...) + return nil, diags + } + + modDir, err := getter.getWithGoGetter(instPath, req.SourceAddr, srcDirAbs) if err != nil { if _, ok := err.(*MaybeRelativePathErr); ok { log.Printf( @@ -588,3 +600,46 @@ func (i *ModuleInstaller) installGoGetterModule(req *earlyconfig.ModuleRequest, func (i *ModuleInstaller) packageInstallPath(modulePath addrs.Module) string { return filepath.Join(i.modsDir, strings.Join(modulePath, ".")) } + +// A called module can exist in a (possibly remote) source code repository, +// and go-getter will attempt to retrieve the modules from the repo if that is +// the case. +// +// There is special case notion of "remote" for Terraform modules that live in +// source code repositories that are "outside of the user's current Terraform +// project", but which are expected to be present on the local file system. In +// particular, Git submodules (see git-submodule(1)) can be used to keep such +// repos nearby, in a location from which they can be cloned relative to +// top-level directory of the current project. +// +// The location to such local repos can only be known in advance by the +// relative path **from the Terraform module** that is "calling" a module +// whose source comes from such a repo: +// +// module "my_module" { +// source = "git::../../../git-submodules/tf-modules/my-tf-module//some/subdir?rev=v1.2.3" +// // ... +// } +// +// To locate such repos, we must provide go-getter with the absolute path to +// the directory for the Terraform module that contains such references. +// +func (i *ModuleInstaller) calledFromSourceDirAbs(req *earlyconfig.ModuleRequest) (string, tfdiags.Diagnostics) { + callingModSourceDir := filepath.Dir(req.CallPos.Filename) // dirname + + var diags tfdiags.Diagnostics + + abs, err := filepath.Abs(callingModSourceDir) + if err != nil { + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Error, + "Unable to resolve absolute filepath of Terraform module", + fmt.Sprintf( + "Terraform tried resolve the absolute filepath of source file \"%s\", but encountered an error: %s", + callingModSourceDir, err, + ), + )) + } + + return abs, diags +}