feat: add service/{service}/{env}/env_vars path

Allow dev values to be loaded in services namespace. Change order of
operations to always load env specific values over base path

Author: mloberg

README.md

@@ -19,22 +19,38 @@ ENTRYPOINT [ "/entrypoint" ]
 
 ## Usage
 
-To load a environment variables, you'll need to set a `SERVICE_ENV` environment
-variable to `prod`, `stage`, `dev`, or `peer`. The following paths get loaded as
-environment variables, but some environments may change this. You can view exact
-paths used in each template.
+To load values from Consul's KV store, you will need to set `CONSUL_ADDR`. It
+will load keys from the following paths, using the basename as the variable name:
 
-* `global/env_vars/*` (_Consul_)
-* `services/${SERVICE_NAME}/env_vars/*` (_Consul_)
-* `secrets/global/env_vars/*` (_Vault_ using the `value` key)
-* `services/${SERVICE_NAME}/env_vars/*` (_Vault_ using the `value` key)
+* `global/env_vars/*`
+* `global/${SERVICE_ENV}/env_vars/*`
+* `services/${SERVICE_NAME}/env_vars/*`
+* `service/${SERVICE_NAME}/${SERVICE_ENV}/env_vars/*`
 
-To load values from Consul, you'll need to make sure `CONSUL_ADDR` is accessible
-from your Docker container.
+For example, `consul kv put services/foo/env_vars/API_SERVICE_URI https://api.priv/v1`
+will load an environment variable `API_SERVICE_URI=https://api.priv/v1`.
 
-To load values from Vault, you'll need to make sure both `CONSUL_ADDR` and `VAULT_ADDR`
-are accessible. You'll also need to authenticate with Vault in one of the following
-ways:
+Any environment variables set previous to calling the script, will not change.
+Paths later in the list will overwrite any previous values. For example,
+`global/env_vars/FOO` will be overwritten by `service/my-service/env_vars/FOO`.
+
+To load values from Vault, you will need to set `VAULT_ADDR` and authenticate with
+Vault (see below). Values from vault will use the `value` key as the variable value.
+Values are read from the following paths:
+
+* `secret/global/env_vars/*` (in `stage` or `prod`)
+* `secret/global/${SERVICE_ENV}/env_vars/*`
+* `secret/services/${SERVICE_NAME}/env_vars/*` (in `stage` or `prod`)
+* `secret/service/${SERVICE_NAME}/${SERVICE_ENV}/env_vars/*`
+
+For example, `vault write secret/foo/env_vars/API_KEY value=secretkey` will load
+an environment variable `API_KEY=secretkey`. Values from Vault will overrwrite
+Consul values, but follow the same rules otherwise.
+
+<details>
+<summary>Vault Authentication</summary>
+
+You can authenticate with Vault in one of the following ways:
 
 * Set `VAULT_TOKEN`
 * Set `ENCRYPTED_VAULT_TOKEN` with a value encrypted by AWS KMS
@@ -43,6 +59,8 @@ ways:
 * If running on AWS ECS or Lambda, use the AWS IAM auth method
   * If Vault role does not match IAM role, set with `VAULT_ROLE`
 
+</details>
+
 ## Development
 
 You'll need to install the following:
@@ -61,5 +79,5 @@ or [commitizen](https://github.com/commitizen-tools/commitizen#installation).
 
 ## Creating a Release
 
-To create a release, create a tag that follows [semver](https://semver.org/) and
-a GitHub Action workflow will take care of creating the release.
+To create a release, create a tag that follows [semver](https://semver.org/). A
+GitHub Action workflow will take care of creating the release.

config.go

@@ -1,6 +1,8 @@
 package main
 
-import "fmt"
+import (
+	"fmt"
+)
 
 type Config struct {
 	Service     string
@@ -12,29 +14,32 @@ type Config struct {
 // ConsulPaths returns the paths from Consul to load
 func (c *Config) ConsulPaths() []string {
 	return []string{
-		fmt.Sprintf("global/%s/env_vars", c.Environment), // DEPRECATED
 		"global/env_vars",
+		fmt.Sprintf("global/%s/env_vars", c.Environment),
 		fmt.Sprintf("products/%s/env_vars", c.Product),               // DEPRECATED
 		fmt.Sprintf("apps/%s/%s/env_vars", c.Service, c.Environment), // DEPRECATED
 		fmt.Sprintf("services/%s/env_vars", c.Service),
+		fmt.Sprintf("services/%s/%s/env_vars", c.Service, c.Environment),
 	}
 }
 
 // VaultPaths returns the paths from Vault to load
 func (c *Config) VaultPaths() []string {
 	if c.Environment == "stage" || c.Environment == "prod" {
 		return []string{
-			fmt.Sprintf("secret/global/%s/env_vars", c.Environment), // DEPRECATED
 			"secret/global/env_vars",
+			fmt.Sprintf("secret/global/%s/env_vars", c.Environment),
 			fmt.Sprintf("secret/products/%s/env_vars", c.Product),               // DEPRECATED
 			fmt.Sprintf("secret/apps/%s/%s/env_vars", c.Service, c.Environment), // DEPRECATED
 			fmt.Sprintf("secret/services/%s/env_vars", c.Service),
+			fmt.Sprintf("secret/services/%s/%s/env_vars", c.Service, c.Environment),
 		}
 	}
 
 	return []string{
-		fmt.Sprintf("secret/global/%s/env_vars", c.Environment),                 // DEPRECATED
+		fmt.Sprintf("secret/global/%s/env_vars", c.Environment),
 		fmt.Sprintf("secret/products/%s/%s/env_vars", c.Product, c.Environment), // DEPRECATED
 		fmt.Sprintf("secret/apps/%s/%s/env_vars", c.Service, c.Environment),     // DEPRECATED
+		fmt.Sprintf("secret/services/%s/%s/env_vars", c.Service, c.Environment),
 	}
 }

config_test.go

@@ -14,11 +14,12 @@ func TestConfig_ConsulPaths(t *testing.T) {
 	}
 
 	assert.Equal(t, []string{
-		"global/stage/env_vars",
 		"global/env_vars",
+		"global/stage/env_vars",
 		"products/bar/env_vars",
 		"apps/foo/stage/env_vars",
 		"services/foo/env_vars",
+		"services/foo/stage/env_vars",
 	}, c.ConsulPaths())
 }
 
@@ -30,26 +31,29 @@ func TestConfig_VaultPaths(t *testing.T) {
 	}
 
 	assert.Equal(t, []string{
-		"secret/global/stage/env_vars",
 		"secret/global/env_vars",
+		"secret/global/stage/env_vars",
 		"secret/products/bar/env_vars",
 		"secret/apps/foo/stage/env_vars",
 		"secret/services/foo/env_vars",
+		"secret/services/foo/stage/env_vars",
 	}, c.VaultPaths())
 
 	c.Environment = "prod"
 	assert.Equal(t, []string{
-		"secret/global/prod/env_vars",
 		"secret/global/env_vars",
+		"secret/global/prod/env_vars",
 		"secret/products/bar/env_vars",
 		"secret/apps/foo/prod/env_vars",
 		"secret/services/foo/env_vars",
+		"secret/services/foo/prod/env_vars",
 	}, c.VaultPaths())
 
 	c.Environment = "dev"
 	assert.Equal(t, []string{
 		"secret/global/dev/env_vars",
 		"secret/products/bar/dev/env_vars",
 		"secret/apps/foo/dev/env_vars",
+		"secret/services/foo/dev/env_vars",
 	}, c.VaultPaths())
 }