From b607ac8ff56abf745cd197bc65c06686db81eb68 Mon Sep 17 00:00:00 2001 From: Matt Loberg Date: Thu, 21 Mar 2024 10:56:08 -0500 Subject: [PATCH 1/2] feat: validate environment variables Allow parsing a service.json file for a list of required and optional variables. Log out any variables that aren't set for the given environment. If at least one required one is missing it will return an error --- README.md | 31 ++++++++++++++- config.go | 18 ++++++--- env.go | 6 +++ main.go | 7 +++- validate.go | 98 ++++++++++++++++++++++++++++++++++++++++++++++ validate_test.go | 100 +++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 252 insertions(+), 8 deletions(-) create mode 100644 validate.go create mode 100644 validate_test.go diff --git a/README.md b/README.md index 8ac3afa..3bdc10a 100644 --- a/README.md +++ b/README.md @@ -61,11 +61,40 @@ You can authenticate with Vault in one of the following ways: +### Environment Variables + +If you want to ensure some environment variables exist before running your command, +you can include a JSON file called `service.json` in the working directory. The +entrypoint will parse this file and check that the configured environment variables +exist and are not empty. + +```json +{ + "dependencies": { + "env_vars": { + "required": [ + "FOO", + "BAR" + ], + "optional": [ + "BAZ" + ] + } + } +} +``` + +If any optional environment variables are missing, it will log that, but continue +to run. + +If any required environment variables are missing, it will log that and then exit +with an exit code of 4. + ## Development You'll need to install the following: -* Go 1.20 +* Go * [golangci-lint](https://golangci-lint.run/) (`brew install golangci-lint`) * [pre-commit](https://pre-commit.com/) (`brew install pre-commit`) * [GoReleaser](https://goreleaser.com/) (_optional_) diff --git a/config.go b/config.go index cc5b088..c52527a 100644 --- a/config.go +++ b/config.go @@ -7,17 +7,19 @@ import ( ) type Config struct { - Service string - Environment string - Region string + Service string + Environment string + Region string + ServiceDefinition string } // NewFromEnv creates a new Config from environment variables and defaults func NewFromEnv() *Config { cfg := &Config{ - Service: os.Getenv("SERVICE_NAME"), - Environment: os.Getenv("SERVICE_ENV"), - Region: os.Getenv("AWS_REGION"), + Service: os.Getenv("SERVICE_NAME"), + Environment: os.Getenv("SERVICE_ENV"), + Region: os.Getenv("AWS_REGION"), + ServiceDefinition: os.Getenv("SERVICE_DEFINITION"), } if cfg.Service == "" { @@ -33,6 +35,10 @@ func NewFromEnv() *Config { cfg.Region = "us-east-1" } + if cfg.ServiceDefinition == "" { + cfg.ServiceDefinition = "service.json" + } + return cfg } diff --git a/env.go b/env.go index 780a423..caf7884 100644 --- a/env.go +++ b/env.go @@ -60,3 +60,9 @@ func (e *EnvMap) Environ() []string { return fmt.Sprintf("%s=%s", k, v) }) } + +// Has returns true if the given key is set and not empty +func (e *EnvMap) Has(key string) bool { + v, ok := e.env[key] + return ok && v != "" +} diff --git a/main.go b/main.go index 4f8d7f3..6bf1100 100644 --- a/main.go +++ b/main.go @@ -64,6 +64,11 @@ func main() { env.Add("SERVICE_ENV", cfg.Environment) env.Add("PROCESSOR_COUNT", strconv.Itoa(runtime.NumCPU())) + if err := validate(ctx, cfg, env, logger); err != nil { + logger.ErrorContext(ctx, "Missing dependencies", "error", err) + os.Exit(4) + } + os.Exit(run(ctx, os.Args[1], os.Args[2:], env.Environ(), logger)) } @@ -139,7 +144,7 @@ func run(ctx context.Context, name string, args, env []string, l *slog.Logger) i return exit.ExitCode() } l.ErrorContext(ctx, "Unknown error while running command", "error", err, "cmd", cmd.String()) - return 1 + return 3 } return 0 diff --git a/validate.go b/validate.go new file mode 100644 index 0000000..4f6d22c --- /dev/null +++ b/validate.go @@ -0,0 +1,98 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "os" + + "github.com/samber/lo" +) + +type ( + serviceConfig struct { + Dependencies struct { + EnvVars struct { + Required []dependency `json:"required"` + Optional []dependency `json:"optional"` + } `json:"env_vars"` + } `json:"dependencies"` + } + dependency struct { + dependencyInner + Partial bool `json:"-"` + } + dependencyInner struct { + Key string `json:"key"` + Regions []string `json:"regions"` + } +) + +var ErrMissingEnvVars = errors.New("missing required environment variables") + +// Required returns true if the dependency is required for the given region +func (d *dependency) Required(region string) bool { + return d.Regions == nil || lo.Contains(d.Regions, region) +} + +// UnmarshalJSON handles the dependency being a string or an object +func (d *dependency) UnmarshalJSON(data []byte) error { + var str string + if err := json.Unmarshal(data, &str); err == nil { + d.Key = str + d.Partial = true + return nil + } + + var dep dependencyInner + if err := json.Unmarshal(data, &dep); err != nil { + return fmt.Errorf("could not decode dependency: %w", err) + } + + d.dependencyInner = dep + return nil +} + +func validate(ctx context.Context, c *Config, e *EnvMap, l *slog.Logger) error { + f, err := os.ReadFile(c.ServiceDefinition) + if os.IsNotExist(err) { + return nil + } else if err != nil { + return fmt.Errorf("could not read service definition: %w", err) + } + + var cfg serviceConfig + if err := json.Unmarshal(f, &cfg); err != nil { + return fmt.Errorf("could not decode service definition: %w", err) + } + + req := missing(cfg.Dependencies.EnvVars.Required, c, e) + opt := missing(cfg.Dependencies.EnvVars.Optional, c, e) + + if len(opt) != 0 { + l.WarnContext(ctx, "Missing optional environment variables", "env_vars", opt) + } + + if len(req) != 0 { + l.ErrorContext(ctx, "Missing required environment variables", "env_vars", req) + return ErrMissingEnvVars + } + + return nil +} + +func missing(deps []dependency, c *Config, e *EnvMap) []string { + res := []string{} + for _, d := range deps { + if !d.Required(c.Region) { + continue + } + + if v := os.Getenv(d.Key); v == "" && !e.Has(d.Key) { + res = append(res, d.Key) + } + } + return res +} diff --git a/validate_test.go b/validate_test.go new file mode 100644 index 0000000..da93dc1 --- /dev/null +++ b/validate_test.go @@ -0,0 +1,100 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDependency_Required(t *testing.T) { + d := dependency{ + dependencyInner: dependencyInner{ + Regions: []string{"us-east-1"}, + }, + } + assert.True(t, d.Required("us-east-1")) + assert.False(t, d.Required("us-west-2")) + + d = dependency{} + assert.True(t, d.Required("us-east-1")) + assert.True(t, d.Required("us-west-2")) +} + +func TestValidate(t *testing.T) { //nolint:funlen + s := filepath.Join(t.TempDir(), "service.json") + require.NoError(t, os.WriteFile(s, []byte(`{ + "dependencies": { + "env_vars": { + "required": [ + "FOO", + { + "key": "BAR", + "regions": ["us-east-1"] + }, + { + "key":"BAZ" + } + ], + "optional": [ + "QUX", + { + "key": "FOOBAR", + "regions": ["eu-central-1"] + }, + { + "key":"FOOBAZ" + } + ] + } + } +}`), 0o600)) + + l, log := testLogger() + c := &Config{ServiceDefinition: s, Region: "us-east-1"} + + e := NewEnvMap() + + err := validate(context.TODO(), c, e, l) + require.ErrorIs(t, err, ErrMissingEnvVars) + assert.Contains( + t, + log.String(), + `"ERROR","msg":"Missing required environment variables","env_vars":["FOO","BAR","BAZ"]`, + ) + assert.Contains(t, log.String(), `"WARN","msg":"Missing optional environment variables","env_vars":["QUX","FOOBAZ"]`) + + // Empty env vars should be considered missing + e.Add("FOO", "") + t.Setenv("BAR", "") + + log.Reset() + err = validate(context.TODO(), c, e, l) + require.ErrorIs(t, err, ErrMissingEnvVars) + assert.Contains(t, log.String(), `Missing required environment variables","env_vars":["FOO","BAR"`) + + // Set all required env vars + c.Region = "eu-central-1" + e.Add("FOO", "foo") + t.Setenv("BAZ", "baz") + + log.Reset() + err = validate(context.TODO(), c, e, l) + require.NoError(t, err) + assert.NotContains(t, log.String(), "Missing required environment variables") + assert.Contains(t, log.String(), "Missing optional environment variables") + + // Set all optional env vars + e.Add("QUX", "qux") + e.Add("FOOBAR", "foobar") + t.Setenv("FOOBAZ", "foobaz") + + log.Reset() + err = validate(context.TODO(), c, e, l) + require.NoError(t, err) + assert.NotContains(t, log.String(), "Missing required environment variables") + assert.NotContains(t, log.String(), "Missing optional environment variables") +} From 2b78117d1ee0e3aaf3bb91dbbee4c35329de505d Mon Sep 17 00:00:00 2001 From: Matt Loberg Date: Thu, 21 Mar 2024 13:52:31 -0500 Subject: [PATCH 2/2] feat: allow skipping env var validation Don't run validation in a test environment or when set to skip. Improve setting debug mode --- config.go | 7 +++++++ main.go | 2 +- validate.go | 4 ++++ validate_test.go | 10 ++++++++++ 4 files changed, 22 insertions(+), 1 deletion(-) diff --git a/config.go b/config.go index c52527a..38c0ad9 100644 --- a/config.go +++ b/config.go @@ -4,6 +4,7 @@ import ( "fmt" "log/slog" "os" + "strconv" ) type Config struct { @@ -11,6 +12,7 @@ type Config struct { Environment string Region string ServiceDefinition string + SkipValidation bool } // NewFromEnv creates a new Config from environment variables and defaults @@ -20,6 +22,11 @@ func NewFromEnv() *Config { Environment: os.Getenv("SERVICE_ENV"), Region: os.Getenv("AWS_REGION"), ServiceDefinition: os.Getenv("SERVICE_DEFINITION"), + SkipValidation: false, + } + + if s, err := strconv.ParseBool(os.Getenv("BOOTSTRAP_SKIP_VALIDATION")); err == nil { + cfg.SkipValidation = s } if cfg.Service == "" { diff --git a/main.go b/main.go index 6bf1100..80735ef 100644 --- a/main.go +++ b/main.go @@ -19,7 +19,7 @@ func main() { logger := slog.New(slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: logLevel})) slog.SetDefault(logger) - if v, ok := os.LookupEnv("DEBUG_BOOTSTRAP"); ok && v != "false" { + if v, err := strconv.ParseBool(os.Getenv("DEBUG_BOOTSTRAP")); err == nil && v { logLevel.Set(slog.LevelDebug) } diff --git a/validate.go b/validate.go index 4f6d22c..4336d41 100644 --- a/validate.go +++ b/validate.go @@ -56,6 +56,10 @@ func (d *dependency) UnmarshalJSON(data []byte) error { } func validate(ctx context.Context, c *Config, e *EnvMap, l *slog.Logger) error { + if c.SkipValidation || c.Environment == "test" { + return nil + } + f, err := os.ReadFile(c.ServiceDefinition) if os.IsNotExist(err) { return nil diff --git a/validate_test.go b/validate_test.go index da93dc1..b6f4e0d 100644 --- a/validate_test.go +++ b/validate_test.go @@ -67,6 +67,16 @@ func TestValidate(t *testing.T) { //nolint:funlen ) assert.Contains(t, log.String(), `"WARN","msg":"Missing optional environment variables","env_vars":["QUX","FOOBAZ"]`) + // Skips validation + c.SkipValidation = true + require.NoError(t, validate(context.TODO(), c, e, l)) + c.SkipValidation = false + + // Skips validation in test environment + c.Environment = "test" + require.NoError(t, validate(context.TODO(), c, e, l)) + c.Environment = "dev" + // Empty env vars should be considered missing e.Add("FOO", "") t.Setenv("BAR", "")