Introduce feature flags to control experiments (#658)

* Introduce feature flags to control experiments

* Feature flag assertion validations

* Only allow enabling experiments through env var

- rename package from "feature" to "experiments"
- report enabled experiments in "version" command output

Co-authored-by: John Ryan <[email protected]>

pkg/cmd/template/cmd_test.go

@@ -4,15 +4,28 @@
 package template_test
 
 import (
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	cmdtpl "github.com/vmware-tanzu/carvel-ytt/pkg/cmd/template"
 	"github.com/vmware-tanzu/carvel-ytt/pkg/cmd/ui"
+	"github.com/vmware-tanzu/carvel-ytt/pkg/experiments"
 	"github.com/vmware-tanzu/carvel-ytt/pkg/files"
 )
 
+// TestMain is invoked when any tests are run in this package, *instead of* those tests being run directly.
+// This allows for setup to occur before *any* test is run.
+func TestMain(m *testing.M) {
+	experiments.ResetForTesting()
+	os.Setenv(experiments.Env, "validations")
+
+	exitVal := m.Run() // execute the specified tests
+
+	os.Exit(exitVal) // required in order to properly report the error level when tests fail.
+}
+
 func TestLoad(t *testing.T) {
 	yamlTplData := []byte(`
 #@ load("@ytt:data", "data")

pkg/cmd/version.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 
 	"github.com/spf13/cobra"
+	"github.com/vmware-tanzu/carvel-ytt/pkg/experiments"
 	"github.com/vmware-tanzu/carvel-ytt/pkg/version"
 )
 
@@ -27,6 +28,8 @@ func NewVersionCmd(o *VersionOptions) *cobra.Command {
 
 func (o *VersionOptions) Run() error {
 	fmt.Printf("ytt version %s\n", version.Version)
-
+	for _, experiment := range experiments.GetEnabled() {
+		fmt.Printf("- experiment %q enabled.\n", experiment)
+	}
 	return nil
 }

pkg/experiments/flags.go

@@ -0,0 +1,88 @@
+// Copyright 2022 VMware, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+/*
+Package experiments provides a global "Feature Flag" facility for
+circuit-breaking pre-GA code.
+
+The intent is to provide a means of selecting a flavor of executable at
+boot; not to toggle experiments on and off. Once settings are loaded from
+the environment variable experiments.Env, they are fixed.
+
+Registering a New Experiment
+
+1. implement a getter on this package `Is<experiment-name>Enabled()` and add the experiment to GetEnabled()
+
+2. circuit-break functionality behind that check:
+
+    if experiments.Is<experiment-name>Enabled() {
+        ...
+    }
+
+3. in tests, enable experiment(s) by setting the environment variable:
+
+    experiments.ResetForTesting()
+    os.Setenv(experiments.Env, "<experiment-name>,<other-experiment-name>,...")
+
+*/
+package experiments
+
+import (
+	"os"
+	"strings"
+)
+
+// Env is the OS environment variable with comma-separated names of experiments to enable.
+const Env = "YTTEXPERIMENTS"
+
+// IsValidationsEnabled reports whether the "validations" experiment was enabled by the user (via the Env).
+func IsValidationsEnabled() bool {
+	return isSet("validations")
+}
+
+// GetEnabled reports the name of all enabled experiments.
+//
+// An experiment is enabled by including its name in the OS environment variable named Env.
+func GetEnabled() []string {
+	experiments := []string{}
+	if IsValidationsEnabled() {
+		experiments = append(experiments, "validations")
+	}
+
+	return experiments
+}
+
+func isSet(flag string) bool {
+	for _, setting := range getSettings() {
+		if setting == flag {
+			return true
+		}
+	}
+	return false
+}
+
+func getSettings() []string {
+	if settings == nil {
+		for _, setting := range strings.Split(os.Getenv(Env), ",") {
+			settings = append(settings, strings.ToLower(strings.TrimSpace(setting)))
+		}
+	}
+	return settings
+}
+
+// settings cached copy of name of experiments that are enabled (cleaned up).
+var settings []string
+
+// isNoopEnabled reports whether the "noop" experiment was enabled.
+//
+// This is for testing purposes only.
+func isNoopEnabled() bool {
+	return isSet("noop")
+}
+
+// ResetForTesting clears the experiment flag settings, forcing reload from the Env on next use.
+//
+// This is for testing purposes only.
+func ResetForTesting() {
+	settings = nil
+}

pkg/experiments/flags_test.go

@@ -0,0 +1,29 @@
+// Copyright 2022 VMware, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package experiments
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+/*
+At runtime, there is a singleton instance of experiments flags.
+This test suite verifies the behavior of of such an instance.
+To avoid test pollution, a fresh instance is created in each test.
+*/
+
+func TestFeaturesAreDisabledByDefault(t *testing.T) {
+	ResetForTesting()
+	os.Setenv(Env, "")
+	assert.False(t, isNoopEnabled())
+}
+
+func TestFeaturesCanBeEnabled(t *testing.T) {
+	ResetForTesting()
+	os.Setenv(Env, "noop")
+	assert.True(t, isNoopEnabled())
+}

pkg/validations/assert.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 
 	"github.com/k14s/starlark-go/starlark"
+	"github.com/vmware-tanzu/carvel-ytt/pkg/experiments"
 	"github.com/vmware-tanzu/carvel-ytt/pkg/template"
 	"github.com/vmware-tanzu/carvel-ytt/pkg/yamlmeta"
 )
@@ -23,9 +24,13 @@ const (
 // When a Node's value is invalid, the errors are collected and returned in an AssertCheck.
 // Otherwise, returns empty AssertCheck and nil error.
 func ProcessAndRunValidations(n yamlmeta.Node, threadName string) (AssertCheck, error) {
+	if !experiments.IsValidationsEnabled() {
+		return AssertCheck{}, nil
+	}
 	if n == nil {
 		return AssertCheck{}, nil
 	}
+
 	err := yamlmeta.Walk(n, &convertAssertAnnsToValidations{})
 	if err != nil {
 		return AssertCheck{}, err