Saas 8261 --values param (#174)

* added --values, --set-value, --set-file to all

* added debugging from codefresh-cli

* values -tst1

* rewrite cmd-template-values

* --values for uninstall and upgrade

* debug with cli-1

* README --values

* fix debug on windows

* wip

* rename

* add windows 386

* fixed-windows build

* fixed-windows build

* fixed venona-ci go build old dependencies

Co-authored-by: Oren Gurfinkel <[email protected]>

Author: kosta709

venona/VERSION

@@ -1 +1 @@
-1.4.19
+1.4.20

venona/go.mod

@@ -3,19 +3,24 @@ module github.com/codefresh-io/go/venona
 go 1.14
 
 require (
-	github.com/codefresh-io/venona/venonactl v0.0.0-20200628143411-3135d91be7ee // indirect
+	github.com/evanphx/json-patch v4.5.0+incompatible // indirect
 	github.com/gin-gonic/gin v1.6.3
 	github.com/go-playground/validator/v10 v10.3.0 // indirect
 	github.com/golang/protobuf v1.4.2 // indirect
 	github.com/inconshreveable/log15 v0.0.0-20200109203555-b30bc20e4fd1
 	github.com/json-iterator/go v1.1.10 // indirect
+	github.com/mattn/go-colorable v0.1.6 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.7.0
+	github.com/stretchr/objx v0.2.0 // indirect
 	github.com/stretchr/testify v1.5.1
+	golang.org/x/crypto v0.0.0-20200128174031-69ecbb4d6d5d // indirect
 	golang.org/x/net v0.0.0-20200226121028-0de0cce0169b // indirect
 	golang.org/x/sys v0.0.0-20200610111108-226ff32320da // indirect
 	google.golang.org/protobuf v1.24.0 // indirect
+	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
 	gopkg.in/yaml.v2 v2.3.0
 	k8s.io/api v0.18.3
 	k8s.io/apimachinery v0.18.3

venona/go.sum

@@ -98,6 +98,12 @@ rules:
   verbs:
   - get
 ```
+#### Install using --values <values.yaml>
+`codefresh runner init --values <values.yaml> [parameters]`
+the values from values.yaml are applied to the templates in [pkg/templates/kubernetes](pkg/templates/kubernetes)
+
+See BuildValues() func in [store.go](pkg/store/store.go) for the format  
+Example with explaination is in [values-example.yaml](values-example.yaml) 
 
 #### Install on GCP
   * Make sure your user has `Kubernetes Engine Cluster Admin` role in google console

venonactl/README.md

@@ -1 +1 @@
-1.4.19
+1.4.20

venonactl/VERSION

@@ -40,12 +40,33 @@ var attachRuntimeCmdOptions struct {
 		context   string
 	}
 	restartAgent bool
+	templateValues       []string
+	templateFileValues   []string
+	templateValueFiles   []string	
 }
 
 var attachRuntimeCmd = &cobra.Command{
 	Use:   "attach",
 	Short: "Attach Codefresh runtime to agent",
 	Run: func(cmd *cobra.Command, args []string) {
+		// get valuesMap from --values <values.yaml> --set-value k=v --set-file k=<context-of file> 
+		templateValuesMap, err := templateValuesToMap(
+			attachRuntimeCmdOptions.templateValueFiles, 
+			attachRuntimeCmdOptions.templateValues, 
+			attachRuntimeCmdOptions.templateFileValues)
+		if err != nil {
+			dieOnError(err)
+		}
+		// Merge cmd options with template
+		mergeValueStr(templateValuesMap, "ConfigPath", &kubeConfigPath)
+		mergeValueStr(templateValuesMap, "ConfigPath", &attachRuntimeCmdOptions.kube.kubePath)
+		mergeValueStr(templateValuesMap, "CodefreshHost", &cfAPIHost)
+		mergeValueStr(templateValuesMap, "Token", &cfAPIToken)
+
+		mergeValueStr(templateValuesMap, "Namespace", &attachRuntimeCmdOptions.kube.namespace)
+		mergeValueStr(templateValuesMap, "Context", &attachRuntimeCmdOptions.kube.context)
+		mergeValueStr(templateValuesMap, "RuntimeEnvironmentName", &attachRuntimeCmdOptions.runtimeEnvironmentName)
+		mergeValueStr(templateValuesMap, "RuntimeServiceAccount", &attachRuntimeCmdOptions.kube.serviceAccount)
 
 		s := store.GetStore()
 		lgr := createLogger("Attach-runtime", verbose, logFormatter)
@@ -96,8 +117,8 @@ var attachRuntimeCmd = &cobra.Command{
 
 		builder.Add(plugins.RuntimeAttachType)
 
-		var err error
 		values := s.BuildValues()
+		values = mergeMaps(values, templateValuesMap)		
 		spn := createSpinner("Attaching runtime to agent (might take a few seconds)", "")
 		spn.Start()
 		for _, p := range builder.Get() {
@@ -129,4 +150,8 @@ func init() {
 	attachRuntimeCmd.Flags().StringVar(&attachRuntimeCmdOptions.kubeVenona.kubePath, "kube-config-path-agent", viper.GetString("kubeconfig-agent"), "Path to kubeconfig file (default is $HOME/.kube/config) for agent [$KUBECONFIG]")
 	attachRuntimeCmd.Flags().BoolVar(&attachRuntimeCmdOptions.restartAgent, "restart-agent", viper.GetBool("restart-agent"), "Restart agent after attach operation")
 
+	attachRuntimeCmd.Flags().StringArrayVar(&attachRuntimeCmdOptions.templateValues, "set-value", []string{}, "Set values for templates --set-value agentId=12345")
+	attachRuntimeCmd.Flags().StringArrayVar(&attachRuntimeCmdOptions.templateFileValues, "set-file", []string{}, "Set values for templates from file")
+	attachRuntimeCmd.Flags().StringArrayVarP(&attachRuntimeCmdOptions.templateValueFiles, "values", "f", []string{}, "specify values in a YAML file")
+
 }

venonactl/cmd/attach.go

@@ -1,7 +1,7 @@
 package cmd
 
 import (
-	"encoding/base64"
+
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -25,7 +25,9 @@ import (
 	k8sApi "k8s.io/api/core/v1"
 	"k8s.io/client-go/tools/clientcmd"
 
-	"helm.sh/helm/v3/pkg/strvals"
+	cliValues "helm.sh/helm/v3/pkg/cli/values"
+	"helm.sh/helm/v3/pkg/getter"
+	"github.com/stretchr/objx"	
 )
 
 var (
@@ -220,38 +222,37 @@ func extendStoreWithAgentAPI(logger logger.Logger, token string, agentID string)
 	s := store.GetStore()
 	logger.Debug("Using agent's token", "Token", token)
 	s.AgentAPI = &store.AgentAPI{
-		Token: base64.StdEncoding.EncodeToString([]byte(token)),
+		Token: token,
 		Id:    agentID,
 	}
 }
 
 // Parsing helpers --set-value , --set-file
 // by https://github.com/helm/helm/blob/ec1d1a3d3eb672232f896f9d3b3d0797e4f519e3/pkg/cli/values/options.go#L41
 
-// parses --set-value options
-func parseSetValues(setValuesOpts []string) (map[string]interface{}, error) {
-	base := map[string]interface{}{}
-	for _, value := range setValuesOpts {
-		if err := strvals.ParseInto(value, base); err != nil {
-			return nil, fmt.Errorf("Cannot parse option --set-value %s", value)
+// templateValuesToMap - processes cmd --values <values-file.yaml> --set-value k=v --set-file v=<context-of-file> 
+// using helm libraries
+func templateValuesToMap(templateValueFiles, templateValues, templateFileValues []string) (map[string]interface{}, error) {
+	valueOpts := &cliValues.Options{}
+	if len(templateValueFiles) > 0 {
+		for _, v := range(templateValueFiles) {
+			valueOpts.ValueFiles = append(valueOpts.ValueFiles, v)
 		}
 	}
-	return base, nil
-}
-
-// parses --set-file options
-func parseSetFiles(setFilesOpts []string) (map[string]interface{}, error) {
-	base := map[string]interface{}{}
-	for _, value := range setFilesOpts {
-		reader := func(rs []rune) (interface{}, error) {
-			bytes, err := ioutil.ReadFile(string(rs))
-			return string(bytes), err
-		}
-		if err := strvals.ParseIntoFile(value, base, reader); err != nil {
-			return nil, fmt.Errorf("Cannot parse option --set-file %s", value)
+	
+	if len(templateValues) > 0 {
+		for _, v := range(templateValues) {
+			valueOpts.Values = append(valueOpts.Values, v)
+		}			
+	}
+	
+	if len(templateFileValues) > 0 {
+		for _, v := range(templateFileValues) {
+			valueOpts.FileValues = append(valueOpts.FileValues, v)
 		}
 	}
-	return base, nil
+	valuesMap, err := valueOpts.MergeValues(getter.Providers{})
+	return valuesMap, err
 }
 
 func mergeMaps(a, b map[string]interface{}) map[string]interface{} {
@@ -272,3 +273,25 @@ func mergeMaps(a, b map[string]interface{}) map[string]interface{} {
 	}
 	return out
 }
+
+// mergeValueStr - for merging cli parameters with mapped parameters
+func mergeValueStr(valuesMap map[string]interface{}, key string, param *string) {
+	mapX := objx.New(valuesMap)
+	if param != nil && *param != "" {
+		mapX.Set(key, *param)
+	    return
+    }
+    val := mapX.Get(key).String()
+    *param = val
+}
+
+// mergeValueBool - for merging cli parameters with mapped parameters
+func mergeValueBool(valuesMap map[string]interface{}, key string, param *bool) {
+	mapX := objx.New(valuesMap)
+	if param != nil ||*param == true {
+		mapX.Set(key, *param)
+		return
+	}
+	val := mapX.Get(key).Bool()
+	*param = val
+ }

venonactl/cmd/cmdutils.go

@@ -25,6 +25,8 @@ import (
 	"github.com/codefresh-io/venona/venonactl/pkg/store"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+	// cliValues "helm.sh/helm/v3/pkg/cli/values"
+	// "helm.sh/helm/v3/pkg/getter"	
 )
 
 var installAgentCmdOptions struct {
@@ -44,15 +46,47 @@ var installAgentCmdOptions struct {
 	tolerations          string
 	envVars              []string
 	dockerRegistry       string
+	templateValues       []string
+	templateFileValues   []string
+	templateValueFiles   []string
 }
 
 var installAgentCmd = &cobra.Command{
 	Use:   "agent",
 	Short: "Install Codefresh's agent ",
 	Run: func(cmd *cobra.Command, args []string) {
+
+		// get valuesMap from --values <values.yaml> --set-value k=v --set-file k=<context-of file> 
+		templateValuesMap, err := templateValuesToMap(
+			installAgentCmdOptions.templateValueFiles, 
+			installAgentCmdOptions.templateValues, 
+			installAgentCmdOptions.templateFileValues)
+		if err != nil {
+			dieOnError(err)
+		}
+		// Merge cmd options with template
+		mergeValueStr(templateValuesMap, "ConfigPath", &kubeConfigPath)
+		mergeValueStr(templateValuesMap, "CodefreshHost", &cfAPIHost)
+		mergeValueStr(templateValuesMap, "Token", &cfAPIToken)		
+		mergeValueStr(templateValuesMap, "Namespace", &installAgentCmdOptions.kube.namespace)
+		mergeValueStr(templateValuesMap, "Context", &installAgentCmdOptions.kube.context)
+		mergeValueStr(templateValuesMap, "NodeSelector", &installAgentCmdOptions.kube.nodeSelector)
+		mergeValueStr(templateValuesMap, "Tolerations", &installAgentCmdOptions.tolerations)
+		mergeValueStr(templateValuesMap, "DockerRegistry", &installAgentCmdOptions.dockerRegistry)
+
+		mergeValueStr(templateValuesMap, "AgentToken", &installAgentCmdOptions.agentToken)
+		mergeValueStr(templateValuesMap, "AgentId", &installAgentCmdOptions.agentID)
+		mergeValueStr(templateValuesMap, "Image.Tag", &installAgentCmdOptions.venona.version)
+
+		//mergeValueStrArray(&installAgentCmdOptions.envVars, "envVars", nil, "More env vars to be declared \"key=value\"")
+	
+		mergeValueBool(templateValuesMap, "InCluster", &installAgentCmdOptions.kube.inCluster)
+		mergeValueBool(templateValuesMap, "kubernetesRunnerType", &installAgentCmdOptions.kubernetesRunnerType)
+
 		s := store.GetStore()
 		lgr := createLogger("Install-agent", verbose, logFormatter)
 		buildBasicStore(lgr)
+
 		extendStoreWithAgentAPI(lgr, installAgentCmdOptions.agentToken, installAgentCmdOptions.agentID)
 		extendStoreWithKubeClient(lgr)
 		fillCodefreshAPI(lgr)
@@ -64,6 +98,9 @@ var installAgentCmd = &cobra.Command{
 			CodefreshHost: cfAPIHost,
 		}
 
+		if installAgentCmdOptions.agentToken == "" {
+			installAgentCmdOptions.agentToken = cfAPIToken	
+		}
 		if installAgentCmdOptions.agentToken == "" {
 			dieOnError(fmt.Errorf("Agent token is required in order to install agent"))
 		}
@@ -115,7 +152,8 @@ var installAgentCmd = &cobra.Command{
 		builder.Add(plugins.VenonaPluginType)
 
 		values := s.BuildValues()
-		var err error
+		values = mergeMaps(values, templateValuesMap)
+		
 		for _, p := range builder.Get() {
 			values, err = p.Install(builderInstallOpt, values)
 			if err != nil {
@@ -144,6 +182,11 @@ func init() {
 	installAgentCmd.Flags().BoolVar(&installAgentCmdOptions.kube.inCluster, "in-cluster", false, "Set flag if venona is been installed from inside a cluster")
 	installAgentCmd.Flags().BoolVar(&installAgentCmdOptions.dryRun, "dry-run", false, "Set to true to simulate installation")
 	installAgentCmd.Flags().BoolVar(&installAgentCmdOptions.kubernetesRunnerType, "kubernetes-runner-type", false, "Set the runner type to kubernetes (alpha feature)")
+
+	installAgentCmd.Flags().StringArrayVar(&installAgentCmdOptions.templateValues, "set-value", []string{}, "Set values for templates --set-value agentId=12345")
+	installAgentCmd.Flags().StringArrayVar(&installAgentCmdOptions.templateFileValues, "set-file", []string{}, "Set values for templates from file")
+	installAgentCmd.Flags().StringArrayVarP(&installAgentCmdOptions.templateValueFiles, "values", "f", []string{}, "specify values in a YAML file")
+
 }
 
 func fillCodefreshAPI(logger logger.Logger) {

venonactl/cmd/install-agent.go

@@ -23,6 +23,7 @@ import (
 	"github.com/codefresh-io/venona/venonactl/pkg/store"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+
 )
 
 var installMonitorAgentCmdOptions struct {
@@ -36,13 +37,36 @@ var installMonitorAgentCmdOptions struct {
 	codefreshToken string
 	codefreshHost  string
 	dockerRegistry string
+	templateValues         []string
+	templateFileValues     []string
+	templateValueFiles     []string
 }
 
 // installK8sAgentCmd represents the install command
 var installMonitorAgentCmd = &cobra.Command{
 	Use:   "monitor",
 	Short: "Install Codefresh's monitor agent on cluster",
 	Run: func(cmd *cobra.Command, args []string) {
+		// get valuesMap from --values <values.yaml> --set-value k=v --set-file k=<context-of file> 
+		templateValuesMap, err := templateValuesToMap(
+			installMonitorAgentCmdOptions.templateValueFiles, 
+			installMonitorAgentCmdOptions.templateValues, 
+			installMonitorAgentCmdOptions.templateFileValues)
+		if err != nil {
+			dieOnError(err)
+		}
+		// Merge cmd options with template
+		mergeValueStr(templateValuesMap, "ConfigPath", &kubeConfigPath)
+		mergeValueStr(templateValuesMap, "CodefreshHost", &cfAPIHost)
+		mergeValueStr(templateValuesMap, "CodefreshHost", &installMonitorAgentCmdOptions.codefreshHost)
+		mergeValueStr(templateValuesMap, "Token", &cfAPIToken)
+		mergeValueStr(templateValuesMap, "Token", &installMonitorAgentCmdOptions.codefreshToken)
+		mergeValueStr(templateValuesMap, "Namespace", &installMonitorAgentCmdOptions.kube.namespace)
+		mergeValueStr(templateValuesMap, "Context", &installMonitorAgentCmdOptions.kube.context)
+
+		mergeValueStr(templateValuesMap, "DockerRegistry", &installMonitorAgentCmdOptions.dockerRegistry)
+		mergeValueStr(templateValuesMap, "ClusterId", &installMonitorAgentCmdOptions.clusterId)
+		mergeValueBool(templateValuesMap, "helm3", &installMonitorAgentCmdOptions.helm3)
 
 		s := store.GetStore()
 
@@ -88,6 +112,7 @@ var installMonitorAgentCmd = &cobra.Command{
 		}
 
 		values := s.BuildValues()
+		values = mergeMaps(values, templateValuesMap)
 
 		for _, p := range builder.Get() {
 			_, err := p.Install(builderInstallOpt, values)
@@ -111,5 +136,8 @@ func init() {
 	installMonitorAgentCmd.Flags().StringVar(&installMonitorAgentCmdOptions.codefreshHost, "codefreshHost", "", "Override codefresh host if you use your own codefresh installation")
 
 	installMonitorAgentCmd.Flags().BoolVar(&installMonitorAgentCmdOptions.helm3, "helm3", false, "Set flag if cluster use helm3")
+	installMonitorAgentCmd.Flags().StringArrayVar(&installMonitorAgentCmdOptions.templateValues, "set-value", []string{}, "Set values for templates, example: --set-value LocalVolumesDir=/mnt/disks/ssd0/codefresh-volumes")
+	installMonitorAgentCmd.Flags().StringArrayVar(&installMonitorAgentCmdOptions.templateFileValues, "set-file", []string{}, "Set values for templates from file, example: --set-file Storage.GoogleServiceAccount=/path/to/service-account.json")
+	installMonitorAgentCmd.Flags().StringArrayVarP(&installMonitorAgentCmdOptions.templateValueFiles, "values", "f", []string{}, "specify values in a YAML file")
 
 }