Refactor parameter loading. Support loading of maps.

Author: leonid-shevtsov

README.md

@@ -45,10 +45,12 @@ err := globalAWS.LoadConfigFromParameterStore(
 
 Supported value types: `string`, `int`, `bool` ("true"/"false").
 
-Complex type should be either a `struct`, or a `slice`. You can arbitrarily nest structs and slices.
+Complex type should be either a `struct`, a `map` or a `slice`. You can arbitrarily nest them.
 
 For structs, use `global` or `json` tag to set field name.
 
+For maps, the key name is the map key (maps must use strings as keys.)
+
 For slices, all subscripts in Parameter Store must be integers.
 
 ### Shorthand for running on AWS ECS or Lambda

aws/aws_param_store.go

@@ -2,13 +2,11 @@ package aws
 
 import (
 	"context"
-	"reflect"
-	"strconv"
-	"strings"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/service/ssm"
 	"github.com/railsware/go-global/v2"
+	"github.com/railsware/go-global/v2/utils"
 )
 
 const paramStoreSeparator = "/"
@@ -31,6 +29,11 @@ func LoadConfigFromParameterStore(awsConfig aws.Config, options LoadConfigOption
 		}
 	}()
 
+	reflectedConfig, err := utils.ReflectConfig(globalConfig)
+	if err != nil {
+		return err
+	}
+
 	paramPaginator := ssm.NewGetParametersByPathPaginator(
 		ssm.NewFromConfig(awsConfig),
 		&ssm.GetParametersByPathInput{
@@ -40,145 +43,27 @@ func LoadConfigFromParameterStore(awsConfig aws.Config, options LoadConfigOption
 		},
 	)
 
-	var paramWarnings []global.Error
+	var params []param
 
 	for paramPaginator.HasMorePages() {
 		page, err := paramPaginator.NextPage(context.Background())
 		if err != nil {
 			return global.NewError("global: failed to load from Parameter Store: %v", err)
 		}
-
-		for _, param := range page.Parameters {
-			paramNameWithoutPrefix := (*param.Name)[len(options.ParamPrefix):]
-			destination, err := findParamDestination(globalConfig, paramNameWithoutPrefix)
-			if err != nil {
-				if !err.Warning() {
-					return global.NewError("global: %s: %v", paramNameWithoutPrefix, err)
-				} else if !options.IgnoreUnmappedParams {
-					paramWarnings = append(paramWarnings, global.NewWarning("%s: %v", paramNameWithoutPrefix, err))
-				}
-				continue
-			}
-
-			err = writeParamToConfig(destination, *param.Value)
-			if err != nil {
-				if err.Warning() {
-					paramWarnings = append(paramWarnings, global.NewWarning("%s: %v", paramNameWithoutPrefix, err))
-				} else {
-					return global.NewError("global: %v", err)
-				}
-			}
+		for _, ssmParam := range page.Parameters {
+			paramNameWithoutPrefix := (*ssmParam.Name)[len(options.ParamPrefix):]
+			params = append(params, param{paramNameWithoutPrefix, *ssmParam.Value})
 		}
 	}
 
-	if paramWarnings != nil {
-		var warningMessages []string
-		for _, warning := range paramWarnings {
-			warningMessages = append(warningMessages, warning.Error())
-		}
+	paramTree := buildParamTree(params)
 
-		return global.NewWarning("global: failed to read some parameters: %s", strings.Join(warningMessages, "; "))
-	}
+	errors := paramTree.Write(reflectedConfig)
 
-	return nil
+	return errors.Join()
 }
 
-func findParamDestination(globalConfig interface{}, name string) (reflect.Value, global.Error) {
-	destination := reflect.ValueOf(globalConfig)
-
-	if destination.Kind() != reflect.Ptr {
-		return reflect.Value{}, global.NewError("config must be a pointer to a structure")
-	}
-
-	destination = destination.Elem()
-
-	if destination.Kind() != reflect.Struct && destination.Kind() != reflect.Array {
-		return reflect.Value{}, global.NewError("config must be a pointer to a structure or array")
-	}
-
-	// find nested field in config struct
-	pathParts := strings.Split(name, paramStoreSeparator)
-	for _, part := range pathParts {
-		if destination.Kind() == reflect.Struct {
-			destination = lookupFieldByName(destination, part)
-		} else if destination.Kind() == reflect.Slice {
-			index, err := strconv.Atoi(part)
-			if err != nil || index < 0 {
-				return reflect.Value{}, global.NewWarning("could not map param to array index")
-			}
-			if destination.Cap() <= index {
-				// grow destination array to match
-				destination.SetLen(destination.Cap())
-				additionalLength := index - destination.Cap() + 1
-				additionalElements := reflect.MakeSlice(destination.Type(), additionalLength, additionalLength)
-				destination.Set(reflect.AppendSlice(destination, additionalElements))
-			} else if destination.Len() <= index {
-				destination.SetLen(index + 1)
-			}
-			destination = destination.Index(index)
-		} else {
-			return reflect.Value{}, global.NewWarning("could not map param to config field")
-		}
-		// resolve pointer, if struct was nil
-		if destination.Kind() == reflect.Ptr {
-			if destination.IsNil() {
-				destination.Set(reflect.New(destination.Type().Elem()))
-			}
-			destination = destination.Elem()
-		}
-	}
-
-	// assign value to field
-	if !destination.IsValid() {
-		return reflect.Value{}, global.NewWarning("could not map param to config field")
-	}
-
-	return destination, nil
-}
-
-func writeParamToConfig(destination reflect.Value, value string) global.Error {
-	if !destination.CanSet() {
-		return global.NewWarning("config key is not writable")
-	} else if destination.Kind() == reflect.String {
-		destination.SetString(value)
-	} else if destination.Kind() == reflect.Int {
-		intval, err := strconv.Atoi(value)
-		if err != nil {
-			return global.NewWarning("cannot read int param value")
-		} else {
-			destination.SetInt(int64(intval))
-		}
-	} else if destination.Kind() == reflect.Bool {
-		if value == "true" {
-			destination.SetBool(true)
-		} else if value == "false" {
-			destination.SetBool(false)
-		} else {
-			return global.NewWarning("cannot read bool param value (must be true or false)")
-		}
-	} else {
-		return global.NewWarning("cannot write param: config key is of unsupported type %s", destination.Kind())
-	}
-
-	return nil
-}
-
-func lookupFieldByName(structure reflect.Value, name string) reflect.Value {
-	fieldByName := structure.FieldByName(name)
-	if fieldByName.IsValid() {
-		return fieldByName
-	}
-
-	// TODO might be inefficient, but fine for one-time loading of a not-crazy-big config
-	for i := 0; i < structure.NumField(); i++ {
-		fieldTag := structure.Type().Field(i).Tag
-		if fieldTag.Get("global") == name {
-			return structure.Field(i)
-		}
-		if fieldTag.Get("json") == name {
-			return structure.Field(i)
-		}
-	}
-
-	return reflect.Value{}
+type param struct {
+	path  string
+	value string
 }

aws/build_tree.go

@@ -0,0 +1,30 @@
+package aws
+
+import (
+	"strings"
+
+	"github.com/railsware/go-global/v2/tree"
+)
+
+// Builds a tree of parameters out of the array
+func buildParamTree(params []param) *tree.Node {
+	paramTree := new(tree.Node)
+
+	for _, param := range params {
+		pathParts := strings.Split(param.path, "/")
+		destination := paramTree
+		for _, part := range pathParts {
+			if destination.Children == nil {
+				destination.Children = make(map[string]*tree.Node)
+			}
+			newDestination, ok := destination.Children[part]
+			if !ok {
+				newDestination = &tree.Node{}
+				destination.Children[part] = newDestination
+			}
+			destination = newDestination
+		}
+		destination.Value = param.value
+	}
+	return paramTree
+}

aws/build_tree_test.go

@@ -0,0 +1,42 @@
+package aws
+
+import (
+	"testing"
+
+	"github.com/railsware/go-global/v2/tree"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBuildTree(t *testing.T) {
+	params := []param{
+		{"String", "string"},
+		{"NestedSimple/Nested", "nested_string"},
+		{"NestedDeep/SecondLevel/Nested", "deep_string"},
+		{"NestedDeep/SecondLevel/Nested2", "deep_string_2"},
+	}
+
+	expectedTree := &tree.Node{
+		Children: map[string]*tree.Node{
+			"String": {Value: "string"},
+			"NestedSimple": {
+				Children: map[string]*tree.Node{
+					"Nested": {Value: "nested_string"},
+				},
+			},
+			"NestedDeep": {
+				Children: map[string]*tree.Node{
+					"SecondLevel": {
+						Children: map[string]*tree.Node{
+							"Nested":  {Value: "deep_string"},
+							"Nested2": {Value: "deep_string_2"},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	paramTree := buildParamTree(params)
+
+	assert.Equal(t, expectedTree, paramTree)
+}