From ab0b5d7a2e8ed6b47439909bda3f82559b930b99 Mon Sep 17 00:00:00 2001 From: frank-hsieh-asj Date: Thu, 17 Feb 2022 14:44:07 +0800 Subject: [PATCH 01/58] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20float=20to=20decimal?= =?UTF-8?q?=20&=20set=20DivisionPrecision=20=3D=208?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/config.go | 7 +++++++ eval.go | 15 +++++++++++---- go.mod | 2 ++ go.sum | 2 ++ jlib/aggregate.go | 14 ++++++++------ jsonata_test.go | 42 +++++++++++++++++++++--------------------- 6 files changed, 51 insertions(+), 31 deletions(-) create mode 100644 config/config.go diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..af2978b --- /dev/null +++ b/config/config.go @@ -0,0 +1,7 @@ +package config + +var defaultDivisionPrecision int32 = 8 + +func GetDivisionPrecision() int32 { + return defaultDivisionPrecision +} diff --git a/eval.go b/eval.go index a66b561..0ae1989 100644 --- a/eval.go +++ b/eval.go @@ -10,9 +10,11 @@ import ( "reflect" "sort" + "github.com/blues/jsonata-go/config" "github.com/blues/jsonata-go/jlib" "github.com/blues/jsonata-go/jparse" "github.com/blues/jsonata-go/jtypes" + "github.com/shopspring/decimal" ) var undefined reflect.Value @@ -990,19 +992,24 @@ func evalNumericOperator(node *jparse.NumericOperatorNode, data reflect.Value, e return undefined, nil } + lhsDecimal, rhsDecimal := decimal.NewFromFloat(lhs), decimal.NewFromFloat(rhs) + var x float64 switch node.Type { case jparse.NumericAdd: - x = lhs + rhs + x = lhsDecimal.Add(rhsDecimal).RoundCeil(config.GetDivisionPrecision()).InexactFloat64() case jparse.NumericSubtract: - x = lhs - rhs + x = lhsDecimal.Sub(rhsDecimal).RoundCeil(config.GetDivisionPrecision()).InexactFloat64() case jparse.NumericMultiply: - x = lhs * rhs + x = lhsDecimal.Mul(rhsDecimal).Truncate(config.GetDivisionPrecision()).InexactFloat64() case jparse.NumericDivide: x = lhs / rhs + if !math.IsInf(x, 0) && !math.IsNaN(x) { + x = lhsDecimal.Div(rhsDecimal).RoundCeil(config.GetDivisionPrecision()).InexactFloat64() + } case jparse.NumericModulo: - x = math.Mod(lhs, rhs) + x = lhsDecimal.Mod(rhsDecimal).Truncate(config.GetDivisionPrecision()).InexactFloat64() default: panicf("unrecognised numeric operator %q", node.Type) } diff --git a/go.mod b/go.mod index 6a0e2e5..4ec3f7b 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,5 @@ module github.com/blues/jsonata-go go 1.16 + +require github.com/shopspring/decimal v1.3.1 diff --git a/go.sum b/go.sum index e69de29..3289fec 100644 --- a/go.sum +++ b/go.sum @@ -0,0 +1,2 @@ +github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= +github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= diff --git a/jlib/aggregate.go b/jlib/aggregate.go index 735ff82..8dc0e67 100644 --- a/jlib/aggregate.go +++ b/jlib/aggregate.go @@ -8,7 +8,9 @@ import ( "fmt" "reflect" + "github.com/blues/jsonata-go/config" "github.com/blues/jsonata-go/jtypes" + "github.com/shopspring/decimal" ) // Sum returns the total of an array of numbers. If the array is @@ -24,17 +26,17 @@ func Sum(v reflect.Value) (float64, error) { v = jtypes.Resolve(v) - var sum float64 + var sum decimal.Decimal for i := 0; i < v.Len(); i++ { n, ok := jtypes.AsNumber(v.Index(i)) if !ok { return 0, fmt.Errorf("cannot call sum on an array with non-number types") } - sum += n + sum = sum.Add(decimal.NewFromFloat(n)) } - return sum, nil + return sum.RoundCeil(config.GetDivisionPrecision()).InexactFloat64(), nil } // Max returns the largest value in an array of numbers. If the @@ -115,15 +117,15 @@ func Average(v reflect.Value) (float64, error) { return 0, jtypes.ErrUndefined } - var sum float64 + var sum decimal.Decimal for i := 0; i < v.Len(); i++ { n, ok := jtypes.AsNumber(v.Index(i)) if !ok { return 0, fmt.Errorf("cannot call average on an array with non-number types") } - sum += n + sum = sum.Add(decimal.NewFromFloat(n)) } - return sum / float64(v.Len()), nil + return sum.Div(decimal.NewFromInt(int64(v.Len()))).RoundCeil(config.GetDivisionPrecision()).InexactFloat64(), nil } diff --git a/jsonata_test.go b/jsonata_test.go index 3267918..e967ebe 100644 --- a/jsonata_test.go +++ b/jsonata_test.go @@ -688,11 +688,11 @@ func TestNumericOperators(t *testing.T) { }, { Expression: "foo.bar / bar", - Output: 0.42857142857142855, + Output: 0.42857143, }, { Expression: "bar / foo.bar", - Output: 2.3333333333333335, + Output: 2.33333334, }, { Expression: "foo.bar % bar", @@ -2308,17 +2308,17 @@ func TestObjectConstructor2(t *testing.T) { { Expression: "Account.Order{OrderID: $sum(Product.(Price*Quantity))}", Output: map[string]interface{}{ - "order103": 90.57000000000001, - "order104": 245.79000000000002, + "order103": 90.57, + "order104": 245.79, }, }, { Expression: "Account.Order.{OrderID: $sum(Product.(Price*Quantity))}", Output: []interface{}{ map[string]interface{}{ - "order103": 90.57000000000001, + "order103": 90.57, }, map[string]interface{}{ - "order104": 245.79000000000002, + "order104": 245.79, }, }, }, @@ -2340,14 +2340,14 @@ func TestObjectConstructor2(t *testing.T) { }`, Output: map[string]interface{}{ "order103": map[string]interface{}{ - "TotalPrice": 90.57000000000001, + "TotalPrice": 90.57, "Items": []interface{}{ "Bowler Hat", "Trilby hat", }, }, "order104": map[string]interface{}{ - "TotalPrice": 245.79000000000002, + "TotalPrice": 245.79, "Items": []interface{}{ "Bowler Hat", "Cloak", @@ -2393,7 +2393,7 @@ func TestObjectConstructor2(t *testing.T) { }, }, }, - "Total Price": 90.57000000000001, + "Total Price": 90.57, }, map[string]interface{}{ "ID": "order104", @@ -2415,7 +2415,7 @@ func TestObjectConstructor2(t *testing.T) { }, }, }, - "Total Price": 245.79000000000002, + "Total Price": 245.79, }, }, }, @@ -4004,16 +4004,16 @@ func TestFuncSum2(t *testing.T) { { Expression: "Account.Order.$sum(Product.(Price * Quantity))", Output: []interface{}{ - 90.57000000000001, - 245.79000000000002, + 90.57, + 245.79, }, }, { Expression: `Account.Order.(OrderID & ": " & $sum(Product.(Price*Quantity)))`, Output: []interface{}{ // TODO: Why does jsonata-js only display to 2dp? - "order103: 90.57000000000001", - "order104: 245.79000000000002", + "order103: 90.57", + "order104: 245.79", }, }, { @@ -4293,16 +4293,16 @@ func TestFuncAverage2(t *testing.T) { { Expression: "Account.Order.$average(Product.(Price * Quantity))", Output: []interface{}{ - 45.285000000000004, - 122.89500000000001, + 45.285, + 122.895, }, }, { Expression: `Account.Order.(OrderID & ": " & $average(Product.(Price*Quantity)))`, Output: []interface{}{ // TODO: Why does jsonata-js only display to 3dp? - "order103: 45.285000000000004", - "order104: 122.89500000000001", + "order103: 45.285", + "order104: 122.895", }, }, }) @@ -5066,7 +5066,7 @@ func TestFuncString(t *testing.T) { }, { Expression: `$string(22/7)`, - Output: "3.142857142857143", // TODO: jsonata-js returns "3.142857142857" + Output: "3.14285715", // TODO: jsonata-js returns "3.142857142857" }, { Expression: `$string(1e100)`, @@ -5176,8 +5176,8 @@ func TestFuncString2(t *testing.T) { Expression: `Account.Order.$string($sum(Product.(Price* Quantity)))`, // TODO: jsonata-js rounds to "90.57" and "245.79" Output: []interface{}{ - "90.57000000000001", - "245.79000000000002", + "90.57", + "245.79", }, }, }) From 574f7f3523af8319ff83dd37bf8de24353534d45 Mon Sep 17 00:00:00 2001 From: Tom <53711814+tbal999@users.noreply.github.com> Date: Thu, 7 Jul 2022 12:06:21 +0100 Subject: [PATCH 02/58] add regexp this enables: - comments in jsonata code - fields with any character in their name --- jsonata.go | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/jsonata.go b/jsonata.go index 7277b6e..58dd1f5 100644 --- a/jsonata.go +++ b/jsonata.go @@ -7,6 +7,7 @@ package jsonata import ( "encoding/json" "fmt" + "regexp" "reflect" "sync" "time" @@ -96,8 +97,9 @@ type Expr struct { // not a valid JSONata expression, Compile returns an error // of type jparse.Error. func Compile(expr string) (*Expr, error) { + cleanExpr := replaceQuotesAndCommentsInPaths(expr) - node, err := jparse.Parse(expr) + node, err := jparse.Parse(cleanExpr) if err != nil { return nil, err } @@ -379,3 +381,31 @@ func isLetter(r rune) bool { func isDigit(r rune) bool { return (r >= '0' && r <= '9') || unicode.IsDigit(r) } + +/* + enables: + - comments in jsonata code + - fields with any character in their name +*/ + +var ( + reQuotedPath = regexp.MustCompile(`([A-Za-z\$\\*\` + "`" + `])\.[\"']([\s\S]+?)[\"']`) + reQuotedPathStart = regexp.MustCompile(`^[\"']([ \.0-9A-Za-z]+?)[\"']\.([A-Za-z\$\*\"\'])`) + commentsPath = regexp.MustCompile(`\/\*([\s\S]*?)\*\/`) +) + +func replaceQuotesAndCommentsInPaths(s string) string { + if reQuotedPathStart.MatchString(s) { + s = reQuotedPathStart.ReplaceAllString(s, "`$1`.$2") + } + + for reQuotedPath.MatchString(s) { + s = reQuotedPath.ReplaceAllString(s, "$1.`$2`") + } + + for commentsPath.MatchString(s) { + s = commentsPath.ReplaceAllString(s, "") + } + + return s +} From 840ede8f512cd83ed1706831ba5d506e8bab0fa8 Mon Sep 17 00:00:00 2001 From: Tom Date: Mon, 19 Sep 2022 13:48:12 +0100 Subject: [PATCH 03/58] add some new features to go-jsonata [xia] --- README.md | 6 +++--- callable.go | 6 +++--- callable_test.go | 4 ++-- env.go | 18 +++++++++++++++--- error.go | 2 +- eval.go | 8 ++++---- eval_test.go | 6 +++--- example_eval_test.go | 2 +- example_exts_test.go | 2 +- go.mod | 2 +- jlib/aggregate.go | 4 ++-- jlib/array.go | 2 +- jlib/boolean.go | 2 +- jlib/date.go | 4 ++-- jlib/date_test.go | 4 ++-- jlib/hof.go | 2 +- jlib/jlib.go | 2 +- jlib/number.go | 2 +- jlib/number_test.go | 4 ++-- jlib/object.go | 2 +- jlib/object_test.go | 4 ++-- jlib/string.go | 4 ++-- jlib/string_test.go | 4 ++-- jlib/unescape.go | 18 ++++++++++++++++++ jparse/jparse_test.go | 2 +- jsonata-server/README.md | 2 +- jsonata-server/bench.go | 2 +- jsonata-server/exts.go | 4 ++-- jsonata-server/main.go | 6 ++---- jsonata-test/main.go | 4 ++-- jsonata.go | 6 +++--- jsonata_test.go | 4 ++-- 32 files changed, 86 insertions(+), 58 deletions(-) create mode 100644 jlib/unescape.go diff --git a/README.md b/README.md index aaa8abf..7e4d61c 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ It currently has feature parity with jsonata-js 1.5.4. As well as a most of the ## Install - go get github.com/blues/jsonata-go + go get github.com/xiatechs/jsonata-go ## Usage @@ -17,7 +17,7 @@ import ( "fmt" "log" - jsonata "github.com/blues/jsonata-go" + jsonata "github.com/xiatechs/jsonata-go" ) const jsonString = ` @@ -56,7 +56,7 @@ func main() { ## JSONata Server A locally hosted version of [JSONata Exerciser](http://try.jsonata.org/) -for testing is [available here](https://github.com/blues/jsonata-go/jsonata-server). +for testing is [available here](https://github.com/xiatechs/jsonata-go/jsonata-server). ## JSONata tests A CLI tool for running jsonata-go against the [JSONata test suite](https://github.com/jsonata-js/jsonata/tree/master/test/test-suite) is [available here](./jsonata-test). diff --git a/callable.go b/callable.go index ec08501..0a78e1c 100644 --- a/callable.go +++ b/callable.go @@ -11,9 +11,9 @@ import ( "regexp" "strings" - "github.com/blues/jsonata-go/jlib" - "github.com/blues/jsonata-go/jparse" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/jlib" + "github.com/xiatechs/jsonata-go/jparse" + "github.com/xiatechs/jsonata-go/jtypes" ) type callableName struct { diff --git a/callable_test.go b/callable_test.go index 75062bc..cf7a96b 100644 --- a/callable_test.go +++ b/callable_test.go @@ -13,8 +13,8 @@ import ( "strings" "testing" - "github.com/blues/jsonata-go/jparse" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/jparse" + "github.com/xiatechs/jsonata-go/jtypes" ) var ( diff --git a/env.go b/env.go index bbbefc5..e64b7c0 100644 --- a/env.go +++ b/env.go @@ -11,9 +11,9 @@ import ( "strings" "unicode/utf8" - "github.com/blues/jsonata-go/jlib" - "github.com/blues/jsonata-go/jparse" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/jlib" + "github.com/xiatechs/jsonata-go/jparse" + "github.com/xiatechs/jsonata-go/jtypes" ) type environment struct { @@ -67,6 +67,18 @@ var ( var baseEnv = initBaseEnv(map[string]Extension{ + /* + EXTENDED START + */ + "unescape": { + Func: jlib.Unescape, + UndefinedHandler: defaultUndefinedHandler, + EvalContextHandler: defaultContextHandler, + }, + /* + EXTENDED END + */ + // String functions "string": { diff --git a/error.go b/error.go index f5d383c..3688ebf 100644 --- a/error.go +++ b/error.go @@ -9,7 +9,7 @@ import ( "fmt" "regexp" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/jtypes" ) // ErrUndefined is returned by the evaluation methods when diff --git a/eval.go b/eval.go index 0ae1989..e4d026b 100644 --- a/eval.go +++ b/eval.go @@ -10,10 +10,10 @@ import ( "reflect" "sort" - "github.com/blues/jsonata-go/config" - "github.com/blues/jsonata-go/jlib" - "github.com/blues/jsonata-go/jparse" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/config" + "github.com/xiatechs/jsonata-go/jlib" + "github.com/xiatechs/jsonata-go/jparse" + "github.com/xiatechs/jsonata-go/jtypes" "github.com/shopspring/decimal" ) diff --git a/eval_test.go b/eval_test.go index 80c9378..019a1d5 100644 --- a/eval_test.go +++ b/eval_test.go @@ -11,9 +11,9 @@ import ( "strings" "testing" - "github.com/blues/jsonata-go/jlib" - "github.com/blues/jsonata-go/jparse" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/jlib" + "github.com/xiatechs/jsonata-go/jparse" + "github.com/xiatechs/jsonata-go/jtypes" ) type evalTestCase struct { diff --git a/example_eval_test.go b/example_eval_test.go index fd35592..fc24a41 100644 --- a/example_eval_test.go +++ b/example_eval_test.go @@ -9,7 +9,7 @@ import ( "fmt" "log" - jsonata "github.com/blues/jsonata-go" + jsonata "github.com/xiatechs/jsonata-go" ) const jsonString = ` diff --git a/example_exts_test.go b/example_exts_test.go index 2c02e18..7a1bd05 100644 --- a/example_exts_test.go +++ b/example_exts_test.go @@ -9,7 +9,7 @@ import ( "log" "strings" - jsonata "github.com/blues/jsonata-go" + jsonata "github.com/xiatechs/jsonata-go" ) // diff --git a/go.mod b/go.mod index 4ec3f7b..d4de1f2 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/blues/jsonata-go +module github.com/xiatechs/jsonata-go go 1.16 diff --git a/jlib/aggregate.go b/jlib/aggregate.go index 8dc0e67..9c94706 100644 --- a/jlib/aggregate.go +++ b/jlib/aggregate.go @@ -8,8 +8,8 @@ import ( "fmt" "reflect" - "github.com/blues/jsonata-go/config" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/config" + "github.com/xiatechs/jsonata-go/jtypes" "github.com/shopspring/decimal" ) diff --git a/jlib/array.go b/jlib/array.go index f16f711..8c1f734 100644 --- a/jlib/array.go +++ b/jlib/array.go @@ -10,7 +10,7 @@ import ( "reflect" "sort" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/jtypes" ) // Count (golint) diff --git a/jlib/boolean.go b/jlib/boolean.go index b4325f4..067e2f9 100644 --- a/jlib/boolean.go +++ b/jlib/boolean.go @@ -7,7 +7,7 @@ package jlib import ( "reflect" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/jtypes" ) // Boolean (golint) diff --git a/jlib/date.go b/jlib/date.go index 70a9a2c..c5ee7d4 100644 --- a/jlib/date.go +++ b/jlib/date.go @@ -10,8 +10,8 @@ import ( "strconv" "time" - "github.com/blues/jsonata-go/jlib/jxpath" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/jlib/jxpath" + "github.com/xiatechs/jsonata-go/jtypes" ) // 2006-01-02T15:04:05.000Z07:00 diff --git a/jlib/date_test.go b/jlib/date_test.go index 7c42c62..540fcb9 100644 --- a/jlib/date_test.go +++ b/jlib/date_test.go @@ -9,8 +9,8 @@ import ( "testing" "time" - "github.com/blues/jsonata-go/jlib" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/jlib" + "github.com/xiatechs/jsonata-go/jtypes" ) func TestFromMillis(t *testing.T) { diff --git a/jlib/hof.go b/jlib/hof.go index 6452a3b..b7b07c4 100644 --- a/jlib/hof.go +++ b/jlib/hof.go @@ -8,7 +8,7 @@ import ( "fmt" "reflect" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/jtypes" ) // Map (golint) diff --git a/jlib/jlib.go b/jlib/jlib.go index 644a044..027a047 100644 --- a/jlib/jlib.go +++ b/jlib/jlib.go @@ -11,7 +11,7 @@ import ( "reflect" "time" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/jtypes" ) func init() { diff --git a/jlib/number.go b/jlib/number.go index 8cb0e46..9cc62b0 100644 --- a/jlib/number.go +++ b/jlib/number.go @@ -13,7 +13,7 @@ import ( "strconv" "strings" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/jtypes" ) var reNumber = regexp.MustCompile(`^-?(([0-9]+))(\.[0-9]+)?([Ee][-+]?[0-9]+)?$`) diff --git a/jlib/number_test.go b/jlib/number_test.go index 51b422f..478855f 100644 --- a/jlib/number_test.go +++ b/jlib/number_test.go @@ -8,8 +8,8 @@ import ( "fmt" "testing" - "github.com/blues/jsonata-go/jlib" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/jlib" + "github.com/xiatechs/jsonata-go/jtypes" ) func TestRound(t *testing.T) { diff --git a/jlib/object.go b/jlib/object.go index ca43f8d..36284ab 100644 --- a/jlib/object.go +++ b/jlib/object.go @@ -8,7 +8,7 @@ import ( "fmt" "reflect" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/jtypes" ) // typeInterfaceMap is the reflect.Type for map[string]interface{}. diff --git a/jlib/object_test.go b/jlib/object_test.go index e2e8dc7..3e97ece 100644 --- a/jlib/object_test.go +++ b/jlib/object_test.go @@ -12,8 +12,8 @@ import ( "strings" "testing" - "github.com/blues/jsonata-go/jlib" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/jlib" + "github.com/xiatechs/jsonata-go/jtypes" ) type eachTest struct { diff --git a/jlib/string.go b/jlib/string.go index cd49c6f..8764825 100644 --- a/jlib/string.go +++ b/jlib/string.go @@ -17,8 +17,8 @@ import ( "strings" "unicode/utf8" - "github.com/blues/jsonata-go/jlib/jxpath" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/jlib/jxpath" + "github.com/xiatechs/jsonata-go/jtypes" ) // String converts a JSONata value to a string. Values that are diff --git a/jlib/string_test.go b/jlib/string_test.go index 21a91c6..5fcb544 100644 --- a/jlib/string_test.go +++ b/jlib/string_test.go @@ -13,8 +13,8 @@ import ( "testing" "unicode/utf8" - "github.com/blues/jsonata-go/jlib" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/jlib" + "github.com/xiatechs/jsonata-go/jtypes" ) var typereplaceCallable = reflect.TypeOf((*replaceCallable)(nil)).Elem() diff --git a/jlib/unescape.go b/jlib/unescape.go new file mode 100644 index 0000000..5597a49 --- /dev/null +++ b/jlib/unescape.go @@ -0,0 +1,18 @@ +package jlib + +import ( + "fmt" + "encoding/json" +) + +// Unescape a string into JSON - simple but powerful +func Unescape(input string) (interface{}, error) { + var output interface{} + + err := json.Unmarshal([]byte(input), &output) + if err != nil { + return output, fmt.Errorf("unescape json unmarshal error: %v", err) + } + + return output, nil +} \ No newline at end of file diff --git a/jparse/jparse_test.go b/jparse/jparse_test.go index f0874e5..a1967ad 100644 --- a/jparse/jparse_test.go +++ b/jparse/jparse_test.go @@ -12,7 +12,7 @@ import ( "testing" "unicode/utf8" - "github.com/blues/jsonata-go/jparse" + "github.com/xiatechs/jsonata-go/jparse" ) type testCase struct { diff --git a/jsonata-server/README.md b/jsonata-server/README.md index 4c67f00..3ef953e 100644 --- a/jsonata-server/README.md +++ b/jsonata-server/README.md @@ -5,7 +5,7 @@ for testing [jsonata-go](https://github.com/blues/jsonata). ## Install - go install github.com/blues/jsonata-go/jsonata-server + go install github.com/xiatechs/jsonata-go/jsonata-server ## Usage diff --git a/jsonata-server/bench.go b/jsonata-server/bench.go index 925ecfc..1a0a19f 100644 --- a/jsonata-server/bench.go +++ b/jsonata-server/bench.go @@ -10,7 +10,7 @@ import ( "encoding/json" - jsonata "github.com/blues/jsonata-go" + jsonata "github.com/xiatechs/jsonata-go" ) var ( diff --git a/jsonata-server/exts.go b/jsonata-server/exts.go index 117beb6..3382a2a 100644 --- a/jsonata-server/exts.go +++ b/jsonata-server/exts.go @@ -5,8 +5,8 @@ package main import ( - "github.com/blues/jsonata-go/jlib" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/jlib" + "github.com/xiatechs/jsonata-go/jtypes" ) // Default format for dates: e.g. 2006-01-02 15:04 MST diff --git a/jsonata-server/main.go b/jsonata-server/main.go index 73ebea0..33f3098 100644 --- a/jsonata-server/main.go +++ b/jsonata-server/main.go @@ -14,8 +14,8 @@ import ( _ "net/http/pprof" "strings" - jsonata "github.com/blues/jsonata-go" - "github.com/blues/jsonata-go/jtypes" + jsonata "github.com/xiatechs/jsonata-go" + "github.com/xiatechs/jsonata-go/jtypes" ) func init() { @@ -52,7 +52,6 @@ func main() { } func evaluate(w http.ResponseWriter, r *http.Request) { - input := strings.TrimSpace(r.FormValue("json")) if input == "" { http.Error(w, "Input is empty", http.StatusBadRequest) @@ -78,7 +77,6 @@ func evaluate(w http.ResponseWriter, r *http.Request) { } func eval(input, expression string) (b []byte, status int, err error) { - defer func() { if r := recover(); r != nil { b = nil diff --git a/jsonata-test/main.go b/jsonata-test/main.go index 937e07e..12078ab 100644 --- a/jsonata-test/main.go +++ b/jsonata-test/main.go @@ -12,8 +12,8 @@ import ( "regexp" "strings" - jsonata "github.com/blues/jsonata-go" - types "github.com/blues/jsonata-go/jtypes" + jsonata "github.com/xiatechs/jsonata-go" + types "github.com/xiatechs/jsonata-go/jtypes" ) type testCase struct { diff --git a/jsonata.go b/jsonata.go index 58dd1f5..1422fb8 100644 --- a/jsonata.go +++ b/jsonata.go @@ -13,9 +13,9 @@ import ( "time" "unicode" - "github.com/blues/jsonata-go/jlib" - "github.com/blues/jsonata-go/jparse" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/jlib" + "github.com/xiatechs/jsonata-go/jparse" + "github.com/xiatechs/jsonata-go/jtypes" ) var ( diff --git a/jsonata_test.go b/jsonata_test.go index e967ebe..140754c 100644 --- a/jsonata_test.go +++ b/jsonata_test.go @@ -19,8 +19,8 @@ import ( "time" "unicode/utf8" - "github.com/blues/jsonata-go/jparse" - "github.com/blues/jsonata-go/jtypes" + "github.com/xiatechs/jsonata-go/jparse" + "github.com/xiatechs/jsonata-go/jtypes" ) type testCase struct { From 68dea732992dd9460b526527161bd1ad48a606c2 Mon Sep 17 00:00:00 2001 From: Tom Date: Mon, 19 Sep 2022 13:48:21 +0100 Subject: [PATCH 04/58] further additions --- env.go | 2 ++ jlib/unescape.go | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/env.go b/env.go index e64b7c0..6e4d727 100644 --- a/env.go +++ b/env.go @@ -70,11 +70,13 @@ var baseEnv = initBaseEnv(map[string]Extension{ /* EXTENDED START */ + "unescape": { Func: jlib.Unescape, UndefinedHandler: defaultUndefinedHandler, EvalContextHandler: defaultContextHandler, }, + /* EXTENDED END */ diff --git a/jlib/unescape.go b/jlib/unescape.go index 5597a49..bc1cc97 100644 --- a/jlib/unescape.go +++ b/jlib/unescape.go @@ -5,7 +5,7 @@ import ( "encoding/json" ) -// Unescape a string into JSON - simple but powerful +// Unescape an escaped json string into JSON (once) func Unescape(input string) (interface{}, error) { var output interface{} From f6d3217c7d89738bd059d54a7d080ac5285e7860 Mon Sep 17 00:00:00 2001 From: Tom Date: Mon, 19 Sep 2022 13:49:13 +0100 Subject: [PATCH 05/58] adjust contributing --- CONTRIBUTING.md | 6 +++--- env.go | 2 +- eval.go | 2 +- jlib/aggregate.go | 2 +- jlib/unescape.go | 4 ++-- jsonata.go | 4 ++-- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3b40568..2376b63 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,4 @@ -# Contributing to blues/jsonata-go +# Contributing to blues/jsonata-go [xiatechs fork] We love pull requests from everyone. By participating in this project, you agree to abide by the Blues Inc [code of conduct]. @@ -18,7 +18,7 @@ clean up inconsistent whitespace ) * by closing [issues][] * by reviewing patches -[issues]: https://github.com/blues/jsonata-go/issues +[issues]: https://github.com/xiatechs/jsonata-go/issues ## Submitting an Issue @@ -55,7 +55,7 @@ clean up inconsistent whitespace ) * If you don't know how to add tests, please put in a PR and leave a comment asking for help. We love helping! -[repo]: https://github.com/blues/jsonata-go/tree/master +[repo]: https://github.com/xiatechs/jsonata-go/tree/master [fork]: https://help.github.com/articles/fork-a-repo/ [branch]: https://help.github.com/articles/creating-and-deleting-branches-within-your-repository/ diff --git a/env.go b/env.go index 6e4d727..29c9fff 100644 --- a/env.go +++ b/env.go @@ -76,7 +76,7 @@ var baseEnv = initBaseEnv(map[string]Extension{ UndefinedHandler: defaultUndefinedHandler, EvalContextHandler: defaultContextHandler, }, - + /* EXTENDED END */ diff --git a/eval.go b/eval.go index e4d026b..0aeb69f 100644 --- a/eval.go +++ b/eval.go @@ -10,11 +10,11 @@ import ( "reflect" "sort" + "github.com/shopspring/decimal" "github.com/xiatechs/jsonata-go/config" "github.com/xiatechs/jsonata-go/jlib" "github.com/xiatechs/jsonata-go/jparse" "github.com/xiatechs/jsonata-go/jtypes" - "github.com/shopspring/decimal" ) var undefined reflect.Value diff --git a/jlib/aggregate.go b/jlib/aggregate.go index 9c94706..475711b 100644 --- a/jlib/aggregate.go +++ b/jlib/aggregate.go @@ -8,9 +8,9 @@ import ( "fmt" "reflect" + "github.com/shopspring/decimal" "github.com/xiatechs/jsonata-go/config" "github.com/xiatechs/jsonata-go/jtypes" - "github.com/shopspring/decimal" ) // Sum returns the total of an array of numbers. If the array is diff --git a/jlib/unescape.go b/jlib/unescape.go index bc1cc97..bbe86ed 100644 --- a/jlib/unescape.go +++ b/jlib/unescape.go @@ -1,8 +1,8 @@ package jlib import ( - "fmt" "encoding/json" + "fmt" ) // Unescape an escaped json string into JSON (once) @@ -15,4 +15,4 @@ func Unescape(input string) (interface{}, error) { } return output, nil -} \ No newline at end of file +} diff --git a/jsonata.go b/jsonata.go index 1422fb8..7285d87 100644 --- a/jsonata.go +++ b/jsonata.go @@ -7,8 +7,8 @@ package jsonata import ( "encoding/json" "fmt" - "regexp" "reflect" + "regexp" "sync" "time" "unicode" @@ -382,7 +382,7 @@ func isDigit(r rune) bool { return (r >= '0' && r <= '9') || unicode.IsDigit(r) } -/* +/* enables: - comments in jsonata code - fields with any character in their name From 1abafe3349ba543a0ce9a6c610993700cae483aa Mon Sep 17 00:00:00 2001 From: Tom Date: Mon, 19 Sep 2022 13:52:18 +0100 Subject: [PATCH 06/58] 1.18 --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index d4de1f2..908d76b 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,5 @@ module github.com/xiatechs/jsonata-go -go 1.16 +go 1.18 require github.com/shopspring/decimal v1.3.1 From f06761b7ef96caa8548bb27a7b505f18ddd4afd9 Mon Sep 17 00:00:00 2001 From: Tom Date: Wed, 21 Sep 2022 20:04:22 +0100 Subject: [PATCH 07/58] add @ to lexer --- jparse/lexer.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/jparse/lexer.go b/jparse/lexer.go index bff6df4..a58713a 100644 --- a/jparse/lexer.go +++ b/jparse/lexer.go @@ -61,6 +61,9 @@ const ( typeAnd typeOr typeIn + + // Join operator + typeJoin ) func (tt tokenType) String() string { @@ -81,6 +84,8 @@ func (tt tokenType) String() string { return "(variable)" case typeRegex: return "(regex)" + case typeJoin: + return "(join)" default: if s := symbolsAndKeywords[tt]; s != "" { return s @@ -114,6 +119,7 @@ var symbols1 = [...]tokenType{ '>': typeGreater, '^': typeSort, '&': typeConcat, + '@': typeJoin, } type runeTokenType struct { From 9af3332177f70489219f3e44fcd24bafb571b3dc Mon Sep 17 00:00:00 2001 From: Tom Date: Wed, 21 Sep 2022 22:13:34 +0100 Subject: [PATCH 08/58] different approach --- env.go | 6 ++++++ jlib/unescape.go | 50 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/env.go b/env.go index 29c9fff..aec7c56 100644 --- a/env.go +++ b/env.go @@ -71,6 +71,12 @@ var baseEnv = initBaseEnv(map[string]Extension{ EXTENDED START */ + "ljoin": { + Func: jlib.LJoin, + UndefinedHandler: defaultUndefinedHandler, + EvalContextHandler: nil, + }, + "unescape": { Func: jlib.Unescape, UndefinedHandler: defaultUndefinedHandler, diff --git a/jlib/unescape.go b/jlib/unescape.go index bbe86ed..495ba76 100644 --- a/jlib/unescape.go +++ b/jlib/unescape.go @@ -3,6 +3,7 @@ package jlib import ( "encoding/json" "fmt" + "reflect" ) // Unescape an escaped json string into JSON (once) @@ -16,3 +17,52 @@ func Unescape(input string) (interface{}, error) { return output, nil } + +func getVal(input interface{}) string { + return fmt.Sprintf("%v", input) +} + +// LJoin (golint) +func LJoin(v, v2 reflect.Value, field1, field2 string) (interface{}, error) { + output := make([]interface{}, 0) + + i1, ok := v.Interface().([]interface{}) + if !ok { + return nil, fmt.Errorf("both objects must be slice of objects") + } + + i2, ok := v2.Interface().([]interface{}) + if !ok { + return nil, fmt.Errorf("both objects must be slice of objects") + } + + for a := range i1 { + item1, ok := i1[a].(map[string]interface{}) + if !ok { + continue + } + + f1 := item1[field1] + + for b := range i2 { + item2, ok := i2[b].(map[string]interface{}) + if !ok { + continue + } + + f2 := item2[field2] + if f1 == f2 { + newitem := make(map[string]interface{}) + for key := range item1 { + newitem[key] = item1[key] + } + for key := range item2 { + newitem[key] = item2[key] + } + output = append(output, newitem) + } + } + } + + return output, nil +} From 74852c424f424fcf00f6ef8be76d03ebafb8be62 Mon Sep 17 00:00:00 2001 From: Tom Date: Thu, 22 Sep 2022 09:02:46 +0100 Subject: [PATCH 09/58] change name for new funcs --- jlib/{unescape.go => new.go} | 1 + 1 file changed, 1 insertion(+) rename jlib/{unescape.go => new.go} (99%) diff --git a/jlib/unescape.go b/jlib/new.go similarity index 99% rename from jlib/unescape.go rename to jlib/new.go index 495ba76..df3a57c 100644 --- a/jlib/unescape.go +++ b/jlib/new.go @@ -22,6 +22,7 @@ func getVal(input interface{}) string { return fmt.Sprintf("%v", input) } + // LJoin (golint) func LJoin(v, v2 reflect.Value, field1, field2 string) (interface{}, error) { output := make([]interface{}, 0) From 36922a6aa64ca12bc0e89ca0087e375ccceac2c0 Mon Sep 17 00:00:00 2001 From: Tom Date: Thu, 22 Sep 2022 10:04:54 +0100 Subject: [PATCH 10/58] add obj merge func --- env.go | 10 ++++++++-- jlib/new.go | 49 ++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 52 insertions(+), 7 deletions(-) diff --git a/env.go b/env.go index aec7c56..dd5bec5 100644 --- a/env.go +++ b/env.go @@ -71,8 +71,14 @@ var baseEnv = initBaseEnv(map[string]Extension{ EXTENDED START */ - "ljoin": { - Func: jlib.LJoin, + "objmerge": { + Func: jlib.ObjMerge, + UndefinedHandler: defaultUndefinedHandler, + EvalContextHandler: nil, + }, + + "sjoin": { + Func: jlib.SimpleJoin, UndefinedHandler: defaultUndefinedHandler, EvalContextHandler: nil, }, diff --git a/jlib/new.go b/jlib/new.go index df3a57c..1bc1ea6 100644 --- a/jlib/new.go +++ b/jlib/new.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "reflect" + "strings" ) // Unescape an escaped json string into JSON (once) @@ -22,9 +23,8 @@ func getVal(input interface{}) string { return fmt.Sprintf("%v", input) } - -// LJoin (golint) -func LJoin(v, v2 reflect.Value, field1, field2 string) (interface{}, error) { +// SimpleJoin - a 1 key left join very simple and useful in certain circumstances +func SimpleJoin(v, v2 reflect.Value, field1, field2 string) (interface{}, error) { output := make([]interface{}, 0) i1, ok := v.Interface().([]interface{}) @@ -37,6 +37,24 @@ func LJoin(v, v2 reflect.Value, field1, field2 string) (interface{}, error) { return nil, fmt.Errorf("both objects must be slice of objects") } + field1Arr := strings.Split(field1, "|") // todo: only works as an OR atm and only 1 dimension deep + + field2Arr := strings.Split(field2, "|") + + if len(field1Arr) != len(field2Arr) { + return nil, fmt.Errorf("field arrays must be same length") + } + + for index := range field1Arr { + output = append(output, addItems(i1, i2, field1Arr[index], field2Arr[index])...) + } + + return output, nil +} + +func addItems(i1, i2 []interface{}, field1, field2 string) []interface{} { + output := make([]interface{}, 0) + for a := range i1 { item1, ok := i1[a].(map[string]interface{}) if !ok { @@ -50,7 +68,7 @@ func LJoin(v, v2 reflect.Value, field1, field2 string) (interface{}, error) { if !ok { continue } - + f2 := item2[field2] if f1 == f2 { newitem := make(map[string]interface{}) @@ -65,5 +83,26 @@ func LJoin(v, v2 reflect.Value, field1, field2 string) (interface{}, error) { } } - return output, nil + return output } + +// ObjMerge - merge two map[string]interface{} objects together - if they have unique keys +func ObjMerge(i1, i2 interface{}) interface{} { + output := make(map[string]interface{}) + + merge1, ok1 := i1.(map[string]interface{}) + merge2, ok2 := i2.(map[string]interface{}) + if !ok1 || !ok2 { + return output + } + + for key := range merge1 { + output[key] = merge1[key] + } + + for key := range merge2 { + output[key] = merge2[key] + } + + return output +} \ No newline at end of file From e1c7a4c2bf9028f71d60f07d869fcff791870074 Mon Sep 17 00:00:00 2001 From: Tom Date: Mon, 26 Sep 2022 19:02:26 +0100 Subject: [PATCH 11/58] make left join --- jlib/new.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/jlib/new.go b/jlib/new.go index 1bc1ea6..785da82 100644 --- a/jlib/new.go +++ b/jlib/new.go @@ -61,6 +61,8 @@ func addItems(i1, i2 []interface{}, field1, field2 string) []interface{} { continue } + var exists bool + f1 := item1[field1] for b := range i2 { @@ -71,6 +73,7 @@ func addItems(i1, i2 []interface{}, field1, field2 string) []interface{} { f2 := item2[field2] if f1 == f2 { + exists = true newitem := make(map[string]interface{}) for key := range item1 { newitem[key] = item1[key] @@ -78,9 +81,14 @@ func addItems(i1, i2 []interface{}, field1, field2 string) []interface{} { for key := range item2 { newitem[key] = item2[key] } + output = append(output, newitem) } } + + if !exists { + output = append(output, item1) + } } return output From c5c577dd00cbf7466630215e967c66f37359ec4f Mon Sep 17 00:00:00 2001 From: Tom Date: Tue, 27 Sep 2022 09:47:29 +0100 Subject: [PATCH 12/58] join enhancement --- jlib/new.go | 167 ++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 143 insertions(+), 24 deletions(-) diff --git a/jlib/new.go b/jlib/new.go index 785da82..22ae7a5 100644 --- a/jlib/new.go +++ b/jlib/new.go @@ -1,6 +1,7 @@ package jlib import ( + "log" "encoding/json" "fmt" "reflect" @@ -23,9 +24,16 @@ func getVal(input interface{}) string { return fmt.Sprintf("%v", input) } -// SimpleJoin - a 1 key left join very simple and useful in certain circumstances +const ( + arrDelimiter = "|" + keyDelimiter = "¬" +) + +// SimpleJoin - a multi-key multi-level full OR join - very simple and useful in certain circumstances func SimpleJoin(v, v2 reflect.Value, field1, field2 string) (interface{}, error) { - output := make([]interface{}, 0) + if !(v.IsValid() && v.CanInterface() && v2.IsValid() && v2.CanInterface()) { + return nil, nil + } i1, ok := v.Interface().([]interface{}) if !ok { @@ -37,61 +45,172 @@ func SimpleJoin(v, v2 reflect.Value, field1, field2 string) (interface{}, error) return nil, fmt.Errorf("both objects must be slice of objects") } - field1Arr := strings.Split(field1, "|") // todo: only works as an OR atm and only 1 dimension deep + field1Arr := strings.Split(field1, arrDelimiter) // todo: only works as an OR atm - field2Arr := strings.Split(field2, "|") + field2Arr := strings.Split(field2, arrDelimiter) if len(field1Arr) != len(field2Arr) { return nil, fmt.Errorf("field arrays must be same length") } + relationMap := make(map[string]*relation) + for index := range field1Arr { - output = append(output, addItems(i1, i2, field1Arr[index], field2Arr[index])...) + addItems(relationMap, i1, i2, field1Arr[index], field2Arr[index]) + } + + output := make([]interface{}, 0) + + for index := range relationMap { + output = append(output, relationMap[index].generateItem()) } return output, nil } -func addItems(i1, i2 []interface{}, field1, field2 string) []interface{} { - output := make([]interface{}, 0) +type relation struct { + object map[string]interface{} + related []interface{} +} + +func newRelation(input map[string]interface{}) *relation { + return &relation{ + object: input, + related: make([]interface{}, 0), + } +} + +func (r *relation) generateItem() map[string]interface{} { + newitem := make(map[string]interface{}) + for key := range r.object { + newitem[key] = r.object[key] + + for index := range r.related { + if val, ok := r.related[index].(map[string]interface{}); ok { + for key := range val { + newitem[key] = val[key] + } + } + } + + + + return newitem +} + +func addItems(relationMap map[string]*relation, i1, i2 []interface{}, field1, field2 string) { for a := range i1 { item1, ok := i1[a].(map[string]interface{}) if !ok { continue } - var exists bool + key := fmt.Sprintf("%v", item1) + + if _, ok := relationMap[key]; !ok { + relationMap[key] = newRelation(item1) + } - f1 := item1[field1] + rel := relationMap[key] + + f1 := getMapStringValue(strings.Split(field1, keyDelimiter), 0, item1) + if f1 == nil { + continue + } for b := range i2 { - item2, ok := i2[b].(map[string]interface{}) - if !ok { + f2 := getMapStringValue(strings.Split(field2, keyDelimiter), 0, i2[b]) + if f2 == nil { continue } - f2 := item2[field2] if f1 == f2 { - exists = true - newitem := make(map[string]interface{}) - for key := range item1 { - newitem[key] = item1[key] - } - for key := range item2 { - newitem[key] = item2[key] + rel.related = append(rel.related, i2[b]) + } + } + + relationMap[key] = rel + } +} + +func outsideRange(fieldArr []string, index int) bool { + return index > len(fieldArr)-1 +} + +func getMapStringValue(fieldArr []string, index int, item interface{}) interface{} { + if outsideRange(fieldArr, index) { + return nil + } + + if obj, ok := item.(map[string]interface{}); ok { + for key := range obj { + log.Println(fieldArr[index], key) + if key == fieldArr[index] { + if len(fieldArr)-1 == index { + return obj[key] + } else { + index++ + new := getMapStringValue(fieldArr, index, obj[key]) + if new != nil { + return new + } } + } + } + } - output = append(output, newitem) + return getArrayValue(fieldArr, index, item) +} + +func getArrayValue(fieldArr []string, index int, item interface{}) interface{} { + if outsideRange(fieldArr, index) { + return nil + } + + if obj, ok := item.([]interface{}); ok { + for value := range obj { + a := fmt.Sprintf("%v", fieldArr[index]) + b := fmt.Sprintf("%v", obj[value]) + log.Println(a, b) + if a == b { + if len(fieldArr)-1 == index { + return item + } else { + index++ + new := getMapStringValue(fieldArr, index, obj) + if new != nil { + return new + } + } } } + } + + return getSingleValue(fieldArr, index, item) +} + + +func getSingleValue(fieldArr []string, index int, item interface{}) interface{} { + if outsideRange(fieldArr, index) { + return nil + } - if !exists { - output = append(output, item1) + a := fmt.Sprintf("%v", fieldArr[index]) + b := fmt.Sprintf("%v", item) + if a == b { + if len(fieldArr)-1 == index { + return item + } else { + index++ + new := getMapStringValue(fieldArr, index, item) + if new != nil { + return new + } } } - return output + return nil } // ObjMerge - merge two map[string]interface{} objects together - if they have unique keys @@ -113,4 +232,4 @@ func ObjMerge(i1, i2 interface{}) interface{} { } return output -} \ No newline at end of file +} From bad44b7e4df481374d386f06caed3925fbef11d3 Mon Sep 17 00:00:00 2001 From: Tom Date: Tue, 27 Sep 2022 09:59:05 +0100 Subject: [PATCH 13/58] fix --- jlib/new.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/jlib/new.go b/jlib/new.go index 22ae7a5..7aed2d0 100644 --- a/jlib/new.go +++ b/jlib/new.go @@ -1,9 +1,9 @@ package jlib import ( - "log" "encoding/json" "fmt" + "log" "reflect" "strings" ) @@ -89,13 +89,13 @@ func (r *relation) generateItem() map[string]interface{} { for index := range r.related { if val, ok := r.related[index].(map[string]interface{}); ok { for key := range val { - newitem[key] = val[key] + newitem[key] = val[key] } } - } - - + } + } + return newitem } @@ -190,7 +190,6 @@ func getArrayValue(fieldArr []string, index int, item interface{}) interface{} { return getSingleValue(fieldArr, index, item) } - func getSingleValue(fieldArr []string, index int, item interface{}) interface{} { if outsideRange(fieldArr, index) { return nil From ef957bf07450c75a91386999ec614f12a174e6d0 Mon Sep 17 00:00:00 2001 From: Tom <53711814+tbal999@users.noreply.github.com> Date: Tue, 27 Sep 2022 17:57:52 +0100 Subject: [PATCH 14/58] add test --- go.mod | 11 ++++++- go.sum | 15 ++++++++++ jlib/new.go | 3 -- jlib/new_test.go | 77 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 102 insertions(+), 4 deletions(-) create mode 100644 jlib/new_test.go diff --git a/go.mod b/go.mod index 908d76b..e1be4ac 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,13 @@ module github.com/xiatechs/jsonata-go go 1.18 -require github.com/shopspring/decimal v1.3.1 +require ( + github.com/shopspring/decimal v1.3.1 + github.com/stretchr/testify v1.8.0 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum index 3289fec..ed1f018 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,17 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/jlib/new.go b/jlib/new.go index 7aed2d0..1356a58 100644 --- a/jlib/new.go +++ b/jlib/new.go @@ -3,7 +3,6 @@ package jlib import ( "encoding/json" "fmt" - "log" "reflect" "strings" ) @@ -145,7 +144,6 @@ func getMapStringValue(fieldArr []string, index int, item interface{}) interface if obj, ok := item.(map[string]interface{}); ok { for key := range obj { - log.Println(fieldArr[index], key) if key == fieldArr[index] { if len(fieldArr)-1 == index { return obj[key] @@ -172,7 +170,6 @@ func getArrayValue(fieldArr []string, index int, item interface{}) interface{} { for value := range obj { a := fmt.Sprintf("%v", fieldArr[index]) b := fmt.Sprintf("%v", obj[value]) - log.Println(a, b) if a == b { if len(fieldArr)-1 == index { return item diff --git a/jlib/new_test.go b/jlib/new_test.go new file mode 100644 index 0000000..eb6e198 --- /dev/null +++ b/jlib/new_test.go @@ -0,0 +1,77 @@ +package jlib + +import ( + "encoding/json" + "reflect" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSJoin(t *testing.T) { + tests := []struct { + description string + object1 string + object2 string + joinStr1 string + joinStr2 string + expectedOutput string + }{ + { + description: "simple join", + object1: `[{"test": { + "id": 1, + "age": 5 + }}]`, + object2: `[{"test": { + "id": 1, + "name": "Tim" + }}]`, + joinStr1: "id", + joinStr2: "id", + expectedOutput: "[{\"test\":{\"age\":5,\"id\":1}}]", + }, + { + description: "nested join", + object1: `[{"test": { + "id": 1, + "age": 5 + }}]`, + object2: `[ + { + "object":{ + "test":{ + "id":1, + "name":"Tim" + } + } + } + ]`, + joinStr1: "id", + joinStr2: "test¬id", + expectedOutput: "[{\"test\":{\"age\":5,\"id\":1}}]", + }, + } + for _, tt := range tests { + tt := tt + + t.Run(tt.description, func(t *testing.T) { + var o1, o2 interface{} + + err := json.Unmarshal([]byte(tt.object1), &o1) + assert.NoError(t, err) + err = json.Unmarshal([]byte(tt.object2), &o2) + assert.NoError(t, err) + + i1 := reflect.ValueOf(o1) + i2 := reflect.ValueOf(o2) + + output, err := SimpleJoin(i1, i2, tt.joinStr1, tt.joinStr2) + assert.NoError(t, err) + + bytes, err := json.Marshal(output) + assert.NoError(t, err) + assert.Equal(t, tt.expectedOutput, string(bytes)) + }) + } +} \ No newline at end of file From e582d2a143ff3b30b83218ff7c2f19c8b070a327 Mon Sep 17 00:00:00 2001 From: Tom Date: Tue, 27 Sep 2022 18:13:37 +0100 Subject: [PATCH 15/58] test & gofmt --- jlib/new.go | 2 +- jlib/new_test.go | 34 +++++++++++++++++----------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/jlib/new.go b/jlib/new.go index 1356a58..b29cb14 100644 --- a/jlib/new.go +++ b/jlib/new.go @@ -94,7 +94,7 @@ func (r *relation) generateItem() map[string]interface{} { } } - + return newitem } diff --git a/jlib/new_test.go b/jlib/new_test.go index eb6e198..a6c98c7 100644 --- a/jlib/new_test.go +++ b/jlib/new_test.go @@ -27,29 +27,29 @@ func TestSJoin(t *testing.T) { "id": 1, "name": "Tim" }}]`, - joinStr1: "id", - joinStr2: "id", + joinStr1: "id", + joinStr2: "id", expectedOutput: "[{\"test\":{\"age\":5,\"id\":1}}]", }, { description: "nested join", - object1: `[{"test": { - "id": 1, - "age": 5 - }}]`, + object1: `[ + { + "age": 5, + "id": 1 + } + ]`, object2: `[ { - "object":{ - "test":{ - "id":1, - "name":"Tim" - } - } + "test": { + "id": 1, + "name": "Tim" + } } - ]`, - joinStr1: "id", - joinStr2: "test¬id", - expectedOutput: "[{\"test\":{\"age\":5,\"id\":1}}]", + ]`, + joinStr1: "id", + joinStr2: "test¬id", + expectedOutput: "[{\"age\":5,\"id\":1,\"test\":{\"id\":1,\"name\":\"Tim\"}}]", }, } for _, tt := range tests { @@ -74,4 +74,4 @@ func TestSJoin(t *testing.T) { assert.Equal(t, tt.expectedOutput, string(bytes)) }) } -} \ No newline at end of file +} From 4cc632d920f8620d1968bde4ef53fc2f9edd8ef4 Mon Sep 17 00:00:00 2001 From: tbal999 Date: Tue, 29 Nov 2022 16:39:42 +0000 Subject: [PATCH 16/58] update dev From 42def18e37c5e84d30e7b1fef02cb0f4dae90b88 Mon Sep 17 00:00:00 2001 From: tbal999 Date: Mon, 12 Dec 2022 15:33:52 +0000 Subject: [PATCH 17/58] add mutex --- callable.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/callable.go b/callable.go index 0a78e1c..266733a 100644 --- a/callable.go +++ b/callable.go @@ -10,6 +10,7 @@ import ( "reflect" "regexp" "strings" + "sync" "github.com/xiatechs/jsonata-go/jlib" "github.com/xiatechs/jsonata-go/jparse" @@ -76,6 +77,7 @@ func newGoCallableParam(typ reflect.Type) goCallableParam { // A goCallable represents a built-in or third party Go function. // It implements the Callable interface. type goCallable struct { + mu sync.Mutex callableName callableMarshaler fn reflect.Value @@ -205,7 +207,9 @@ func makeGoCallableParams(typ reflect.Type) []goCallableParam { } func (c *goCallable) SetContext(context reflect.Value) { + c.mu.Lock() c.context = context + c.mu.Unlock() } func (c *goCallable) ParamCount() int { From bb14382e1d38bce08b80b10bf8aaf2c67f800599 Mon Sep 17 00:00:00 2001 From: tbal999 Date: Mon, 12 Dec 2022 15:37:20 +0000 Subject: [PATCH 18/58] another --- callable.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/callable.go b/callable.go index 266733a..cb01e22 100644 --- a/callable.go +++ b/callable.go @@ -26,6 +26,8 @@ func (n callableName) Name() string { } func (n *callableName) SetName(s string) { + c.mu.Lock() + defer c.mu.Unlock() n.name = s } @@ -208,8 +210,8 @@ func makeGoCallableParams(typ reflect.Type) []goCallableParam { func (c *goCallable) SetContext(context reflect.Value) { c.mu.Lock() + defer c.mu.Unlock() c.context = context - c.mu.Unlock() } func (c *goCallable) ParamCount() int { From 03374d4260e62e8a787c79454b9a3761d1bc2aa4 Mon Sep 17 00:00:00 2001 From: tbal999 Date: Mon, 12 Dec 2022 15:38:23 +0000 Subject: [PATCH 19/58] and another --- callable.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/callable.go b/callable.go index cb01e22..6945fc4 100644 --- a/callable.go +++ b/callable.go @@ -18,6 +18,7 @@ import ( ) type callableName struct { + mu sync.Mutex name string } @@ -26,8 +27,8 @@ func (n callableName) Name() string { } func (n *callableName) SetName(s string) { - c.mu.Lock() - defer c.mu.Unlock() + n.mu.Lock() + defer n.mu.Unlock() n.name = s } From 45655f20fffae37f56667ec092142e6638d0c5c3 Mon Sep 17 00:00:00 2001 From: tbal999 Date: Mon, 12 Dec 2022 15:42:23 +0000 Subject: [PATCH 20/58] fix tests etc --- callable.go | 2 +- callable_test.go | 37 +++++++++++++++++++++++++------------ eval.go | 2 ++ 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/callable.go b/callable.go index 6945fc4..0c29004 100644 --- a/callable.go +++ b/callable.go @@ -18,7 +18,7 @@ import ( ) type callableName struct { - mu sync.Mutex + mu sync.Mutex name string } diff --git a/callable_test.go b/callable_test.go index cf7a96b..9a1b217 100644 --- a/callable_test.go +++ b/callable_test.go @@ -11,6 +11,7 @@ import ( "regexp" "sort" "strings" + "sync" "testing" "github.com/xiatechs/jsonata-go/jparse" @@ -2371,6 +2372,7 @@ func TestRegexCallable(t *testing.T) { "groups": []string{}, "next": &matchCallable{ callableName: callableName{ + sync.Mutex{}, "next", }, match: "ac", @@ -2378,23 +2380,26 @@ func TestRegexCallable(t *testing.T) { end: 5, groups: []string{}, next: &matchCallable{ - callableName: callableName{ - "next", - }, + callableName: callableName{ + sync.Mutex{}, + "next", + }, match: "ad", start: 5, end: 7, groups: []string{}, next: &matchCallable{ - callableName: callableName{ - "next", - }, + callableName: callableName{ + sync.Mutex{}, + "next", + }, match: "ab", start: 7, end: 9, groups: []string{}, next: &matchCallable{ callableName: callableName{ + sync.Mutex{}, "next", }, match: "a", @@ -2425,6 +2430,7 @@ func TestRegexCallable(t *testing.T) { }, "next": &matchCallable{ callableName: callableName{ + sync.Mutex{}, "next", }, match: "ac", @@ -2435,6 +2441,7 @@ func TestRegexCallable(t *testing.T) { }, next: &matchCallable{ callableName: callableName{ + sync.Mutex{}, "next", }, match: "ad", @@ -2444,9 +2451,10 @@ func TestRegexCallable(t *testing.T) { "d", }, next: &matchCallable{ - callableName: callableName{ - "next", - }, + callableName: callableName{ + sync.Mutex{}, + "next", + }, match: "ab", start: 7, end: 9, @@ -2455,6 +2463,7 @@ func TestRegexCallable(t *testing.T) { }, next: &matchCallable{ callableName: callableName{ + sync.Mutex{}, "next", }, match: "a", @@ -2491,6 +2500,7 @@ func TestRegexCallable(t *testing.T) { }, "next": &matchCallable{ callableName: callableName{ + sync.Mutex{}, "next", }, match: "ac", @@ -2502,6 +2512,7 @@ func TestRegexCallable(t *testing.T) { }, next: &matchCallable{ callableName: callableName{ + sync.Mutex{}, "next", }, match: "ad", @@ -2512,9 +2523,10 @@ func TestRegexCallable(t *testing.T) { "", // undefined in jsonata-js }, next: &matchCallable{ - callableName: callableName{ - "next", - }, + callableName: callableName{ + sync.Mutex{}, + "next", + }, match: "ab", start: 7, end: 9, @@ -2524,6 +2536,7 @@ func TestRegexCallable(t *testing.T) { }, next: &matchCallable{ callableName: callableName{ + sync.Mutex{}, "next", }, match: "a", diff --git a/eval.go b/eval.go index 0aeb69f..406e314 100644 --- a/eval.go +++ b/eval.go @@ -9,6 +9,7 @@ import ( "math" "reflect" "sort" + "sync" "github.com/shopspring/decimal" "github.com/xiatechs/jsonata-go/config" @@ -831,6 +832,7 @@ func evalTypedLambda(node *jparse.TypedLambdaNode, data reflect.Value, env *envi func evalObjectTransformation(node *jparse.ObjectTransformationNode, data reflect.Value, env *environment) (reflect.Value, error) { f := &transformationCallable{ callableName: callableName{ + sync.Mutex{}, "transform", }, pattern: node.Pattern, From 71c66fe12a0198563ded1f7451a9bcdb2683870d Mon Sep 17 00:00:00 2001 From: Tom <53711814+tbal999@users.noreply.github.com> Date: Mon, 12 Dec 2022 15:45:13 +0000 Subject: [PATCH 21/58] thread safety (#7) * update dev * add mutex * another * and another * fix tests etc Co-authored-by: tbal999 --- callable.go | 7 +++++++ callable_test.go | 37 +++++++++++++++++++++++++------------ eval.go | 2 ++ 3 files changed, 34 insertions(+), 12 deletions(-) diff --git a/callable.go b/callable.go index 0a78e1c..0c29004 100644 --- a/callable.go +++ b/callable.go @@ -10,6 +10,7 @@ import ( "reflect" "regexp" "strings" + "sync" "github.com/xiatechs/jsonata-go/jlib" "github.com/xiatechs/jsonata-go/jparse" @@ -17,6 +18,7 @@ import ( ) type callableName struct { + mu sync.Mutex name string } @@ -25,6 +27,8 @@ func (n callableName) Name() string { } func (n *callableName) SetName(s string) { + n.mu.Lock() + defer n.mu.Unlock() n.name = s } @@ -76,6 +80,7 @@ func newGoCallableParam(typ reflect.Type) goCallableParam { // A goCallable represents a built-in or third party Go function. // It implements the Callable interface. type goCallable struct { + mu sync.Mutex callableName callableMarshaler fn reflect.Value @@ -205,6 +210,8 @@ func makeGoCallableParams(typ reflect.Type) []goCallableParam { } func (c *goCallable) SetContext(context reflect.Value) { + c.mu.Lock() + defer c.mu.Unlock() c.context = context } diff --git a/callable_test.go b/callable_test.go index cf7a96b..9a1b217 100644 --- a/callable_test.go +++ b/callable_test.go @@ -11,6 +11,7 @@ import ( "regexp" "sort" "strings" + "sync" "testing" "github.com/xiatechs/jsonata-go/jparse" @@ -2371,6 +2372,7 @@ func TestRegexCallable(t *testing.T) { "groups": []string{}, "next": &matchCallable{ callableName: callableName{ + sync.Mutex{}, "next", }, match: "ac", @@ -2378,23 +2380,26 @@ func TestRegexCallable(t *testing.T) { end: 5, groups: []string{}, next: &matchCallable{ - callableName: callableName{ - "next", - }, + callableName: callableName{ + sync.Mutex{}, + "next", + }, match: "ad", start: 5, end: 7, groups: []string{}, next: &matchCallable{ - callableName: callableName{ - "next", - }, + callableName: callableName{ + sync.Mutex{}, + "next", + }, match: "ab", start: 7, end: 9, groups: []string{}, next: &matchCallable{ callableName: callableName{ + sync.Mutex{}, "next", }, match: "a", @@ -2425,6 +2430,7 @@ func TestRegexCallable(t *testing.T) { }, "next": &matchCallable{ callableName: callableName{ + sync.Mutex{}, "next", }, match: "ac", @@ -2435,6 +2441,7 @@ func TestRegexCallable(t *testing.T) { }, next: &matchCallable{ callableName: callableName{ + sync.Mutex{}, "next", }, match: "ad", @@ -2444,9 +2451,10 @@ func TestRegexCallable(t *testing.T) { "d", }, next: &matchCallable{ - callableName: callableName{ - "next", - }, + callableName: callableName{ + sync.Mutex{}, + "next", + }, match: "ab", start: 7, end: 9, @@ -2455,6 +2463,7 @@ func TestRegexCallable(t *testing.T) { }, next: &matchCallable{ callableName: callableName{ + sync.Mutex{}, "next", }, match: "a", @@ -2491,6 +2500,7 @@ func TestRegexCallable(t *testing.T) { }, "next": &matchCallable{ callableName: callableName{ + sync.Mutex{}, "next", }, match: "ac", @@ -2502,6 +2512,7 @@ func TestRegexCallable(t *testing.T) { }, next: &matchCallable{ callableName: callableName{ + sync.Mutex{}, "next", }, match: "ad", @@ -2512,9 +2523,10 @@ func TestRegexCallable(t *testing.T) { "", // undefined in jsonata-js }, next: &matchCallable{ - callableName: callableName{ - "next", - }, + callableName: callableName{ + sync.Mutex{}, + "next", + }, match: "ab", start: 7, end: 9, @@ -2524,6 +2536,7 @@ func TestRegexCallable(t *testing.T) { }, next: &matchCallable{ callableName: callableName{ + sync.Mutex{}, "next", }, match: "a", diff --git a/eval.go b/eval.go index 0aeb69f..406e314 100644 --- a/eval.go +++ b/eval.go @@ -9,6 +9,7 @@ import ( "math" "reflect" "sort" + "sync" "github.com/shopspring/decimal" "github.com/xiatechs/jsonata-go/config" @@ -831,6 +832,7 @@ func evalTypedLambda(node *jparse.TypedLambdaNode, data reflect.Value, env *envi func evalObjectTransformation(node *jparse.ObjectTransformationNode, data reflect.Value, env *environment) (reflect.Value, error) { f := &transformationCallable{ callableName: callableName{ + sync.Mutex{}, "transform", }, pattern: node.Pattern, From 5befb7dd24ea4a76976103f7b6cfb57d5cd30a20 Mon Sep 17 00:00:00 2001 From: Tom <53711814+tbal999@users.noreply.github.com> Date: Tue, 14 Mar 2023 15:53:48 +0000 Subject: [PATCH 22/58] add eval functionality (#9) Co-authored-by: tbal999 --- env.go | 7 ++++++- jsonata.go | 30 ++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/env.go b/env.go index dd5bec5..dc18a68 100644 --- a/env.go +++ b/env.go @@ -70,7 +70,6 @@ var baseEnv = initBaseEnv(map[string]Extension{ /* EXTENDED START */ - "objmerge": { Func: jlib.ObjMerge, UndefinedHandler: defaultUndefinedHandler, @@ -411,6 +410,12 @@ var baseEnv = initBaseEnv(map[string]Extension{ UndefinedHandler: nil, EvalContextHandler: nil, }, + + "eval": { + Func: RunEval, + UndefinedHandler: defaultUndefinedHandler, + EvalContextHandler: defaultContextHandler, + }, }) func initBaseEnv(exts map[string]Extension) *environment { diff --git a/jsonata.go b/jsonata.go index 7285d87..c795278 100644 --- a/jsonata.go +++ b/jsonata.go @@ -180,6 +180,36 @@ func (e *Expr) EvalBytes(data []byte) ([]byte, error) { return json.Marshal(v) } +func RunEval(expression string) (interface{}, error) { + var s evaluator + + s = simple{} + + return s.Eval(expression) +} + +type evaluator interface { + Eval(expression string) (interface{}, error) +} + +type simple struct { + +} + +func (s simple) Eval(expression string) (interface{}, error) { + expr, err := Compile(expression) + if err != nil { + return nil, err + } + + result, err := expr.Eval(``) + if err != nil { + return nil, err + } + + return result, nil +} + // RegisterExts registers custom functions for use during // evaluation. Custom functions registered with this method // are only available to this Expr object. To make custom From 427648316e951b8ab206178cd963b508fc234830 Mon Sep 17 00:00:00 2001 From: Tom <53711814+tbal999@users.noreply.github.com> Date: Wed, 15 Mar 2023 13:11:39 +0000 Subject: [PATCH 23/58] Feature/eval (#10) * add eval functionality * a more fancy eval * fixes * fully fleshed out eval * update --------- Co-authored-by: tbal999 --- env.go | 13 ++++++------- jsonata.go | 55 +++++++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 56 insertions(+), 12 deletions(-) diff --git a/env.go b/env.go index dc18a68..0d1e08e 100644 --- a/env.go +++ b/env.go @@ -82,6 +82,12 @@ var baseEnv = initBaseEnv(map[string]Extension{ EvalContextHandler: nil, }, + "eval": { + Func: RunEval, + UndefinedHandler: defaultUndefinedHandler, + EvalContextHandler: defaultContextHandler, + }, + "unescape": { Func: jlib.Unescape, UndefinedHandler: defaultUndefinedHandler, @@ -410,12 +416,6 @@ var baseEnv = initBaseEnv(map[string]Extension{ UndefinedHandler: nil, EvalContextHandler: nil, }, - - "eval": { - Func: RunEval, - UndefinedHandler: defaultUndefinedHandler, - EvalContextHandler: defaultContextHandler, - }, }) func initBaseEnv(exts map[string]Extension) *environment { @@ -473,7 +473,6 @@ func undefinedHandlerAppend(argv []reflect.Value) bool { // Context handlers func contextHandlerSubstring(argv []reflect.Value) bool { - // If substring() is called with one or two numeric arguments, // use the evaluation context as the first argument. switch len(argv) { diff --git a/jsonata.go b/jsonata.go index c795278..0a3111c 100644 --- a/jsonata.go +++ b/jsonata.go @@ -180,29 +180,74 @@ func (e *Expr) EvalBytes(data []byte) ([]byte, error) { return json.Marshal(v) } -func RunEval(expression string) (interface{}, error) { +func RunEval(initialContext reflect.Value, expression ...interface{}) (interface{}, error) { var s evaluator s = simple{} - return s.Eval(expression) + var result interface{} + + var err error + + if len(expression) == 0 { + result, err = s.InitialEval(initialContext.Interface(), "$$") + if err != nil { + return nil, err + } + } + + for index := range expression { + expressionStr, ok := expression[index].(string) + if !ok { + return nil, fmt.Errorf("%v not able to be used as a string in eval statement", expression[index]) + } + if index == 0 { + result, err = s.InitialEval(initialContext.Interface(), expressionStr) + if err != nil { + return nil, err + } + continue + } + + result, err = s.InitialEval(result, expressionStr) + if err != nil { + return nil, err + } + } + + return result, nil } type evaluator interface { - Eval(expression string) (interface{}, error) + InitialEval(item interface{}, expression string) (interface{}, error) + Eval(override, expression string) (interface{}, error) } type simple struct { } -func (s simple) Eval(expression string) (interface{}, error) { +func (s simple) InitialEval(item interface{}, expression string) (interface{}, error) { + expr, err := Compile(expression) + if err != nil { + return nil, err + } + + result, err := expr.Eval(item) + if err != nil { + return nil, err + } + + return result, nil +} + +func (s simple) Eval(override, expression string) (interface{}, error) { expr, err := Compile(expression) if err != nil { return nil, err } - result, err := expr.Eval(``) + result, err := expr.Eval(override) if err != nil { return nil, err } From 42e6f559aa80b86d083e88d592e20835ac8ead3a Mon Sep 17 00:00:00 2001 From: Tom <53711814+tbal999@users.noreply.github.com> Date: Fri, 17 Mar 2023 08:01:09 +0000 Subject: [PATCH 24/58] add go timezones (#11) Co-authored-by: tbal999 --- jlib/date.go | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/jlib/date.go b/jlib/date.go index c5ee7d4..7c2ad03 100644 --- a/jlib/date.go +++ b/jlib/date.go @@ -17,7 +17,12 @@ import ( // 2006-01-02T15:04:05.000Z07:00 const defaultFormatTimeLayout = "[Y]-[M01]-[D01]T[H01]:[m]:[s].[f001][Z01:01t]" +// added some new timezones var defaultParseTimeLayouts = []string{ + "2006-01-02T15:04:05", + "2006-01-02T15:04:05.000", + "15:04:05 AM 2006-01-02", + "2006-01-02 15:04:05 AM", "[Y]-[M01]-[D01]T[H01]:[m]:[s][Z01:01t]", "[Y]-[M01]-[D01]T[H01]:[m]:[s][Z0100t]", "[Y]-[M01]-[D01]T[H01]:[m]:[s]", @@ -107,26 +112,15 @@ func ToMillis(s string, picture jtypes.OptionalString, tz jtypes.OptionalString) } } - return 0, fmt.Errorf("could not parse time %q", s) + return 0, fmt.Errorf("ToMillis: could not parse time %q | picture: %s", s, picture.String) } var reMinus7 = regexp.MustCompile("-(0*7)") func parseTime(s string, picture string) (time.Time, error) { - // Go's reference time: Mon Jan 2 15:04:05 MST 2006 - refTime := time.Date(2006, time.January, 2, 15, 4, 5, 0, time.FixedZone("MST", -7*60*60)) - - layout, err := jxpath.FormatTime(refTime, picture) - if err != nil { - return time.Time{}, fmt.Errorf("the second argument of the toMillis function must be a valid date format") - } - - // Replace -07:00 with Z07:00 - layout = reMinus7.ReplaceAllString(layout, "Z$1") - - t, err := time.Parse(layout, s) + t, err := time.Parse(picture, s) if err != nil { - return time.Time{}, fmt.Errorf("could not parse time %q", s) + return time.Time{}, fmt.Errorf("parseTime: could not parse time %q", s) } return t, nil From 7957ab7f7256cd6dead2e91a34d1370d34755a1d Mon Sep 17 00:00:00 2001 From: Nick Pocock Date: Thu, 23 Mar 2023 10:36:06 +0000 Subject: [PATCH 25/58] Modify date time string to match the layout provided (#12) * Add a failing test for time parsing * Remove idea * test * Revert old error change * Fix * Add another test * Fix * Add fix incase there were spaces * Fix a few more edge cases * More edge cases * Finaly fixes * Remove milliseconds when a date time comes in * Remove test from the root --- .gitignore | 1 + jlib/date.go | 104 +++++++++++++++++++++++++++++---- jlib/date_test.go | 104 ++++++++++++++++++++++++++++++++- jlib/jxpath/formatdate_test.go | 21 ++++++- jparse/jparse.go | 1 - 5 files changed, 214 insertions(+), 17 deletions(-) diff --git a/.gitignore b/.gitignore index 66fd13c..0ca2a00 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ *.dll *.so *.dylib +.idea # Test binary, built with `go test -c` *.test diff --git a/jlib/date.go b/jlib/date.go index 7c2ad03..69fcaf8 100644 --- a/jlib/date.go +++ b/jlib/date.go @@ -8,7 +8,9 @@ import ( "fmt" "regexp" "strconv" + "strings" "time" + "unicode" "github.com/xiatechs/jsonata-go/jlib/jxpath" "github.com/xiatechs/jsonata-go/jtypes" @@ -17,22 +19,23 @@ import ( // 2006-01-02T15:04:05.000Z07:00 const defaultFormatTimeLayout = "[Y]-[M01]-[D01]T[H01]:[m]:[s].[f001][Z01:01t]" -// added some new timezones +const amSuffix = "am" +const pmSuffix = "pm" + var defaultParseTimeLayouts = []string{ - "2006-01-02T15:04:05", - "2006-01-02T15:04:05.000", - "15:04:05 AM 2006-01-02", - "2006-01-02 15:04:05 AM", "[Y]-[M01]-[D01]T[H01]:[m]:[s][Z01:01t]", "[Y]-[M01]-[D01]T[H01]:[m]:[s][Z0100t]", "[Y]-[M01]-[D01]T[H01]:[m]:[s]", + "[Y0001]-[M01]-[D01]", "[Y]-[M01]-[D01]", + "[Y0001]-[M01]-[D01] [H01]:[m01]:[s01]", + "[Y0001]-[M01]-[D01] [H01]:[m01]:[s01] [P]", + "[H01]", "[Y]", } // FromMillis (golint) func FromMillis(ms int64, picture jtypes.OptionalString, tz jtypes.OptionalString) (string, error) { - t := msToTime(ms).UTC() if tz.String != "" { @@ -99,33 +102,112 @@ func parseTimeZone(tz string) (*time.Location, error) { // ToMillis (golint) func ToMillis(s string, picture jtypes.OptionalString, tz jtypes.OptionalString) (int64, error) { + var err error + var t time.Time + layouts := defaultParseTimeLayouts if picture.String != "" { layouts = []string{picture.String} } // TODO: How are timezones used for parsing? - for _, l := range layouts { - if t, err := parseTime(s, l); err == nil { + if t, err = parseTime(s, l); err == nil { return timeToMS(t), nil } } - return 0, fmt.Errorf("ToMillis: could not parse time %q | picture: %s", s, picture.String) + return 0, err } var reMinus7 = regexp.MustCompile("-(0*7)") func parseTime(s string, picture string) (time.Time, error) { - t, err := time.Parse(picture, s) + // Go's reference time: Mon Jan 2 15:04:05 MST 2006 + refTime := time.Date( + 2006, + time.January, + 2, + 15, + 4, + 5, + 0, + time.FixedZone("MST", -7*60*60), + ) + + layout, err := jxpath.FormatTime(refTime, picture) if err != nil { - return time.Time{}, fmt.Errorf("parseTime: could not parse time %q", s) + return time.Time{}, fmt.Errorf("the second argument of the toMillis function must be a valid date format") + } + + // Replace -07:00 with Z07:00 + layout = reMinus7.ReplaceAllString(layout, "Z$1") + + // First remove the milliseconds from the date time string as it messes up our layouts + splitString := strings.Split(s, ".") + var dateTimeWithoutMilli = splitString[0] + + var formattedTime = dateTimeWithoutMilli + switch layout { + case time.DateOnly: + formattedTime = formattedTime[:len(time.DateOnly)] + case time.RFC3339: + // If the layout contains a time zone but the date string doesn't, lets remove it. + if !strings.Contains(formattedTime, "Z") { + layout = layout[:len(time.DateTime)] + } + } + + // Occasionally date time strings contain a T in the string and the layout doesn't, if that's the + // case, lets remove it. + if strings.Contains(formattedTime, "T") && !strings.Contains(layout, "T") { + formattedTime = strings.ReplaceAll(formattedTime, "T", "") + } else if !strings.Contains(formattedTime, "T") && strings.Contains(layout, "T") { + layout = strings.ReplaceAll(layout, "T", "") + } + + sanitisedLayout := strings.ToLower(stripSpaces(layout)) + sanitisedDateTime := strings.ToLower(stripSpaces(formattedTime)) + + sanitisedLayout = addSuffixIfNotExists(sanitisedLayout, sanitisedDateTime) + sanitisedDateTime = addSuffixIfNotExists(sanitisedDateTime, sanitisedLayout) + + t, err := time.Parse(sanitisedLayout, sanitisedDateTime) + if err != nil { + return time.Time{}, fmt.Errorf( + "could not parse time %q due to inconsistency in layout and date time string, date %s layout %s", + s, + sanitisedDateTime, + sanitisedLayout, + ) } return t, nil } +// It isn't consistent that both the date time string and format have a PM/AM suffix. If we find the suffix +// on one of the strings, add it to the other. Sometimes we can have conflicting suffixes for example the layout +// is always in PM 2006-01-0215:04:05pm but the actual date time string could be AM 2023-01-3110:44:59am. +// If this is the case, just ignore it as the time will parse correctly. +func addSuffixIfNotExists(s string, target string) string { + if strings.HasSuffix(target, amSuffix) && !strings.HasSuffix(s, amSuffix) && !strings.HasSuffix(s, pmSuffix) { + return s + amSuffix + } + + return s +} + +func stripSpaces(str string) string { + return strings.Map(func(r rune) rune { + if unicode.IsSpace(r) { + // if the character is a space, drop it + return -1 + } + // else keep it in the string + return r + }, str) +} + func msToTime(ms int64) time.Time { return time.Unix(ms/1000, (ms%1000)*int64(time.Millisecond)) } diff --git a/jlib/date_test.go b/jlib/date_test.go index 540fcb9..f956dab 100644 --- a/jlib/date_test.go +++ b/jlib/date_test.go @@ -14,7 +14,6 @@ import ( ) func TestFromMillis(t *testing.T) { - date := time.Date(2018, time.September, 30, 15, 58, 5, int(762*time.Millisecond), time.UTC) input := date.UnixNano() / int64(time.Millisecond) @@ -28,10 +27,10 @@ func TestFromMillis(t *testing.T) { Picture: "[Y0001]-[M01]-[D01]", Output: "2018-09-30", }, - /*{ + { Picture: "[[[Y0001]-[M01]-[D01]]]", Output: "[2018-09-30]", - },*/ + }, { Picture: "[M]-[D]-[Y]", Output: "9-30-2018", @@ -117,3 +116,102 @@ func TestFromMillis(t *testing.T) { } } } + +func TestToMillis(t *testing.T) { + var picture jtypes.OptionalString + var tz jtypes.OptionalString + + t.Run("2023-01-31T10:44:59.800 is truncated to [Y0001]-[M01]-[D01]", func(t *testing.T) { + picture.Set(reflect.ValueOf("[Y0001]-[M01]-[D01]")) + + // time string is cut down to match the layout provided + _, err := jlib.ToMillis("2023-01-31T10:44:59.800", picture, tz) + if err != nil { + t.Fatal(err) + } + }) + + t.Run("2023-01-31T10:44:59.800 can be parsed", func(t *testing.T) { + picture.Set(reflect.ValueOf("[Y0001]-[M01]-[D01]T[H01]:[m01]:[s01]")) + + // time string is cut down to match the layout provided + _, err := jlib.ToMillis("2023-01-31T10:44:59.800", picture, tz) + if err != nil { + t.Fatal(err) + } + }) + + t.Run("Whitespace is trimmed to ensure layout and time string match", func(t *testing.T) { + picture.Set(reflect.ValueOf("[Y0001]-[M01]-[D01] [H01]:[m01]:[s01]")) + + // time string is cut down to match the layout provided + _, err := jlib.ToMillis("2023-01-3110:44:59", picture, tz) + if err != nil { + t.Error(err.Error()) + } + }) + + t.Run("Milliseconds are ignored from the date time string", func(t *testing.T) { + picture.Set(reflect.ValueOf("[Y0001]-[M01]-[D01][H01]:[m01]:[s01]")) + + // time string is cut down to match the layout provided + _, err := jlib.ToMillis("2023-01-3110:44:59.100", picture, tz) + if err != nil { + t.Error(err.Error()) + } + }) + + t.Run("T is removed from date time string if it doesn't appear in the layout", func(t *testing.T) { + picture.Set(reflect.ValueOf("[Y0001]-[M01]-[D01] [H01]:[m01]:[s01]")) + + // time string is cut down to match the layout provided + _, err := jlib.ToMillis("2023-01-31T10:44:59.800", picture, tz) + if err != nil { + t.Error(err.Error()) + } + }) + + t.Run("T is removed from layout string if it doesn't appear in the date time", func(t *testing.T) { + picture.Set(reflect.ValueOf("[Y0001]-[M01]-[D01]T[H01]:[m01]:[s01]")) + + // time string is cut down to match the layout provided + _, err := jlib.ToMillis("2023-01-31 10:44:59.800", picture, tz) + if err != nil { + t.Error(err.Error()) + } + }) + + t.Run("No picture is passed to the to millis function", func(t *testing.T) { + // time string is cut down to match the layout provided + _, err := jlib.ToMillis("2023-01-31T10:47:06.260", picture, tz) + if err != nil { + t.Error(err.Error()) + } + }) + + t.Run("Picture contains timezone (using RFC3339 format) but no timezone provided in date time string", func(t *testing.T) { + picture.Set(reflect.ValueOf("[Y0001]-[M01]-[D01]T[H01]:[m01]:[s01][Z]")) + _, err := jlib.ToMillis("2023-01-31T10:47:06.260", picture, tz) + if err != nil { + t.Error(err.Error()) + } + }) + + t.Run("[P] placeholder within date format & date time string", func(t *testing.T) { + picture.Set(reflect.ValueOf("[Y0001]-[M01]-[D01] [H01]:[m01]:[s01] [P]")) + // time string is cut down to match the layout provided + _, err := jlib.ToMillis("2023-01-31 10:44:59 AM", picture, tz) + if err != nil { + t.Error(err.Error()) + } + }) + + t.Run("AM present on date time string but not in the layout", func(t *testing.T) { + picture.Set(reflect.ValueOf("[Y0001]-[M01]-[D01] [H01]:[m01]:[s01]")) + // time string is cut down to match the layout provided + _, err := jlib.ToMillis("2023-01-31 10:44:59 AM", picture, tz) + if err != nil { + t.Error(err.Error()) + } + }) +} diff --git a/jlib/jxpath/formatdate_test.go b/jlib/jxpath/formatdate_test.go index 0ceaa9d..c673266 100644 --- a/jlib/jxpath/formatdate_test.go +++ b/jlib/jxpath/formatdate_test.go @@ -11,7 +11,6 @@ import ( ) func TestFormatYear(t *testing.T) { - input := time.Date(2018, time.April, 1, 12, 0, 0, 0, time.UTC) data := []struct { @@ -24,6 +23,10 @@ func TestFormatYear(t *testing.T) { Picture: "[Y]", Output: "2018", }, + { + Picture: "[Y0001] [M01] [D01]", + Output: "2018 04 01", + }, { Picture: "[Y1]", Output: "2018", @@ -81,8 +84,22 @@ func TestFormatYear(t *testing.T) { } } -func TestFormatTimezone(t *testing.T) { +func TestFormatYearAndTimezone(t *testing.T) { + location, _ := time.LoadLocation("Europe/Rome") + input := time.Date(2018, time.April, 1, 12, 0, 0, 0, location) + + picture := "[Y0001]-[M01]-[D01] [H01]:[m01]:[s01] [P]" + got, err := FormatTime(input, picture) + if err != nil { + t.Errorf("unable to format time %+v", err) + } + if got != "2018-04-01 12:00:00 pm" { + t.Errorf("got %s expected %s", got, "2018-04-01 12:00:00 pm") + } +} + +func TestFormatTimezone(t *testing.T) { const minutes = 60 const hours = 60 * minutes diff --git a/jparse/jparse.go b/jparse/jparse.go index 01d405a..2eb8513 100644 --- a/jparse/jparse.go +++ b/jparse/jparse.go @@ -170,7 +170,6 @@ func lookupBp(tt tokenType) int { // and returns the root node. If the provided expression is not // valid, Parse returns an error of type Error. func Parse(expr string) (root Node, err error) { - // Handle panics from parseExpression. defer func() { if r := recover(); r != nil { From 30f0186c2fee06635634bbb455c34742a623df72 Mon Sep 17 00:00:00 2001 From: Nick Pocock Date: Thu, 23 Mar 2023 13:28:53 +0000 Subject: [PATCH 26/58] Timeformats fix (#13) * add panic recover * Fix error string within From Millis function * Add to millis fix * Use MST --------- Co-authored-by: tbal999 --- jlib/date.go | 24 ++++++++++++++++-------- jsonata_test.go | 7 ++++++- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/jlib/date.go b/jlib/date.go index 69fcaf8..40b905b 100644 --- a/jlib/date.go +++ b/jlib/date.go @@ -19,8 +19,11 @@ import ( // 2006-01-02T15:04:05.000Z07:00 const defaultFormatTimeLayout = "[Y]-[M01]-[D01]T[H01]:[m]:[s].[f001][Z01:01t]" -const amSuffix = "am" -const pmSuffix = "pm" +const ( + amSuffix = "am" + pmSuffix = "pm" + MST = "07:00" +) var defaultParseTimeLayouts = []string{ "[Y]-[M01]-[D01]T[H01]:[m]:[s][Z01:01t]", @@ -143,18 +146,23 @@ func parseTime(s string, picture string) (time.Time, error) { // Replace -07:00 with Z07:00 layout = reMinus7.ReplaceAllString(layout, "Z$1") - // First remove the milliseconds from the date time string as it messes up our layouts - splitString := strings.Split(s, ".") - var dateTimeWithoutMilli = splitString[0] - - var formattedTime = dateTimeWithoutMilli + var formattedTime = s switch layout { case time.DateOnly: - formattedTime = formattedTime[:len(time.DateOnly)] + if len(formattedTime) > len(time.DateOnly) { + formattedTime = formattedTime[:len(time.DateOnly)] + } case time.RFC3339: // If the layout contains a time zone but the date string doesn't, lets remove it. + // Otherwise, if the layout contains a timezone and the time string doesn't add a default + // The default is currently MST which is GMT -7. if !strings.Contains(formattedTime, "Z") { layout = layout[:len(time.DateTime)] + } else { + formattedTimeWithTimeZone := strings.Split(formattedTime, "Z") + if len(formattedTimeWithTimeZone) == 2 { + formattedTime += MST + } } } diff --git a/jsonata_test.go b/jsonata_test.go index 140754c..f474b3a 100644 --- a/jsonata_test.go +++ b/jsonata_test.go @@ -7612,6 +7612,11 @@ func TestFuncMillis2(t *testing.T) { } func TestFuncToMillis(t *testing.T) { + defer func() { // added this to help with the test as it panics and that is annoying + if r := recover(); r != nil { + fmt.Println("Recovered in f", r) + } + }() runTestCases(t, nil, []*testCase{ { @@ -7628,7 +7633,7 @@ func TestFuncToMillis(t *testing.T) { }, { Expression: `$toMillis("foo")`, - Error: fmt.Errorf(`could not parse time "foo"`), + Error: fmt.Errorf(`could not parse time "foo" due to inconsistency in layout and date time string, date foo layout 2006`), }, }) } From ed4855fae49078f0b75ef1a59893aaa75fa17589 Mon Sep 17 00:00:00 2001 From: Nick Pocock Date: Fri, 24 Mar 2023 10:34:47 +0000 Subject: [PATCH 27/58] Update to go 1.19 --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index e1be4ac..bf07301 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/xiatechs/jsonata-go -go 1.18 +go 1.19 require ( github.com/shopspring/decimal v1.3.1 From aac0abdda5916dc8a38e72da0e8712739132a5f7 Mon Sep 17 00:00:00 2001 From: Nick Pocock Date: Fri, 24 Mar 2023 11:39:07 +0000 Subject: [PATCH 28/58] Bump to Go 1.20 --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index bf07301..3ac84f7 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/xiatechs/jsonata-go -go 1.19 +go 1.20 require ( github.com/shopspring/decimal v1.3.1 From 928dc023f994e27eafdd2e1cebeda9530640e3ff Mon Sep 17 00:00:00 2001 From: Tom <53711814+tbal999@users.noreply.github.com> Date: Tue, 28 Mar 2023 13:01:25 +0100 Subject: [PATCH 29/58] Time/localconsts (#14) * add local consts * add test to CI --------- Co-authored-by: tbal999 --- .github/workflows/test.yml | 40 +++++++++++++++++--------------------- jlib/date.go | 13 +++++++++---- 2 files changed, 27 insertions(+), 26 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 37bc6d3..7697949 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,26 +1,22 @@ -on: [push] +on: + push: + branches: '*' + pull_request: + branches: '*' + name: Test jobs: test: - strategy: - matrix: - go-version: [1.16.x] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} + name: Unit Tests + runs-on: ubuntu-latest steps: - - name: Install Go - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go-version }} - - name: Checkout code - uses: actions/checkout@v2 - - name: Install dependencies - run: | - go get -u honnef.co/go/tools/cmd/staticcheck@latest - go get -u golang.org/x/tools/cmd/goimports - - name: Run staticcheck - run: staticcheck ./... - - name: Check code formatting - run: test -z $(goimports -l .) - - name: Run Test - run: go test ./... + - name: Install Go + uses: actions/setup-go@v4 + with: + go-version: '1.20.1' + + - name: Checkout code + uses: actions/checkout@v2 + + - name: Unit tests + run: go test ./... -race \ No newline at end of file diff --git a/jlib/date.go b/jlib/date.go index 40b905b..bcc47d1 100644 --- a/jlib/date.go +++ b/jlib/date.go @@ -125,6 +125,11 @@ func ToMillis(s string, picture jtypes.OptionalString, tz jtypes.OptionalString) var reMinus7 = regexp.MustCompile("-(0*7)") +const ( + timeDateOnly = "2006-01-02" + timeDateTime = "2006-01-02 15:04:05" +) + func parseTime(s string, picture string) (time.Time, error) { // Go's reference time: Mon Jan 2 15:04:05 MST 2006 refTime := time.Date( @@ -148,16 +153,16 @@ func parseTime(s string, picture string) (time.Time, error) { var formattedTime = s switch layout { - case time.DateOnly: - if len(formattedTime) > len(time.DateOnly) { - formattedTime = formattedTime[:len(time.DateOnly)] + case timeDateOnly: + if len(formattedTime) > len(timeDateOnly) { + formattedTime = formattedTime[:len(timeDateOnly)] } case time.RFC3339: // If the layout contains a time zone but the date string doesn't, lets remove it. // Otherwise, if the layout contains a timezone and the time string doesn't add a default // The default is currently MST which is GMT -7. if !strings.Contains(formattedTime, "Z") { - layout = layout[:len(time.DateTime)] + layout = layout[:len(timeDateTime)] } else { formattedTimeWithTimeZone := strings.Split(formattedTime, "Z") if len(formattedTimeWithTimeZone) == 2 { From 49373f6a991db22b5eddc9a2eac5cee63d1ee4fb Mon Sep 17 00:00:00 2001 From: Tom <53711814+tbal999@users.noreply.github.com> Date: Thu, 13 Apr 2023 16:52:07 +0100 Subject: [PATCH 30/58] fix null bug (#15) Co-authored-by: tbal999 --- jlib/jlib.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/jlib/jlib.go b/jlib/jlib.go index 027a047..37483eb 100644 --- a/jlib/jlib.go +++ b/jlib/jlib.go @@ -57,7 +57,17 @@ func (s StringCallable) toInterface() interface{} { // TypeOf implements the jsonata $type function that returns the data type of // the argument func TypeOf(x interface{}) (string, error) { + if fmt.Sprintf("%v", x) == "" { + return "null", nil + } + v := reflect.ValueOf(x) + + switch x.(type) { + case *interface{}: + return "null", nil + } + if jtypes.IsCallable(v) { return "function", nil } @@ -77,11 +87,6 @@ func TypeOf(x interface{}) (string, error) { return "object", nil } - switch x.(type) { - case *interface{}: - return "null", nil - } - xType := reflect.TypeOf(x).String() return "", fmt.Errorf("unknown type %s", xType) } From 4a6a7c0700e62fa0961edb94c671e7976af34cce Mon Sep 17 00:00:00 2001 From: stefan Date: Thu, 18 May 2023 10:56:18 +0300 Subject: [PATCH 31/58] test --- .github/workflows/test.yml | 40 ++++--- .idea/.gitignore | 8 ++ .idea/jsonata-go.iml | 9 ++ .idea/modules.xml | 8 ++ .idea/vcs.xml | 6 ++ callable.go | 6 +- callable_test.go | 37 +++---- env.go | 8 +- error.go | 8 +- errrors_test.go | 190 +++++++++++++++++++++++++++++++++ eval.go | 55 ++++++---- eval_test.go | 9 +- go.mod | 2 +- jlib/date.go | 103 ++++++++++++++++-- jlib/date_test.go | 104 +++++++++++++++++- jlib/jlib.go | 15 ++- jparse/error.go | 3 +- jparse/jparse.go | 6 +- jparse/lexer_test.go | 7 +- jparse/node.go | 211 +++++++++++++++++++++++++++++++++++-- jsonata.go | 76 ++++++++++++- jsonata_test.go | 7 +- 22 files changed, 812 insertions(+), 106 deletions(-) create mode 100644 .idea/.gitignore create mode 100644 .idea/jsonata-go.iml create mode 100644 .idea/modules.xml create mode 100644 .idea/vcs.xml create mode 100644 errrors_test.go diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 37bc6d3..7697949 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,26 +1,22 @@ -on: [push] +on: + push: + branches: '*' + pull_request: + branches: '*' + name: Test jobs: test: - strategy: - matrix: - go-version: [1.16.x] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} + name: Unit Tests + runs-on: ubuntu-latest steps: - - name: Install Go - uses: actions/setup-go@v2 - with: - go-version: ${{ matrix.go-version }} - - name: Checkout code - uses: actions/checkout@v2 - - name: Install dependencies - run: | - go get -u honnef.co/go/tools/cmd/staticcheck@latest - go get -u golang.org/x/tools/cmd/goimports - - name: Run staticcheck - run: staticcheck ./... - - name: Check code formatting - run: test -z $(goimports -l .) - - name: Run Test - run: go test ./... + - name: Install Go + uses: actions/setup-go@v4 + with: + go-version: '1.20.1' + + - name: Checkout code + uses: actions/checkout@v2 + + - name: Unit tests + run: go test ./... -race \ No newline at end of file diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..13566b8 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml diff --git a/.idea/jsonata-go.iml b/.idea/jsonata-go.iml new file mode 100644 index 0000000..5e764c4 --- /dev/null +++ b/.idea/jsonata-go.iml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..0514d04 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..35eb1dd --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/callable.go b/callable.go index 0c29004..ad7763c 100644 --- a/callable.go +++ b/callable.go @@ -689,7 +689,7 @@ func (f *transformationCallable) Call(argv []reflect.Value) (reflect.Value, erro obj, err := f.clone(argv[0]) if err != nil { - return undefined, newEvalError(ErrClone, nil, nil) + return undefined, newEvalError(ErrClone, nil, nil, 0) } if obj == undefined { @@ -746,7 +746,7 @@ func (f *transformationCallable) updateEntries(item reflect.Value) error { } if !jtypes.IsMap(updates) { - return newEvalError(ErrIllegalUpdate, f.updates, nil) + return newEvalError(ErrIllegalUpdate, f.updates, nil, 0) } for _, key := range updates.MapKeys() { @@ -766,7 +766,7 @@ func (f *transformationCallable) deleteEntries(item reflect.Value) error { deletes = arrayify(deletes) if !jtypes.IsArrayOf(deletes, jtypes.IsString) { - return newEvalError(ErrIllegalDelete, f.deletes, nil) + return newEvalError(ErrIllegalDelete, f.deletes, nil, 0) } for i := 0; i < deletes.Len(); i++ { diff --git a/callable_test.go b/callable_test.go index 9a1b217..bac016f 100644 --- a/callable_test.go +++ b/callable_test.go @@ -6,6 +6,7 @@ package jsonata import ( "errors" + "github.com/stretchr/testify/assert" "math" "reflect" "regexp" @@ -2329,8 +2330,8 @@ func testTransformationCallable(t *testing.T, tests []transformationCallableTest } } - if !reflect.DeepEqual(err, test.Error) { - t.Errorf("transform %d: expected error %v, got %v", i+1, test.Error, err) + if err != nil && test.Error != nil { + assert.EqualError(t, err, test.Error.Error()) } } } @@ -2380,19 +2381,19 @@ func TestRegexCallable(t *testing.T) { end: 5, groups: []string{}, next: &matchCallable{ - callableName: callableName{ - sync.Mutex{}, - "next", - }, + callableName: callableName{ + sync.Mutex{}, + "next", + }, match: "ad", start: 5, end: 7, groups: []string{}, next: &matchCallable{ - callableName: callableName{ - sync.Mutex{}, - "next", - }, + callableName: callableName{ + sync.Mutex{}, + "next", + }, match: "ab", start: 7, end: 9, @@ -2451,10 +2452,10 @@ func TestRegexCallable(t *testing.T) { "d", }, next: &matchCallable{ - callableName: callableName{ - sync.Mutex{}, - "next", - }, + callableName: callableName{ + sync.Mutex{}, + "next", + }, match: "ab", start: 7, end: 9, @@ -2523,10 +2524,10 @@ func TestRegexCallable(t *testing.T) { "", // undefined in jsonata-js }, next: &matchCallable{ - callableName: callableName{ - sync.Mutex{}, - "next", - }, + callableName: callableName{ + sync.Mutex{}, + "next", + }, match: "ab", start: 7, end: 9, diff --git a/env.go b/env.go index dd5bec5..0d1e08e 100644 --- a/env.go +++ b/env.go @@ -70,7 +70,6 @@ var baseEnv = initBaseEnv(map[string]Extension{ /* EXTENDED START */ - "objmerge": { Func: jlib.ObjMerge, UndefinedHandler: defaultUndefinedHandler, @@ -83,6 +82,12 @@ var baseEnv = initBaseEnv(map[string]Extension{ EvalContextHandler: nil, }, + "eval": { + Func: RunEval, + UndefinedHandler: defaultUndefinedHandler, + EvalContextHandler: defaultContextHandler, + }, + "unescape": { Func: jlib.Unescape, UndefinedHandler: defaultUndefinedHandler, @@ -468,7 +473,6 @@ func undefinedHandlerAppend(argv []reflect.Value) bool { // Context handlers func contextHandlerSubstring(argv []reflect.Value) bool { - // If substring() is called with one or two numeric arguments, // use the evaluation context as the first argument. switch len(argv) { diff --git a/error.go b/error.go index 3688ebf..3e51fd9 100644 --- a/error.go +++ b/error.go @@ -80,9 +80,10 @@ type EvalError struct { Type ErrType Token string Value string + Pos int } -func newEvalError(typ ErrType, token interface{}, value interface{}) *EvalError { +func newEvalError(typ ErrType, token interface{}, value interface{}, pos int) *EvalError { stringify := func(v interface{}) string { switch v := v.(type) { @@ -99,6 +100,7 @@ func newEvalError(typ ErrType, token interface{}, value interface{}) *EvalError Type: typ, Token: stringify(token), Value: stringify(value), + Pos: pos, } } @@ -112,9 +114,9 @@ func (e EvalError) Error() string { return reErrMsg.ReplaceAllStringFunc(s, func(match string) string { switch match { case "{{token}}": - return e.Token + return fmt.Sprintf("token:%v, position: %v", e.Token, e.Pos) case "{{value}}": - return e.Value + return fmt.Sprintf("value:%v, position: %v", e.Value, e.Pos) default: return match } diff --git a/errrors_test.go b/errrors_test.go new file mode 100644 index 0000000..d127316 --- /dev/null +++ b/errrors_test.go @@ -0,0 +1,190 @@ +package jsonata + +import ( + "encoding/json" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestErrors(t *testing.T) { + // your JSON data + data1 := `{ + "employees": [ + { + "firstName": "John", + "lastName": "Doe", + "department": "Sales", + "salary": 50000, + "joining_date": "2020-05-12", + "details": { + "address": "123 Main St", + "city": "New York", + "state": "NY" + } + }, + { + "firstName": "Anna", + "lastName": "Smith", + "department": "Marketing", + "salary": 60000, + "joining_date": "2019-07-01", + "details": { + "address": "456 Market St", + "city": "San Francisco", + "state": "CA" + } + }, + { + "firstName": "Peter", + "lastName": "Jones", + "department": "Sales", + "salary": 70000, + "joining_date": "2021-01-20", + "details": { + "address": "789 Broad St", + "city": "Los Angeles", + "state": "CA" + } + } + ] +}` + var data interface{} + + // Decode JSON. + err := json.Unmarshal([]byte(data1), &data) + assert.NoError(t, err) + t.Run("wrong arithmetic errors", func(t *testing.T) { + + // Create expression. + e := MustCompile("employees.firstName + 5") + + // Evaluate. + _, err := e.Eval(data) + assert.Error(t, err, "left side of the \"value:+, position: 20\" operator must evaluate to a number") + }) + t.Run("Cannot call non-function token:", func(t *testing.T) { + + // Create expression. + e := MustCompile("employees.details.state.$address()") + + // Evaluate. + _, err := e.Eval(data) + assert.EqualError(t, err, "cannot call non-function token:$address, position: 25") + + }) + t.Run("Trying to get the maximum of a string field:", func(t *testing.T) { + + // Create expression. + e := MustCompile("$max(employees.firstName)") + + // Evaluate. + _, err := e.Eval(data) + assert.EqualError(t, err, "err: cannot call max on an array with non-number types, possition: 1, arguments: number:0 value:[John Anna Peter] ") + + }) + t.Run("Invalid Function Call on a non-array field:", func(t *testing.T) { + + // Create expression. + e := MustCompile("employees.department.$count()") + + // Evaluate. + _, err := e.Eval(data) + assert.EqualError(t, err, "err: function \"count\" takes 1 argument(s), got 0, possition: 22, arguments: ") + + }) + t.Run("Cannot use wildcard on non-object type:", func(t *testing.T) { + + // Create expression. + e := MustCompile("employees.*.salary") + + // Evaluate. + _, err := e.Eval(data) + assert.EqualError(t, err, "err:no results found, token: employees.*.salary, possition: 0") + }) + t.Run("Indexing on non-array type:", func(t *testing.T) { + + // Create expression. + e := MustCompile("employees.firstName[1]") + + // Evaluate. + _, err := e.Eval(data) + assert.EqualError(t, err, "err:no results found, token: employees.firstName[1], possition: 0") + + }) + t.Run("Use of an undefined variable:", func(t *testing.T) { + + // Create expression. + e := MustCompile("$undefinedVariable") + + // Evaluate. + _, err := e.Eval(data) + assert.EqualError(t, err, "err:no results found, token: $undefinedVariable, possition: 1") + + }) + t.Run("Use of an undefined function:", func(t *testing.T) { + + // Create expression. + e := MustCompile("$undefinedFunction()") + + // Evaluate. + _, err := e.Eval(data) + assert.EqualError(t, err, "cannot call non-function token:$undefinedFunction, position: 1") + + }) + t.Run("Comparison of incompatible types:", func(t *testing.T) { + + // Create expression. + e := MustCompile("employees.firstName > employees.salary") + + // Evaluate. + _, err := e.Eval(data) + assert.EqualError(t, err, "left side of the \"value:>, position: 0\" operator must evaluate to a number or string") + + }) + t.Run("Use of an invalid JSONata operator:", func(t *testing.T) { + + // Create expression. + _, err := Compile("employees ! employees") + assert.EqualError(t, err, "syntax error: '', character position 10") + }) + t.Run("Incorrect use of the reduce function:", func(t *testing.T) { + + // Create expression. + e := MustCompile("$reduce(employees.firstName, function($acc, $val) { $acc + $val })") + + // Evaluate. + _, err := e.Eval(data) + assert.ErrorContains(t, err, "err: left side of the \"value:+, position: 57\" operator must evaluate to a number, possition: 1, arguments: number:0 value:[John Anna Peter]") + + }) + t.Run("Incorrect use of the map function:", func(t *testing.T) { + + // Create expression. + e := MustCompile("$map(employees, function($employee) { $employee.firstName + 5 })") + + // Evaluate. + _, err := e.Eval(data) + assert.ErrorContains(t, err, "err: left side of the \"value:+, position: 58\" operator must evaluate to a number, possition: 1, arguments: number:0 value:[map[department:Sales details:map[address:123 Main St city:New York state:NY] firstName:John joining_date:2020-05-12") + + }) + t.Run("Incorrect use of the filter function:", func(t *testing.T) { + + // Create expression. + e := MustCompile("$filter(employees, function($employee) { $employee.salary.$uppercase() })") + + // Evaluate. + _, err := e.Eval(data) + assert.ErrorContains(t, err, "err: err: argument 1 of function \"uppercase\" does not match function signature, possition: 59, arguments: , possition: 1, arguments: number:0 value:[map[department:Sales details:map[address:123 Main St city:New York state:NY] firstName:John joining_date:2020-05-12 lastName:Doe salary:50000") + + }) + t.Run("Incorrect use of the join function:", func(t *testing.T) { + + // Create expression. + e := MustCompile("$join(employees.firstName, 5)") + + // Evaluate. + _, err := e.Eval(data) + assert.ErrorContains(t, err, "err: argument 2 of function \"join\" does not match function signature, possition: 1, arguments: number:0 value:[John Anna Peter] number:1 value:5") + + }) +} diff --git a/eval.go b/eval.go index 406e314..f395f31 100644 --- a/eval.go +++ b/eval.go @@ -315,7 +315,7 @@ func evalNegation(node *jparse.NegationNode, data reflect.Value, env *environmen n, ok := jtypes.AsNumber(rhs) if !ok { - return undefined, newEvalError(ErrNonNumberRHS, node.RHS, "-") + return undefined, newEvalError(ErrNonNumberRHS, node.RHS, "-", 0) } return reflect.ValueOf(-n), nil @@ -356,11 +356,11 @@ func evalRange(node *jparse.RangeNode, data reflect.Value, env *environment) (re // If either side is not an integer, return an error. if lhsOK && !lhsInteger { - return undefined, newEvalError(ErrNonIntegerLHS, node.LHS, "..") + return undefined, newEvalError(ErrNonIntegerLHS, node.LHS, "..", 0) } if rhsOK && !rhsInteger { - return undefined, newEvalError(ErrNonIntegerRHS, node.RHS, "..") + return undefined, newEvalError(ErrNonIntegerRHS, node.RHS, "..", 0) } // If either side is undefined or the left side is greater @@ -373,7 +373,7 @@ func evalRange(node *jparse.RangeNode, data reflect.Value, env *environment) (re // Check for integer overflow or an array size that exceeds // our upper bound. if size < 0 || size > maxRangeItems { - return undefined, newEvalError(ErrMaxRangeItems, "..", nil) + return undefined, newEvalError(ErrMaxRangeItems, "..", nil, 0) } results := reflect.MakeSlice(typeInterfaceSlice, size, size) @@ -478,7 +478,7 @@ func groupItemsByKey(obj *jparse.ObjectNode, items reflect.Value, env *environme key := s.Value if _, ok := results[key]; ok { - return nil, newEvalError(ErrDuplicateKey, keyNode, key) + return nil, newEvalError(ErrDuplicateKey, keyNode, key, 0) } results[key] = keyIndexes{ @@ -496,7 +496,7 @@ func groupItemsByKey(obj *jparse.ObjectNode, items reflect.Value, env *environme key, ok := jtypes.AsString(v) if !ok { - return nil, newEvalError(ErrIllegalKey, keyNode, nil) + return nil, newEvalError(ErrIllegalKey, keyNode, nil, 0) } idx, ok := results[key] @@ -509,7 +509,7 @@ func groupItemsByKey(obj *jparse.ObjectNode, items reflect.Value, env *environme } if idx.pair != i { - return nil, newEvalError(ErrDuplicateKey, keyNode, key) + return nil, newEvalError(ErrDuplicateKey, keyNode, key, 0) } idx.items = append(idx.items, j) @@ -718,20 +718,20 @@ func buildSortInfo(items reflect.Value, terms []jparse.SortTerm, env *environmen switch { case jtypes.IsNumber(v): if isStringTerm[j] { - return nil, newEvalError(ErrSortMismatch, term.Expr, nil) + return nil, newEvalError(ErrSortMismatch, term.Expr, nil, 0) } values[j] = v isNumberTerm[j] = true case jtypes.IsString(v): if isNumberTerm[j] { - return nil, newEvalError(ErrSortMismatch, term.Expr, nil) + return nil, newEvalError(ErrSortMismatch, term.Expr, nil, 0) } values[j] = v isStringTerm[j] = true default: - return nil, newEvalError(ErrNonSortable, term.Expr, nil) + return nil, newEvalError(ErrNonSortable, term.Expr, nil, 0) } } @@ -852,7 +852,7 @@ func evalPartial(node *jparse.PartialNode, data reflect.Value, env *environment) fn, ok := jtypes.AsCallable(v) if !ok { - return undefined, newEvalError(ErrNonCallablePartial, node.Func, nil) + return undefined, newEvalError(ErrNonCallablePartial, node.Func, nil, 0) } f := &partialCallable{ @@ -884,7 +884,7 @@ func evalFunctionCall(node *jparse.FunctionCallNode, data reflect.Value, env *en fn, ok := jtypes.AsCallable(v) if !ok { - return undefined, newEvalError(ErrNonCallable, node.Func, nil) + return undefined, newEvalError(ErrNonCallable, node.Func, reflect.ValueOf(data), node.Func.Pos()) } if setter, ok := fn.(nameSetter); ok { @@ -907,8 +907,19 @@ func evalFunctionCall(node *jparse.FunctionCallNode, data reflect.Value, env *en argv[i] = v } + res, err := fn.Call(argv) + if err != nil { + return res, fmt.Errorf("err: %v, possition: %v, arguments: %v", err, node.Func.Pos(), transformArgsToString(argv)) + } + return res, nil +} - return fn.Call(argv) +func transformArgsToString(argv []reflect.Value) string { + argvString := "" + for i, value := range argv { + argvString += fmt.Sprintf("number:%v value:%v ", i, value.Interface()) + } + return argvString } func evalFunctionApplication(node *jparse.FunctionApplicationNode, data reflect.Value, env *environment) (reflect.Value, error) { @@ -935,7 +946,7 @@ func evalFunctionApplication(node *jparse.FunctionApplicationNode, data reflect. // Check that the right hand side is callable. f2, ok := jtypes.AsCallable(rhs) if !ok { - return undefined, newEvalError(ErrNonCallableApply, node.RHS, "~>") + return undefined, newEvalError(ErrNonCallableApply, node.RHS, "~>", 0) } // If the left hand side is not callable, call the right @@ -981,12 +992,14 @@ func evalNumericOperator(node *jparse.NumericOperatorNode, data reflect.Value, e } // Return an error if either side is not a number. + if lhsOK && !lhsNumber { - return undefined, newEvalError(ErrNonNumberLHS, node.LHS, node.Type) + fmt.Println(reflect.ValueOf(data)) + return undefined, newEvalError(ErrNonNumberLHS, node.LHS, node.Type, node.Pos()) } if rhsOK && !rhsNumber { - return undefined, newEvalError(ErrNonNumberRHS, node.RHS, node.Type) + return undefined, newEvalError(ErrNonNumberRHS, node.RHS, node.Type, 0) } // Return undefined if either side is undefined. @@ -1017,11 +1030,11 @@ func evalNumericOperator(node *jparse.NumericOperatorNode, data reflect.Value, e } if math.IsInf(x, 0) { - return undefined, newEvalError(ErrNumberInf, nil, node.Type) + return undefined, newEvalError(ErrNumberInf, nil, node.Type, 0) } if math.IsNaN(x) { - return undefined, newEvalError(ErrNumberNaN, nil, node.Type) + return undefined, newEvalError(ErrNumberNaN, nil, node.Type, 0) } return reflect.ValueOf(x), nil @@ -1056,16 +1069,16 @@ func evalComparisonOperator(node *jparse.ComparisonOperatorNode, data reflect.Va // left side type does not equal right side type. if needComparableTypes(node.Type) { if lhs != undefined && !lhsNumber && !lhsString { - return undefined, newEvalError(ErrNonComparableLHS, node.LHS, node.Type) + return undefined, newEvalError(ErrNonComparableLHS, node.LHS, node.Type, 0) } if rhs != undefined && !rhsNumber && !rhsString { - return undefined, newEvalError(ErrNonComparableRHS, node.RHS, node.Type) + return undefined, newEvalError(ErrNonComparableRHS, node.RHS, node.Type, 0) } if lhs != undefined && rhs != undefined && (lhsNumber != rhsNumber || lhsString != rhsString) { - return undefined, newEvalError(ErrTypeMismatch, nil, node.Type) + return undefined, newEvalError(ErrTypeMismatch, nil, node.Type, 0) } } diff --git a/eval_test.go b/eval_test.go index 019a1d5..515a10c 100644 --- a/eval_test.go +++ b/eval_test.go @@ -5,6 +5,7 @@ package jsonata import ( + "github.com/stretchr/testify/assert" "math" "reflect" "regexp" @@ -4297,7 +4298,6 @@ func testEvalTestCases(t *testing.T, tests []evalTestCase) { } v, err := eval(test.Input, reflect.ValueOf(test.Data), env) - var output interface{} if v.IsValid() && v.CanInterface() { output = v.Interface() @@ -4309,11 +4309,12 @@ func testEvalTestCases(t *testing.T, tests []evalTestCase) { } if !equal(output, test.Output) { - t.Errorf("%s: Expected %v, got %v", test.Input, test.Output, output) + t.Errorf("%s: Expected: %v, got: %v", test.Input, test.Output, output) } - if !reflect.DeepEqual(err, test.Error) { - t.Errorf("%s: Expected error %v, got %v", test.Input, test.Error, err) + if err != nil && test.Error != nil { + assert.EqualError(t, err, test.Error.Error()) } + } } diff --git a/go.mod b/go.mod index e1be4ac..3ac84f7 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/xiatechs/jsonata-go -go 1.18 +go 1.20 require ( github.com/shopspring/decimal v1.3.1 diff --git a/jlib/date.go b/jlib/date.go index c5ee7d4..bcc47d1 100644 --- a/jlib/date.go +++ b/jlib/date.go @@ -8,7 +8,9 @@ import ( "fmt" "regexp" "strconv" + "strings" "time" + "unicode" "github.com/xiatechs/jsonata-go/jlib/jxpath" "github.com/xiatechs/jsonata-go/jtypes" @@ -17,17 +19,26 @@ import ( // 2006-01-02T15:04:05.000Z07:00 const defaultFormatTimeLayout = "[Y]-[M01]-[D01]T[H01]:[m]:[s].[f001][Z01:01t]" +const ( + amSuffix = "am" + pmSuffix = "pm" + MST = "07:00" +) + var defaultParseTimeLayouts = []string{ "[Y]-[M01]-[D01]T[H01]:[m]:[s][Z01:01t]", "[Y]-[M01]-[D01]T[H01]:[m]:[s][Z0100t]", "[Y]-[M01]-[D01]T[H01]:[m]:[s]", + "[Y0001]-[M01]-[D01]", "[Y]-[M01]-[D01]", + "[Y0001]-[M01]-[D01] [H01]:[m01]:[s01]", + "[Y0001]-[M01]-[D01] [H01]:[m01]:[s01] [P]", + "[H01]", "[Y]", } // FromMillis (golint) func FromMillis(ms int64, picture jtypes.OptionalString, tz jtypes.OptionalString) (string, error) { - t := msToTime(ms).UTC() if tz.String != "" { @@ -94,27 +105,43 @@ func parseTimeZone(tz string) (*time.Location, error) { // ToMillis (golint) func ToMillis(s string, picture jtypes.OptionalString, tz jtypes.OptionalString) (int64, error) { + var err error + var t time.Time + layouts := defaultParseTimeLayouts if picture.String != "" { layouts = []string{picture.String} } // TODO: How are timezones used for parsing? - for _, l := range layouts { - if t, err := parseTime(s, l); err == nil { + if t, err = parseTime(s, l); err == nil { return timeToMS(t), nil } } - return 0, fmt.Errorf("could not parse time %q", s) + return 0, err } var reMinus7 = regexp.MustCompile("-(0*7)") +const ( + timeDateOnly = "2006-01-02" + timeDateTime = "2006-01-02 15:04:05" +) + func parseTime(s string, picture string) (time.Time, error) { // Go's reference time: Mon Jan 2 15:04:05 MST 2006 - refTime := time.Date(2006, time.January, 2, 15, 4, 5, 0, time.FixedZone("MST", -7*60*60)) + refTime := time.Date( + 2006, + time.January, + 2, + 15, + 4, + 5, + 0, + time.FixedZone("MST", -7*60*60), + ) layout, err := jxpath.FormatTime(refTime, picture) if err != nil { @@ -124,14 +151,76 @@ func parseTime(s string, picture string) (time.Time, error) { // Replace -07:00 with Z07:00 layout = reMinus7.ReplaceAllString(layout, "Z$1") - t, err := time.Parse(layout, s) + var formattedTime = s + switch layout { + case timeDateOnly: + if len(formattedTime) > len(timeDateOnly) { + formattedTime = formattedTime[:len(timeDateOnly)] + } + case time.RFC3339: + // If the layout contains a time zone but the date string doesn't, lets remove it. + // Otherwise, if the layout contains a timezone and the time string doesn't add a default + // The default is currently MST which is GMT -7. + if !strings.Contains(formattedTime, "Z") { + layout = layout[:len(timeDateTime)] + } else { + formattedTimeWithTimeZone := strings.Split(formattedTime, "Z") + if len(formattedTimeWithTimeZone) == 2 { + formattedTime += MST + } + } + } + + // Occasionally date time strings contain a T in the string and the layout doesn't, if that's the + // case, lets remove it. + if strings.Contains(formattedTime, "T") && !strings.Contains(layout, "T") { + formattedTime = strings.ReplaceAll(formattedTime, "T", "") + } else if !strings.Contains(formattedTime, "T") && strings.Contains(layout, "T") { + layout = strings.ReplaceAll(layout, "T", "") + } + + sanitisedLayout := strings.ToLower(stripSpaces(layout)) + sanitisedDateTime := strings.ToLower(stripSpaces(formattedTime)) + + sanitisedLayout = addSuffixIfNotExists(sanitisedLayout, sanitisedDateTime) + sanitisedDateTime = addSuffixIfNotExists(sanitisedDateTime, sanitisedLayout) + + t, err := time.Parse(sanitisedLayout, sanitisedDateTime) if err != nil { - return time.Time{}, fmt.Errorf("could not parse time %q", s) + return time.Time{}, fmt.Errorf( + "could not parse time %q due to inconsistency in layout and date time string, date %s layout %s", + s, + sanitisedDateTime, + sanitisedLayout, + ) } return t, nil } +// It isn't consistent that both the date time string and format have a PM/AM suffix. If we find the suffix +// on one of the strings, add it to the other. Sometimes we can have conflicting suffixes for example the layout +// is always in PM 2006-01-0215:04:05pm but the actual date time string could be AM 2023-01-3110:44:59am. +// If this is the case, just ignore it as the time will parse correctly. +func addSuffixIfNotExists(s string, target string) string { + if strings.HasSuffix(target, amSuffix) && !strings.HasSuffix(s, amSuffix) && !strings.HasSuffix(s, pmSuffix) { + return s + amSuffix + } + + return s +} + +func stripSpaces(str string) string { + return strings.Map(func(r rune) rune { + if unicode.IsSpace(r) { + // if the character is a space, drop it + return -1 + } + // else keep it in the string + return r + }, str) +} + func msToTime(ms int64) time.Time { return time.Unix(ms/1000, (ms%1000)*int64(time.Millisecond)) } diff --git a/jlib/date_test.go b/jlib/date_test.go index 540fcb9..f956dab 100644 --- a/jlib/date_test.go +++ b/jlib/date_test.go @@ -14,7 +14,6 @@ import ( ) func TestFromMillis(t *testing.T) { - date := time.Date(2018, time.September, 30, 15, 58, 5, int(762*time.Millisecond), time.UTC) input := date.UnixNano() / int64(time.Millisecond) @@ -28,10 +27,10 @@ func TestFromMillis(t *testing.T) { Picture: "[Y0001]-[M01]-[D01]", Output: "2018-09-30", }, - /*{ + { Picture: "[[[Y0001]-[M01]-[D01]]]", Output: "[2018-09-30]", - },*/ + }, { Picture: "[M]-[D]-[Y]", Output: "9-30-2018", @@ -117,3 +116,102 @@ func TestFromMillis(t *testing.T) { } } } + +func TestToMillis(t *testing.T) { + var picture jtypes.OptionalString + var tz jtypes.OptionalString + + t.Run("2023-01-31T10:44:59.800 is truncated to [Y0001]-[M01]-[D01]", func(t *testing.T) { + picture.Set(reflect.ValueOf("[Y0001]-[M01]-[D01]")) + + // time string is cut down to match the layout provided + _, err := jlib.ToMillis("2023-01-31T10:44:59.800", picture, tz) + if err != nil { + t.Fatal(err) + } + }) + + t.Run("2023-01-31T10:44:59.800 can be parsed", func(t *testing.T) { + picture.Set(reflect.ValueOf("[Y0001]-[M01]-[D01]T[H01]:[m01]:[s01]")) + + // time string is cut down to match the layout provided + _, err := jlib.ToMillis("2023-01-31T10:44:59.800", picture, tz) + if err != nil { + t.Fatal(err) + } + }) + + t.Run("Whitespace is trimmed to ensure layout and time string match", func(t *testing.T) { + picture.Set(reflect.ValueOf("[Y0001]-[M01]-[D01] [H01]:[m01]:[s01]")) + + // time string is cut down to match the layout provided + _, err := jlib.ToMillis("2023-01-3110:44:59", picture, tz) + if err != nil { + t.Error(err.Error()) + } + }) + + t.Run("Milliseconds are ignored from the date time string", func(t *testing.T) { + picture.Set(reflect.ValueOf("[Y0001]-[M01]-[D01][H01]:[m01]:[s01]")) + + // time string is cut down to match the layout provided + _, err := jlib.ToMillis("2023-01-3110:44:59.100", picture, tz) + if err != nil { + t.Error(err.Error()) + } + }) + + t.Run("T is removed from date time string if it doesn't appear in the layout", func(t *testing.T) { + picture.Set(reflect.ValueOf("[Y0001]-[M01]-[D01] [H01]:[m01]:[s01]")) + + // time string is cut down to match the layout provided + _, err := jlib.ToMillis("2023-01-31T10:44:59.800", picture, tz) + if err != nil { + t.Error(err.Error()) + } + }) + + t.Run("T is removed from layout string if it doesn't appear in the date time", func(t *testing.T) { + picture.Set(reflect.ValueOf("[Y0001]-[M01]-[D01]T[H01]:[m01]:[s01]")) + + // time string is cut down to match the layout provided + _, err := jlib.ToMillis("2023-01-31 10:44:59.800", picture, tz) + if err != nil { + t.Error(err.Error()) + } + }) + + t.Run("No picture is passed to the to millis function", func(t *testing.T) { + // time string is cut down to match the layout provided + _, err := jlib.ToMillis("2023-01-31T10:47:06.260", picture, tz) + if err != nil { + t.Error(err.Error()) + } + }) + + t.Run("Picture contains timezone (using RFC3339 format) but no timezone provided in date time string", func(t *testing.T) { + picture.Set(reflect.ValueOf("[Y0001]-[M01]-[D01]T[H01]:[m01]:[s01][Z]")) + _, err := jlib.ToMillis("2023-01-31T10:47:06.260", picture, tz) + if err != nil { + t.Error(err.Error()) + } + }) + + t.Run("[P] placeholder within date format & date time string", func(t *testing.T) { + picture.Set(reflect.ValueOf("[Y0001]-[M01]-[D01] [H01]:[m01]:[s01] [P]")) + // time string is cut down to match the layout provided + _, err := jlib.ToMillis("2023-01-31 10:44:59 AM", picture, tz) + if err != nil { + t.Error(err.Error()) + } + }) + + t.Run("AM present on date time string but not in the layout", func(t *testing.T) { + picture.Set(reflect.ValueOf("[Y0001]-[M01]-[D01] [H01]:[m01]:[s01]")) + // time string is cut down to match the layout provided + _, err := jlib.ToMillis("2023-01-31 10:44:59 AM", picture, tz) + if err != nil { + t.Error(err.Error()) + } + }) +} diff --git a/jlib/jlib.go b/jlib/jlib.go index 027a047..37483eb 100644 --- a/jlib/jlib.go +++ b/jlib/jlib.go @@ -57,7 +57,17 @@ func (s StringCallable) toInterface() interface{} { // TypeOf implements the jsonata $type function that returns the data type of // the argument func TypeOf(x interface{}) (string, error) { + if fmt.Sprintf("%v", x) == "" { + return "null", nil + } + v := reflect.ValueOf(x) + + switch x.(type) { + case *interface{}: + return "null", nil + } + if jtypes.IsCallable(v) { return "function", nil } @@ -77,11 +87,6 @@ func TypeOf(x interface{}) (string, error) { return "object", nil } - switch x.(type) { - case *interface{}: - return "null", nil - } - xType := reflect.TypeOf(x).String() return "", fmt.Errorf("unknown type %s", xType) } diff --git a/jparse/error.go b/jparse/error.go index 8bc1d57..054f1e9 100644 --- a/jparse/error.go +++ b/jparse/error.go @@ -85,7 +85,8 @@ type Error struct { } func newError(typ ErrType, tok token) error { - return newErrorHint(typ, tok, "") + return fmt.Errorf("%v, character position %d", newErrorHint(typ, tok, ""), tok.Position) + } func newErrorHint(typ ErrType, tok token, hint string) error { diff --git a/jparse/jparse.go b/jparse/jparse.go index 01d405a..cb36a74 100644 --- a/jparse/jparse.go +++ b/jparse/jparse.go @@ -4,6 +4,8 @@ package jparse +import "fmt" + // The JSONata parser is based on Pratt's Top Down Operator // Precededence algorithm (see https://tdop.github.io/). Given // a series of tokens representing a JSONata expression and the @@ -170,7 +172,6 @@ func lookupBp(tt tokenType) int { // and returns the root node. If the provided expression is not // valid, Parse returns an error of type Error. func Parse(expr string) (root Node, err error) { - // Handle panics from parseExpression. defer func() { if r := recover(); r != nil { @@ -302,7 +303,8 @@ func (p *parser) consume(expected tokenType, allowRegex bool) { typ = ErrMissingToken } - panic(newErrorHint(typ, p.token, expected.String())) + // syntax errors now tell you exact character position where error failed - which you can find using software or local editor + panic(newErrorHint(typ, p.token, fmt.Sprintf("'%s' at character position: %d", expected.String(), p.token.Position))) } p.advance(allowRegex) diff --git a/jparse/lexer_test.go b/jparse/lexer_test.go index 8a64f00..2281f3e 100644 --- a/jparse/lexer_test.go +++ b/jparse/lexer_test.go @@ -5,7 +5,8 @@ package jparse import ( - "reflect" + "fmt" + "github.com/stretchr/testify/assert" "testing" ) @@ -392,10 +393,8 @@ func compareTokens(t *testing.T, prefix string, exp, got token) { } func compareErrors(t *testing.T, prefix string, exp, got error) { + assert.EqualError(t, exp, fmt.Sprintf("%v", got)) - if !reflect.DeepEqual(exp, got) { - t.Errorf("%s: expected error %v, got %v", prefix, exp, got) - } } func tok(typ tokenType, value string, position int) token { diff --git a/jparse/node.go b/jparse/node.go index 6d2bbe4..19590ba 100644 --- a/jparse/node.go +++ b/jparse/node.go @@ -18,11 +18,21 @@ import ( type Node interface { String() string optimize() (Node, error) + Pos() int +} + +func (n *NumberNode) Pos() int { + return n.pos } // A StringNode represents a string literal. type StringNode struct { Value string + pos int +} + +func (n *StringNode) Pos() int { + return n.pos } func parseString(p *parser, t token) (Node, error) { @@ -39,6 +49,7 @@ func parseString(p *parser, t token) (Node, error) { return &StringNode{ Value: s, + pos: t.Position, }, nil } @@ -53,6 +64,7 @@ func (n StringNode) String() string { // A NumberNode represents a number literal. type NumberNode struct { Value float64 + pos int } func parseNumber(p *parser, t token) (Node, error) { @@ -69,6 +81,7 @@ func parseNumber(p *parser, t token) (Node, error) { return &NumberNode{ Value: n, + pos: t.Position, }, nil } @@ -83,6 +96,11 @@ func (n NumberNode) String() string { // A BooleanNode represents the boolean constant true or false. type BooleanNode struct { Value bool + pos int +} + +func (n *BooleanNode) Pos() int { + return n.pos } func parseBoolean(p *parser, t token) (Node, error) { @@ -100,6 +118,7 @@ func parseBoolean(p *parser, t token) (Node, error) { return &BooleanNode{ Value: b, + pos: t.Position, }, nil } @@ -112,10 +131,17 @@ func (n BooleanNode) String() string { } // A NullNode represents the JSON null value. -type NullNode struct{} +type NullNode struct { + pos int +} +func (n *NullNode) Pos() int { + return n.pos +} func parseNull(p *parser, t token) (Node, error) { - return &NullNode{}, nil + return &NullNode{ + pos: t.Position, + }, nil } func (n *NullNode) optimize() (Node, error) { @@ -129,6 +155,11 @@ func (NullNode) String() string { // A RegexNode represents a regular expression. type RegexNode struct { Value *regexp.Regexp + pos int +} + +func (n *RegexNode) Pos() int { + return n.pos } func parseRegex(p *parser, t token) (Node, error) { @@ -149,6 +180,7 @@ func parseRegex(p *parser, t token) (Node, error) { return &RegexNode{ Value: re, + pos: t.Position, }, nil } @@ -167,11 +199,17 @@ func (n RegexNode) String() string { // A VariableNode represents a JSONata variable. type VariableNode struct { Name string + pos int +} + +func (n *VariableNode) Pos() int { + return n.pos } func parseVariable(p *parser, t token) (Node, error) { return &VariableNode{ Name: t.Value, + pos: t.Position, }, nil } @@ -187,11 +225,17 @@ func (n VariableNode) String() string { type NameNode struct { Value string escaped bool + pos int +} + +func (n *NameNode) Pos() int { + return n.pos } func parseName(p *parser, t token) (Node, error) { return &NameNode{ Value: t.Value, + pos: t.Position, }, nil } @@ -199,12 +243,14 @@ func parseEscapedName(p *parser, t token) (Node, error) { return &NameNode{ Value: t.Value, escaped: true, + pos: t.Position, }, nil } func (n *NameNode) optimize() (Node, error) { return &PathNode{ Steps: []Node{n}, + pos: n.Pos(), }, nil } @@ -228,6 +274,11 @@ func (n NameNode) Escaped() bool { type PathNode struct { Steps []Node KeepArrays bool + pos int +} + +func (n *PathNode) Pos() int { + return n.pos } func (n *PathNode) optimize() (Node, error) { @@ -245,11 +296,17 @@ func (n PathNode) String() string { // A NegationNode represents a numeric negation operation. type NegationNode struct { RHS Node + pos int +} + +func (n *NegationNode) Pos() int { + return n.pos } func parseNegation(p *parser, t token) (Node, error) { return &NegationNode{ RHS: p.parseExpression(p.bp(t.Type)), + pos: t.Position, }, nil } @@ -267,6 +324,7 @@ func (n *NegationNode) optimize() (Node, error) { if number, ok := n.RHS.(*NumberNode); ok { return &NumberNode{ Value: -number.Value, + pos: number.Pos(), }, nil } @@ -281,6 +339,11 @@ func (n NegationNode) String() string { type RangeNode struct { LHS Node RHS Node + pos int +} + +func (n *RangeNode) Pos() int { + return n.pos } func (n *RangeNode) optimize() (Node, error) { @@ -307,8 +370,12 @@ func (n RangeNode) String() string { // An ArrayNode represents an array of items. type ArrayNode struct { Items []Node + pos int } +func (n *ArrayNode) Pos() int { + return n.pos +} func parseArray(p *parser, t token) (Node, error) { var items []Node @@ -324,6 +391,7 @@ func parseArray(p *parser, t token) (Node, error) { item = &RangeNode{ LHS: item, RHS: p.parseExpression(0), + pos: p.token.Position, } } @@ -339,6 +407,7 @@ func parseArray(p *parser, t token) (Node, error) { return &ArrayNode{ Items: items, + pos: t.Position, }, nil } @@ -364,6 +433,11 @@ func (n ArrayNode) String() string { // key-value pairs. type ObjectNode struct { Pairs [][2]Node + pos int +} + +func (n *ObjectNode) Pos() int { + return n.pos } func parseObject(p *parser, t token) (Node, error) { @@ -388,6 +462,7 @@ func parseObject(p *parser, t token) (Node, error) { return &ObjectNode{ Pairs: pairs, + pos: t.Position, }, nil } @@ -421,6 +496,11 @@ func (n ObjectNode) String() string { // A BlockNode represents a block expression. type BlockNode struct { Exprs []Node + pos int +} + +func (n *BlockNode) Pos() int { + return n.pos } func parseBlock(p *parser, t token) (Node, error) { @@ -441,6 +521,7 @@ func parseBlock(p *parser, t token) (Node, error) { return &BlockNode{ Exprs: exprs, + pos: t.Position, }, nil } @@ -463,10 +544,17 @@ func (n BlockNode) String() string { } // A WildcardNode represents the wildcard operator. -type WildcardNode struct{} +type WildcardNode struct { + pos int +} +func (n *WildcardNode) Pos() int { + return n.pos +} func parseWildcard(p *parser, t token) (Node, error) { - return &WildcardNode{}, nil + return &WildcardNode{ + pos: t.Position, + }, nil } func (n *WildcardNode) optimize() (Node, error) { @@ -478,10 +566,17 @@ func (WildcardNode) String() string { } // A DescendentNode represents the descendent operator. -type DescendentNode struct{} +type DescendentNode struct { + pos int +} +func (n *DescendentNode) Pos() int { + return n.pos +} func parseDescendent(p *parser, t token) (Node, error) { - return &DescendentNode{}, nil + return &DescendentNode{ + pos: t.Position, + }, nil } func (n *DescendentNode) optimize() (Node, error) { @@ -498,6 +593,11 @@ type ObjectTransformationNode struct { Pattern Node Updates Node Deletes Node + pos int +} + +func (n *ObjectTransformationNode) Pos() int { + return n.pos } func parseObjectTransformation(p *parser, t token) (Node, error) { @@ -517,6 +617,7 @@ func parseObjectTransformation(p *parser, t token) (Node, error) { Pattern: pattern, Updates: updates, Deletes: deletes, + pos: t.Position, }, nil } @@ -833,8 +934,12 @@ type LambdaNode struct { Body Node ParamNames []string shorthand bool + pos int } +func (n *LambdaNode) Pos() int { + return n.pos +} func (n *LambdaNode) optimize() (Node, error) { var err error @@ -876,8 +981,12 @@ type TypedLambdaNode struct { *LambdaNode In []Param Out []Param + pos int } +func (n *TypedLambdaNode) Pos() int { + return n.pos +} func (n *TypedLambdaNode) optimize() (Node, error) { node, err := n.LambdaNode.optimize() @@ -913,6 +1022,11 @@ func (n TypedLambdaNode) String() string { type PartialNode struct { Func Node Args []Node + pos int +} + +func (n *PartialNode) Pos() int { + return n.pos } func (n *PartialNode) optimize() (Node, error) { @@ -940,8 +1054,13 @@ func (n PartialNode) String() string { // A PlaceholderNode represents a placeholder argument // in a partially applied function. -type PlaceholderNode struct{} +type PlaceholderNode struct { + pos int +} +func (n *PlaceholderNode) Pos() int { + return n.pos +} func (n *PlaceholderNode) optimize() (Node, error) { return n, nil } @@ -954,6 +1073,11 @@ func (PlaceholderNode) String() string { type FunctionCallNode struct { Func Node Args []Node + pos int +} + +func (n *FunctionCallNode) Pos() int { + return n.pos } const typePlaceholder = typeCondition @@ -993,12 +1117,14 @@ func parseFunctionCall(p *parser, t token, lhs Node) (Node, error) { return &PartialNode{ Func: lhs, Args: args, + pos: t.Position, }, nil } return &FunctionCallNode{ Func: lhs, Args: args, + pos: t.Position, }, nil } @@ -1062,6 +1188,7 @@ func parseLambdaDefinition(p *parser, shorthand bool) (Node, error) { Body: body, ParamNames: paramNames, shorthand: shorthand, + pos: body.Pos(), } if !isTyped { @@ -1071,6 +1198,7 @@ func parseLambdaDefinition(p *parser, shorthand bool) (Node, error) { return &TypedLambdaNode{ LambdaNode: lambda, In: params, + pos: lambda.Pos(), }, nil } @@ -1149,6 +1277,11 @@ Loop: type PredicateNode struct { Expr Node Filters []Node + pos int +} + +func (n *PredicateNode) Pos() int { + return n.pos } func (n *PredicateNode) optimize() (Node, error) { @@ -1163,6 +1296,7 @@ func (n PredicateNode) String() string { type GroupNode struct { Expr Node *ObjectNode + pos int } func parseGroup(p *parser, t token, lhs Node) (Node, error) { @@ -1175,6 +1309,7 @@ func parseGroup(p *parser, t token, lhs Node) (Node, error) { return &GroupNode{ Expr: lhs, ObjectNode: obj.(*ObjectNode), + pos: t.Position, }, nil } @@ -1212,8 +1347,12 @@ type ConditionalNode struct { If Node Then Node Else Node + pos int } +func (n *ConditionalNode) Pos() int { + return n.pos +} func parseConditional(p *parser, t token, lhs Node) (Node, error) { var els Node @@ -1228,6 +1367,7 @@ func parseConditional(p *parser, t token, lhs Node) (Node, error) { If: lhs, Then: rhs, Else: els, + pos: t.Position, }, nil } @@ -1269,6 +1409,11 @@ func (n ConditionalNode) String() string { type AssignmentNode struct { Name string Value Node + pos int +} + +func (n *AssignmentNode) Pos() int { + return n.pos } func parseAssignment(p *parser, t token, lhs Node) (Node, error) { @@ -1281,6 +1426,7 @@ func parseAssignment(p *parser, t token, lhs Node) (Node, error) { return &AssignmentNode{ Name: v.Name, Value: p.parseExpression(p.bp(t.Type) - 1), // right-associative + pos: t.Position, }, nil } @@ -1336,8 +1482,12 @@ type NumericOperatorNode struct { Type NumericOperator LHS Node RHS Node + pos int } +func (n *NumericOperatorNode) Pos() int { + return n.pos +} func parseNumericOperator(p *parser, t token, lhs Node) (Node, error) { var op NumericOperator @@ -1361,6 +1511,7 @@ func parseNumericOperator(p *parser, t token, lhs Node) (Node, error) { Type: op, LHS: lhs, RHS: p.parseExpression(p.bp(t.Type)), + pos: t.Position, }, nil } @@ -1426,8 +1577,12 @@ type ComparisonOperatorNode struct { Type ComparisonOperator LHS Node RHS Node + pos int } +func (n *ComparisonOperatorNode) Pos() int { + return n.pos +} func parseComparisonOperator(p *parser, t token, lhs Node) (Node, error) { var op ComparisonOperator @@ -1455,6 +1610,7 @@ func parseComparisonOperator(p *parser, t token, lhs Node) (Node, error) { Type: op, LHS: lhs, RHS: p.parseExpression(p.bp(t.Type)), + pos: t.Position, }, nil } @@ -1506,6 +1662,11 @@ type BooleanOperatorNode struct { Type BooleanOperator LHS Node RHS Node + pos int +} + +func (n *BooleanOperatorNode) Pos() int { + return n.pos } func parseBooleanOperator(p *parser, t token, lhs Node) (Node, error) { @@ -1518,13 +1679,14 @@ func parseBooleanOperator(p *parser, t token, lhs Node) (Node, error) { case typeOr: op = BooleanOr default: // should be unreachable - panicf("parseBooleanOperator: unexpected operator %q", t.Value) + panicf("parseBooleanOperator: unexpected operator %q at position %d", t.Value, p.token.Position) } return &BooleanOperatorNode{ Type: op, LHS: lhs, RHS: p.parseExpression(p.bp(t.Type)), + pos: t.Position, }, nil } @@ -1554,12 +1716,17 @@ func (n BooleanOperatorNode) String() string { type StringConcatenationNode struct { LHS Node RHS Node + pos int } +func (n *StringConcatenationNode) Pos() int { + return n.pos +} func parseStringConcatenation(p *parser, t token, lhs Node) (Node, error) { return &StringConcatenationNode{ LHS: lhs, RHS: p.parseExpression(p.bp(t.Type)), + pos: t.Position, }, nil } @@ -1605,8 +1772,12 @@ type SortTerm struct { type SortNode struct { Expr Node Terms []SortTerm + pos int } +func (n *SortNode) Pos() int { + return n.pos +} func parseSort(p *parser, t token, lhs Node) (Node, error) { var terms []SortTerm @@ -1641,6 +1812,7 @@ func parseSort(p *parser, t token, lhs Node) (Node, error) { return &SortNode{ Expr: lhs, Terms: terms, + pos: t.Position, }, nil } @@ -1689,12 +1861,18 @@ func (n SortNode) String() string { type FunctionApplicationNode struct { LHS Node RHS Node + pos int +} + +func (n *FunctionApplicationNode) Pos() int { + return n.pos } func parseFunctionApplication(p *parser, t token, lhs Node) (Node, error) { return &FunctionApplicationNode{ LHS: lhs, RHS: p.parseExpression(p.bp(t.Type)), + pos: t.Position, }, nil } @@ -1725,12 +1903,18 @@ func (n FunctionApplicationNode) String() string { type dotNode struct { lhs Node rhs Node + pos int +} + +func (n *dotNode) Pos() int { + return n.pos } func parseDot(p *parser, t token, lhs Node) (Node, error) { return &dotNode{ lhs: lhs, rhs: p.parseExpression(p.bp(t.Type)), + pos: t.Position, }, nil } @@ -1792,6 +1976,11 @@ func (n dotNode) String() string { // and gets converted into a PathNode during optimization. type singletonArrayNode struct { lhs Node + pos int +} + +func (n *singletonArrayNode) Pos() int { + return n.pos } func (n *singletonArrayNode) optimize() (Node, error) { @@ -1823,8 +2012,12 @@ func (n singletonArrayNode) String() string { type predicateNode struct { lhs Node // the context for this predicate rhs Node // the predicate expression + pos int } +func (n *predicateNode) Pos() int { + return n.pos +} func parsePredicate(p *parser, t token, lhs Node) (Node, error) { if p.token.Type == typeBracketClose { @@ -1834,6 +2027,7 @@ func parsePredicate(p *parser, t token, lhs Node) (Node, error) { // flatten singleton arrays into single values. return &singletonArrayNode{ lhs: lhs, + pos: t.Position, }, nil } @@ -1843,6 +2037,7 @@ func parsePredicate(p *parser, t token, lhs Node) (Node, error) { return &predicateNode{ lhs: lhs, rhs: rhs, + pos: t.Position, }, nil } diff --git a/jsonata.go b/jsonata.go index 7285d87..28f3633 100644 --- a/jsonata.go +++ b/jsonata.go @@ -147,7 +147,7 @@ func (e *Expr) Eval(data interface{}) (interface{}, error) { } if !result.IsValid() { - return nil, ErrUndefined + return nil, fmt.Errorf("err:%v, token: %v, possition: %v", ErrUndefined, e.node.String(), e.node.Pos()) } if !result.CanInterface() { @@ -180,6 +180,80 @@ func (e *Expr) EvalBytes(data []byte) ([]byte, error) { return json.Marshal(v) } +func RunEval(initialContext reflect.Value, expression ...interface{}) (interface{}, error) { + var s evaluator + + s = simple{} + + var result interface{} + + var err error + + if len(expression) == 0 { + result, err = s.InitialEval(initialContext.Interface(), "$$") + if err != nil { + return nil, err + } + } + + for index := range expression { + expressionStr, ok := expression[index].(string) + if !ok { + return nil, fmt.Errorf("%v not able to be used as a string in eval statement", expression[index]) + } + if index == 0 { + result, err = s.InitialEval(initialContext.Interface(), expressionStr) + if err != nil { + return nil, err + } + continue + } + + result, err = s.InitialEval(result, expressionStr) + if err != nil { + return nil, err + } + } + + return result, nil +} + +type evaluator interface { + InitialEval(item interface{}, expression string) (interface{}, error) + Eval(override, expression string) (interface{}, error) +} + +type simple struct { +} + +func (s simple) InitialEval(item interface{}, expression string) (interface{}, error) { + expr, err := Compile(expression) + if err != nil { + return nil, err + } + + result, err := expr.Eval(item) + if err != nil { + return nil, err + } + + return result, nil +} + +func (s simple) Eval(override, expression string) (interface{}, error) { + expr, err := Compile(expression) + if err != nil { + return nil, err + } + + result, err := expr.Eval(override) + if err != nil { + return nil, err + } + + return result, nil +} + // RegisterExts registers custom functions for use during // evaluation. Custom functions registered with this method // are only available to this Expr object. To make custom diff --git a/jsonata_test.go b/jsonata_test.go index 140754c..f474b3a 100644 --- a/jsonata_test.go +++ b/jsonata_test.go @@ -7612,6 +7612,11 @@ func TestFuncMillis2(t *testing.T) { } func TestFuncToMillis(t *testing.T) { + defer func() { // added this to help with the test as it panics and that is annoying + if r := recover(); r != nil { + fmt.Println("Recovered in f", r) + } + }() runTestCases(t, nil, []*testCase{ { @@ -7628,7 +7633,7 @@ func TestFuncToMillis(t *testing.T) { }, { Expression: `$toMillis("foo")`, - Error: fmt.Errorf(`could not parse time "foo"`), + Error: fmt.Errorf(`could not parse time "foo" due to inconsistency in layout and date time string, date foo layout 2006`), }, }) } From 6775490e42ee2190832688215b80742e0365d60e Mon Sep 17 00:00:00 2001 From: stefan Date: Thu, 18 May 2023 14:53:43 +0300 Subject: [PATCH 32/58] improve the errors --- error.go | 67 +++++++++-------- errrors_test.go | 102 ++++++++++++------------- eval.go | 17 ++++- jparse/error.go | 60 ++++++++------- jparse/jparse.go | 4 +- jparse/jparse_test.go | 17 ++--- jparse/lexer.go | 2 +- jparse/lexer_test.go | 8 +- jsonata.go | 2 +- jsonata_test.go | 171 +++++++++++++++++++++++++++++------------- 10 files changed, 268 insertions(+), 182 deletions(-) diff --git a/error.go b/error.go index 3e51fd9..3020e5c 100644 --- a/error.go +++ b/error.go @@ -50,37 +50,38 @@ const ( ) var errmsgs = map[ErrType]string{ - ErrNonIntegerLHS: `left side of the "{{value}}" operator must evaluate to an integer`, - ErrNonIntegerRHS: `right side of the "{{value}}" operator must evaluate to an integer`, - ErrNonNumberLHS: `left side of the "{{value}}" operator must evaluate to a number`, - ErrNonNumberRHS: `right side of the "{{value}}" operator must evaluate to a number`, - ErrNonComparableLHS: `left side of the "{{value}}" operator must evaluate to a number or string`, - ErrNonComparableRHS: `right side of the "{{value}}" operator must evaluate to a number or string`, - ErrTypeMismatch: `both sides of the "{{value}}" operator must have the same type`, - ErrNonCallable: `cannot call non-function {{token}}`, - ErrNonCallableApply: `cannot use function application with non-function {{token}}`, - ErrNonCallablePartial: `cannot partially apply non-function {{token}}`, - ErrNumberInf: `result of the "{{value}}" operator is out of range`, - ErrNumberNaN: `result of the "{{value}}" operator is not a valid number`, - ErrMaxRangeItems: `range operator has too many items`, - ErrIllegalKey: `object key {{token}} does not evaluate to a string`, - ErrDuplicateKey: `multiple object keys evaluate to the value "{{value}}"`, - ErrClone: `object transformation: cannot make a copy of the object`, - ErrIllegalUpdate: `the insert/update clause of an object transformation must evaluate to an object`, - ErrIllegalDelete: `the delete clause of an object transformation must evaluate to an array of strings`, - ErrNonSortable: `expressions in a sort term must evaluate to strings or numbers`, - ErrSortMismatch: `expressions in a sort term must have the same type`, + ErrNonIntegerLHS: `left side of the "{{value}}" operator must evaluate to an integer, position:{{position}}, arguments: {{arguments}}`, + ErrNonIntegerRHS: `right side of the "{{value}}" operator must evaluate to an integer, position:{{position}}, arguments: {{arguments}}`, + ErrNonNumberLHS: `left side of the "{{value}}" operator must evaluate to a number, position:{{position}}, arguments: {{arguments}}`, + ErrNonNumberRHS: `right side of the "{{value}}" operator must evaluate to a number, position:{{position}}, arguments: {{arguments}}`, + ErrNonComparableLHS: `left side of the "{{value}}" operator must evaluate to a number or string, position:{{position}}, arguments: {{arguments}}`, + ErrNonComparableRHS: `right side of the "{{value}}" operator must evaluate to a number or string, position:{{position}}, arguments: {{arguments}}`, + ErrTypeMismatch: `both sides of the "{{value}}" operator must have the same type, position:{{position}}, arguments: {{arguments}}`, + ErrNonCallable: `cannot call non-function {{token}}, position:{{position}}, arguments: {{arguments}}`, + ErrNonCallableApply: `cannot use function application with non-function {{token}}, position:{{position}}, arguments: {{arguments}}`, + ErrNonCallablePartial: `cannot partially apply non-function {{token}}, position:{{position}}, arguments: {{arguments}}`, + ErrNumberInf: `result of the "{{value}}" operator is out of range, position:{{position}}, arguments: {{arguments}}`, + ErrNumberNaN: `result of the "{{value}}" operator is not a valid number, position:{{position}}, arguments: {{arguments}}`, + ErrMaxRangeItems: `range operator has too many items, position:{{position}}, arguments: {{arguments}}`, + ErrIllegalKey: `object key {{token}} does not evaluate to a string, position:{{position}}, arguments: {{arguments}}`, + ErrDuplicateKey: `multiple object keys evaluate to the value "{{value}}", position:{{position}}, arguments: {{arguments}}`, + ErrClone: `object transformation: cannot make a copy of the object, position:{{position}}, arguments: {{arguments}}`, + ErrIllegalUpdate: `the insert/update clause of an object transformation must evaluate to an object, position:{{position}}, arguments: {{arguments}}`, + ErrIllegalDelete: `the delete clause of an object transformation must evaluate to an array of strings, position:{{position}}, arguments: {{arguments}}`, + ErrNonSortable: `expressions in a sort term must evaluate to strings or numbers, position:{{position}}, arguments: {{arguments}}`, + ErrSortMismatch: `expressions in a sort term must have the same type, position:{{position}}, arguments: {{arguments}}`, } -var reErrMsg = regexp.MustCompile("{{(token|value)}}") +var reErrMsg = regexp.MustCompile("{{(token|value|position|arguments)}}") // An EvalError represents an error during evaluation of a // JSONata expression. type EvalError struct { - Type ErrType - Token string - Value string - Pos int + Type ErrType + Token string + Value string + Pos int + Arguments string } func newEvalError(typ ErrType, token interface{}, value interface{}, pos int) *EvalError { @@ -114,9 +115,13 @@ func (e EvalError) Error() string { return reErrMsg.ReplaceAllStringFunc(s, func(match string) string { switch match { case "{{token}}": - return fmt.Sprintf("token:%v, position: %v", e.Token, e.Pos) + return e.Token case "{{value}}": - return fmt.Sprintf("value:%v, position: %v", e.Value, e.Pos) + return e.Value + case "{{arguments}}": + return e.Arguments + case "{{position}}": + return fmt.Sprintf("%v", e.Pos) default: return match } @@ -148,8 +153,10 @@ func (e ArgCountError) Error() string { // expression contains a function call with the wrong argument // type. type ArgTypeError struct { - Func string - Which int + Func string + Which int + Pos int + Arguments string } func newArgTypeError(f jtypes.Callable, which int) *ArgTypeError { @@ -160,5 +167,5 @@ func newArgTypeError(f jtypes.Callable, which int) *ArgTypeError { } func (e ArgTypeError) Error() string { - return fmt.Sprintf("argument %d of function %q does not match function signature", e.Which, e.Func) + return fmt.Sprintf("argument %d of function %q does not match function signature, position: %v, arguments: %v", e.Which, e.Func, e.Pos, e.Arguments) } diff --git a/errrors_test.go b/errrors_test.go index d127316..55232d9 100644 --- a/errrors_test.go +++ b/errrors_test.go @@ -9,44 +9,44 @@ import ( func TestErrors(t *testing.T) { // your JSON data data1 := `{ - "employees": [ - { - "firstName": "John", - "lastName": "Doe", - "department": "Sales", - "salary": 50000, - "joining_date": "2020-05-12", - "details": { - "address": "123 Main St", - "city": "New York", - "state": "NY" - } - }, - { - "firstName": "Anna", - "lastName": "Smith", - "department": "Marketing", - "salary": 60000, - "joining_date": "2019-07-01", - "details": { - "address": "456 Market St", - "city": "San Francisco", - "state": "CA" - } - }, - { - "firstName": "Peter", - "lastName": "Jones", - "department": "Sales", - "salary": 70000, - "joining_date": "2021-01-20", - "details": { - "address": "789 Broad St", - "city": "Los Angeles", - "state": "CA" - } - } - ] + "employees": [ + { + "firstName": "John", + "lastName": "Doe", + "department": "Sales", + "salary": 50000, + "joining_date": "2020-05-12", + "details": { + "address": "123 Main St", + "city": "New York", + "state": "NY" + } + }, + { + "firstName": "Anna", + "lastName": "Smith", + "department": "Marketing", + "salary": 60000, + "joining_date": "2019-07-01", + "details": { + "address": "456 Market St", + "city": "San Francisco", + "state": "CA" + } + }, + { + "firstName": "Peter", + "lastName": "Jones", + "department": "Sales", + "salary": 70000, + "joining_date": "2021-01-20", + "details": { + "address": "789 Broad St", + "city": "Los Angeles", + "state": "CA" + } + } + ] }` var data interface{} @@ -69,7 +69,7 @@ func TestErrors(t *testing.T) { // Evaluate. _, err := e.Eval(data) - assert.EqualError(t, err, "cannot call non-function token:$address, position: 25") + assert.EqualError(t, err, "cannot call non-function $address, position:25, arguments: ") }) t.Run("Trying to get the maximum of a string field:", func(t *testing.T) { @@ -79,7 +79,7 @@ func TestErrors(t *testing.T) { // Evaluate. _, err := e.Eval(data) - assert.EqualError(t, err, "err: cannot call max on an array with non-number types, possition: 1, arguments: number:0 value:[John Anna Peter] ") + assert.EqualError(t, err, "cannot call max on an array with non-number types, position: 1, arguments: number:0 value:[John Anna Peter] ") }) t.Run("Invalid Function Call on a non-array field:", func(t *testing.T) { @@ -89,7 +89,7 @@ func TestErrors(t *testing.T) { // Evaluate. _, err := e.Eval(data) - assert.EqualError(t, err, "err: function \"count\" takes 1 argument(s), got 0, possition: 22, arguments: ") + assert.EqualError(t, err, "function \"count\" takes 1 argument(s), got 0, position: 22, arguments: ") }) t.Run("Cannot use wildcard on non-object type:", func(t *testing.T) { @@ -99,7 +99,7 @@ func TestErrors(t *testing.T) { // Evaluate. _, err := e.Eval(data) - assert.EqualError(t, err, "err:no results found, token: employees.*.salary, possition: 0") + assert.EqualError(t, err, "no results found") }) t.Run("Indexing on non-array type:", func(t *testing.T) { @@ -108,7 +108,7 @@ func TestErrors(t *testing.T) { // Evaluate. _, err := e.Eval(data) - assert.EqualError(t, err, "err:no results found, token: employees.firstName[1], possition: 0") + assert.EqualError(t, err, "no results found") }) t.Run("Use of an undefined variable:", func(t *testing.T) { @@ -118,7 +118,7 @@ func TestErrors(t *testing.T) { // Evaluate. _, err := e.Eval(data) - assert.EqualError(t, err, "err:no results found, token: $undefinedVariable, possition: 1") + assert.EqualError(t, err, "no results found") }) t.Run("Use of an undefined function:", func(t *testing.T) { @@ -128,7 +128,7 @@ func TestErrors(t *testing.T) { // Evaluate. _, err := e.Eval(data) - assert.EqualError(t, err, "cannot call non-function token:$undefinedFunction, position: 1") + assert.EqualError(t, err, "cannot call non-function $undefinedFunction, position:1, arguments: ") }) t.Run("Comparison of incompatible types:", func(t *testing.T) { @@ -138,14 +138,14 @@ func TestErrors(t *testing.T) { // Evaluate. _, err := e.Eval(data) - assert.EqualError(t, err, "left side of the \"value:>, position: 0\" operator must evaluate to a number or string") + assert.EqualError(t, err, "left side of the \">\" operator must evaluate to a number or string, position:0, arguments: ") }) t.Run("Use of an invalid JSONata operator:", func(t *testing.T) { // Create expression. _, err := Compile("employees ! employees") - assert.EqualError(t, err, "syntax error: '', character position 10") + assert.EqualError(t, err, "syntax error: '', position: 10") }) t.Run("Incorrect use of the reduce function:", func(t *testing.T) { @@ -154,7 +154,7 @@ func TestErrors(t *testing.T) { // Evaluate. _, err := e.Eval(data) - assert.ErrorContains(t, err, "err: left side of the \"value:+, position: 57\" operator must evaluate to a number, possition: 1, arguments: number:0 value:[John Anna Peter]") + assert.ErrorContains(t, err, "left side of the \"+\" operator must evaluate to a number, position:57, arguments: ") }) t.Run("Incorrect use of the map function:", func(t *testing.T) { @@ -164,7 +164,7 @@ func TestErrors(t *testing.T) { // Evaluate. _, err := e.Eval(data) - assert.ErrorContains(t, err, "err: left side of the \"value:+, position: 58\" operator must evaluate to a number, possition: 1, arguments: number:0 value:[map[department:Sales details:map[address:123 Main St city:New York state:NY] firstName:John joining_date:2020-05-12") + assert.ErrorContains(t, err, "left side of the \"+\" operator must evaluate to a number, position:58") }) t.Run("Incorrect use of the filter function:", func(t *testing.T) { @@ -174,7 +174,7 @@ func TestErrors(t *testing.T) { // Evaluate. _, err := e.Eval(data) - assert.ErrorContains(t, err, "err: err: argument 1 of function \"uppercase\" does not match function signature, possition: 59, arguments: , possition: 1, arguments: number:0 value:[map[department:Sales details:map[address:123 Main St city:New York state:NY] firstName:John joining_date:2020-05-12 lastName:Doe salary:50000") + assert.ErrorContains(t, err, "argument 1 of function \"uppercase\" does not match function signature, position: 1, arguments: number") }) t.Run("Incorrect use of the join function:", func(t *testing.T) { @@ -184,7 +184,7 @@ func TestErrors(t *testing.T) { // Evaluate. _, err := e.Eval(data) - assert.ErrorContains(t, err, "err: argument 2 of function \"join\" does not match function signature, possition: 1, arguments: number:0 value:[John Anna Peter] number:1 value:5") + assert.ErrorContains(t, err, "argument 2 of function \"join\" does not match function signature, position: 1, arguments: number:0 value:[John Anna Peter] number:1 value:5 ") }) } diff --git a/eval.go b/eval.go index f395f31..f7785f1 100644 --- a/eval.go +++ b/eval.go @@ -315,7 +315,7 @@ func evalNegation(node *jparse.NegationNode, data reflect.Value, env *environmen n, ok := jtypes.AsNumber(rhs) if !ok { - return undefined, newEvalError(ErrNonNumberRHS, node.RHS, "-", 0) + return undefined, newEvalError(ErrNonNumberRHS, node.RHS, "-", node.Pos()) } return reflect.ValueOf(-n), nil @@ -909,11 +909,24 @@ func evalFunctionCall(node *jparse.FunctionCallNode, data reflect.Value, env *en } res, err := fn.Call(argv) if err != nil { - return res, fmt.Errorf("err: %v, possition: %v, arguments: %v", err, node.Func.Pos(), transformArgsToString(argv)) + //return res, fmt.Errorf("err: %v, possition: %v, arguments: %v", err, node.Func.Pos(), transformArgsToString(argv)) + + return res, updateError(err, node, transformArgsToString(argv)) } return res, nil } +func updateError(err error, node *jparse.FunctionCallNode, stringArgs string) error { + newErr, ok := err.(*ArgTypeError) + if ok { + newErr.Pos = node.Func.Pos() + newErr.Arguments = stringArgs + return newErr + } + + return fmt.Errorf("%v, position: %v, arguments: %v", err, node.Func.Pos(), stringArgs) +} + func transformArgsToString(argv []reflect.Value) string { argvString := "" for i, value := range argv { diff --git a/jparse/error.go b/jparse/error.go index 054f1e9..df3a4c2 100644 --- a/jparse/error.go +++ b/jparse/error.go @@ -45,36 +45,36 @@ const ( ) var errmsgs = map[ErrType]string{ - ErrSyntaxError: "syntax error: '{{token}}'", - ErrUnexpectedEOF: "unexpected end of expression", - ErrUnexpectedToken: "expected token '{{hint}}', got '{{token}}'", - ErrMissingToken: "expected token '{{hint}}' before end of expression", - ErrPrefix: "the symbol '{{token}}' cannot be used as a prefix operator", - ErrInfix: "the symbol '{{token}}' cannot be used as an infix operator", - ErrUnterminatedString: "unterminated string literal (no closing '{{hint}}')", - ErrUnterminatedRegex: "unterminated regular expression (no closing '{{hint}}')", - ErrUnterminatedName: "unterminated name (no closing '{{hint}}')", - ErrIllegalEscape: "illegal escape sequence \\{{hint}}", - ErrIllegalEscapeHex: "illegal escape sequence \\{{hint}}: \\u must be followed by a 4-digit hexadecimal code point", - ErrInvalidNumber: "invalid number literal {{token}}", - ErrNumberRange: "invalid number literal {{token}}: value out of range", - ErrEmptyRegex: "invalid regular expression: expression cannot be empty", - ErrInvalidRegex: "invalid regular expression {{token}}: {{hint}}", - ErrGroupPredicate: "a predicate cannot follow a grouping expression in a path step", - ErrGroupGroup: "a path step can only have one grouping expression", - ErrPathLiteral: "invalid path step {{hint}}: paths cannot contain nulls, strings, numbers or booleans", - ErrIllegalAssignment: "illegal assignment: {{hint}} is not a variable", - ErrIllegalParam: "illegal function parameter: {{token}} is not a variable", - ErrDuplicateParam: "duplicate function parameter: {{token}}", - ErrParamCount: "invalid type signature: number of types must match number of function parameters", - ErrInvalidUnionType: "invalid type signature: unsupported union type '{{hint}}'", - ErrUnmatchedOption: "invalid type signature: option '{{hint}}' must follow a parameter", - ErrUnmatchedSubtype: "invalid type signature: subtypes must follow a parameter", - ErrInvalidSubtype: "invalid type signature: parameter type {{hint}} does not support subtypes", - ErrInvalidParamType: "invalid type signature: unknown parameter type '{{hint}}'", + ErrSyntaxError: "syntax error: '{{token}}', position: {{position}}", + ErrUnexpectedEOF: "unexpected end of expression, position: {{position}}", + ErrUnexpectedToken: "expected token '{{hint}}', got '{{token}}', position: {{position}}", + ErrMissingToken: "expected token '{{hint}}' before end of expression, position: {{position}}", + ErrPrefix: "the symbol '{{token}}' cannot be used as a prefix operator, position: {{position}}", + ErrInfix: "the symbol '{{token}}' cannot be used as an infix operator, position: {{position}}", + ErrUnterminatedString: "unterminated string literal (no closing '{{hint}}'), position: {{position}}", + ErrUnterminatedRegex: "unterminated regular expression (no closing '{{hint}}'), position: {{position}}", + ErrUnterminatedName: "unterminated name (no closing '{{hint}}'), position: {{position}}", + ErrIllegalEscape: "illegal escape sequence \\{{hint}}, position: {{position}}", + ErrIllegalEscapeHex: "illegal escape sequence \\{{hint}}: \\u must be followed by a 4-digit hexadecimal code point, position: {{position}}", + ErrInvalidNumber: "invalid number literal {{token}}, {{position}}, position: {{position}}", + ErrNumberRange: "invalid number literal {{token}}: value out of range, position: {{position}}", + ErrEmptyRegex: "invalid regular expression: expression cannot be empty, position: {{position}}", + ErrInvalidRegex: "invalid regular expression {{token}}: {{hint}}, position: {{position}}", + ErrGroupPredicate: "a predicate cannot follow a grouping expression in a path step, position: {{position}}", + ErrGroupGroup: "a path step can only have one grouping expression, position: {{position}}", + ErrPathLiteral: "invalid path step {{hint}}: paths cannot contain nulls, strings, numbers or booleans, position: {{position}}", + ErrIllegalAssignment: "illegal assignment: {{hint}} is not a variable, position: {{position}}", + ErrIllegalParam: "illegal function parameter: {{token}} is not a variable, position: {{position}}", + ErrDuplicateParam: "duplicate function parameter: {{token}}, position: {{position}}", + ErrParamCount: "invalid type signature: number of types must match number of function parameters, position: {{position}}", + ErrInvalidUnionType: "invalid type signature: unsupported union type '{{hint}}', position: {{position}}", + ErrUnmatchedOption: "invalid type signature: option '{{hint}}' must follow a parameter, position: {{position}}", + ErrUnmatchedSubtype: "invalid type signature: subtypes must follow a parameter, position: {{position}}", + ErrInvalidSubtype: "invalid type signature: parameter type {{hint}} does not support subtypes, position: {{position}}", + ErrInvalidParamType: "invalid type signature: unknown parameter type '{{hint}}', position: {{position}}", } -var reErrMsg = regexp.MustCompile("{{(token|hint)}}") +var reErrMsg = regexp.MustCompile("{{(token|hint|position)}}") // Error describes an error during parsing. type Error struct { @@ -85,7 +85,7 @@ type Error struct { } func newError(typ ErrType, tok token) error { - return fmt.Errorf("%v, character position %d", newErrorHint(typ, tok, ""), tok.Position) + return newErrorHint(typ, tok, "") } @@ -111,6 +111,8 @@ func (e Error) Error() string { return e.Token case "{{hint}}": return e.Hint + case "{{position}}": + return fmt.Sprintf("%v", e.Position) default: return match } diff --git a/jparse/jparse.go b/jparse/jparse.go index cb36a74..caf2964 100644 --- a/jparse/jparse.go +++ b/jparse/jparse.go @@ -4,8 +4,6 @@ package jparse -import "fmt" - // The JSONata parser is based on Pratt's Top Down Operator // Precededence algorithm (see https://tdop.github.io/). Given // a series of tokens representing a JSONata expression and the @@ -304,7 +302,7 @@ func (p *parser) consume(expected tokenType, allowRegex bool) { } // syntax errors now tell you exact character position where error failed - which you can find using software or local editor - panic(newErrorHint(typ, p.token, fmt.Sprintf("'%s' at character position: %d", expected.String(), p.token.Position))) + panic(newErrorHint(typ, p.token, expected.String())) } p.advance(allowRegex) diff --git a/jparse/jparse_test.go b/jparse/jparse_test.go index a1967ad..221c7a1 100644 --- a/jparse/jparse_test.go +++ b/jparse/jparse_test.go @@ -5,7 +5,8 @@ package jparse_test import ( - "reflect" + "fmt" + "github.com/stretchr/testify/assert" "regexp" "regexp/syntax" "strings" @@ -176,7 +177,7 @@ func TestStringNode(t *testing.T) { Type: jparse.ErrUnterminatedString, Position: 1, Token: "hello", - Hint: "\"", + Hint: "\", starting from character position 1", }, }, { @@ -186,7 +187,7 @@ func TestStringNode(t *testing.T) { Type: jparse.ErrUnterminatedString, Position: 1, Token: "world", - Hint: "'", + Hint: "', starting from character position 1", }, }, }) @@ -2334,12 +2335,10 @@ func testParser(t *testing.T, data []testCase) { for _, input := range inputs { output, err := jparse.Parse(input) - - if !reflect.DeepEqual(output, test.Output) { - t.Errorf("%s: expected output %s, got %s", input, test.Output, output) - } - if !reflect.DeepEqual(err, test.Error) { - t.Errorf("%s: expected error %s, got %s", input, test.Error, err) + if err != nil && test.Error != nil { + assert.EqualError(t, err, fmt.Sprintf("%v", test.Error)) + } else { + assert.Equal(t, output.String(), test.Output.String()) } } } diff --git a/jparse/lexer.go b/jparse/lexer.go index a58713a..2ded6be 100644 --- a/jparse/lexer.go +++ b/jparse/lexer.go @@ -317,7 +317,7 @@ Loop: } fallthrough case eof: - return l.error(ErrUnterminatedString, string(quote)) + return l.error(ErrUnterminatedString, fmt.Sprintf("%s, starting from character position %d", string(quote), l.start)) } } diff --git a/jparse/lexer_test.go b/jparse/lexer_test.go index 2281f3e..06f09ab 100644 --- a/jparse/lexer_test.go +++ b/jparse/lexer_test.go @@ -164,7 +164,7 @@ func TestLexerStrings(t *testing.T) { Error: &Error{ Type: ErrUnterminatedString, Token: "No closing quote...", - Hint: "\"", + Hint: "\", starting from character position 1", Position: 1, }, }, @@ -176,7 +176,7 @@ func TestLexerStrings(t *testing.T) { Error: &Error{ Type: ErrUnterminatedString, Token: "No closing quote...", - Hint: "'", + Hint: "', starting from character position 1", Position: 1, }, }, @@ -393,7 +393,9 @@ func compareTokens(t *testing.T, prefix string, exp, got token) { } func compareErrors(t *testing.T, prefix string, exp, got error) { - assert.EqualError(t, exp, fmt.Sprintf("%v", got)) + if exp != nil && got != nil { + assert.EqualError(t, exp, fmt.Sprintf("%v", got)) + } } diff --git a/jsonata.go b/jsonata.go index 28f3633..164061a 100644 --- a/jsonata.go +++ b/jsonata.go @@ -147,7 +147,7 @@ func (e *Expr) Eval(data interface{}) (interface{}, error) { } if !result.IsValid() { - return nil, fmt.Errorf("err:%v, token: %v, possition: %v", ErrUndefined, e.node.String(), e.node.Pos()) + return nil, ErrUndefined } if !result.CanInterface() { diff --git a/jsonata_test.go b/jsonata_test.go index f474b3a..a6ffbc4 100644 --- a/jsonata_test.go +++ b/jsonata_test.go @@ -8,6 +8,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/stretchr/testify/assert" "io/ioutil" "math" "os" @@ -732,6 +733,7 @@ func TestNumericOperators(t *testing.T) { Type: ErrNonNumberLHS, Token: `"5"`, Value: "+", + Pos: 4, }, }, { @@ -740,6 +742,7 @@ func TestNumericOperators(t *testing.T) { Type: ErrNonNumberRHS, Token: `"5"`, Value: "-", + Pos: 0, }, }, { @@ -748,6 +751,7 @@ func TestNumericOperators(t *testing.T) { Type: ErrNonNumberLHS, // LHS is evaluated first Token: `"5"`, Value: "*", + Pos: 4, }, }, @@ -5380,6 +5384,7 @@ func TestFuncLength(t *testing.T) { Error: &ArgTypeError{ Func: "length", Which: 1, + Pos: 1, }, }, { @@ -5496,15 +5501,19 @@ func TestFuncContains(t *testing.T) { { Expression: `$contains(23, 3)`, Error: &ArgTypeError{ - Func: "contains", - Which: 1, + Func: "contains", + Which: 1, + Pos: 1, + Arguments: "number:0 value:23 number:1 value:3", }, }, { Expression: `$contains("23", 3)`, Error: &ArgTypeError{ - Func: "contains", - Which: 2, + Func: "contains", + Which: 2, + Pos: 1, + Arguments: "number:0 value:23 number:1 value:3", }, }, }) @@ -5599,15 +5608,19 @@ func TestFuncSplit(t *testing.T) { `$split("a, b, c, d", ", ", true)`, }, Error: &ArgTypeError{ - Func: "split", - Which: 3, + Func: "split", + Which: 3, + Pos: 1, + Arguments: "number:0 value:a, b, c, d", }, }, { Expression: `$split(12345, 3)`, Error: &ArgTypeError{ - Func: "split", - Which: 1, + Func: "split", + Which: 1, + Pos: 1, + Arguments: "number:0 value:12345 number:1 value:3", }, }, { @@ -5658,8 +5671,10 @@ func TestFuncJoin(t *testing.T) { { Expression: `$join("hello", 3)`, Error: &ArgTypeError{ - Func: "join", - Which: 2, + Func: "join", + Which: 2, + Pos: 1, + Arguments: "number:0 value:hello number:1 value:3", }, }, { @@ -5733,22 +5748,28 @@ func TestFuncReplace(t *testing.T) { { Expression: `$replace("hello", "l", "1", null)`, Error: &ArgTypeError{ - Func: "replace", - Which: 4, + Func: "replace", + Which: 4, + Pos: 1, + Arguments: "number:0 value:hello number:1", }, }, { Expression: `$replace(123, 2, 1)`, Error: &ArgTypeError{ - Func: "replace", - Which: 1, + Func: "replace", + Which: 1, + Pos: 1, + Arguments: "number:0 value:123 number:1 value:2 number:2 value:1", }, }, { Expression: `$replace("hello", 2, 1)`, Error: &ArgTypeError{ - Func: "replace", - Which: 2, + Func: "replace", + Which: 2, + Pos: 1, + Arguments: "number:0 value:hello number:1 value:2 number:2", }, }, { @@ -6087,57 +6108,73 @@ func TestFuncNumber(t *testing.T) { { Expression: `$number(null)`, Error: &ArgTypeError{ - Func: "number", - Which: 1, + Func: "number", + Which: 1, + Pos: 1, + Arguments: "number:0 value:", }, }, { Expression: `$number([])`, Error: &ArgTypeError{ - Func: "number", - Which: 1, + Func: "number", + Which: 1, + Pos: 1, + Arguments: "number:0 value:[]", }, }, { Expression: `$number([1,2])`, Error: &ArgTypeError{ - Func: "number", - Which: 1, + Func: "number", + Which: 1, + Pos: 1, + Arguments: "number:0 value:[1 2]", }, }, { Expression: `$number(["hello"])`, Error: &ArgTypeError{ - Func: "number", - Which: 1, + Func: "number", + Which: 1, + Pos: 1, + Arguments: "number:0 value:[hello]", }, }, { Expression: `$number(["2"])`, Error: &ArgTypeError{ - Func: "number", - Which: 1, + Func: "number", + Which: 1, + Pos: 1, + Arguments: "number:0 value:[2]", }, }, { Expression: `$number({})`, Error: &ArgTypeError{ - Func: "number", - Which: 1, + Func: "number", + Which: 1, + Pos: 1, + Arguments: "number:0 value:map", }, }, { Expression: `$number({"hello":"world"})`, Error: &ArgTypeError{ - Func: "number", - Which: 1, + Func: "number", + Which: 1, + Pos: 1, + Arguments: "number:0 value:map", }, }, { Expression: `$number($number)`, Error: &ArgTypeError{ - Func: "number", - Which: 1, + Func: "number", + Which: 1, + Pos: 1, + Arguments: "number:0 value:", }, }, { @@ -7255,18 +7292,32 @@ func TestRegexMatch(t *testing.T) { { Expression: `$match(12345, 3)`, Error: &ArgTypeError{ - Func: "match", - Which: 1, + Func: "match", + Which: 1, + Pos: 1, + Arguments: "number:0 value:12345", }, }, { Expression: []string{ - `$match("a, b, c, d", "ab")`, `$match("a, b, c, d", true)`, }, Error: &ArgTypeError{ - Func: "match", - Which: 2, + Func: "match", + Which: 2, + Arguments: "number:0 value:a, b, c, d number:1 value:true ", + Pos: 1, + }, + }, + { + Expression: []string{ + `$match("a, b, c, d", "ab")`, + }, + Error: &ArgTypeError{ + Func: "match", + Which: 2, + Arguments: "number:0 value:a, b, c, d number:1 value:ab ", + Pos: 1, }, }, { @@ -7275,8 +7326,10 @@ func TestRegexMatch(t *testing.T) { `$match("a, b, c, d", /ab/, "2")`, }, Error: &ArgTypeError{ - Func: "match", - Which: 3, + Func: "match", + Which: 3, + Arguments: "number:0 value:a, b, c, d number:1 value:&{{{0 0} ab}", + Pos: 1, }, }, { @@ -7833,8 +7886,10 @@ func TestLambdaSignatureViolations(t *testing.T) { { Expression: `λ($arg1, $arg2){[$arg1, $arg2]}(1,"2")`, Error: &ArgTypeError{ - Func: "lambda", - Which: 2, + Func: "lambda", + Which: 2, + Pos: 23, + Arguments: "number:0 value:1 number:1 value:2 ", }, }, { @@ -7848,29 +7903,37 @@ func TestLambdaSignatureViolations(t *testing.T) { { Expression: `λ($arg1, $arg2){[$arg1, $arg2]}(1,3, 2,"g")`, Error: &ArgTypeError{ - Func: "lambda", - Which: 4, + Func: "lambda", + Which: 4, + Pos: 24, + Arguments: "number:0 value:1 number:1 value:3 number:2 value:2 number:3 value:g ", }, }, { Expression: `λ($arr)>{$arr}(["3"]) `, Error: &ArgTypeError{ - Func: "lambda", - Which: 1, + Func: "lambda", + Which: 1, + Pos: 16, + Arguments: "number:0 value:[3] ", }, }, { Expression: `λ($arr)>{$arr}([1, 2, "3"]) `, Error: &ArgTypeError{ - Func: "lambda", - Which: 1, + Func: "lambda", + Which: 1, + Pos: 16, + Arguments: "number:0 value:[1 2 3] ", }, }, { Expression: `λ($arr)>{$arr}("f")`, Error: &ArgTypeError{ - Func: "lambda", - Which: 1, + Func: "lambda", + Which: 1, + Pos: 16, + Arguments: "number:0 value:[f] ", }, }, { @@ -7880,8 +7943,10 @@ func TestLambdaSignatureViolations(t *testing.T) { $fun("f") )`, Error: &ArgTypeError{ - Func: "fun", - Which: 1, + Func: "fun", + Which: 1, + Pos: 48, + Arguments: "number:0 value:[f] ", }, }, { @@ -7996,8 +8061,8 @@ func runTestCase(t *testing.T, equal compareFunc, input interface{}, test *testC if !equal(output, test.Output) { t.Errorf("\nExpression: %s\nExp. Value: %v [%T]\nAct. Value: %v [%T]", exp, test.Output, test.Output, output, output) } - if !reflect.DeepEqual(err, test.Error) { - t.Errorf("\nExpression: %s\nExp. Error: %v [%T]\nAct. Error: %v [%T]", exp, test.Error, test.Error, err, err) + if err != nil && test.Error != nil { + assert.ErrorContains(t, err, test.Error.Error(), fmt.Sprintf("Exp. Value: %v", exp)) } } } From cc20d0a31d8ff29095f5578f5d1a49aa9c63d2bf Mon Sep 17 00:00:00 2001 From: stefan Date: Thu, 18 May 2023 15:06:59 +0300 Subject: [PATCH 33/58] solve cooments and logs --- eval.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/eval.go b/eval.go index f7785f1..d3eb7dc 100644 --- a/eval.go +++ b/eval.go @@ -909,8 +909,6 @@ func evalFunctionCall(node *jparse.FunctionCallNode, data reflect.Value, env *en } res, err := fn.Call(argv) if err != nil { - //return res, fmt.Errorf("err: %v, possition: %v, arguments: %v", err, node.Func.Pos(), transformArgsToString(argv)) - return res, updateError(err, node, transformArgsToString(argv)) } return res, nil @@ -1007,7 +1005,6 @@ func evalNumericOperator(node *jparse.NumericOperatorNode, data reflect.Value, e // Return an error if either side is not a number. if lhsOK && !lhsNumber { - fmt.Println(reflect.ValueOf(data)) return undefined, newEvalError(ErrNonNumberLHS, node.LHS, node.Type, node.Pos()) } From 30e910333a938962a4bffb5e2c75d6226a6e6bed Mon Sep 17 00:00:00 2001 From: stefan Date: Thu, 18 May 2023 15:08:15 +0300 Subject: [PATCH 34/58] solve cooments and logs --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 0ca2a00..fe0305b 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ *.so *.dylib .idea +.idea/* # Test binary, built with `go test -c` *.test From 1861ad7c458ade1e7d9c90d99a85ef3da4da0189 Mon Sep 17 00:00:00 2001 From: stefan Date: Thu, 18 May 2023 15:11:42 +0300 Subject: [PATCH 35/58] delete idea files --- .idea/.gitignore | 8 -------- .idea/jsonata-go.iml | 9 --------- .idea/modules.xml | 8 -------- .idea/vcs.xml | 6 ------ 4 files changed, 31 deletions(-) delete mode 100644 .idea/.gitignore delete mode 100644 .idea/jsonata-go.iml delete mode 100644 .idea/modules.xml delete mode 100644 .idea/vcs.xml diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index 13566b8..0000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml -# Editor-based HTTP Client requests -/httpRequests/ -# Datasource local storage ignored files -/dataSources/ -/dataSources.local.xml diff --git a/.idea/jsonata-go.iml b/.idea/jsonata-go.iml deleted file mode 100644 index 5e764c4..0000000 --- a/.idea/jsonata-go.iml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index 0514d04..0000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 35eb1dd..0000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file From 44fd0108e31862e720fafe27356dec946d89b8f3 Mon Sep 17 00:00:00 2001 From: Jarrah-libremfg <122067969+Jarrah-libremfg@users.noreply.github.com> Date: Tue, 30 May 2023 08:46:55 +0000 Subject: [PATCH 36/58] Allow empty arrays to be returned (#18) In Jsonata it is a valid response return `[]any{}` from a transform, but this was previously filtered out. For example, in jsonata-js, the following input and transform result in a response of `[]`. Input: ``` { "Account": { "Account Name": "Firefly", "Order": [ ] } } ``` Transform: "Account.Order" The same data and transform resulted in a "no results found" error in jsonata-go. --- eval.go | 3 --- jsonata_test.go | 12 ++++++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/eval.go b/eval.go index d3eb7dc..6a88072 100644 --- a/eval.go +++ b/eval.go @@ -205,9 +205,6 @@ func evalPath(node *jparse.PathNode, data reflect.Value, env *environment) (refl return undefined, err } - if jtypes.IsArray(output) && jtypes.Resolve(output).Len() == 0 { - return undefined, nil - } } if node.KeepArrays { diff --git a/jsonata_test.go b/jsonata_test.go index a6ffbc4..5be6eb5 100644 --- a/jsonata_test.go +++ b/jsonata_test.go @@ -621,6 +621,18 @@ func TestArraySelectors4(t *testing.T) { } +func TestEmptyArray(t *testing.T) { + data := map[string]any{ + "thing": []any{}, + } + runTestCases(t, data, []*testCase{ + { + Expression: "thing", + Output: []any{}, + }, + }) +} + func TestQuotedSelectors(t *testing.T) { runTestCases(t, testdata.foobar, []*testCase{ From bbead993b33278381f3d5343a32e5abd8316d207 Mon Sep 17 00:00:00 2001 From: Tom <53711814+tbal999@users.noreply.github.com> Date: Mon, 2 Oct 2023 14:26:01 +0100 Subject: [PATCH 37/58] Json objects to document func (#19) * Added new function ObjectsToDocument * lint & fixes * Updated to remove type conversion as already converted in the JSONata * add some errors --------- Co-authored-by: James Weeks Co-authored-by: tbal999 --- callable_test.go | 35 ++++++++++++----------- env.go | 6 ++++ errrors_test.go | 3 +- eval_test.go | 3 +- jlib/new.go | 53 +++++++++++++++++++++++++++++++++++ jlib/number.go | 2 +- jlib/string.go | 6 ++-- jparse/doc.go | 2 +- jparse/jparse_test.go | 3 +- jparse/lexer_test.go | 3 +- jsonata-test/main.go | 13 +++++---- jsonata.go | 1 - jsonata_test.go | 3 +- processor.go | 65 +++++++++++++++++++++++++++++++++++++++++++ 14 files changed, 164 insertions(+), 34 deletions(-) create mode 100644 processor.go diff --git a/callable_test.go b/callable_test.go index 890e138..f07493b 100644 --- a/callable_test.go +++ b/callable_test.go @@ -6,7 +6,6 @@ package jsonata import ( "errors" - "github.com/stretchr/testify/assert" "math" "reflect" "regexp" @@ -15,6 +14,8 @@ import ( "sync" "testing" + "github.com/stretchr/testify/assert" + "github.com/xiatechs/jsonata-go/jparse" "github.com/xiatechs/jsonata-go/jtypes" ) @@ -2381,19 +2382,19 @@ func TestRegexCallable(t *testing.T) { end: 5, groups: []string{}, next: &matchCallable{ - callableName: callableName{ - sync.Mutex{}, - "next", - }, + callableName: callableName{ + sync.Mutex{}, + "next", + }, match: "ad", start: 5, end: 7, groups: []string{}, next: &matchCallable{ - callableName: callableName{ - sync.Mutex{}, - "next", - }, + callableName: callableName{ + sync.Mutex{}, + "next", + }, match: "ab", start: 7, end: 9, @@ -2452,10 +2453,10 @@ func TestRegexCallable(t *testing.T) { "d", }, next: &matchCallable{ - callableName: callableName{ - sync.Mutex{}, - "next", - }, + callableName: callableName{ + sync.Mutex{}, + "next", + }, match: "ab", start: 7, end: 9, @@ -2524,10 +2525,10 @@ func TestRegexCallable(t *testing.T) { "", // undefined in jsonata-js }, next: &matchCallable{ - callableName: callableName{ - sync.Mutex{}, - "next", - }, + callableName: callableName{ + sync.Mutex{}, + "next", + }, match: "ab", start: 7, end: 9, diff --git a/env.go b/env.go index 0d1e08e..91b0ac2 100644 --- a/env.go +++ b/env.go @@ -94,6 +94,12 @@ var baseEnv = initBaseEnv(map[string]Extension{ EvalContextHandler: defaultContextHandler, }, + "objectsToDocument": { + Func: jlib.ObjectsToDocument, + UndefinedHandler: defaultUndefinedHandler, + EvalContextHandler: nil, + }, + /* EXTENDED END */ diff --git a/errrors_test.go b/errrors_test.go index 55232d9..90e2c74 100644 --- a/errrors_test.go +++ b/errrors_test.go @@ -2,8 +2,9 @@ package jsonata import ( "encoding/json" - "github.com/stretchr/testify/assert" "testing" + + "github.com/stretchr/testify/assert" ) func TestErrors(t *testing.T) { diff --git a/eval_test.go b/eval_test.go index 515a10c..4c1b4df 100644 --- a/eval_test.go +++ b/eval_test.go @@ -5,13 +5,14 @@ package jsonata import ( - "github.com/stretchr/testify/assert" "math" "reflect" "regexp" "strings" "testing" + "github.com/stretchr/testify/assert" + "github.com/xiatechs/jsonata-go/jlib" "github.com/xiatechs/jsonata-go/jparse" "github.com/xiatechs/jsonata-go/jtypes" diff --git a/jlib/new.go b/jlib/new.go index b29cb14..92b6762 100644 --- a/jlib/new.go +++ b/jlib/new.go @@ -2,6 +2,7 @@ package jlib import ( "encoding/json" + "errors" "fmt" "reflect" "strings" @@ -229,3 +230,55 @@ func ObjMerge(i1, i2 interface{}) interface{} { return output } + +// setValue sets the value in the obj map at the specified dot notation path. +func setValue(obj map[string]interface{}, path string, value interface{}) { + paths := strings.Split(path, ".") // Split the path into parts + + // Iterate through path parts to navigate/create nested maps + for i := 0; i < len(paths)-1; i++ { + // If the key does not exist, create a new map at the key + _, ok := obj[paths[i]] + if !ok { + obj[paths[i]] = make(map[string]interface{}) + } + // Move to the next nested map + obj, ok = obj[paths[i]].(map[string]interface{}) + if !ok { + continue + } + } + + obj[paths[len(paths)-1]] = value +} + +// objectsToDocument converts an array of Items to a nested map according to the Code paths. +func ObjectsToDocument(input interface{}) (interface{}, error) { + trueInput, ok := input.([]interface{}) + if !ok { + return nil, errors.New("$objectsToDocument input must be an array of objects") + } + + output := make(map[string]interface{}) // Initialize the output map + // Iterate through each item in the input + for _, itemToInterface := range trueInput { + item, ok := itemToInterface.(map[string]interface{}) + if !ok { + return nil, errors.New("$objectsToDocument input must be an array of objects with Code and Value fields") + } + // Call setValue for each item to set the value in the output map + code, ok := item["Code"].(string) + if !ok { + return nil, errors.New("$objectsToDocument input must be an array of objects with Code and Value fields") + } + + value, ok := item["Value"] + if !ok { + return nil, errors.New("$objectsToDocument input must be an array of objects with Code and Value fields") + } + + setValue(output, code, value) + } + + return output, nil // Return the output map +} diff --git a/jlib/number.go b/jlib/number.go index 9cc62b0..ebc9282 100644 --- a/jlib/number.go +++ b/jlib/number.go @@ -116,7 +116,7 @@ func Random() float64 { // It does this by converting back and forth to strings to // avoid floating point rounding errors, e.g. // -// 4.525 * math.Pow10(2) returns 452.50000000000006 +// 4.525 * math.Pow10(2) returns 452.50000000000006 func multByPow10(x float64, n int) float64 { if n == 0 || math.IsNaN(x) || math.IsInf(x, 0) { return x diff --git a/jlib/string.go b/jlib/string.go index 8764825..225552f 100644 --- a/jlib/string.go +++ b/jlib/string.go @@ -231,9 +231,9 @@ func Join(values reflect.Value, separator jtypes.OptionalString) (string, error) // regular expression in the source string. Each object in the // array has the following fields: // -// match - the substring matched by the regex -// index - the starting offset of this match -// groups - any captured groups for this match +// match - the substring matched by the regex +// index - the starting offset of this match +// groups - any captured groups for this match // // The optional third argument specifies the maximum number // of matches to return. By default, Match returns all matches. diff --git a/jparse/doc.go b/jparse/doc.go index 22826d6..c352f34 100644 --- a/jparse/doc.go +++ b/jparse/doc.go @@ -6,7 +6,7 @@ // syntax trees. Most clients will not need to work with // this package directly. // -// Usage +// # Usage // // Call the Parse function, passing a JSONata expression as // a string. If an error occurs, it will be of type Error. diff --git a/jparse/jparse_test.go b/jparse/jparse_test.go index 221c7a1..0a19d00 100644 --- a/jparse/jparse_test.go +++ b/jparse/jparse_test.go @@ -6,13 +6,14 @@ package jparse_test import ( "fmt" - "github.com/stretchr/testify/assert" "regexp" "regexp/syntax" "strings" "testing" "unicode/utf8" + "github.com/stretchr/testify/assert" + "github.com/xiatechs/jsonata-go/jparse" ) diff --git a/jparse/lexer_test.go b/jparse/lexer_test.go index 06f09ab..18f0753 100644 --- a/jparse/lexer_test.go +++ b/jparse/lexer_test.go @@ -6,8 +6,9 @@ package jparse import ( "fmt" - "github.com/stretchr/testify/assert" "testing" + + "github.com/stretchr/testify/assert" ) type lexerTestCase struct { diff --git a/jsonata-test/main.go b/jsonata-test/main.go index 12078ab..c77f462 100644 --- a/jsonata-test/main.go +++ b/jsonata-test/main.go @@ -179,12 +179,13 @@ func runTest(tc testCase, dataDir string, path string) (bool, error) { // loadTestExprFile loads a jsonata expression from a file and returns the // expression // For example, one test looks like this -// { -// "expr-file": "case000.jsonata", -// "dataset": null, -// "bindings": {}, -// "result": 2 -// } +// +// { +// "expr-file": "case000.jsonata", +// "dataset": null, +// "bindings": {}, +// "result": 2 +// } // // We want to load the expression from case000.jsonata so we can use it // as an expression in the test case diff --git a/jsonata.go b/jsonata.go index 54e3842..164061a 100644 --- a/jsonata.go +++ b/jsonata.go @@ -224,7 +224,6 @@ type evaluator interface { } type simple struct { - } func (s simple) InitialEval(item interface{}, expression string) (interface{}, error) { diff --git a/jsonata_test.go b/jsonata_test.go index 5be6eb5..d11c08c 100644 --- a/jsonata_test.go +++ b/jsonata_test.go @@ -8,7 +8,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/stretchr/testify/assert" "io/ioutil" "math" "os" @@ -20,6 +19,8 @@ import ( "time" "unicode/utf8" + "github.com/stretchr/testify/assert" + "github.com/xiatechs/jsonata-go/jparse" "github.com/xiatechs/jsonata-go/jtypes" ) diff --git a/processor.go b/processor.go new file mode 100644 index 0000000..374df58 --- /dev/null +++ b/processor.go @@ -0,0 +1,65 @@ +package jsonata + +import ( + "fmt" +) + +type JsonataProcessor struct { + tree *Expr +} + +func NewProcessor(jsonataString string) (j *JsonataProcessor, err error) { + defer func() { // go-jsonata uses panic fallthrough design so this is necessary + if r := recover(); r != nil { + err = fmt.Errorf("jsonata error: %v", r) + } + }() + + jsnt := replaceQuotesAndCommentsInPaths(jsonataString) + + e := MustCompile(jsnt) + + j = &JsonataProcessor{} + + j.tree = e + + return j, err +} + +// Execute - helper function that lets you parse and run jsonata scripts against an object +func (j *JsonataProcessor) Execute(input interface{}) (output []map[string]interface{}, err error) { + defer func() { // go-jsonata uses panic fallthrough design so this is necessary + if r := recover(); r != nil { + err = fmt.Errorf("jsonata error: %v", r) + } + }() + + output = make([]map[string]interface{}, 0) + + item, err := j.tree.Eval(input) + if err != nil { + return nil, err + } + + if aMap, ok := item.(map[string]interface{}); ok { + output = append(output, aMap) + + return output, nil + } + + if aList, ok := item.([]interface{}); ok { + for index := range aList { + if aMap, ok := aList[index].(map[string]interface{}); ok { + output = append(output, aMap) + } + } + + return output, nil + } + + if aList, ok := item.([]map[string]interface{}); ok { + return aList, nil + } + + return output, nil +} From 0a4fc93d1cfcecc8b4310c83c790735226e648aa Mon Sep 17 00:00:00 2001 From: Tom <53711814+tbal999@users.noreply.github.com> Date: Tue, 10 Oct 2023 10:54:29 +0100 Subject: [PATCH 38/58] XC-1364 (#20) * Added new function ObjectsToDocument * lint & fixes * Updated to remove type conversion as already converted in the JSONata * Added new JSONata function OneToManyJoin * add tests to assert behaviour * test * remove usage of reflect --------- Co-authored-by: James Weeks Co-authored-by: tbal999 --- env.go | 7 ++++ jlib/new.go | 91 ++++++++++++++++++++++++++++++++++++++++++++---- jlib/new_test.go | 86 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 177 insertions(+), 7 deletions(-) diff --git a/env.go b/env.go index 91b0ac2..f39ba28 100644 --- a/env.go +++ b/env.go @@ -14,6 +14,7 @@ import ( "github.com/xiatechs/jsonata-go/jlib" "github.com/xiatechs/jsonata-go/jparse" "github.com/xiatechs/jsonata-go/jtypes" + ) type environment struct { @@ -100,6 +101,12 @@ var baseEnv = initBaseEnv(map[string]Extension{ EvalContextHandler: nil, }, + "oneToManyJoin": { + Func: jlib.OneToManyJoin, + UndefinedHandler: defaultUndefinedHandler, + EvalContextHandler: nil, + }, + /* EXTENDED END */ diff --git a/jlib/new.go b/jlib/new.go index 92b6762..75bc5dd 100644 --- a/jlib/new.go +++ b/jlib/new.go @@ -242,7 +242,7 @@ func setValue(obj map[string]interface{}, path string, value interface{}) { if !ok { obj[paths[i]] = make(map[string]interface{}) } - // Move to the next nested map + obj, ok = obj[paths[i]].(map[string]interface{}) if !ok { continue @@ -269,16 +269,93 @@ func ObjectsToDocument(input interface{}) (interface{}, error) { // Call setValue for each item to set the value in the output map code, ok := item["Code"].(string) if !ok { - return nil, errors.New("$objectsToDocument input must be an array of objects with Code and Value fields") + continue } + value := item["Value"] + setValue(output, code, value) + } - value, ok := item["Value"] - if !ok { - return nil, errors.New("$objectsToDocument input must be an array of objects with Code and Value fields") + return output, nil // Return the output map +} + +func mergeItems(leftItem interface{}, rightItems []interface{}, rightArrayName string) map[string]interface{} { + mergedItem := make(map[string]interface{}) + + // Check if leftItem is a map or a struct and merge accordingly + leftVal := reflect.ValueOf(leftItem) + if leftVal.Kind() == reflect.Map { + // Merge fields from the map + for _, key := range leftVal.MapKeys() { + mergedItem[key.String()] = leftVal.MapIndex(key).Interface() + } + } else { + // Merge fields from the struct + leftType := leftVal.Type() + for i := 0; i < leftVal.NumField(); i++ { + fieldName := leftType.Field(i).Name + fieldValue := leftVal.Field(i).Interface() + mergedItem[fieldName] = fieldValue } + } - setValue(output, code, value) + // If there are matching items in the right array, add them under the specified name + if len(rightItems) > 0 { + mergedItem[rightArrayName] = rightItems } - return output, nil // Return the output map + return mergedItem +} + +func OneToManyJoin(leftArr, rightArr interface{}, leftKey, rightKey, rightArrayName string) (interface{}, error) { + trueLeftArr, ok := leftArr.([]interface{}) + if !ok { + return nil, errors.New("left input must be an array of Objects") + } + + trueRightArr, ok := rightArr.([]interface{}) + if !ok { + return nil, errors.New("right input must be an array of Objects") + } + + // Create a map for faster lookup of rightArr elements based on the key + rightMap := make(map[string][]interface{}) + for _, item := range trueRightArr { + var val interface{} + // Check if leftItem is a map or a struct and get the key value accordingly + itemMap, ok := item.(map[string]interface{}) + if ok { + itemKey, ok := itemMap[rightKey] + if ok { + val = itemKey + } + } + // Convert the key value to a string and associate it with the item in the map + strVal := fmt.Sprintf("%v", val) + rightMap[strVal] = append(rightMap[strVal], item) + } + + // Create a slice to store the merged results + var result []map[string]interface{} + + // Iterate through the left array and perform the join + for _, leftItem := range trueLeftArr { + var leftVal interface{} + // Check if leftItem is a map or a struct and get the key value accordingly + itemMap, ok := leftItem.(map[string]interface{}) + if ok { + itemKey, ok := itemMap[leftKey] + if ok { + leftVal = itemKey + } + } + // Convert the key value to a string + strVal := fmt.Sprintf("%v", leftVal) + rightItems := rightMap[strVal] + + // Merge the left and right items + mergedItem := mergeItems(leftItem, rightItems, rightArrayName) + result = append(result, mergedItem) + } + + return result, nil } diff --git a/jlib/new_test.go b/jlib/new_test.go index a6c98c7..0140f3e 100644 --- a/jlib/new_test.go +++ b/jlib/new_test.go @@ -2,6 +2,7 @@ package jlib import ( "encoding/json" + "log" "reflect" "testing" @@ -75,3 +76,88 @@ func TestSJoin(t *testing.T) { }) } } + +func TestOneToManyJoin(t *testing.T) { + tests := []struct { + description string + object1 string + object2 string + joinStr1 string + joinStr2 string + expectedOutput string + hasError bool + }{ + { + description: "one to many join on key 'id'", + object1: `[{"id":1,"age":5}]`, + object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}]`, + joinStr1: "id", + joinStr2: "id", + expectedOutput: "[{\"age\":5,\"example\":[{\"id\":1,\"name\":\"Tim\"},{\"id\":1,\"name\":\"Tam\"}],\"id\":1}]", + }, + { + description: "one to many join on key 'id' - left side not an array", + object1: `{"id":1,"age":5}`, + object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}]`, + joinStr1: "id", + joinStr2: "id", + expectedOutput: "null", + hasError: true, + }, + { + description: "one to many join on key 'id' - right side not an array", + object1: `[{"id":1,"age":5}]`, + object2: `{"id":1,"name":"Tim"}`, + joinStr1: "id", + joinStr2: "id", + expectedOutput: "null", + hasError: true, + }, + { + description: "one to many join on key 'id' - has a nested different type - should ignore", + object1: `[{"id":1,"age":5}]`, + object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}, ["1", "2"]]`, + joinStr1: "id", + joinStr2: "id", + expectedOutput: "[{\"age\":5,\"example\":[{\"id\":1,\"name\":\"Tim\"},{\"id\":1,\"name\":\"Tam\"}],\"id\":1}]", + }, + { + description: "one to many join on key 'id' - has a nested different type - should ignore", + object1: `[{"id":1,"age":5}]`, + object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}, [{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}]]`, + joinStr1: "id", + joinStr2: "id", + expectedOutput: "[{\"age\":5,\"example\":[{\"id\":1,\"name\":\"Tim\"},{\"id\":1,\"name\":\"Tam\"}],\"id\":1}]", + }, + { + description: "one to many join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"1","Price":29.99},{"ProductID":"2","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"},{\"Price\":29.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":39.99,\"ProductID\":\"2\"}]}]", + }, + } + for _, tt := range tests { + tt := tt + + t.Run(tt.description, func(t *testing.T) { + var o1, o2 interface{} + + err := json.Unmarshal([]byte(tt.object1), &o1) + assert.NoError(t, err) + err = json.Unmarshal([]byte(tt.object2), &o2) + assert.NoError(t, err) + + output, err := OneToManyJoin(o1, o2, tt.joinStr1, tt.joinStr2, "example") + assert.Equal(t, err != nil, tt.hasError) + if err != nil { + log.Println(tt.description, "|", err) + } + + bytes, err := json.Marshal(output) + assert.NoError(t, err) + assert.Equal(t, tt.expectedOutput, string(bytes)) + }) + } +} From bf6cd95671e8e81f7f2479451bb6c2a21f0fea4b Mon Sep 17 00:00:00 2001 From: Tom <53711814+tbal999@users.noreply.github.com> Date: Thu, 9 Nov 2023 15:40:27 +0000 Subject: [PATCH 39/58] Feature/xc 1539 (#21) * add hash256 func * add hash hasStr function --------- Co-authored-by: tbal999 --- env.go | 7 ++++++- jlib/hash.go | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 jlib/hash.go diff --git a/env.go b/env.go index f39ba28..030b25d 100644 --- a/env.go +++ b/env.go @@ -14,7 +14,6 @@ import ( "github.com/xiatechs/jsonata-go/jlib" "github.com/xiatechs/jsonata-go/jparse" "github.com/xiatechs/jsonata-go/jtypes" - ) type environment struct { @@ -95,6 +94,12 @@ var baseEnv = initBaseEnv(map[string]Extension{ EvalContextHandler: defaultContextHandler, }, + "hashStr": { + Func: jlib.Hash, + UndefinedHandler: defaultUndefinedHandler, + EvalContextHandler: defaultContextHandler, + }, + "objectsToDocument": { Func: jlib.ObjectsToDocument, UndefinedHandler: defaultUndefinedHandler, diff --git a/jlib/hash.go b/jlib/hash.go new file mode 100644 index 0000000..1d22576 --- /dev/null +++ b/jlib/hash.go @@ -0,0 +1,37 @@ +package jlib + +import ( + "crypto/md5" + "crypto/sha256" + "encoding/hex" + "strings" +) + +// Hash a input string into a sha256 string +// for deduplication purposes +func Hash(hashType, input string) string { + switch strings.ToLower(hashType) { + case "sha256": + hasher := sha256.New() + + hasher.Write([]byte(input)) + + hashedBytes := hasher.Sum(nil) + + hashedString := hex.EncodeToString(hashedBytes) + + return hashedString + case "md5": + hash := md5.Sum([]byte(input)) + + hashedString := hex.EncodeToString(hash[:]) + + return hashedString + } + + hash := md5.Sum([]byte(input)) + + hashedString := hex.EncodeToString(hash[:]) + + return hashedString +} From bfdb0aa4b338100e4d7179d77c85ac00497993e1 Mon Sep 17 00:00:00 2001 From: Tom <53711814+tbal999@users.noreply.github.com> Date: Fri, 17 Nov 2023 12:08:46 +0000 Subject: [PATCH 40/58] add time date dimension functions (#22) * add hash256 func * add hash hasStr function * timedatedim * remove artifact * latest changes * adjustments * passes * correction * update hash * add UTC field todo * remove old field * erase comments * cleanup * tests * bolt on timedatedim * correction * add require * add timeparse / raw constr * abstract * remove outputdata debug data * add gitignore * test * update to integers * adjustments * add test data * fix * fix * added test case * added comment * add lite version * add raw value * fix * add * gofmt * add january test case * fix * remove Z * add more tests --------- Co-authored-by: tbal999 --- .gitignore | 4 + env.go | 29 ++++- jlib/hash.go | 32 ++--- jlib/timeparse/testdata.json | 191 ++++++++++++++++++++++++++++++ jlib/timeparse/testdata_lite.json | 191 ++++++++++++++++++++++++++++++ jlib/timeparse/timeparse.go | 146 +++++++++++++++++++++++ jlib/timeparse/timeparse_test.go | 128 ++++++++++++++++++++ jlib/timeparse/timeparselite.go | 55 +++++++++ jlib/timeparse/timesince.go | 27 +++++ 9 files changed, 782 insertions(+), 21 deletions(-) create mode 100644 jlib/timeparse/testdata.json create mode 100644 jlib/timeparse/testdata_lite.json create mode 100644 jlib/timeparse/timeparse.go create mode 100644 jlib/timeparse/timeparse_test.go create mode 100644 jlib/timeparse/timeparselite.go create mode 100644 jlib/timeparse/timesince.go diff --git a/.gitignore b/.gitignore index fe0305b..fd2129c 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,7 @@ # Dependency directories (remove the comment below to include it) # vendor/ + +# specific files +jlib/timeparse/outputdata.json +jlib/timeparse/outputdata_lite.json \ No newline at end of file diff --git a/env.go b/env.go index 030b25d..2fcc107 100644 --- a/env.go +++ b/env.go @@ -12,6 +12,7 @@ import ( "unicode/utf8" "github.com/xiatechs/jsonata-go/jlib" + "github.com/xiatechs/jsonata-go/jlib/timeparse" "github.com/xiatechs/jsonata-go/jparse" "github.com/xiatechs/jsonata-go/jtypes" ) @@ -94,8 +95,32 @@ var baseEnv = initBaseEnv(map[string]Extension{ EvalContextHandler: defaultContextHandler, }, - "hashStr": { - Func: jlib.Hash, + "hashmd5": { + Func: jlib.HashMD5, + UndefinedHandler: defaultUndefinedHandler, + EvalContextHandler: defaultContextHandler, + }, + + "hash256": { + Func: jlib.Hash256, + UndefinedHandler: defaultUndefinedHandler, + EvalContextHandler: defaultContextHandler, + }, + + "dateTimeDim": { + Func: timeparse.TimeDateDimensions, + UndefinedHandler: defaultUndefinedHandler, + EvalContextHandler: defaultContextHandler, + }, + + "dateTimeDimLite": { + Func: timeparse.TimeDateDimensionsLite, + UndefinedHandler: defaultUndefinedHandler, + EvalContextHandler: defaultContextHandler, + }, + + "timeSince": { + Func: timeparse.Since, UndefinedHandler: defaultUndefinedHandler, EvalContextHandler: defaultContextHandler, }, diff --git a/jlib/hash.go b/jlib/hash.go index 1d22576..712dcdb 100644 --- a/jlib/hash.go +++ b/jlib/hash.go @@ -4,34 +4,28 @@ import ( "crypto/md5" "crypto/sha256" "encoding/hex" - "strings" ) -// Hash a input string into a sha256 string +// Hash a input string into a md5 string // for deduplication purposes -func Hash(hashType, input string) string { - switch strings.ToLower(hashType) { - case "sha256": - hasher := sha256.New() - - hasher.Write([]byte(input)) - - hashedBytes := hasher.Sum(nil) +func HashMD5(input string) string { + hash := md5.Sum([]byte(input)) - hashedString := hex.EncodeToString(hashedBytes) + hashedString := hex.EncodeToString(hash[:]) - return hashedString - case "md5": - hash := md5.Sum([]byte(input)) + return hashedString +} - hashedString := hex.EncodeToString(hash[:]) +// Hash a input string into a sha256 string +// for deduplication purposes +func Hash256(input string) string { + hasher := sha256.New() - return hashedString - } + hasher.Write([]byte(input)) - hash := md5.Sum([]byte(input)) + hashedBytes := hasher.Sum(nil) - hashedString := hex.EncodeToString(hash[:]) + hashedString := hex.EncodeToString(hashedBytes) return hashedString } diff --git a/jlib/timeparse/testdata.json b/jlib/timeparse/testdata.json new file mode 100644 index 0000000..bf5c023 --- /dev/null +++ b/jlib/timeparse/testdata.json @@ -0,0 +1,191 @@ +[ + { + "testDesc": "/* source has explicit input UTC, output UTC - this will convert XF official formatting */", + "input_srcTs": "2023-08-06T00:23:41Z", + "input_srcFormat": "2006-01-02T15:04:05Z", + "input_srcTz": "UTC", + "output_srcTz": "UTC", + "DateDim": { + "TimeZone": "UTC", + "TimeZoneOffset": "+00:00", + "YearMonth": 202308, + "YearWeek": 202330, + "YearIsoWeek": 202331, + "YearDay": 2023218, + "DateId": "Dates_20230806", + "DateKey": "20230806", + "HourId": "Hours_2023080600", + "HourKey": "2023080600", + "Millis": 1691281421000, + "RawValue": "2023-08-06T00:23:41Z", + "UTC": "2023-08-06T00:23:41.000Z", + "DateUTC": "2023-08-06", + "HourUTC": 0, + "Local": "2023-08-06T00:23:41.000+00:00", + "DateLocal": "2023-08-06", + "HourLocal": 0 + } + }, + { + "testDesc": "/* source explicit UTC, Local = 2023-08-06T00:23:41.454Z+01:00 */", + "input_srcTs": "2023-08-06T00:23:41.454Z", + "input_srcFormat": "2006-01-02T15:04:05Z", + "input_srcTz": "UTC", + "output_srcTz": "Europe/London", + "DateDim": { + "TimeZone": "Europe/London", + "TimeZoneOffset": "+00:00", + "YearMonth": 202308, + "YearWeek": 202330, + "YearIsoWeek": 202331, + "YearDay": 2023218, + "DateId": "Dates_20230806", + "DateKey": "20230806", + "HourId": "Hours_2023080601", + "HourKey": "2023080601", + "Millis": 1691281421454, + "RawValue": "2023-08-06T00:23:41.454Z", + "UTC": "2023-08-06T00:23:41.454Z", + "DateUTC": "2023-08-06", + "HourUTC": 0, + "Local": "2023-08-06T01:23:41.454+01:00", + "DateLocal": "2023-08-06", + "HourLocal": 1 + } + }, + { + "testDesc": "/* source explicit src TZ, UTC = 2023-08-05T23:23:41.454 (NOTE: Day before!), Local = 2023-08-06T00:23:41.454Z+01:00 */", + "input_srcTs": "2023-08-06T00:23:41.454+01:00", + "input_srcFormat": "2006-01-02T15:04:05.999999999Z07:00", + "input_srcTz": "Europe/London", + "output_srcTz": "Europe/London", + "DateDim": { + "TimeZone": "Europe/London", + "TimeZoneOffset": "+00:00", + "YearMonth": 202308, + "YearWeek": 202330, + "YearIsoWeek": 202331, + "YearDay": 2023218, + "DateId": "Dates_20230806", + "DateKey": "20230806", + "HourId": "Hours_2023080600", + "HourKey": "2023080600", + "Millis": 1691277821454, + "RawValue": "2023-08-06T00:23:41.454+01:00", + "UTC": "2023-08-05T23:23:41.454Z", + "DateUTC": "2023-08-05", + "HourUTC": 23, + "Local": "2023-08-06T00:23:41.454+01:00", + "DateLocal": "2023-08-06", + "HourLocal": 0 + } + }, + { + "testDesc": "/* source impliciti TZ, bad use of a 'Z' whici is meant to meant UTC - same output as above as equiv */", + "input_srcTs": "2023-08-06T00:23:41.454Z", + "input_srcFormat": "2006-01-02T15:04:05.000Z", + "input_srcTz": "Europe/London", + "output_srcTz": "Europe/London", + "DateDim": { + "TimeZone": "Europe/London", + "TimeZoneOffset": "+00:00", + "YearMonth": 202308, + "YearWeek": 202330, + "YearIsoWeek": 202331, + "YearDay": 2023218, + "DateId": "Dates_20230806", + "DateKey": "20230806", + "HourId": "Hours_2023080600", + "HourKey": "2023080600", + "Millis": 1691277821454, + "RawValue": "2023-08-06T00:23:41.454Z", + "UTC": "2023-08-05T23:23:41.454Z", + "DateUTC": "2023-08-05", + "HourUTC": 23, + "Local": "2023-08-06T00:23:41.454+01:00", + "DateLocal": "2023-08-06", + "HourLocal": 0 + } + }, + { + "testDesc": "/* utc to america/new_york */", + "input_srcTs": "2023-08-06T00:23:41.454Z", + "input_srcFormat": "2006-01-02T15:04:05.000Z", + "input_srcTz": "Europe/London", + "output_srcTz": "America/New_York", + "DateDim": { + "TimeZone": "America/New_York", + "TimeZoneOffset": "+00:00", + "YearMonth": 202308, + "YearWeek": 202330, + "YearIsoWeek": 202331, + "YearDay": 2023217, + "DateId": "Dates_20230805", + "DateKey": "20230805", + "HourId": "Hours_2023080519", + "HourKey": "2023080519", + "Millis": 1691277821454, + "RawValue": "2023-08-06T00:23:41.454Z", + "UTC": "2023-08-05T23:23:41.454Z", + "DateUTC": "2023-08-05", + "HourUTC": 23, + "Local": "2023-08-05T19:23:41.454-04:00", + "DateLocal": "2023-08-05", + "HourLocal": 19 + } + }, + { + "testDesc": "/* same output as above, note the lack of 'Z'*/", + "input_srcTs": "2023-08-06T00:23:41.454", + "input_srcFormat": "2006-01-02T15:04:05.000", + "input_srcTz": "Europe/London", + "output_srcTz": "Europe/London", + "DateDim": { + "TimeZone": "Europe/London", + "TimeZoneOffset": "+00:00", + "YearMonth": 202308, + "YearWeek": 202330, + "YearIsoWeek": 202331, + "YearDay": 2023218, + "DateId": "Dates_20230806", + "DateKey": "20230806", + "HourId": "Hours_2023080600", + "HourKey": "2023080600", + "Millis": 1691277821454, + "RawValue": "2023-08-06T00:23:41.454", + "UTC": "2023-08-05T23:23:41.454Z", + "DateUTC": "2023-08-05", + "HourUTC": 23, + "Local": "2023-08-06T00:23:41.454+01:00", + "DateLocal": "2023-08-06", + "HourLocal": 0 + } + }, + { + "testDesc": "/* same output as above, note the lack of 'Z' - january*/", + "input_srcTs": "2023-01-06T00:23:41.454", + "input_srcFormat": "2006-01-02T15:04:05.000", + "input_srcTz": "Europe/London", + "output_srcTz": "Europe/London", + "DateDim": { + "TimeZone": "Europe/London", + "TimeZoneOffset": "+00:00", + "YearMonth": 202301, + "YearWeek": 202300, + "YearIsoWeek": 202301, + "YearDay": 2023006, + "DateId": "Dates_20230106", + "DateKey": "20230106", + "HourId": "Hours_2023010600", + "HourKey": "2023010600", + "Millis": 1672964621454, + "RawValue": "2023-01-06T00:23:41.454", + "UTC": "2023-01-06T00:23:41.454Z", + "DateUTC": "2023-01-06", + "HourUTC": 0, + "Local": "2023-01-06T00:23:41.454+00:00", + "DateLocal": "2023-01-06", + "HourLocal": 0 + } + } + ] \ No newline at end of file diff --git a/jlib/timeparse/testdata_lite.json b/jlib/timeparse/testdata_lite.json new file mode 100644 index 0000000..0a0fc0e --- /dev/null +++ b/jlib/timeparse/testdata_lite.json @@ -0,0 +1,191 @@ +[ + { + "testDesc": "/* source has explicit input UTC, output UTC - this will convert XF official formatting */", + "input_srcTs": "2023-08-06T00:23:41Z", + "input_srcFormat": "2006-01-02T15:04:05Z", + "input_srcTz": "UTC", + "output_srcTz": "UTC", + "DateDim": { + "TimeZone": "UTC", + "TimeZoneOffset": "+00:00", + "YearMonth": 0, + "YearWeek": 0, + "YearIsoWeek": 0, + "YearDay": 0, + "DateId": "Dates_20230806", + "DateKey": "20230806", + "HourId": "", + "HourKey": "", + "Millis": 1691281421000, + "RawValue": "2023-08-06T00:23:41Z", + "UTC": "2023-08-06T00:23:41.000Z", + "DateUTC": "2023-08-06", + "HourUTC": 0, + "Local": "2023-08-06T00:23:41.000+00:00", + "DateLocal": "2023-08-06", + "HourLocal": 0 + } + }, + { + "testDesc": "/* source explicit UTC, Local = 2023-08-06T00:23:41.454Z+01:00 */", + "input_srcTs": "2023-08-06T00:23:41.454Z", + "input_srcFormat": "2006-01-02T15:04:05Z", + "input_srcTz": "UTC", + "output_srcTz": "Europe/London", + "DateDim": { + "TimeZone": "Europe/London", + "TimeZoneOffset": "+00:00", + "YearMonth": 0, + "YearWeek": 0, + "YearIsoWeek": 0, + "YearDay": 0, + "DateId": "Dates_20230806", + "DateKey": "20230806", + "HourId": "", + "HourKey": "", + "Millis": 1691281421454, + "RawValue": "2023-08-06T00:23:41.454Z", + "UTC": "2023-08-06T00:23:41.454Z", + "DateUTC": "2023-08-06", + "HourUTC": 0, + "Local": "2023-08-06T01:23:41.454+01:00", + "DateLocal": "2023-08-06", + "HourLocal": 0 + } + }, + { + "testDesc": "/* source explicit src TZ, UTC = 2023-08-05T23:23:41.454 (NOTE: Day before!), Local = 2023-08-06T00:23:41.454Z+01:00 */", + "input_srcTs": "2023-08-06T00:23:41.454+01:00", + "input_srcFormat": "2006-01-02T15:04:05.999999999Z07:00", + "input_srcTz": "Europe/London", + "output_srcTz": "Europe/London", + "DateDim": { + "TimeZone": "Europe/London", + "TimeZoneOffset": "+00:00", + "YearMonth": 0, + "YearWeek": 0, + "YearIsoWeek": 0, + "YearDay": 0, + "DateId": "Dates_20230806", + "DateKey": "20230806", + "HourId": "", + "HourKey": "", + "Millis": 1691277821454, + "RawValue": "2023-08-06T00:23:41.454+01:00", + "UTC": "2023-08-05T23:23:41.454Z", + "DateUTC": "2023-08-05", + "HourUTC": 0, + "Local": "2023-08-06T00:23:41.454+01:00", + "DateLocal": "2023-08-06", + "HourLocal": 0 + } + }, + { + "testDesc": "/* source impliciti TZ, bad use of a 'Z' whici is meant to meant UTC - same output as above as equiv */", + "input_srcTs": "2023-08-06T00:23:41.454Z", + "input_srcFormat": "2006-01-02T15:04:05.000Z", + "input_srcTz": "Europe/London", + "output_srcTz": "Europe/London", + "DateDim": { + "TimeZone": "Europe/London", + "TimeZoneOffset": "+00:00", + "YearMonth": 0, + "YearWeek": 0, + "YearIsoWeek": 0, + "YearDay": 0, + "DateId": "Dates_20230806", + "DateKey": "20230806", + "HourId": "", + "HourKey": "", + "Millis": 1691277821454, + "RawValue": "2023-08-06T00:23:41.454Z", + "UTC": "2023-08-05T23:23:41.454Z", + "DateUTC": "2023-08-05", + "HourUTC": 0, + "Local": "2023-08-06T00:23:41.454+01:00", + "DateLocal": "2023-08-06", + "HourLocal": 0 + } + }, + { + "testDesc": "/* utc to america/new_york */", + "input_srcTs": "2023-08-06T00:23:41.454Z", + "input_srcFormat": "2006-01-02T15:04:05.000Z", + "input_srcTz": "Europe/London", + "output_srcTz": "America/New_York", + "DateDim": { + "TimeZone": "America/New_York", + "TimeZoneOffset": "+00:00", + "YearMonth": 0, + "YearWeek": 0, + "YearIsoWeek": 0, + "YearDay": 0, + "DateId": "Dates_20230805", + "DateKey": "20230805", + "HourId": "", + "HourKey": "", + "Millis": 1691277821454, + "RawValue": "2023-08-06T00:23:41.454Z", + "UTC": "2023-08-05T23:23:41.454Z", + "DateUTC": "2023-08-05", + "HourUTC": 0, + "Local": "2023-08-05T19:23:41.454-04:00", + "DateLocal": "2023-08-05", + "HourLocal": 0 + } + }, + { + "testDesc": "/* same output as above, note the lack of 'Z'*/", + "input_srcTs": "2023-08-06T00:23:41.454", + "input_srcFormat": "2006-01-02T15:04:05.000", + "input_srcTz": "Europe/London", + "output_srcTz": "Europe/London", + "DateDim": { + "TimeZone": "Europe/London", + "TimeZoneOffset": "+00:00", + "YearMonth": 0, + "YearWeek": 0, + "YearIsoWeek": 0, + "YearDay": 0, + "DateId": "Dates_20230806", + "DateKey": "20230806", + "HourId": "", + "HourKey": "", + "Millis": 1691277821454, + "RawValue": "2023-08-06T00:23:41.454", + "UTC": "2023-08-05T23:23:41.454Z", + "DateUTC": "2023-08-05", + "HourUTC": 0, + "Local": "2023-08-06T00:23:41.454+01:00", + "DateLocal": "2023-08-06", + "HourLocal": 0 + } + }, + { + "testDesc": "/* same output as above, note the lack of 'Z' - january*/", + "input_srcTs": "2023-01-06T00:23:41.454", + "input_srcFormat": "2006-01-02T15:04:05.000", + "input_srcTz": "Europe/London", + "output_srcTz": "Europe/London", + "DateDim": { + "TimeZone": "Europe/London", + "TimeZoneOffset": "+00:00", + "YearMonth": 0, + "YearWeek": 0, + "YearIsoWeek": 0, + "YearDay": 0, + "DateId": "Dates_20230106", + "DateKey": "20230106", + "HourId": "", + "HourKey": "", + "Millis": 1672964621454, + "RawValue": "2023-01-06T00:23:41.454", + "UTC": "2023-01-06T00:23:41.454Z", + "DateUTC": "2023-01-06", + "HourUTC": 0, + "Local": "2023-01-06T00:23:41.454+00:00", + "DateLocal": "2023-01-06", + "HourLocal": 0 + } + } + ] \ No newline at end of file diff --git a/jlib/timeparse/timeparse.go b/jlib/timeparse/timeparse.go new file mode 100644 index 0000000..34c84b2 --- /dev/null +++ b/jlib/timeparse/timeparse.go @@ -0,0 +1,146 @@ +package timeparse + +import ( + "fmt" + "strconv" + "strings" + "time" +) + +// DateDim is the date dimension object returned from the timeparse function +type DateDim struct { + // Other + TimeZone string `json:"TimeZone"` // lite + TimeZoneOffset string `json:"TimeZoneOffset"` // lite + YearMonth int `json:"YearMonth"` // int + YearWeek int `json:"YearWeek"` // int + YearIsoWeek int `json:"YearIsoWeek"` // int + YearDay int `json:"YearDay"` // int + DateID string `json:"DateId"` // lite + DateKey string `json:"DateKey"` // lite + HourID string `json:"HourId"` + HourKey string `json:"HourKey"` + Millis int `json:"Millis"` // lite + RawValue string `json:"RawValue"` // lite + + // UTC + UTC string `json:"UTC"` // lite + DateUTC string `json:"DateUTC"` // lite + HourUTC int `json:"HourUTC"` + + // Local + Local string `json:"Local"` // lite + DateLocal string `json:"DateLocal"` // lite + HourLocal int `json:"HourLocal"` +} + +// TimeDateDimensions generates a JSON object dependent on input source timestamp, input source format and input source timezone +// using golang time formats +func TimeDateDimensions(inputSrcTs, inputSrcFormat, inputSrcTz, requiredTz string) (*DateDim, error) { + inputLocation, err := time.LoadLocation(inputSrcTz) + if err != nil { + return nil, err + } + + // Since the source timestamp is implied to be in local time ("Europe/London"), + // we parse it with the location set to Europe/London + inputTime, err := time.ParseInLocation(inputSrcFormat, inputSrcTs, inputLocation) + if err != nil { + return nil, err + } + + outputLocation, err := time.LoadLocation(requiredTz) + if err != nil { + return nil, err + } + + localTime := inputTime.In(outputLocation) + + // convert the parsed time into a UTC time for UTC calculations + utcTime := localTime.UTC() + + // UTC TIME values + + utcAsYearMonthDay := utcTime.Format("2006-01-02") + + // Input time stamp TIME values (we confirmed there need to be a seperate set of UTC values) + dateID := localTime.Format("20060102") + + year, week := localTime.ISOWeek() + + yearDay, err := strconv.Atoi(localTime.Format("2006") + localTime.Format("002")) + if err != nil { + return nil, err + } + + hourKeyStr := localTime.Format("2006010215") + + mondayWeek, err := getWeekOfYearString(localTime) + if err != nil { + return nil, err + } + + yearIsoWeekInt, err := strconv.Atoi(fmt.Sprintf("%d%02d", year, week)) + if err != nil { + return nil, err + } + + yearMonthInt, err := strconv.Atoi(localTime.Format("200601")) + if err != nil { + return nil, err + } + + localTimeStamp := localTime.Format("2006-01-02T15:04:05.000-07:00") + // construct the date dimension structure + dateDim := &DateDim{ + RawValue: inputSrcTs, + TimeZoneOffset: getOffsetString(localTimeStamp), + YearWeek: mondayWeek, + YearDay: yearDay, + YearIsoWeek: yearIsoWeekInt, + YearMonth: yearMonthInt, + Millis: int(localTime.UnixMilli()), + HourLocal: localTime.Hour(), + HourKey: hourKeyStr, + HourID: "Hours_" + hourKeyStr, + DateLocal: localTime.Format("2006-01-02"), + TimeZone: localTime.Location().String(), + Local: localTimeStamp, + DateKey: dateID, + DateID: "Dates_" + dateID, + DateUTC: utcAsYearMonthDay, + UTC: utcTime.Format("2006-01-02T15:04:05.000Z"), + HourUTC: utcTime.Hour(), + } + + return dateDim, nil +} + +func getOffsetString(input string) string { + znegArr := strings.Split(input, "Z-") + if len(znegArr) == 2 { + return "-" + znegArr[1] + } + + zposArr := strings.Split(input, "Z+") + if len(zposArr) == 2 { + return "+" + zposArr[1] + } + + return "+00:00" +} + +func getWeekOfYearString(date time.Time) (int, error) { + _, week := date.ISOWeek() + + firstWednesday := date.AddDate(0, 0, -int(date.Weekday())+1) + if firstWednesday.Weekday() != time.Wednesday { + firstWednesday = firstWednesday.AddDate(0, 0, 7-int(firstWednesday.Weekday())+int(time.Wednesday)) + } + + if date.Weekday() == time.Sunday || date.Before(firstWednesday) { + week-- + } + + return strconv.Atoi(fmt.Sprintf("%04d%02d", date.Year(), week)) +} diff --git a/jlib/timeparse/timeparse_test.go b/jlib/timeparse/timeparse_test.go new file mode 100644 index 0000000..d6c8539 --- /dev/null +++ b/jlib/timeparse/timeparse_test.go @@ -0,0 +1,128 @@ +package timeparse_test + +import ( + "encoding/json" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + jsonatatime "github.com/xiatechs/jsonata-go/jlib/timeparse" +) + +type TestCase struct { + TestDesc string `json:"testDesc"` + InputSrcTs string `json:"input_srcTs"` + InputSrcFormat string `json:"input_srcFormat"` + InputSrcTz string `json:"input_srcTz"` + OutputSrcTz string `json:"output_srcTz"` + DateDim struct { + // Other + TimeZone string `json:"TimeZone"` // lite + TimeZoneOffset string `json:"TimeZoneOffset"` // lite + YearMonth int `json:"YearMonth"` // int + YearWeek int `json:"YearWeek"` // int + YearIsoWeek int `json:"YearIsoWeek"` // int + YearDay int `json:"YearDay"` // int + DateID string `json:"DateId"` // lite + DateKey string `json:"DateKey"` // lite + HourID string `json:"HourId"` + HourKey string `json:"HourKey"` + Millis int `json:"Millis"` // lite + RawValue string `json:"RawValue"` // lite + + // UTC + UTC string `json:"UTC"` // lite + DateUTC string `json:"DateUTC"` // lite + HourUTC int `json:"HourUTC"` + + // Local + Local string `json:"Local"` // lite + DateLocal string `json:"DateLocal"` // lite + HourLocal int `json:"HourLocal"` + } `json:"DateDim"` +} + +func TestTime(t *testing.T) { + tests := []TestCase{} + fileBytes, err := os.ReadFile("testdata.json") + require.NoError(t, err) + err = json.Unmarshal(fileBytes, &tests) + require.NoError(t, err) + + output := make([]interface{}, 0) + + for _, tc := range tests { + tc := tc // race protection + + t.Run(tc.TestDesc, func(t *testing.T) { + result, err := jsonatatime.TimeDateDimensions(tc.InputSrcTs, tc.InputSrcFormat, tc.InputSrcTz, tc.OutputSrcTz) + require.NoError(t, err) + + testObj := tc + + expectedByts, err := json.Marshal(tc.DateDim) + require.NoError(t, err) + + expectedDateDim := jsonatatime.DateDim{} + + actualByts, err := json.Marshal(result) + require.NoError(t, err) + + actualDateDim := jsonatatime.DateDim{} + + err = json.Unmarshal(actualByts, &actualDateDim) + require.NoError(t, err) + + testObj.DateDim = actualDateDim + output = append(output, testObj) + err = json.Unmarshal(expectedByts, &expectedDateDim) + assert.Equal(t, expectedDateDim, actualDateDim) + }) + } + + outputbytes, _ := json.MarshalIndent(output, "", " ") + _ = os.WriteFile("outputdata.json", outputbytes, os.ModePerm) +} + +func TestTimeLite(t *testing.T) { + tests := []TestCase{} + fileBytes, err := os.ReadFile("testdata_lite.json") + require.NoError(t, err) + err = json.Unmarshal(fileBytes, &tests) + require.NoError(t, err) + + output := make([]interface{}, 0) + + for _, tc := range tests { + tc := tc // race protection + + t.Run(tc.TestDesc, func(t *testing.T) { + result, err := jsonatatime.TimeDateDimensionsLite(tc.InputSrcTs, tc.InputSrcFormat, tc.InputSrcTz, tc.OutputSrcTz) + require.NoError(t, err) + + testObj := tc + + expectedByts, err := json.Marshal(tc.DateDim) + require.NoError(t, err) + + expectedDateDim := jsonatatime.DateDim{} + + actualByts, err := json.Marshal(result) + require.NoError(t, err) + + actualDateDim := jsonatatime.DateDim{} + + err = json.Unmarshal(actualByts, &actualDateDim) + require.NoError(t, err) + + testObj.DateDim = actualDateDim + output = append(output, testObj) + err = json.Unmarshal(expectedByts, &expectedDateDim) + assert.Equal(t, expectedDateDim, actualDateDim) + }) + } + + outputbytes, _ := json.MarshalIndent(output, "", " ") + _ = os.WriteFile("outputdata_lite.json", outputbytes, os.ModePerm) +} diff --git a/jlib/timeparse/timeparselite.go b/jlib/timeparse/timeparselite.go new file mode 100644 index 0000000..eba5226 --- /dev/null +++ b/jlib/timeparse/timeparselite.go @@ -0,0 +1,55 @@ +package timeparse + +import ( + "time" +) + +// TimeDateDimensionsLite generates a JSON object dependent on input source timestamp, input source format and input source timezone +// using golang time formats +func TimeDateDimensionsLite(inputSrcTs, inputSrcFormat, inputSrcTz, requiredTz string) (*DateDim, error) { + inputLocation, err := time.LoadLocation(inputSrcTz) + if err != nil { + return nil, err + } + + // Since the source timestamp is implied to be in local time ("Europe/London"), + // we parse it with the location set to Europe/London + inputTime, err := time.ParseInLocation(inputSrcFormat, inputSrcTs, inputLocation) + if err != nil { + return nil, err + } + + outputLocation, err := time.LoadLocation(requiredTz) + if err != nil { + return nil, err + } + + localTime := inputTime.In(outputLocation) + + // convert the parsed time into a UTC time for UTC calculations + utcTime := localTime.UTC() + + // UTC TIME values + + utcAsYearMonthDay := utcTime.Format("2006-01-02") + + // Input time stamp TIME values (we confirmed there need to be a seperate set of UTC values) + dateID := localTime.Format("20060102") + + localTimeStamp := localTime.Format("2006-01-02T15:04:05.000-07:00") + // construct the date dimension structure + dateDim := &DateDim{ + RawValue: inputSrcTs, + TimeZoneOffset: getOffsetString(localTimeStamp), + Millis: int(localTime.UnixMilli()), + DateLocal: localTime.Format("2006-01-02"), + TimeZone: localTime.Location().String(), + Local: localTimeStamp, + DateKey: dateID, + DateID: "Dates_" + dateID, + DateUTC: utcAsYearMonthDay, + UTC: utcTime.Format("2006-01-02T15:04:05.000Z"), + } + + return dateDim, nil +} diff --git a/jlib/timeparse/timesince.go b/jlib/timeparse/timesince.go new file mode 100644 index 0000000..516f5a8 --- /dev/null +++ b/jlib/timeparse/timesince.go @@ -0,0 +1,27 @@ +package timeparse + +import "time" + +func Since(time1, time1format, time1location, time2, time2format, time2location string) (float64, error) { + inputLocation, err := time.LoadLocation(time1location) + if err != nil { + return 0, err + } + + firstTime, err := time.ParseInLocation(time1format, time1, inputLocation) + if err != nil { + return 0, err + } + + outputLocation, err := time.LoadLocation(time2location) + if err != nil { + return 0, err + } + + secondTime, err := time.ParseInLocation(time2format, time2, outputLocation) + if err != nil { + return 0, err + } + + return firstTime.Sub(secondTime).Seconds(), nil +} From 48c87cfdaa97b71cab6677b829f329be7bd53e66 Mon Sep 17 00:00:00 2001 From: Tom <53711814+tbal999@users.noreply.github.com> Date: Mon, 20 Nov 2023 12:11:53 +0000 Subject: [PATCH 41/58] add adjustments (#23) Co-authored-by: tbal999 --- jlib/jlib.go | 3 +- jlib/timeparse/testdata.json | 35 +++++++----- jlib/timeparse/testdata_lite.json | 91 +++++++------------------------ jlib/timeparse/timeparse.go | 27 +++++++-- jlib/timeparse/timeparse_test.go | 41 +++++--------- jlib/timeparse/timeparselite.go | 54 +++++++++++++++++- 6 files changed, 131 insertions(+), 120 deletions(-) diff --git a/jlib/jlib.go b/jlib/jlib.go index 37483eb..3a976c7 100644 --- a/jlib/jlib.go +++ b/jlib/jlib.go @@ -88,5 +88,6 @@ func TypeOf(x interface{}) (string, error) { } xType := reflect.TypeOf(x).String() - return "", fmt.Errorf("unknown type %s", xType) + + return xType, nil } diff --git a/jlib/timeparse/testdata.json b/jlib/timeparse/testdata.json index bf5c023..e40444e 100644 --- a/jlib/timeparse/testdata.json +++ b/jlib/timeparse/testdata.json @@ -13,9 +13,10 @@ "YearIsoWeek": 202331, "YearDay": 2023218, "DateId": "Dates_20230806", - "DateKey": "20230806", + "DateKey": 20230806, + "DateTimeKey": 20230806002341000, "HourId": "Hours_2023080600", - "HourKey": "2023080600", + "HourKey": 2023080600, "Millis": 1691281421000, "RawValue": "2023-08-06T00:23:41Z", "UTC": "2023-08-06T00:23:41.000Z", @@ -40,9 +41,10 @@ "YearIsoWeek": 202331, "YearDay": 2023218, "DateId": "Dates_20230806", - "DateKey": "20230806", + "DateKey": 20230806, + "DateTimeKey": 20230806012341000, "HourId": "Hours_2023080601", - "HourKey": "2023080601", + "HourKey": 2023080601, "Millis": 1691281421454, "RawValue": "2023-08-06T00:23:41.454Z", "UTC": "2023-08-06T00:23:41.454Z", @@ -67,9 +69,10 @@ "YearIsoWeek": 202331, "YearDay": 2023218, "DateId": "Dates_20230806", - "DateKey": "20230806", + "DateKey": 20230806, + "DateTimeKey": 20230806002341000, "HourId": "Hours_2023080600", - "HourKey": "2023080600", + "HourKey": 2023080600, "Millis": 1691277821454, "RawValue": "2023-08-06T00:23:41.454+01:00", "UTC": "2023-08-05T23:23:41.454Z", @@ -94,9 +97,10 @@ "YearIsoWeek": 202331, "YearDay": 2023218, "DateId": "Dates_20230806", - "DateKey": "20230806", + "DateKey": 20230806, + "DateTimeKey": 20230806002341000, "HourId": "Hours_2023080600", - "HourKey": "2023080600", + "HourKey": 2023080600, "Millis": 1691277821454, "RawValue": "2023-08-06T00:23:41.454Z", "UTC": "2023-08-05T23:23:41.454Z", @@ -121,9 +125,10 @@ "YearIsoWeek": 202331, "YearDay": 2023217, "DateId": "Dates_20230805", - "DateKey": "20230805", + "DateKey": 20230805, + "DateTimeKey": 20230805192341000, "HourId": "Hours_2023080519", - "HourKey": "2023080519", + "HourKey": 2023080519, "Millis": 1691277821454, "RawValue": "2023-08-06T00:23:41.454Z", "UTC": "2023-08-05T23:23:41.454Z", @@ -148,9 +153,10 @@ "YearIsoWeek": 202331, "YearDay": 2023218, "DateId": "Dates_20230806", - "DateKey": "20230806", + "DateKey": 20230806, + "DateTimeKey": 20230806002341000, "HourId": "Hours_2023080600", - "HourKey": "2023080600", + "HourKey": 2023080600, "Millis": 1691277821454, "RawValue": "2023-08-06T00:23:41.454", "UTC": "2023-08-05T23:23:41.454Z", @@ -175,9 +181,10 @@ "YearIsoWeek": 202301, "YearDay": 2023006, "DateId": "Dates_20230106", - "DateKey": "20230106", + "DateKey": 20230106, + "DateTimeKey": 20230106002341000, "HourId": "Hours_2023010600", - "HourKey": "2023010600", + "HourKey": 2023010600, "Millis": 1672964621454, "RawValue": "2023-01-06T00:23:41.454", "UTC": "2023-01-06T00:23:41.454Z", diff --git a/jlib/timeparse/testdata_lite.json b/jlib/timeparse/testdata_lite.json index 0a0fc0e..71d5a53 100644 --- a/jlib/timeparse/testdata_lite.json +++ b/jlib/timeparse/testdata_lite.json @@ -8,22 +8,15 @@ "DateDim": { "TimeZone": "UTC", "TimeZoneOffset": "+00:00", - "YearMonth": 0, - "YearWeek": 0, - "YearIsoWeek": 0, - "YearDay": 0, "DateId": "Dates_20230806", - "DateKey": "20230806", - "HourId": "", - "HourKey": "", + "DateKey": 20230806, + "DateTimeKey": 20230806002341000, "Millis": 1691281421000, "RawValue": "2023-08-06T00:23:41Z", "UTC": "2023-08-06T00:23:41.000Z", "DateUTC": "2023-08-06", - "HourUTC": 0, "Local": "2023-08-06T00:23:41.000+00:00", - "DateLocal": "2023-08-06", - "HourLocal": 0 + "DateLocal": "2023-08-06" } }, { @@ -35,22 +28,15 @@ "DateDim": { "TimeZone": "Europe/London", "TimeZoneOffset": "+00:00", - "YearMonth": 0, - "YearWeek": 0, - "YearIsoWeek": 0, - "YearDay": 0, "DateId": "Dates_20230806", - "DateKey": "20230806", - "HourId": "", - "HourKey": "", + "DateKey": 20230806, + "DateTimeKey": 20230806012341000, "Millis": 1691281421454, "RawValue": "2023-08-06T00:23:41.454Z", "UTC": "2023-08-06T00:23:41.454Z", "DateUTC": "2023-08-06", - "HourUTC": 0, "Local": "2023-08-06T01:23:41.454+01:00", - "DateLocal": "2023-08-06", - "HourLocal": 0 + "DateLocal": "2023-08-06" } }, { @@ -62,22 +48,15 @@ "DateDim": { "TimeZone": "Europe/London", "TimeZoneOffset": "+00:00", - "YearMonth": 0, - "YearWeek": 0, - "YearIsoWeek": 0, - "YearDay": 0, "DateId": "Dates_20230806", - "DateKey": "20230806", - "HourId": "", - "HourKey": "", + "DateKey": 20230806, + "DateTimeKey": 20230806002341000, "Millis": 1691277821454, "RawValue": "2023-08-06T00:23:41.454+01:00", "UTC": "2023-08-05T23:23:41.454Z", "DateUTC": "2023-08-05", - "HourUTC": 0, "Local": "2023-08-06T00:23:41.454+01:00", - "DateLocal": "2023-08-06", - "HourLocal": 0 + "DateLocal": "2023-08-06" } }, { @@ -89,22 +68,15 @@ "DateDim": { "TimeZone": "Europe/London", "TimeZoneOffset": "+00:00", - "YearMonth": 0, - "YearWeek": 0, - "YearIsoWeek": 0, - "YearDay": 0, "DateId": "Dates_20230806", - "DateKey": "20230806", - "HourId": "", - "HourKey": "", + "DateKey": 20230806, + "DateTimeKey": 20230806002341000, "Millis": 1691277821454, "RawValue": "2023-08-06T00:23:41.454Z", "UTC": "2023-08-05T23:23:41.454Z", "DateUTC": "2023-08-05", - "HourUTC": 0, "Local": "2023-08-06T00:23:41.454+01:00", - "DateLocal": "2023-08-06", - "HourLocal": 0 + "DateLocal": "2023-08-06" } }, { @@ -116,22 +88,15 @@ "DateDim": { "TimeZone": "America/New_York", "TimeZoneOffset": "+00:00", - "YearMonth": 0, - "YearWeek": 0, - "YearIsoWeek": 0, - "YearDay": 0, "DateId": "Dates_20230805", - "DateKey": "20230805", - "HourId": "", - "HourKey": "", + "DateKey": 20230805, + "DateTimeKey": 20230805192341000, "Millis": 1691277821454, "RawValue": "2023-08-06T00:23:41.454Z", "UTC": "2023-08-05T23:23:41.454Z", "DateUTC": "2023-08-05", - "HourUTC": 0, "Local": "2023-08-05T19:23:41.454-04:00", - "DateLocal": "2023-08-05", - "HourLocal": 0 + "DateLocal": "2023-08-05" } }, { @@ -143,22 +108,15 @@ "DateDim": { "TimeZone": "Europe/London", "TimeZoneOffset": "+00:00", - "YearMonth": 0, - "YearWeek": 0, - "YearIsoWeek": 0, - "YearDay": 0, "DateId": "Dates_20230806", - "DateKey": "20230806", - "HourId": "", - "HourKey": "", + "DateKey": 20230806, + "DateTimeKey": 20230806002341000, "Millis": 1691277821454, "RawValue": "2023-08-06T00:23:41.454", "UTC": "2023-08-05T23:23:41.454Z", "DateUTC": "2023-08-05", - "HourUTC": 0, "Local": "2023-08-06T00:23:41.454+01:00", - "DateLocal": "2023-08-06", - "HourLocal": 0 + "DateLocal": "2023-08-06" } }, { @@ -170,22 +128,15 @@ "DateDim": { "TimeZone": "Europe/London", "TimeZoneOffset": "+00:00", - "YearMonth": 0, - "YearWeek": 0, - "YearIsoWeek": 0, - "YearDay": 0, "DateId": "Dates_20230106", - "DateKey": "20230106", - "HourId": "", - "HourKey": "", + "DateKey": 20230106, + "DateTimeKey": 20230106002341000, "Millis": 1672964621454, "RawValue": "2023-01-06T00:23:41.454", "UTC": "2023-01-06T00:23:41.454Z", "DateUTC": "2023-01-06", - "HourUTC": 0, "Local": "2023-01-06T00:23:41.454+00:00", - "DateLocal": "2023-01-06", - "HourLocal": 0 + "DateLocal": "2023-01-06" } } ] \ No newline at end of file diff --git a/jlib/timeparse/timeparse.go b/jlib/timeparse/timeparse.go index 34c84b2..e8d7372 100644 --- a/jlib/timeparse/timeparse.go +++ b/jlib/timeparse/timeparse.go @@ -17,9 +17,10 @@ type DateDim struct { YearIsoWeek int `json:"YearIsoWeek"` // int YearDay int `json:"YearDay"` // int DateID string `json:"DateId"` // lite - DateKey string `json:"DateKey"` // lite + DateKey int `json:"DateKey"` // lite + DateTimeKey int `json:"DateTimeKey"` // lite HourID string `json:"HourId"` - HourKey string `json:"HourKey"` + HourKey int `json:"HourKey"` Millis int `json:"Millis"` // lite RawValue string `json:"RawValue"` // lite @@ -90,6 +91,23 @@ func TimeDateDimensions(inputSrcTs, inputSrcFormat, inputSrcTz, requiredTz strin return nil, err } + dateKeyInt, err := strconv.Atoi(dateID) + if err != nil { + return nil, err + } + + dateTimeID := localTime.Format("20060102150405000") + + dateTimeKeyInt, err := strconv.Atoi(dateTimeID) + if err != nil { + return nil, err + } + + hourKeyInt, err := strconv.Atoi(hourKeyStr) + if err != nil { + return nil, err + } + localTimeStamp := localTime.Format("2006-01-02T15:04:05.000-07:00") // construct the date dimension structure dateDim := &DateDim{ @@ -101,12 +119,13 @@ func TimeDateDimensions(inputSrcTs, inputSrcFormat, inputSrcTz, requiredTz strin YearMonth: yearMonthInt, Millis: int(localTime.UnixMilli()), HourLocal: localTime.Hour(), - HourKey: hourKeyStr, + HourKey: hourKeyInt, HourID: "Hours_" + hourKeyStr, DateLocal: localTime.Format("2006-01-02"), TimeZone: localTime.Location().String(), Local: localTimeStamp, - DateKey: dateID, + DateKey: dateKeyInt, + DateTimeKey: dateTimeKeyInt, DateID: "Dates_" + dateID, DateUTC: utcAsYearMonthDay, UTC: utcTime.Format("2006-01-02T15:04:05.000Z"), diff --git a/jlib/timeparse/timeparse_test.go b/jlib/timeparse/timeparse_test.go index d6c8539..d5ec477 100644 --- a/jlib/timeparse/timeparse_test.go +++ b/jlib/timeparse/timeparse_test.go @@ -16,31 +16,7 @@ type TestCase struct { InputSrcFormat string `json:"input_srcFormat"` InputSrcTz string `json:"input_srcTz"` OutputSrcTz string `json:"output_srcTz"` - DateDim struct { - // Other - TimeZone string `json:"TimeZone"` // lite - TimeZoneOffset string `json:"TimeZoneOffset"` // lite - YearMonth int `json:"YearMonth"` // int - YearWeek int `json:"YearWeek"` // int - YearIsoWeek int `json:"YearIsoWeek"` // int - YearDay int `json:"YearDay"` // int - DateID string `json:"DateId"` // lite - DateKey string `json:"DateKey"` // lite - HourID string `json:"HourId"` - HourKey string `json:"HourKey"` - Millis int `json:"Millis"` // lite - RawValue string `json:"RawValue"` // lite - - // UTC - UTC string `json:"UTC"` // lite - DateUTC string `json:"DateUTC"` // lite - HourUTC int `json:"HourUTC"` - - // Local - Local string `json:"Local"` // lite - DateLocal string `json:"DateLocal"` // lite - HourLocal int `json:"HourLocal"` - } `json:"DateDim"` + DateDim jsonatatime.DateDim `json:"DateDim"` } func TestTime(t *testing.T) { @@ -85,8 +61,17 @@ func TestTime(t *testing.T) { _ = os.WriteFile("outputdata.json", outputbytes, os.ModePerm) } +type TestCaseLite struct { + TestDesc string `json:"testDesc"` + InputSrcTs string `json:"input_srcTs"` + InputSrcFormat string `json:"input_srcFormat"` + InputSrcTz string `json:"input_srcTz"` + OutputSrcTz string `json:"output_srcTz"` + DateDim jsonatatime.DateDimLite `json:"DateDim"` +} + func TestTimeLite(t *testing.T) { - tests := []TestCase{} + tests := []TestCaseLite{} fileBytes, err := os.ReadFile("testdata_lite.json") require.NoError(t, err) err = json.Unmarshal(fileBytes, &tests) @@ -106,12 +91,12 @@ func TestTimeLite(t *testing.T) { expectedByts, err := json.Marshal(tc.DateDim) require.NoError(t, err) - expectedDateDim := jsonatatime.DateDim{} + expectedDateDim := jsonatatime.DateDimLite{} actualByts, err := json.Marshal(result) require.NoError(t, err) - actualDateDim := jsonatatime.DateDim{} + actualDateDim := jsonatatime.DateDimLite{} err = json.Unmarshal(actualByts, &actualDateDim) require.NoError(t, err) diff --git a/jlib/timeparse/timeparselite.go b/jlib/timeparse/timeparselite.go index eba5226..9150d91 100644 --- a/jlib/timeparse/timeparselite.go +++ b/jlib/timeparse/timeparselite.go @@ -1,12 +1,46 @@ package timeparse import ( + "strconv" "time" ) +/* + RawValue: inputSrcTs, + TimeZoneOffset: getOffsetString(localTimeStamp), + Millis: int(localTime.UnixMilli()), + DateLocal: localTime.Format("2006-01-02"), + TimeZone: localTime.Location().String(), + Local: localTimeStamp, + DateKey: dateID, + DateID: "Dates_" + dateID, + DateUTC: utcAsYearMonthDay, + UTC: utcTime.Format("2006-01-02T15:04:05.000Z"), +*/ + +// DateDimLite is the date dimension object returned from the timeparse function (light version) +type DateDimLite struct { + // Other + TimeZone string `json:"TimeZone"` // lite + TimeZoneOffset string `json:"TimeZoneOffset"` // lite + DateID string `json:"DateId"` // lite + DateKey int `json:"DateKey"` // lite + DateTimeKey int `json:"DateTimeKey"` // lite + Millis int `json:"Millis"` // lite + RawValue string `json:"RawValue"` // lite + + // UTC + UTC string `json:"UTC"` // lite + DateUTC string `json:"DateUTC"` // lite + + // Local + Local string `json:"Local"` // lite + DateLocal string `json:"DateLocal"` // lite +} + // TimeDateDimensionsLite generates a JSON object dependent on input source timestamp, input source format and input source timezone // using golang time formats -func TimeDateDimensionsLite(inputSrcTs, inputSrcFormat, inputSrcTz, requiredTz string) (*DateDim, error) { +func TimeDateDimensionsLite(inputSrcTs, inputSrcFormat, inputSrcTz, requiredTz string) (*DateDimLite, error) { inputLocation, err := time.LoadLocation(inputSrcTz) if err != nil { return nil, err @@ -36,16 +70,30 @@ func TimeDateDimensionsLite(inputSrcTs, inputSrcFormat, inputSrcTz, requiredTz s // Input time stamp TIME values (we confirmed there need to be a seperate set of UTC values) dateID := localTime.Format("20060102") + dateKeyInt, err := strconv.Atoi(dateID) + if err != nil { + return nil, err + } + + // Input time stamp TIME values (we confirmed there need to be a seperate set of UTC values) + dateTimeID := localTime.Format("20060102150405000") + + dateTimeKeyInt, err := strconv.Atoi(dateTimeID) + if err != nil { + return nil, err + } + localTimeStamp := localTime.Format("2006-01-02T15:04:05.000-07:00") // construct the date dimension structure - dateDim := &DateDim{ + dateDim := &DateDimLite{ RawValue: inputSrcTs, TimeZoneOffset: getOffsetString(localTimeStamp), Millis: int(localTime.UnixMilli()), DateLocal: localTime.Format("2006-01-02"), TimeZone: localTime.Location().String(), Local: localTimeStamp, - DateKey: dateID, + DateKey: dateKeyInt, + DateTimeKey: dateTimeKeyInt, DateID: "Dates_" + dateID, DateUTC: utcAsYearMonthDay, UTC: utcTime.Format("2006-01-02T15:04:05.000Z"), From 11a33a1932c4b2976655de82b457b56e76206e07 Mon Sep 17 00:00:00 2001 From: Tom <53711814+tbal999@users.noreply.github.com> Date: Mon, 27 Nov 2023 15:51:43 +0000 Subject: [PATCH 42/58] Enhancement to oneToMany function to add sql like join types (#25) * Enhancement to oneToMany function to add sql like join types * migrate & add non-reflect methods * fix --------- Co-authored-by: JamesXiatech Co-authored-by: tbal999 --- env.go | 3 +- jlib/jlib.go | 2 +- jlib/join/join.go | 255 +++++++++++++++++++++++++++++ jlib/join/join_bench_test.go | 273 +++++++++++++++++++++++++++++++ jlib/join/join_test.go | 265 ++++++++++++++++++++++++++++++ jlib/new.go | 82 ---------- jlib/new_test.go | 86 ---------- jlib/timeparse/timeparse_test.go | 20 +-- 8 files changed, 806 insertions(+), 180 deletions(-) create mode 100644 jlib/join/join.go create mode 100644 jlib/join/join_bench_test.go create mode 100644 jlib/join/join_test.go diff --git a/env.go b/env.go index 2fcc107..dd84e14 100644 --- a/env.go +++ b/env.go @@ -12,6 +12,7 @@ import ( "unicode/utf8" "github.com/xiatechs/jsonata-go/jlib" + "github.com/xiatechs/jsonata-go/jlib/join" "github.com/xiatechs/jsonata-go/jlib/timeparse" "github.com/xiatechs/jsonata-go/jparse" "github.com/xiatechs/jsonata-go/jtypes" @@ -132,7 +133,7 @@ var baseEnv = initBaseEnv(map[string]Extension{ }, "oneToManyJoin": { - Func: jlib.OneToManyJoin, + Func: join.OneToManyJoin, UndefinedHandler: defaultUndefinedHandler, EvalContextHandler: nil, }, diff --git a/jlib/jlib.go b/jlib/jlib.go index 3a976c7..9147665 100644 --- a/jlib/jlib.go +++ b/jlib/jlib.go @@ -88,6 +88,6 @@ func TypeOf(x interface{}) (string, error) { } xType := reflect.TypeOf(x).String() - + return xType, nil } diff --git a/jlib/join/join.go b/jlib/join/join.go new file mode 100644 index 0000000..ad1f048 --- /dev/null +++ b/jlib/join/join.go @@ -0,0 +1,255 @@ +package join + +import ( + "errors" + "fmt" + "reflect" +) + +// OneToManyJoin performs a join operation between two slices of maps/structs based on specified keys. +// It supports different types of joins: left, right, inner, and full. +func OneToManyJoin(leftArr, rightArr interface{}, leftKey, rightKey, rightArrayName, joinType string) (interface{}, error) { + // Convert input to slices of interfaces + trueLeftArr, ok := leftArr.([]interface{}) + if !ok { + return nil, errors.New("left input must be an array of Objects") + } + + trueRightArr, ok := rightArr.([]interface{}) + if !ok { + return nil, errors.New("right input must be an array of Objects") + } + + // Maps for tracking processed items + alreadyProcessed := make(map[string]bool) + rightProcessed := make(map[string]bool) + + // Create a map for faster lookup of rightArr elements based on the key + rightMap := make(map[string][]interface{}) + for _, item := range trueRightArr { + itemMap, ok := item.(map[string]interface{}) + if ok { + if itemKey, ok := itemMap[rightKey]; ok { + strVal := fmt.Sprintf("%v", itemKey) + rightMap[strVal] = append(rightMap[strVal], item) + } + } + } + + // Slice to store the merged results + var result []map[string]interface{} + leftMatched := make(map[string]interface{}) + + // Iterate through the left array and perform the join + for _, leftItem := range trueLeftArr { + itemMap, ok := leftItem.(map[string]interface{}) + if ok { + if itemKey, ok := itemMap[leftKey]; ok { + strVal := fmt.Sprintf("%v", itemKey) + + // Determine the right items to join + rightItems := rightMap[strVal] + + // Perform the join based on the join type + if joinType == "left" || joinType == "full" || (joinType == "inner" && len(rightItems) > 0) { + mergedItem := mergeItems(leftItem, rightItems, rightArrayName) + result = append(result, mergedItem) + } + + // Mark items as processed + leftMatched[strVal] = leftItem + alreadyProcessed[strVal] = true + } + } + } + + // Add items from the right array for right or full join + if joinType == "right" || joinType == "full" { + for _, rightItem := range trueRightArr { + itemMap, ok := rightItem.(map[string]interface{}) + if ok { + if itemKey, ok := itemMap[rightKey]; ok { + strVal := fmt.Sprintf("%v", itemKey) + + // Determine the left item to merge with + var leftItemToMerge interface{} + if leftMatch, ok := leftMatched[strVal]; ok { + leftItemToMerge = leftMatch + } else { + leftItemToMerge = map[string]interface{}{rightKey: itemKey} + } + + // Handle right and full join separately to avoid duplication + if joinType == "right" && !rightProcessed[strVal] { + mergedItem := mergeItems(leftItemToMerge, rightMap[strVal], rightArrayName) + result = append(result, mergedItem) + rightProcessed[strVal] = true + } else if joinType == "full" && !rightProcessed[strVal] && !alreadyProcessed[strVal] { + mergedItem := mergeItems(leftItemToMerge, rightMap[strVal], rightArrayName) + result = append(result, mergedItem) + rightProcessed[strVal] = true + } + } + } + } + } + + return result, nil +} + +func mergeItems(leftItem interface{}, rightItems []interface{}, rightArrayName string) map[string]interface{} { + mergedItem := make(map[string]interface{}) + + // Check if leftItem is a map or a struct and merge accordingly + leftVal := reflect.ValueOf(leftItem) + if leftVal.Kind() == reflect.Map { + // Merge fields from the map + for _, key := range leftVal.MapKeys() { + mergedItem[key.String()] = leftVal.MapIndex(key).Interface() + } + } else { + // Merge fields from the struct + leftType := leftVal.Type() + for i := 0; i < leftVal.NumField(); i++ { + fieldName := leftType.Field(i).Name + fieldValue := leftVal.Field(i).Interface() + mergedItem[fieldName] = fieldValue + } + } + + // If there are matching items in the right array, add them under the specified name + if len(rightItems) > 0 { + mergedItem[rightArrayName] = rightItems + } + + return mergedItem +} + +// OneToManyJoin2 performs a join operation between two slices of maps/structs based on specified keys. +// It supports different types of joins: left, right, inner, and full. +func OneToManyJoin2(leftArr, rightArr interface{}, leftKey, rightKey, rightArrayName, joinType string) (interface{}, error) { + // Convert input to slices of interfaces + trueLeftArr, ok := leftArr.([]interface{}) + if !ok { + return nil, errors.New("left input must be an array of Objects") + } + + trueRightArr, ok := rightArr.([]interface{}) + if !ok { + return nil, errors.New("right input must be an array of Objects") + } + + // Maps for tracking processed items + alreadyProcessed := make(map[string]bool) + rightProcessed := make(map[string]bool) + + // Create a map for faster lookup of rightArr elements based on the key + rightMap := make(map[string][]interface{}) + for _, item := range trueRightArr { + itemMap, ok := item.(map[string]interface{}) + if ok { + if itemKey, ok := itemMap[rightKey]; ok { + strVal := fmt.Sprintf("%v", itemKey) + rightMap[strVal] = append(rightMap[strVal], item) + } + } + } + + // Slice to store the merged results + var result []map[string]interface{} + leftMatched := make(map[string]interface{}) + + // Iterate through the left array and perform the join + for _, leftItem := range trueLeftArr { + itemMap, ok := leftItem.(map[string]interface{}) + if ok { + if itemKey, ok := itemMap[leftKey]; ok { + strVal := fmt.Sprintf("%v", itemKey) + + // Determine the right items to join + rightItems := rightMap[strVal] + + // Perform the join based on the join type + if joinType == "left" || joinType == "full" || (joinType == "inner" && len(rightItems) > 0) { + mergedItem := mergeItemsNew(leftItem, rightItems, rightArrayName) + result = append(result, mergedItem) + } + + // Mark items as processed + leftMatched[strVal] = leftItem + alreadyProcessed[strVal] = true + } + } + } + + // Add items from the right array for right or full join + if joinType == "right" || joinType == "full" { + for _, rightItem := range trueRightArr { + itemMap, ok := rightItem.(map[string]interface{}) + if ok { + if itemKey, ok := itemMap[rightKey]; ok { + strVal := fmt.Sprintf("%v", itemKey) + + // Determine the left item to merge with + var leftItemToMerge interface{} + if leftMatch, ok := leftMatched[strVal]; ok { + leftItemToMerge = leftMatch + } else { + leftItemToMerge = map[string]interface{}{rightKey: itemKey} + } + + // Handle right and full join separately to avoid duplication + if joinType == "right" && !rightProcessed[strVal] { + mergedItem := mergeItemsNew(leftItemToMerge, rightMap[strVal], rightArrayName) + result = append(result, mergedItem) + rightProcessed[strVal] = true + } else if joinType == "full" && !rightProcessed[strVal] && !alreadyProcessed[strVal] { + mergedItem := mergeItemsNew(leftItemToMerge, rightMap[strVal], rightArrayName) + result = append(result, mergedItem) + rightProcessed[strVal] = true + } + } + } + } + } + + return result, nil +} + +// use reflect sparingly to avoid performance issues +func mergeStruct(item interface{}) map[string]interface{} { + mergedItem := make(map[string]interface{}) + val := reflect.ValueOf(item) + + for i := 0; i < val.NumField(); i++ { + fieldName := val.Type().Field(i).Name + fieldValue := val.Field(i).Interface() + mergedItem[fieldName] = fieldValue + } + + return mergedItem +} + +func mergeItemsNew(leftItem interface{}, rightItems []interface{}, rightArrayName string) map[string]interface{} { + mergedItem := make(map[string]interface{}) + + switch left := leftItem.(type) { + case map[string]interface{}: + for key, value := range left { + mergedItem[key] = value + } + case nil: + // skip + default: + structFields := mergeStruct(left) + for key, value := range structFields { + mergedItem[key] = value + } + } + + if len(rightItems) > 0 { + mergedItem[rightArrayName] = rightItems + } + + return mergedItem +} diff --git a/jlib/join/join_bench_test.go b/jlib/join/join_bench_test.go new file mode 100644 index 0000000..3c3c3a8 --- /dev/null +++ b/jlib/join/join_bench_test.go @@ -0,0 +1,273 @@ +package join + +import ( + "encoding/json" + "log" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func BenchmarkOldJoin(t *testing.B) { + tests := []struct { + description string + object1 string + object2 string + joinStr1 string + joinStr2 string + joinType string + expectedOutput string + hasError bool + }{ + { + description: "one to many join on key 'id'", + object1: `[{"id":1,"age":5}]`, + object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}]`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "[{\"age\":5,\"example\":[{\"id\":1,\"name\":\"Tim\"},{\"id\":1,\"name\":\"Tam\"}],\"id\":1}]", + }, + { + description: "one to many join on key 'id' - left side not an array", + object1: `{"id":1,"age":5}`, + object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}]`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "null", + hasError: true, + }, + { + description: "one to many join on key 'id' - right side not an array", + object1: `[{"id":1,"age":5}]`, + object2: `{"id":1,"name":"Tim"}`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "null", + hasError: true, + }, + { + description: "one to many join on key 'id' - has a nested different type - should ignore", + object1: `[{"id":1,"age":5}]`, + object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}, ["1", "2"]]`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "[{\"age\":5,\"example\":[{\"id\":1,\"name\":\"Tim\"},{\"id\":1,\"name\":\"Tam\"}],\"id\":1}]", + }, + { + description: "one to many join on key 'id' - has a nested different type - should ignore", + object1: `[{"id":1,"age":5}]`, + object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}, [{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}]]`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "[{\"age\":5,\"example\":[{\"id\":1,\"name\":\"Tim\"},{\"id\":1,\"name\":\"Tam\"}],\"id\":1}]", + }, + { + description: "one to many join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"1","Price":29.99},{"ProductID":"2","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "left", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"},{\"Price\":29.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":39.99,\"ProductID\":\"2\"}]}]", + }, + { + description: "one to many left join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"},{"ID":4,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"2","Price":29.99},{"ProductID":"2","Price":12.99},{"ProductID":"3","Price":24.99},{"ProductID":"3","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "left", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":29.99,\"ProductID\":\"2\"},{\"Price\":12.99,\"ProductID\":\"2\"}]},{\"ID\":4,\"Name\":\"Item2\"}]", + }, + { + description: "one to many right join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"},{"ID":4,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"2","Price":29.99},{"ProductID":"2","Price":12.99},{"ProductID":"3","Price":24.99},{"ProductID":"3","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "right", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":29.99,\"ProductID\":\"2\"},{\"Price\":12.99,\"ProductID\":\"2\"}]},{\"ProductID\":\"3\",\"example\":[{\"Price\":24.99,\"ProductID\":\"3\"},{\"Price\":39.99,\"ProductID\":\"3\"}]}]", + }, + { + description: "one to many full join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"},{"ID":4,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"2","Price":29.99},{"ProductID":"2","Price":12.99},{"ProductID":"3","Price":24.99},{"ProductID":"3","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "full", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":29.99,\"ProductID\":\"2\"},{\"Price\":12.99,\"ProductID\":\"2\"}]},{\"ID\":4,\"Name\":\"Item2\"},{\"ProductID\":\"3\",\"example\":[{\"Price\":24.99,\"ProductID\":\"3\"},{\"Price\":39.99,\"ProductID\":\"3\"}]}]", + }, + { + description: "one to many inner join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"},{"ID":4,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"2","Price":29.99},{"ProductID":"2","Price":12.99},{"ProductID":"3","Price":24.99},{"ProductID":"3","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "inner", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":29.99,\"ProductID\":\"2\"},{\"Price\":12.99,\"ProductID\":\"2\"}]}]", + }, + } + for _, tt := range tests { + tt := tt + + t.Run(tt.description, func(t *testing.B) { + for i := 0; i < t.N; i++ { + var o1, o2 interface{} + + err := json.Unmarshal([]byte(tt.object1), &o1) + assert.NoError(t, err) + err = json.Unmarshal([]byte(tt.object2), &o2) + assert.NoError(t, err) + + output, err := OneToManyJoin(o1, o2, tt.joinStr1, tt.joinStr2, "example", tt.joinType) + assert.Equal(t, err != nil, tt.hasError) + if err != nil { + log.Println(tt.description, "|", err) + } + + bytes, err := json.Marshal(output) + assert.NoError(t, err) + assert.Equal(t, tt.expectedOutput, string(bytes)) + } + }) + } + + log.Println("done") + time.Sleep(2 * time.Second) +} + +func BenchmarkNewJoin(t *testing.B) { + tests := []struct { + description string + object1 string + object2 string + joinStr1 string + joinStr2 string + joinType string + expectedOutput string + hasError bool + }{ + { + description: "one to many join on key 'id'", + object1: `[{"id":1,"age":5}]`, + object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}]`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "[{\"age\":5,\"example\":[{\"id\":1,\"name\":\"Tim\"},{\"id\":1,\"name\":\"Tam\"}],\"id\":1}]", + }, + { + description: "one to many join on key 'id' - left side not an array", + object1: `{"id":1,"age":5}`, + object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}]`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "null", + hasError: true, + }, + { + description: "one to many join on key 'id' - right side not an array", + object1: `[{"id":1,"age":5}]`, + object2: `{"id":1,"name":"Tim"}`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "null", + hasError: true, + }, + { + description: "one to many join on key 'id' - has a nested different type - should ignore", + object1: `[{"id":1,"age":5}]`, + object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}, ["1", "2"]]`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "[{\"age\":5,\"example\":[{\"id\":1,\"name\":\"Tim\"},{\"id\":1,\"name\":\"Tam\"}],\"id\":1}]", + }, + { + description: "one to many join on key 'id' - has a nested different type - should ignore", + object1: `[{"id":1,"age":5}]`, + object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}, [{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}]]`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "[{\"age\":5,\"example\":[{\"id\":1,\"name\":\"Tim\"},{\"id\":1,\"name\":\"Tam\"}],\"id\":1}]", + }, + { + description: "one to many join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"1","Price":29.99},{"ProductID":"2","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "left", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"},{\"Price\":29.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":39.99,\"ProductID\":\"2\"}]}]", + }, + { + description: "one to many left join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"},{"ID":4,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"2","Price":29.99},{"ProductID":"2","Price":12.99},{"ProductID":"3","Price":24.99},{"ProductID":"3","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "left", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":29.99,\"ProductID\":\"2\"},{\"Price\":12.99,\"ProductID\":\"2\"}]},{\"ID\":4,\"Name\":\"Item2\"}]", + }, + { + description: "one to many right join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"},{"ID":4,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"2","Price":29.99},{"ProductID":"2","Price":12.99},{"ProductID":"3","Price":24.99},{"ProductID":"3","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "right", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":29.99,\"ProductID\":\"2\"},{\"Price\":12.99,\"ProductID\":\"2\"}]},{\"ProductID\":\"3\",\"example\":[{\"Price\":24.99,\"ProductID\":\"3\"},{\"Price\":39.99,\"ProductID\":\"3\"}]}]", + }, + { + description: "one to many full join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"},{"ID":4,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"2","Price":29.99},{"ProductID":"2","Price":12.99},{"ProductID":"3","Price":24.99},{"ProductID":"3","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "full", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":29.99,\"ProductID\":\"2\"},{\"Price\":12.99,\"ProductID\":\"2\"}]},{\"ID\":4,\"Name\":\"Item2\"},{\"ProductID\":\"3\",\"example\":[{\"Price\":24.99,\"ProductID\":\"3\"},{\"Price\":39.99,\"ProductID\":\"3\"}]}]", + }, + { + description: "one to many inner join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"},{"ID":4,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"2","Price":29.99},{"ProductID":"2","Price":12.99},{"ProductID":"3","Price":24.99},{"ProductID":"3","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "inner", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":29.99,\"ProductID\":\"2\"},{\"Price\":12.99,\"ProductID\":\"2\"}]}]", + }, + } + for _, tt := range tests { + tt := tt + + t.Run(tt.description, func(t *testing.B) { + for i := 0; i < t.N; i++ { + var o1, o2 interface{} + + err := json.Unmarshal([]byte(tt.object1), &o1) + assert.NoError(t, err) + err = json.Unmarshal([]byte(tt.object2), &o2) + assert.NoError(t, err) + + output, err := OneToManyJoin2(o1, o2, tt.joinStr1, tt.joinStr2, "example", tt.joinType) + assert.Equal(t, err != nil, tt.hasError) + if err != nil { + log.Println(tt.description, "|", err) + } + + bytes, err := json.Marshal(output) + assert.NoError(t, err) + assert.Equal(t, tt.expectedOutput, string(bytes)) + } + }) + } +} diff --git a/jlib/join/join_test.go b/jlib/join/join_test.go new file mode 100644 index 0000000..4959c21 --- /dev/null +++ b/jlib/join/join_test.go @@ -0,0 +1,265 @@ +package join + +import ( + "encoding/json" + "log" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestOneToManyJoin(t *testing.T) { + tests := []struct { + description string + object1 string + object2 string + joinStr1 string + joinStr2 string + joinType string + expectedOutput string + hasError bool + }{ + { + description: "one to many join on key 'id'", + object1: `[{"id":1,"age":5}]`, + object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}]`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "[{\"age\":5,\"example\":[{\"id\":1,\"name\":\"Tim\"},{\"id\":1,\"name\":\"Tam\"}],\"id\":1}]", + }, + { + description: "one to many join on key 'id' - left side not an array", + object1: `{"id":1,"age":5}`, + object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}]`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "null", + hasError: true, + }, + { + description: "one to many join on key 'id' - right side not an array", + object1: `[{"id":1,"age":5}]`, + object2: `{"id":1,"name":"Tim"}`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "null", + hasError: true, + }, + { + description: "one to many join on key 'id' - has a nested different type - should ignore", + object1: `[{"id":1,"age":5}]`, + object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}, ["1", "2"]]`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "[{\"age\":5,\"example\":[{\"id\":1,\"name\":\"Tim\"},{\"id\":1,\"name\":\"Tam\"}],\"id\":1}]", + }, + { + description: "one to many join on key 'id' - has a nested different type - should ignore", + object1: `[{"id":1,"age":5}]`, + object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}, [{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}]]`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "[{\"age\":5,\"example\":[{\"id\":1,\"name\":\"Tim\"},{\"id\":1,\"name\":\"Tam\"}],\"id\":1}]", + }, + { + description: "one to many join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"1","Price":29.99},{"ProductID":"2","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "left", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"},{\"Price\":29.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":39.99,\"ProductID\":\"2\"}]}]", + }, + { + description: "one to many left join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"},{"ID":4,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"2","Price":29.99},{"ProductID":"2","Price":12.99},{"ProductID":"3","Price":24.99},{"ProductID":"3","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "left", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":29.99,\"ProductID\":\"2\"},{\"Price\":12.99,\"ProductID\":\"2\"}]},{\"ID\":4,\"Name\":\"Item2\"}]", + }, + { + description: "one to many right join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"},{"ID":4,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"2","Price":29.99},{"ProductID":"2","Price":12.99},{"ProductID":"3","Price":24.99},{"ProductID":"3","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "right", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":29.99,\"ProductID\":\"2\"},{\"Price\":12.99,\"ProductID\":\"2\"}]},{\"ProductID\":\"3\",\"example\":[{\"Price\":24.99,\"ProductID\":\"3\"},{\"Price\":39.99,\"ProductID\":\"3\"}]}]", + }, + { + description: "one to many full join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"},{"ID":4,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"2","Price":29.99},{"ProductID":"2","Price":12.99},{"ProductID":"3","Price":24.99},{"ProductID":"3","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "full", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":29.99,\"ProductID\":\"2\"},{\"Price\":12.99,\"ProductID\":\"2\"}]},{\"ID\":4,\"Name\":\"Item2\"},{\"ProductID\":\"3\",\"example\":[{\"Price\":24.99,\"ProductID\":\"3\"},{\"Price\":39.99,\"ProductID\":\"3\"}]}]", + }, + { + description: "one to many inner join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"},{"ID":4,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"2","Price":29.99},{"ProductID":"2","Price":12.99},{"ProductID":"3","Price":24.99},{"ProductID":"3","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "inner", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":29.99,\"ProductID\":\"2\"},{\"Price\":12.99,\"ProductID\":\"2\"}]}]", + }, + } + for _, tt := range tests { + tt := tt + + t.Run(tt.description, func(t *testing.T) { + var o1, o2 interface{} + + err := json.Unmarshal([]byte(tt.object1), &o1) + assert.NoError(t, err) + err = json.Unmarshal([]byte(tt.object2), &o2) + assert.NoError(t, err) + + output, err := OneToManyJoin(o1, o2, tt.joinStr1, tt.joinStr2, "example", tt.joinType) + assert.Equal(t, err != nil, tt.hasError) + if err != nil { + log.Println(tt.description, "|", err) + } + + bytes, err := json.Marshal(output) + assert.NoError(t, err) + assert.Equal(t, tt.expectedOutput, string(bytes)) + }) + } +} + +func TestOneToManyJoinNew(t *testing.T) { + tests := []struct { + description string + object1 string + object2 string + joinStr1 string + joinStr2 string + joinType string + expectedOutput string + hasError bool + }{ + { + description: "one to many join on key 'id'", + object1: `[{"id":1,"age":5}]`, + object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}]`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "[{\"age\":5,\"example\":[{\"id\":1,\"name\":\"Tim\"},{\"id\":1,\"name\":\"Tam\"}],\"id\":1}]", + }, + { + description: "one to many join on key 'id' - left side not an array", + object1: `{"id":1,"age":5}`, + object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}]`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "null", + hasError: true, + }, + { + description: "one to many join on key 'id' - right side not an array", + object1: `[{"id":1,"age":5}]`, + object2: `{"id":1,"name":"Tim"}`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "null", + hasError: true, + }, + { + description: "one to many join on key 'id' - has a nested different type - should ignore", + object1: `[{"id":1,"age":5}]`, + object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}, ["1", "2"]]`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "[{\"age\":5,\"example\":[{\"id\":1,\"name\":\"Tim\"},{\"id\":1,\"name\":\"Tam\"}],\"id\":1}]", + }, + { + description: "one to many join on key 'id' - has a nested different type - should ignore", + object1: `[{"id":1,"age":5}]`, + object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}, [{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}]]`, + joinStr1: "id", + joinStr2: "id", + joinType: "left", + expectedOutput: "[{\"age\":5,\"example\":[{\"id\":1,\"name\":\"Tim\"},{\"id\":1,\"name\":\"Tam\"}],\"id\":1}]", + }, + { + description: "one to many join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"1","Price":29.99},{"ProductID":"2","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "left", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"},{\"Price\":29.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":39.99,\"ProductID\":\"2\"}]}]", + }, + { + description: "one to many left join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"},{"ID":4,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"2","Price":29.99},{"ProductID":"2","Price":12.99},{"ProductID":"3","Price":24.99},{"ProductID":"3","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "left", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":29.99,\"ProductID\":\"2\"},{\"Price\":12.99,\"ProductID\":\"2\"}]},{\"ID\":4,\"Name\":\"Item2\"}]", + }, + { + description: "one to many right join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"},{"ID":4,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"2","Price":29.99},{"ProductID":"2","Price":12.99},{"ProductID":"3","Price":24.99},{"ProductID":"3","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "right", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":29.99,\"ProductID\":\"2\"},{\"Price\":12.99,\"ProductID\":\"2\"}]},{\"ProductID\":\"3\",\"example\":[{\"Price\":24.99,\"ProductID\":\"3\"},{\"Price\":39.99,\"ProductID\":\"3\"}]}]", + }, + { + description: "one to many full join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"},{"ID":4,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"2","Price":29.99},{"ProductID":"2","Price":12.99},{"ProductID":"3","Price":24.99},{"ProductID":"3","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "full", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":29.99,\"ProductID\":\"2\"},{\"Price\":12.99,\"ProductID\":\"2\"}]},{\"ID\":4,\"Name\":\"Item2\"},{\"ProductID\":\"3\",\"example\":[{\"Price\":24.99,\"ProductID\":\"3\"},{\"Price\":39.99,\"ProductID\":\"3\"}]}]", + }, + { + description: "one to many inner join - complex", + object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"},{"ID":4,"Name":"Item2"}]`, + object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"2","Price":29.99},{"ProductID":"2","Price":12.99},{"ProductID":"3","Price":24.99},{"ProductID":"3","Price":39.99}]`, + joinStr1: "ID", + joinStr2: "ProductID", + joinType: "inner", + expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":29.99,\"ProductID\":\"2\"},{\"Price\":12.99,\"ProductID\":\"2\"}]}]", + }, + } + for _, tt := range tests { + tt := tt + + t.Run(tt.description, func(t *testing.T) { + var o1, o2 interface{} + + err := json.Unmarshal([]byte(tt.object1), &o1) + assert.NoError(t, err) + err = json.Unmarshal([]byte(tt.object2), &o2) + assert.NoError(t, err) + + output, err := OneToManyJoin2(o1, o2, tt.joinStr1, tt.joinStr2, "example", tt.joinType) + assert.Equal(t, err != nil, tt.hasError) + if err != nil { + log.Println(tt.description, "|", err) + } + + bytes, err := json.Marshal(output) + assert.NoError(t, err) + assert.Equal(t, tt.expectedOutput, string(bytes)) + }) + } +} diff --git a/jlib/new.go b/jlib/new.go index 75bc5dd..aaaceee 100644 --- a/jlib/new.go +++ b/jlib/new.go @@ -277,85 +277,3 @@ func ObjectsToDocument(input interface{}) (interface{}, error) { return output, nil // Return the output map } - -func mergeItems(leftItem interface{}, rightItems []interface{}, rightArrayName string) map[string]interface{} { - mergedItem := make(map[string]interface{}) - - // Check if leftItem is a map or a struct and merge accordingly - leftVal := reflect.ValueOf(leftItem) - if leftVal.Kind() == reflect.Map { - // Merge fields from the map - for _, key := range leftVal.MapKeys() { - mergedItem[key.String()] = leftVal.MapIndex(key).Interface() - } - } else { - // Merge fields from the struct - leftType := leftVal.Type() - for i := 0; i < leftVal.NumField(); i++ { - fieldName := leftType.Field(i).Name - fieldValue := leftVal.Field(i).Interface() - mergedItem[fieldName] = fieldValue - } - } - - // If there are matching items in the right array, add them under the specified name - if len(rightItems) > 0 { - mergedItem[rightArrayName] = rightItems - } - - return mergedItem -} - -func OneToManyJoin(leftArr, rightArr interface{}, leftKey, rightKey, rightArrayName string) (interface{}, error) { - trueLeftArr, ok := leftArr.([]interface{}) - if !ok { - return nil, errors.New("left input must be an array of Objects") - } - - trueRightArr, ok := rightArr.([]interface{}) - if !ok { - return nil, errors.New("right input must be an array of Objects") - } - - // Create a map for faster lookup of rightArr elements based on the key - rightMap := make(map[string][]interface{}) - for _, item := range trueRightArr { - var val interface{} - // Check if leftItem is a map or a struct and get the key value accordingly - itemMap, ok := item.(map[string]interface{}) - if ok { - itemKey, ok := itemMap[rightKey] - if ok { - val = itemKey - } - } - // Convert the key value to a string and associate it with the item in the map - strVal := fmt.Sprintf("%v", val) - rightMap[strVal] = append(rightMap[strVal], item) - } - - // Create a slice to store the merged results - var result []map[string]interface{} - - // Iterate through the left array and perform the join - for _, leftItem := range trueLeftArr { - var leftVal interface{} - // Check if leftItem is a map or a struct and get the key value accordingly - itemMap, ok := leftItem.(map[string]interface{}) - if ok { - itemKey, ok := itemMap[leftKey] - if ok { - leftVal = itemKey - } - } - // Convert the key value to a string - strVal := fmt.Sprintf("%v", leftVal) - rightItems := rightMap[strVal] - - // Merge the left and right items - mergedItem := mergeItems(leftItem, rightItems, rightArrayName) - result = append(result, mergedItem) - } - - return result, nil -} diff --git a/jlib/new_test.go b/jlib/new_test.go index 0140f3e..a6c98c7 100644 --- a/jlib/new_test.go +++ b/jlib/new_test.go @@ -2,7 +2,6 @@ package jlib import ( "encoding/json" - "log" "reflect" "testing" @@ -76,88 +75,3 @@ func TestSJoin(t *testing.T) { }) } } - -func TestOneToManyJoin(t *testing.T) { - tests := []struct { - description string - object1 string - object2 string - joinStr1 string - joinStr2 string - expectedOutput string - hasError bool - }{ - { - description: "one to many join on key 'id'", - object1: `[{"id":1,"age":5}]`, - object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}]`, - joinStr1: "id", - joinStr2: "id", - expectedOutput: "[{\"age\":5,\"example\":[{\"id\":1,\"name\":\"Tim\"},{\"id\":1,\"name\":\"Tam\"}],\"id\":1}]", - }, - { - description: "one to many join on key 'id' - left side not an array", - object1: `{"id":1,"age":5}`, - object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}]`, - joinStr1: "id", - joinStr2: "id", - expectedOutput: "null", - hasError: true, - }, - { - description: "one to many join on key 'id' - right side not an array", - object1: `[{"id":1,"age":5}]`, - object2: `{"id":1,"name":"Tim"}`, - joinStr1: "id", - joinStr2: "id", - expectedOutput: "null", - hasError: true, - }, - { - description: "one to many join on key 'id' - has a nested different type - should ignore", - object1: `[{"id":1,"age":5}]`, - object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}, ["1", "2"]]`, - joinStr1: "id", - joinStr2: "id", - expectedOutput: "[{\"age\":5,\"example\":[{\"id\":1,\"name\":\"Tim\"},{\"id\":1,\"name\":\"Tam\"}],\"id\":1}]", - }, - { - description: "one to many join on key 'id' - has a nested different type - should ignore", - object1: `[{"id":1,"age":5}]`, - object2: `[{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}, [{"id":1,"name":"Tim"},{"id":1,"name":"Tam"}]]`, - joinStr1: "id", - joinStr2: "id", - expectedOutput: "[{\"age\":5,\"example\":[{\"id\":1,\"name\":\"Tim\"},{\"id\":1,\"name\":\"Tam\"}],\"id\":1}]", - }, - { - description: "one to many join - complex", - object1: `[{"ID":1,"Name":"Item1"},{"ID":2,"Name":"Item2"}]`, - object2: `[{"ProductID":"1","Price":19.99},{"ProductID":"1","Price":29.99},{"ProductID":"2","Price":39.99}]`, - joinStr1: "ID", - joinStr2: "ProductID", - expectedOutput: "[{\"ID\":1,\"Name\":\"Item1\",\"example\":[{\"Price\":19.99,\"ProductID\":\"1\"},{\"Price\":29.99,\"ProductID\":\"1\"}]},{\"ID\":2,\"Name\":\"Item2\",\"example\":[{\"Price\":39.99,\"ProductID\":\"2\"}]}]", - }, - } - for _, tt := range tests { - tt := tt - - t.Run(tt.description, func(t *testing.T) { - var o1, o2 interface{} - - err := json.Unmarshal([]byte(tt.object1), &o1) - assert.NoError(t, err) - err = json.Unmarshal([]byte(tt.object2), &o2) - assert.NoError(t, err) - - output, err := OneToManyJoin(o1, o2, tt.joinStr1, tt.joinStr2, "example") - assert.Equal(t, err != nil, tt.hasError) - if err != nil { - log.Println(tt.description, "|", err) - } - - bytes, err := json.Marshal(output) - assert.NoError(t, err) - assert.Equal(t, tt.expectedOutput, string(bytes)) - }) - } -} diff --git a/jlib/timeparse/timeparse_test.go b/jlib/timeparse/timeparse_test.go index d5ec477..7f4e33a 100644 --- a/jlib/timeparse/timeparse_test.go +++ b/jlib/timeparse/timeparse_test.go @@ -11,11 +11,11 @@ import ( ) type TestCase struct { - TestDesc string `json:"testDesc"` - InputSrcTs string `json:"input_srcTs"` - InputSrcFormat string `json:"input_srcFormat"` - InputSrcTz string `json:"input_srcTz"` - OutputSrcTz string `json:"output_srcTz"` + TestDesc string `json:"testDesc"` + InputSrcTs string `json:"input_srcTs"` + InputSrcFormat string `json:"input_srcFormat"` + InputSrcTz string `json:"input_srcTz"` + OutputSrcTz string `json:"output_srcTz"` DateDim jsonatatime.DateDim `json:"DateDim"` } @@ -62,11 +62,11 @@ func TestTime(t *testing.T) { } type TestCaseLite struct { - TestDesc string `json:"testDesc"` - InputSrcTs string `json:"input_srcTs"` - InputSrcFormat string `json:"input_srcFormat"` - InputSrcTz string `json:"input_srcTz"` - OutputSrcTz string `json:"output_srcTz"` + TestDesc string `json:"testDesc"` + InputSrcTs string `json:"input_srcTs"` + InputSrcFormat string `json:"input_srcFormat"` + InputSrcTz string `json:"input_srcTz"` + OutputSrcTz string `json:"output_srcTz"` DateDim jsonatatime.DateDimLite `json:"DateDim"` } From 834811cd2ace08372cc7001e02769ff9b1cffcd0 Mon Sep 17 00:00:00 2001 From: Tom <53711814+tbal999@users.noreply.github.com> Date: Wed, 29 Nov 2023 10:59:13 +0000 Subject: [PATCH 43/58] fix offset (#26) Co-authored-by: tbal999 --- jlib/timeparse/testdata.json | 10 +++++----- jlib/timeparse/testdata_lite.json | 10 +++++----- jlib/timeparse/timeparse.go | 18 ++---------------- jlib/timeparse/timeparselite.go | 3 ++- 4 files changed, 14 insertions(+), 27 deletions(-) diff --git a/jlib/timeparse/testdata.json b/jlib/timeparse/testdata.json index e40444e..f97c929 100644 --- a/jlib/timeparse/testdata.json +++ b/jlib/timeparse/testdata.json @@ -35,7 +35,7 @@ "output_srcTz": "Europe/London", "DateDim": { "TimeZone": "Europe/London", - "TimeZoneOffset": "+00:00", + "TimeZoneOffset": "+01:00", "YearMonth": 202308, "YearWeek": 202330, "YearIsoWeek": 202331, @@ -63,7 +63,7 @@ "output_srcTz": "Europe/London", "DateDim": { "TimeZone": "Europe/London", - "TimeZoneOffset": "+00:00", + "TimeZoneOffset": "+01:00", "YearMonth": 202308, "YearWeek": 202330, "YearIsoWeek": 202331, @@ -91,7 +91,7 @@ "output_srcTz": "Europe/London", "DateDim": { "TimeZone": "Europe/London", - "TimeZoneOffset": "+00:00", + "TimeZoneOffset": "+01:00", "YearMonth": 202308, "YearWeek": 202330, "YearIsoWeek": 202331, @@ -119,7 +119,7 @@ "output_srcTz": "America/New_York", "DateDim": { "TimeZone": "America/New_York", - "TimeZoneOffset": "+00:00", + "TimeZoneOffset": "-04:00", "YearMonth": 202308, "YearWeek": 202330, "YearIsoWeek": 202331, @@ -147,7 +147,7 @@ "output_srcTz": "Europe/London", "DateDim": { "TimeZone": "Europe/London", - "TimeZoneOffset": "+00:00", + "TimeZoneOffset": "+01:00", "YearMonth": 202308, "YearWeek": 202330, "YearIsoWeek": 202331, diff --git a/jlib/timeparse/testdata_lite.json b/jlib/timeparse/testdata_lite.json index 71d5a53..455db4c 100644 --- a/jlib/timeparse/testdata_lite.json +++ b/jlib/timeparse/testdata_lite.json @@ -27,7 +27,7 @@ "output_srcTz": "Europe/London", "DateDim": { "TimeZone": "Europe/London", - "TimeZoneOffset": "+00:00", + "TimeZoneOffset": "+01:00", "DateId": "Dates_20230806", "DateKey": 20230806, "DateTimeKey": 20230806012341000, @@ -47,7 +47,7 @@ "output_srcTz": "Europe/London", "DateDim": { "TimeZone": "Europe/London", - "TimeZoneOffset": "+00:00", + "TimeZoneOffset": "+01:00", "DateId": "Dates_20230806", "DateKey": 20230806, "DateTimeKey": 20230806002341000, @@ -67,7 +67,7 @@ "output_srcTz": "Europe/London", "DateDim": { "TimeZone": "Europe/London", - "TimeZoneOffset": "+00:00", + "TimeZoneOffset": "+01:00", "DateId": "Dates_20230806", "DateKey": 20230806, "DateTimeKey": 20230806002341000, @@ -87,7 +87,7 @@ "output_srcTz": "America/New_York", "DateDim": { "TimeZone": "America/New_York", - "TimeZoneOffset": "+00:00", + "TimeZoneOffset": "-04:00", "DateId": "Dates_20230805", "DateKey": 20230805, "DateTimeKey": 20230805192341000, @@ -107,7 +107,7 @@ "output_srcTz": "Europe/London", "DateDim": { "TimeZone": "Europe/London", - "TimeZoneOffset": "+00:00", + "TimeZoneOffset": "+01:00", "DateId": "Dates_20230806", "DateKey": 20230806, "DateTimeKey": 20230806002341000, diff --git a/jlib/timeparse/timeparse.go b/jlib/timeparse/timeparse.go index e8d7372..e40af54 100644 --- a/jlib/timeparse/timeparse.go +++ b/jlib/timeparse/timeparse.go @@ -3,7 +3,6 @@ package timeparse import ( "fmt" "strconv" - "strings" "time" ) @@ -109,10 +108,11 @@ func TimeDateDimensions(inputSrcTs, inputSrcFormat, inputSrcTz, requiredTz strin } localTimeStamp := localTime.Format("2006-01-02T15:04:05.000-07:00") + offsetStr := localTime.Format("-07:00") // construct the date dimension structure dateDim := &DateDim{ RawValue: inputSrcTs, - TimeZoneOffset: getOffsetString(localTimeStamp), + TimeZoneOffset: offsetStr, YearWeek: mondayWeek, YearDay: yearDay, YearIsoWeek: yearIsoWeekInt, @@ -135,20 +135,6 @@ func TimeDateDimensions(inputSrcTs, inputSrcFormat, inputSrcTz, requiredTz strin return dateDim, nil } -func getOffsetString(input string) string { - znegArr := strings.Split(input, "Z-") - if len(znegArr) == 2 { - return "-" + znegArr[1] - } - - zposArr := strings.Split(input, "Z+") - if len(zposArr) == 2 { - return "+" + zposArr[1] - } - - return "+00:00" -} - func getWeekOfYearString(date time.Time) (int, error) { _, week := date.ISOWeek() diff --git a/jlib/timeparse/timeparselite.go b/jlib/timeparse/timeparselite.go index 9150d91..dd4318a 100644 --- a/jlib/timeparse/timeparselite.go +++ b/jlib/timeparse/timeparselite.go @@ -84,10 +84,11 @@ func TimeDateDimensionsLite(inputSrcTs, inputSrcFormat, inputSrcTz, requiredTz s } localTimeStamp := localTime.Format("2006-01-02T15:04:05.000-07:00") + offsetStr := localTime.Format("-07:00") // construct the date dimension structure dateDim := &DateDimLite{ RawValue: inputSrcTs, - TimeZoneOffset: getOffsetString(localTimeStamp), + TimeZoneOffset: offsetStr, Millis: int(localTime.UnixMilli()), DateLocal: localTime.Format("2006-01-02"), TimeZone: localTime.Location().String(), From d1badc053c7dc0995587fa765d4ea995d85bd794 Mon Sep 17 00:00:00 2001 From: JamesXiatech <66721487+JamesXiatech@users.noreply.github.com> Date: Tue, 12 Dec 2023 10:17:14 +0000 Subject: [PATCH 44/58] Rename keys (#29) * Added new function to rename object keys * Added unit test for RenameKeys --- env.go | 6 ++++ jlib/new.go | 77 ++++++++++++++++++++++++++++++++++++++++++++++++ jlib/new_test.go | 45 ++++++++++++++++++++++++++++ 3 files changed, 128 insertions(+) diff --git a/env.go b/env.go index dd84e14..e9aac55 100644 --- a/env.go +++ b/env.go @@ -138,6 +138,12 @@ var baseEnv = initBaseEnv(map[string]Extension{ EvalContextHandler: nil, }, + "renameKeys": { + Func: jlib.RenameKeys, + UndefinedHandler: defaultUndefinedHandler, + EvalContextHandler: nil, + }, + /* EXTENDED END */ diff --git a/jlib/new.go b/jlib/new.go index aaaceee..914fad1 100644 --- a/jlib/new.go +++ b/jlib/new.go @@ -277,3 +277,80 @@ func ObjectsToDocument(input interface{}) (interface{}, error) { return output, nil // Return the output map } + +// TransformRule defines a transformation rule with a search substring and a new name. +type TransformRule struct { + SearchSubstring string + NewName string +} + +// RenameKeys applies a series of transformations to the keys in a JSON-compatible data structure. +// 'data' is the original data where keys need to be transformed. +// 'rulesInterface' is expected to be a slice of interface{}, where each element is a slice containing two strings: +// the substring to search for in the keys, and the new name to replace the key with. +func RenameKeys(data interface{}, rulesInterface interface{}) (interface{}, error) { + // Attempt to assert rulesInterface as a slice of interface{} + rulesRaw, ok := rulesInterface.([]interface{}) + if !ok { + return nil, fmt.Errorf("rules must be a slice of interface{}") + } + + // Process each rule, converting it into a TransformRule + var rules []TransformRule + for _, r := range rulesRaw { + rule, ok := r.([]interface{}) + if !ok || len(rule) != 2 { + return nil, fmt.Errorf("each rule must be an array of two strings") + } + + searchSubstring, ok1 := rule[0].(string) + newName, ok2 := rule[1].(string) + if !ok1 || !ok2 { + return nil, fmt.Errorf("each rule must be an array of two strings") + } + + rules = append(rules, TransformRule{SearchSubstring: searchSubstring, NewName: newName}) + } + + // Marshal the original data into JSON + jsonData, err := json.Marshal(data) + if err != nil { + return nil, err + } + + // Unmarshal the JSON into a map for easy manipulation + var mapData map[string]interface{} + err = json.Unmarshal(jsonData, &mapData) + if err != nil { + return nil, err + } + + // Create a new map to store the modified data + newData := make(map[string]interface{}) + for key, value := range mapData { + newKey := key // Default to the original key + // Apply transformation rules + for _, rule := range rules { + if strings.Contains(key, rule.SearchSubstring) { + newKey = rule.NewName // Update the key if rule matches + break + } + } + newData[newKey] = value // Store the value with the new key + } + + // Re-marshal to JSON to maintain the same data type as the input + resultJSON, err := json.Marshal(newData) + if err != nil { + return nil, err + } + + // Unmarshal the JSON back into an interface{} for the return value + var result interface{} + err = json.Unmarshal(resultJSON, &result) + if err != nil { + return nil, err + } + + return result, nil +} diff --git a/jlib/new_test.go b/jlib/new_test.go index a6c98c7..9d26230 100644 --- a/jlib/new_test.go +++ b/jlib/new_test.go @@ -75,3 +75,48 @@ func TestSJoin(t *testing.T) { }) } } + +func TestRenameKeys(t *testing.T) { + tests := []struct { + description string + object1 string + object2 string + expectedOutput string + }{ + { + description: "Rename Keys", + object1: `{ + "itemLineId": "1", + "unitPrice": 104.5, + "percentageDiscountValue": 5, + "discountedLinePrice": 104.5, + "name": "LD Wrong Price", + "discountAmount": 5.5, + "discountType": "AMOUNT", + "discountReasonCode": "9901", + "discountReasonName": "LD Wrong Price" + }`, + object2: `[["ReasonCode","reasonCode"],["ReasonName","reasonName"],["DiscountValue","value"],["Amount","amount"],["Type","type"],["LinePrice","linePrice"]]`, + expectedOutput: "{\"amount\":5.5,\"itemLineId\":\"1\",\"linePrice\":104.5,\"name\":\"LD Wrong Price\",\"reasonCode\":\"9901\",\"reasonName\":\"LD Wrong Price\",\"type\":\"AMOUNT\",\"unitPrice\":104.5,\"value\":5}", + }, + } + for _, tt := range tests { + tt := tt + + t.Run(tt.description, func(t *testing.T) { + var o1, o2 interface{} + + err := json.Unmarshal([]byte(tt.object1), &o1) + assert.NoError(t, err) + err = json.Unmarshal([]byte(tt.object2), &o2) + assert.NoError(t, err) + + output, err := RenameKeys(o1, o2) + assert.NoError(t, err) + + bytes, err := json.Marshal(output) + assert.NoError(t, err) + assert.Equal(t, tt.expectedOutput, string(bytes)) + }) + } +} From 933f57f6507d7111eeb054cb9af7eb798e705760 Mon Sep 17 00:00:00 2001 From: Tom <53711814+tbal999@users.noreply.github.com> Date: Tue, 12 Dec 2023 11:21:29 +0000 Subject: [PATCH 45/58] fix bug (#30) Co-authored-by: tbal999 --- eval.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eval.go b/eval.go index 6a88072..6141596 100644 --- a/eval.go +++ b/eval.go @@ -925,7 +925,9 @@ func updateError(err error, node *jparse.FunctionCallNode, stringArgs string) er func transformArgsToString(argv []reflect.Value) string { argvString := "" for i, value := range argv { - argvString += fmt.Sprintf("number:%v value:%v ", i, value.Interface()) + if value.IsValid() && value.CanInterface() { + argvString += fmt.Sprintf("number:%v value:%v ", i, value.Interface()) + } } return argvString } From 5c4e36c009047174f27e2323ee8b49e530b3536a Mon Sep 17 00:00:00 2001 From: Tom <53711814+tbal999@users.noreply.github.com> Date: Wed, 13 Dec 2023 11:35:53 +0000 Subject: [PATCH 46/58] add fold function & goccy (#31) * add fold function logic * replace with goccy * test --------- Co-authored-by: tbal999 --- README.md | 2 +- callable.go | 2 +- errrors_test.go | 2 +- example_eval_test.go | 2 +- go.mod | 1 + go.sum | 2 ++ jlib/fold.go | 19 +++++++++++++++ jlib/fold_test.go | 40 ++++++++++++++++++++++++++++++++ jlib/join/join_bench_test.go | 2 +- jlib/join/join_test.go | 2 +- jlib/new.go | 2 +- jlib/new_test.go | 2 +- jlib/string.go | 2 +- jlib/timeparse/timeparse_test.go | 2 +- jsonata-server/bench.go | 2 +- jsonata-server/main.go | 2 +- jsonata-test/main.go | 2 +- jsonata_test.go | 4 ++-- 18 files changed, 77 insertions(+), 15 deletions(-) create mode 100644 jlib/fold.go create mode 100644 jlib/fold_test.go diff --git a/README.md b/README.md index 7e4d61c..c60d015 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ It currently has feature parity with jsonata-js 1.5.4. As well as a most of the ```Go import ( - "encoding/json" + "github.com/goccy/go-json" "fmt" "log" diff --git a/callable.go b/callable.go index ad7763c..b54c82f 100644 --- a/callable.go +++ b/callable.go @@ -5,7 +5,7 @@ package jsonata import ( - "encoding/json" + "github.com/goccy/go-json" "fmt" "reflect" "regexp" diff --git a/errrors_test.go b/errrors_test.go index 90e2c74..8b23059 100644 --- a/errrors_test.go +++ b/errrors_test.go @@ -1,7 +1,7 @@ package jsonata import ( - "encoding/json" + "github.com/goccy/go-json" "testing" "github.com/stretchr/testify/assert" diff --git a/example_eval_test.go b/example_eval_test.go index fc24a41..d713791 100644 --- a/example_eval_test.go +++ b/example_eval_test.go @@ -5,7 +5,7 @@ package jsonata_test import ( - "encoding/json" + "github.com/goccy/go-json" "fmt" "log" diff --git a/go.mod b/go.mod index 3ac84f7..8666435 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/xiatechs/jsonata-go go 1.20 require ( + github.com/goccy/go-json v0.10.2 github.com/shopspring/decimal v1.3.1 github.com/stretchr/testify v1.8.0 ) diff --git a/go.sum b/go.sum index ed1f018..fcf2da5 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= diff --git a/jlib/fold.go b/jlib/fold.go new file mode 100644 index 0000000..112824c --- /dev/null +++ b/jlib/fold.go @@ -0,0 +1,19 @@ +package jlib + +import "errors" + +func FoldArray(input interface{}) ([][]interface{}, error) { + inputSlice, ok := input.([]interface{}) + if !ok { + return nil, errors.New("input for $foldarray was not an []interface type") + } + + result := make([][]interface{}, len(inputSlice)) + + for i := range inputSlice { + result[i] = make([]interface{}, i+1) + copy(result[i], inputSlice[:i+1]) + } + + return result, nil +} diff --git a/jlib/fold_test.go b/jlib/fold_test.go new file mode 100644 index 0000000..ab236c5 --- /dev/null +++ b/jlib/fold_test.go @@ -0,0 +1,40 @@ +package jlib + +import ( + "github.com/goccy/go-json" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestFold(t *testing.T) { + t.Run("fold a []interface", func(t *testing.T) { + input := []interface{}{ + map[string]interface{}{"Amount": 8}, + map[string]interface{}{"Amount": -1}, + map[string]interface{}{"Amount": 0}, + map[string]interface{}{"Amount": -3}, + map[string]interface{}{"Amount": -4}, + } + + output, err := FoldArray(input) + assert.NoError(t, err) + + outputJSON, err := json.Marshal(output) + assert.NoError(t, err) + + assert.Equal(t, string(outputJSON), `[[{"Amount":8}],[{"Amount":8},{"Amount":-1}],[{"Amount":8},{"Amount":-1},{"Amount":0}],[{"Amount":8},{"Amount":-1},{"Amount":0},{"Amount":-3}],[{"Amount":8},{"Amount":-1},{"Amount":0},{"Amount":-3},{"Amount":-4}]]`) + }) + + t.Run("fold - not an []interface{}", func(t *testing.T) { + input := "testing" + + output, err := FoldArray(input) + assert.Error(t, err) + + outputJSON, err := json.Marshal(output) + assert.NoError(t, err) + + assert.Equal(t, string(outputJSON), `null`) + }) +} diff --git a/jlib/join/join_bench_test.go b/jlib/join/join_bench_test.go index 3c3c3a8..fbb36f1 100644 --- a/jlib/join/join_bench_test.go +++ b/jlib/join/join_bench_test.go @@ -1,7 +1,7 @@ package join import ( - "encoding/json" + "github.com/goccy/go-json" "log" "testing" "time" diff --git a/jlib/join/join_test.go b/jlib/join/join_test.go index 4959c21..6a9a953 100644 --- a/jlib/join/join_test.go +++ b/jlib/join/join_test.go @@ -1,7 +1,7 @@ package join import ( - "encoding/json" + "github.com/goccy/go-json" "log" "testing" diff --git a/jlib/new.go b/jlib/new.go index 914fad1..b4fadf2 100644 --- a/jlib/new.go +++ b/jlib/new.go @@ -1,7 +1,7 @@ package jlib import ( - "encoding/json" + "github.com/goccy/go-json" "errors" "fmt" "reflect" diff --git a/jlib/new_test.go b/jlib/new_test.go index 9d26230..9c70e12 100644 --- a/jlib/new_test.go +++ b/jlib/new_test.go @@ -1,7 +1,7 @@ package jlib import ( - "encoding/json" + "github.com/goccy/go-json" "reflect" "testing" diff --git a/jlib/string.go b/jlib/string.go index 225552f..2f63deb 100644 --- a/jlib/string.go +++ b/jlib/string.go @@ -7,7 +7,7 @@ package jlib import ( "bytes" "encoding/base64" - "encoding/json" + "github.com/goccy/go-json" "fmt" "math" "net/url" diff --git a/jlib/timeparse/timeparse_test.go b/jlib/timeparse/timeparse_test.go index 7f4e33a..f3e4be2 100644 --- a/jlib/timeparse/timeparse_test.go +++ b/jlib/timeparse/timeparse_test.go @@ -1,7 +1,7 @@ package timeparse_test import ( - "encoding/json" + "github.com/goccy/go-json" "os" "testing" diff --git a/jsonata-server/bench.go b/jsonata-server/bench.go index 1a0a19f..64dbf6c 100644 --- a/jsonata-server/bench.go +++ b/jsonata-server/bench.go @@ -8,7 +8,7 @@ import ( "log" "net/http" - "encoding/json" + "github.com/goccy/go-json" jsonata "github.com/xiatechs/jsonata-go" ) diff --git a/jsonata-server/main.go b/jsonata-server/main.go index 33f3098..ad42c53 100644 --- a/jsonata-server/main.go +++ b/jsonata-server/main.go @@ -6,7 +6,7 @@ package main import ( "bytes" - "encoding/json" + "github.com/goccy/go-json" "flag" "fmt" "log" diff --git a/jsonata-test/main.go b/jsonata-test/main.go index c77f462..de768bf 100644 --- a/jsonata-test/main.go +++ b/jsonata-test/main.go @@ -1,7 +1,7 @@ package main import ( - "encoding/json" + "github.com/goccy/go-json" "flag" "fmt" "io" diff --git a/jsonata_test.go b/jsonata_test.go index d11c08c..687a8dc 100644 --- a/jsonata_test.go +++ b/jsonata_test.go @@ -5,7 +5,7 @@ package jsonata import ( - "encoding/json" + "github.com/goccy/go-json" "errors" "fmt" "io/ioutil" @@ -5099,7 +5099,7 @@ func TestFuncString(t *testing.T) { }, { Expression: `$string(1e-7)`, - Output: "1e-7", + Output: "1e-07", }, { Expression: `$string(1e+20)`, From 353b4d4e9d76749194ddfc31400e2a868ed1fd82 Mon Sep 17 00:00:00 2001 From: Tom <53711814+tbal999@users.noreply.github.com> Date: Wed, 13 Dec 2023 15:11:13 +0000 Subject: [PATCH 47/58] Update env.go --- env.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/env.go b/env.go index e9aac55..c2847da 100644 --- a/env.go +++ b/env.go @@ -138,6 +138,12 @@ var baseEnv = initBaseEnv(map[string]Extension{ EvalContextHandler: nil, }, + "accumulatingSlice": { + Func: jlib.FoldArray, + UndefinedHandler: defaultUndefinedHandler, + EvalContextHandler: nil, + }, + "renameKeys": { Func: jlib.RenameKeys, UndefinedHandler: defaultUndefinedHandler, From 98098e0fc0c5005ab34a9c96547860452e55b4a2 Mon Sep 17 00:00:00 2001 From: Tom <53711814+tbal999@users.noreply.github.com> Date: Wed, 20 Mar 2024 09:37:03 +0000 Subject: [PATCH 48/58] fix bug - 000 end to timestamp (#33) * remove the 000 timestmp * remove wednesday checking --------- Co-authored-by: tbal999 --- jlib/timeparse/testdata.json | 18 +++---- jlib/timeparse/testdata_lite.json | 84 +++++++++++++++++++++++++------ jlib/timeparse/timeparse.go | 12 ++--- jlib/timeparse/timeparse_test.go | 3 +- jlib/timeparse/timeparselite.go | 5 +- 5 files changed, 90 insertions(+), 32 deletions(-) diff --git a/jlib/timeparse/testdata.json b/jlib/timeparse/testdata.json index f97c929..c239026 100644 --- a/jlib/timeparse/testdata.json +++ b/jlib/timeparse/testdata.json @@ -30,7 +30,7 @@ { "testDesc": "/* source explicit UTC, Local = 2023-08-06T00:23:41.454Z+01:00 */", "input_srcTs": "2023-08-06T00:23:41.454Z", - "input_srcFormat": "2006-01-02T15:04:05Z", + "input_srcFormat": "2006-01-02T15:04:05.000Z", "input_srcTz": "UTC", "output_srcTz": "Europe/London", "DateDim": { @@ -42,7 +42,7 @@ "YearDay": 2023218, "DateId": "Dates_20230806", "DateKey": 20230806, - "DateTimeKey": 20230806012341000, + "DateTimeKey": 20230806012341454, "HourId": "Hours_2023080601", "HourKey": 2023080601, "Millis": 1691281421454, @@ -70,7 +70,7 @@ "YearDay": 2023218, "DateId": "Dates_20230806", "DateKey": 20230806, - "DateTimeKey": 20230806002341000, + "DateTimeKey": 20230806002341454, "HourId": "Hours_2023080600", "HourKey": 2023080600, "Millis": 1691277821454, @@ -98,7 +98,7 @@ "YearDay": 2023218, "DateId": "Dates_20230806", "DateKey": 20230806, - "DateTimeKey": 20230806002341000, + "DateTimeKey": 20230806002341454, "HourId": "Hours_2023080600", "HourKey": 2023080600, "Millis": 1691277821454, @@ -121,12 +121,12 @@ "TimeZone": "America/New_York", "TimeZoneOffset": "-04:00", "YearMonth": 202308, - "YearWeek": 202330, + "YearWeek": 202331, "YearIsoWeek": 202331, "YearDay": 2023217, "DateId": "Dates_20230805", "DateKey": 20230805, - "DateTimeKey": 20230805192341000, + "DateTimeKey": 20230805192341454, "HourId": "Hours_2023080519", "HourKey": 2023080519, "Millis": 1691277821454, @@ -154,7 +154,7 @@ "YearDay": 2023218, "DateId": "Dates_20230806", "DateKey": 20230806, - "DateTimeKey": 20230806002341000, + "DateTimeKey": 20230806002341454, "HourId": "Hours_2023080600", "HourKey": 2023080600, "Millis": 1691277821454, @@ -177,12 +177,12 @@ "TimeZone": "Europe/London", "TimeZoneOffset": "+00:00", "YearMonth": 202301, - "YearWeek": 202300, + "YearWeek": 202301, "YearIsoWeek": 202301, "YearDay": 2023006, "DateId": "Dates_20230106", "DateKey": 20230106, - "DateTimeKey": 20230106002341000, + "DateTimeKey": 20230106002341454, "HourId": "Hours_2023010600", "HourKey": 2023010600, "Millis": 1672964621454, diff --git a/jlib/timeparse/testdata_lite.json b/jlib/timeparse/testdata_lite.json index 455db4c..c239026 100644 --- a/jlib/timeparse/testdata_lite.json +++ b/jlib/timeparse/testdata_lite.json @@ -8,35 +8,51 @@ "DateDim": { "TimeZone": "UTC", "TimeZoneOffset": "+00:00", + "YearMonth": 202308, + "YearWeek": 202330, + "YearIsoWeek": 202331, + "YearDay": 2023218, "DateId": "Dates_20230806", "DateKey": 20230806, "DateTimeKey": 20230806002341000, + "HourId": "Hours_2023080600", + "HourKey": 2023080600, "Millis": 1691281421000, "RawValue": "2023-08-06T00:23:41Z", "UTC": "2023-08-06T00:23:41.000Z", "DateUTC": "2023-08-06", + "HourUTC": 0, "Local": "2023-08-06T00:23:41.000+00:00", - "DateLocal": "2023-08-06" + "DateLocal": "2023-08-06", + "HourLocal": 0 } }, { "testDesc": "/* source explicit UTC, Local = 2023-08-06T00:23:41.454Z+01:00 */", "input_srcTs": "2023-08-06T00:23:41.454Z", - "input_srcFormat": "2006-01-02T15:04:05Z", + "input_srcFormat": "2006-01-02T15:04:05.000Z", "input_srcTz": "UTC", "output_srcTz": "Europe/London", "DateDim": { "TimeZone": "Europe/London", "TimeZoneOffset": "+01:00", + "YearMonth": 202308, + "YearWeek": 202330, + "YearIsoWeek": 202331, + "YearDay": 2023218, "DateId": "Dates_20230806", "DateKey": 20230806, - "DateTimeKey": 20230806012341000, + "DateTimeKey": 20230806012341454, + "HourId": "Hours_2023080601", + "HourKey": 2023080601, "Millis": 1691281421454, "RawValue": "2023-08-06T00:23:41.454Z", "UTC": "2023-08-06T00:23:41.454Z", "DateUTC": "2023-08-06", + "HourUTC": 0, "Local": "2023-08-06T01:23:41.454+01:00", - "DateLocal": "2023-08-06" + "DateLocal": "2023-08-06", + "HourLocal": 1 } }, { @@ -48,15 +64,23 @@ "DateDim": { "TimeZone": "Europe/London", "TimeZoneOffset": "+01:00", + "YearMonth": 202308, + "YearWeek": 202330, + "YearIsoWeek": 202331, + "YearDay": 2023218, "DateId": "Dates_20230806", "DateKey": 20230806, - "DateTimeKey": 20230806002341000, + "DateTimeKey": 20230806002341454, + "HourId": "Hours_2023080600", + "HourKey": 2023080600, "Millis": 1691277821454, "RawValue": "2023-08-06T00:23:41.454+01:00", "UTC": "2023-08-05T23:23:41.454Z", "DateUTC": "2023-08-05", + "HourUTC": 23, "Local": "2023-08-06T00:23:41.454+01:00", - "DateLocal": "2023-08-06" + "DateLocal": "2023-08-06", + "HourLocal": 0 } }, { @@ -68,15 +92,23 @@ "DateDim": { "TimeZone": "Europe/London", "TimeZoneOffset": "+01:00", + "YearMonth": 202308, + "YearWeek": 202330, + "YearIsoWeek": 202331, + "YearDay": 2023218, "DateId": "Dates_20230806", "DateKey": 20230806, - "DateTimeKey": 20230806002341000, + "DateTimeKey": 20230806002341454, + "HourId": "Hours_2023080600", + "HourKey": 2023080600, "Millis": 1691277821454, "RawValue": "2023-08-06T00:23:41.454Z", "UTC": "2023-08-05T23:23:41.454Z", "DateUTC": "2023-08-05", + "HourUTC": 23, "Local": "2023-08-06T00:23:41.454+01:00", - "DateLocal": "2023-08-06" + "DateLocal": "2023-08-06", + "HourLocal": 0 } }, { @@ -88,15 +120,23 @@ "DateDim": { "TimeZone": "America/New_York", "TimeZoneOffset": "-04:00", + "YearMonth": 202308, + "YearWeek": 202331, + "YearIsoWeek": 202331, + "YearDay": 2023217, "DateId": "Dates_20230805", "DateKey": 20230805, - "DateTimeKey": 20230805192341000, + "DateTimeKey": 20230805192341454, + "HourId": "Hours_2023080519", + "HourKey": 2023080519, "Millis": 1691277821454, "RawValue": "2023-08-06T00:23:41.454Z", "UTC": "2023-08-05T23:23:41.454Z", "DateUTC": "2023-08-05", + "HourUTC": 23, "Local": "2023-08-05T19:23:41.454-04:00", - "DateLocal": "2023-08-05" + "DateLocal": "2023-08-05", + "HourLocal": 19 } }, { @@ -108,15 +148,23 @@ "DateDim": { "TimeZone": "Europe/London", "TimeZoneOffset": "+01:00", + "YearMonth": 202308, + "YearWeek": 202330, + "YearIsoWeek": 202331, + "YearDay": 2023218, "DateId": "Dates_20230806", "DateKey": 20230806, - "DateTimeKey": 20230806002341000, + "DateTimeKey": 20230806002341454, + "HourId": "Hours_2023080600", + "HourKey": 2023080600, "Millis": 1691277821454, "RawValue": "2023-08-06T00:23:41.454", "UTC": "2023-08-05T23:23:41.454Z", "DateUTC": "2023-08-05", + "HourUTC": 23, "Local": "2023-08-06T00:23:41.454+01:00", - "DateLocal": "2023-08-06" + "DateLocal": "2023-08-06", + "HourLocal": 0 } }, { @@ -128,15 +176,23 @@ "DateDim": { "TimeZone": "Europe/London", "TimeZoneOffset": "+00:00", + "YearMonth": 202301, + "YearWeek": 202301, + "YearIsoWeek": 202301, + "YearDay": 2023006, "DateId": "Dates_20230106", "DateKey": 20230106, - "DateTimeKey": 20230106002341000, + "DateTimeKey": 20230106002341454, + "HourId": "Hours_2023010600", + "HourKey": 2023010600, "Millis": 1672964621454, "RawValue": "2023-01-06T00:23:41.454", "UTC": "2023-01-06T00:23:41.454Z", "DateUTC": "2023-01-06", + "HourUTC": 0, "Local": "2023-01-06T00:23:41.454+00:00", - "DateLocal": "2023-01-06" + "DateLocal": "2023-01-06", + "HourLocal": 0 } } ] \ No newline at end of file diff --git a/jlib/timeparse/timeparse.go b/jlib/timeparse/timeparse.go index e40af54..179f1da 100644 --- a/jlib/timeparse/timeparse.go +++ b/jlib/timeparse/timeparse.go @@ -3,6 +3,7 @@ package timeparse import ( "fmt" "strconv" + "strings" "time" ) @@ -95,7 +96,9 @@ func TimeDateDimensions(inputSrcTs, inputSrcFormat, inputSrcTz, requiredTz strin return nil, err } - dateTimeID := localTime.Format("20060102150405000") + dateTimeID := localTime.Format("20060102150405.000") + + dateTimeID = strings.ReplaceAll(dateTimeID, ".", "") dateTimeKeyInt, err := strconv.Atoi(dateTimeID) if err != nil { @@ -138,12 +141,7 @@ func TimeDateDimensions(inputSrcTs, inputSrcFormat, inputSrcTz, requiredTz strin func getWeekOfYearString(date time.Time) (int, error) { _, week := date.ISOWeek() - firstWednesday := date.AddDate(0, 0, -int(date.Weekday())+1) - if firstWednesday.Weekday() != time.Wednesday { - firstWednesday = firstWednesday.AddDate(0, 0, 7-int(firstWednesday.Weekday())+int(time.Wednesday)) - } - - if date.Weekday() == time.Sunday || date.Before(firstWednesday) { + if date.Weekday() == time.Sunday { week-- } diff --git a/jlib/timeparse/timeparse_test.go b/jlib/timeparse/timeparse_test.go index f3e4be2..c7efc5e 100644 --- a/jlib/timeparse/timeparse_test.go +++ b/jlib/timeparse/timeparse_test.go @@ -1,10 +1,11 @@ package timeparse_test import ( - "github.com/goccy/go-json" "os" "testing" + "github.com/goccy/go-json" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" jsonatatime "github.com/xiatechs/jsonata-go/jlib/timeparse" diff --git a/jlib/timeparse/timeparselite.go b/jlib/timeparse/timeparselite.go index dd4318a..72afb75 100644 --- a/jlib/timeparse/timeparselite.go +++ b/jlib/timeparse/timeparselite.go @@ -2,6 +2,7 @@ package timeparse import ( "strconv" + "strings" "time" ) @@ -76,7 +77,9 @@ func TimeDateDimensionsLite(inputSrcTs, inputSrcFormat, inputSrcTz, requiredTz s } // Input time stamp TIME values (we confirmed there need to be a seperate set of UTC values) - dateTimeID := localTime.Format("20060102150405000") + dateTimeID := localTime.Format("20060102150405.000") + + dateTimeID = strings.ReplaceAll(dateTimeID, ".", "") dateTimeKeyInt, err := strconv.Atoi(dateTimeID) if err != nil { From 61ce7d5db841ad78784c444f9c20db32dc98ca8c Mon Sep 17 00:00:00 2001 From: Jarrah-libremfg <122067969+Jarrah-libremfg@users.noreply.github.com> Date: Wed, 17 Apr 2024 16:30:20 +1000 Subject: [PATCH 49/58] Fix panic calling $distinct on array of arrays (#34) This fixes the following panic: ``` panic: runtime error: hash of unhashable type []interface {} goroutine 176 [running]: http://github.com/xiatechs/jsonata-go/jlib.Distinct ({0x1087cc0?, 0xc000661368?, 0x0?}) /go/pkg/mod/github.com/xiatechs/jsonata-go@v1.8.4/jlib/array.go:61 +0x3e5 reflect.Value.call({0x10c10a0?, 0x1404a18?, 0x4cadbf?}, {0x1318275, 0x4}, {0xc000661590, 0x1, 0x0?}) /usr/local/go/src/reflect/value.go:596 +0xce5 reflect.Value.Call({0x10c10a0?, 0x1404a18?, 0x1?}, {0xc000661590?, 0xc000661590?, 0x1087d00?}) /usr/local/go/src/reflect/value.go:380 +0xb9 ``` --- jlib/array.go | 10 +++++----- jsonata_test.go | 22 ++++++++++++++++++++++ 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/jlib/array.go b/jlib/array.go index 8c1f734..690bc70 100644 --- a/jlib/array.go +++ b/jlib/array.go @@ -45,14 +45,14 @@ func Distinct(v reflect.Value) interface{} { for i := 0; i < items.Len(); i++ { item := jtypes.Resolve(items.Index(i)) - if jtypes.IsMap(item) { - // We can't hash a map, so convert it to a + if jtypes.IsMap(item) || jtypes.IsArray(item) { + // We can't hash a map or array, so convert it to a // string that is hashable - mapItem := fmt.Sprint(item.Interface()) - if _, ok := visited[mapItem]; ok { + unhashableItem := fmt.Sprint(item.Interface()) + if _, ok := visited[unhashableItem]; ok { continue } - visited[mapItem] = struct{}{} + visited[unhashableItem] = struct{}{} distinctValues = reflect.Append(distinctValues, item) continue diff --git a/jsonata_test.go b/jsonata_test.go index 687a8dc..2965827 100644 --- a/jsonata_test.go +++ b/jsonata_test.go @@ -8028,6 +8028,28 @@ func TestTransform(t *testing.T) { }) } +func TestUnhashableDistinct(t *testing.T) { + runTestCases(t, testdata.address, []*testCase{ + { + Expression: `$distinct([["a", "b"]])`, + Output: []interface{}{ + []interface{}{ + "a", + "b", + }, + }, + }, + { + Expression: `$distinct([{"a": "b"}])`, + Output: []interface{}{ + map[string]interface{}{ + "a": "b", + }, + }, + }, + }) +} + // Helper functions type compareFunc func(interface{}, interface{}) bool From 72bb1f209d00d0bfb078ff1d57efc8a228f52072 Mon Sep 17 00:00:00 2001 From: Tom <53711814+tbal999@users.noreply.github.com> Date: Fri, 26 Apr 2024 15:58:49 +0100 Subject: [PATCH 50/58] add monday-weekday (#35) * add monday-weekday * chore: just use isoweek twice * chore: fix test * chore: pushed * update * chore: update tests * chore: remove uneeded func * chore: update testdata --------- Co-authored-by: tbal999 --- go.mod | 1 + go.sum | 2 + jlib/timeparse/testdata.json | 234 ++++++++++++++++++++++++++++++++++- jlib/timeparse/timeparse.go | 30 +++-- 4 files changed, 249 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 8666435..fbee8af 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.20 require ( github.com/goccy/go-json v0.10.2 + github.com/ncruces/go-strftime v0.1.9 github.com/shopspring/decimal v1.3.1 github.com/stretchr/testify v1.8.0 ) diff --git a/go.sum b/go.sum index fcf2da5..7deb495 100644 --- a/go.sum +++ b/go.sum @@ -3,6 +3,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= diff --git a/jlib/timeparse/testdata.json b/jlib/timeparse/testdata.json index c239026..f1327eb 100644 --- a/jlib/timeparse/testdata.json +++ b/jlib/timeparse/testdata.json @@ -1,4 +1,228 @@ [ + { + "testDesc": "test id 62", + "input_srcTs": "2023-08-05T23:23:41.454Z", + "input_srcFormat": "2006-01-02T15:04:05.000Z", + "input_srcTz": "Europe/London", + "output_srcTz": "Europe/London", + "DateDim": { + "TimeZone": "Europe/London", + "TimeZoneOffset": "+01:00", + "YearMonth": 202308, + "YearWeek": 202331, + "YearIsoWeek": 202331, + "YearDay": 2023217, + "DateId": "Dates_20230805", + "DateKey": 20230805, + "DateTimeKey": 20230805232341454, + "HourId": "Hours_2023080523", + "HourKey": 2023080523, + "Millis": 1691274221454, + "RawValue": "2023-08-05T23:23:41.454Z", + "UTC": "2023-08-05T22:23:41.454Z", + "DateUTC": "2023-08-05", + "HourUTC": 22, + "Local": "2023-08-05T23:23:41.454+01:00", + "DateLocal": "2023-08-05", + "HourLocal": 23 + } + }, + { + "testDesc": "test id 61", + "input_srcTs": "2023-08-05T23:23:41.454Z", + "input_srcFormat": "2006-01-02T15:04:05.000Z", + "input_srcTz": "UTC", + "output_srcTz": "Europe/London", + "DateDim": { + "TimeZone": "Europe/London", + "TimeZoneOffset": "+01:00", + "YearMonth": 202308, + "YearWeek": 202331, + "YearIsoWeek": 202331, + "YearDay": 2023218, + "DateId": "Dates_20230806", + "DateKey": 20230806, + "DateTimeKey": 20230806002341454, + "HourId": "Hours_2023080600", + "HourKey": 2023080600, + "Millis": 1691277821454, + "RawValue": "2023-08-05T23:23:41.454Z", + "UTC": "2023-08-05T23:23:41.454Z", + "DateUTC": "2023-08-05", + "HourUTC": 23, + "Local": "2023-08-06T00:23:41.454+01:00", + "DateLocal": "2023-08-06", + "HourLocal": 0 + } + }, + { + "testDesc": "test id 54", + "input_srcTs": "2024-10-26T12:34:56Z", + "input_srcFormat": "2006-01-02T15:04:05Z", + "input_srcTz": "Europe/London", + "output_srcTz": "UTC", + "DateDim": { + "TimeZone": "UTC", + "TimeZoneOffset": "+00:00", + "YearMonth": 202410, + "YearWeek": 202443, + "YearIsoWeek": 202443, + "YearDay": 2024300, + "DateId": "Dates_20241026", + "DateKey": 20241026, + "DateTimeKey": 20241026113456000, + "HourId": "Hours_2024102611", + "HourKey": 2024102611, + "Millis": 1729942496000, + "RawValue": "2024-10-26T12:34:56Z", + "UTC": "2024-10-26T11:34:56.000Z", + "DateUTC": "2024-10-26", + "HourUTC": 11, + "Local": "2024-10-26T11:34:56.000+00:00", + "DateLocal": "2024-10-26", + "HourLocal": 11 + } + }, + { + "testDesc": "test id 53", + "input_srcTs": "2024-10-26T12:34:56Z", + "input_srcFormat": "2006-01-02T15:04:05Z", + "input_srcTz": "Europe/London", + "output_srcTz": "UTC", + "DateDim": { + "TimeZone": "UTC", + "TimeZoneOffset": "+00:00", + "YearMonth": 202410, + "YearWeek": 202443, + "YearIsoWeek": 202443, + "YearDay": 2024300, + "DateId": "Dates_20241026", + "DateKey": 20241026, + "DateTimeKey": 20241026113456000, + "HourId": "Hours_2024102611", + "HourKey": 2024102611, + "Millis": 1729942496000, + "RawValue": "2024-10-26T12:34:56Z", + "UTC": "2024-10-26T11:34:56.000Z", + "DateUTC": "2024-10-26", + "HourUTC": 11, + "Local": "2024-10-26T11:34:56.000+00:00", + "DateLocal": "2024-10-26", + "HourLocal": 11 + } + }, + { + "testDesc": "test id 52", + "input_srcTs": "2024-03-30T12:34:56Z", + "input_srcFormat": "2006-01-02T15:04:05Z", + "input_srcTz": "Europe/London", + "output_srcTz": "UTC", + "DateDim": { + "TimeZone": "UTC", + "TimeZoneOffset": "+00:00", + "YearMonth": 202403, + "YearWeek": 202413, + "YearIsoWeek": 202413, + "YearDay": 2024090, + "DateId": "Dates_20240330", + "DateKey": 20240330, + "DateTimeKey": 20240330123456000, + "HourId": "Hours_2024033012", + "HourKey": 2024033012, + "Millis": 1711802096000, + "RawValue": "2024-03-30T12:34:56Z", + "UTC": "2024-03-30T12:34:56.000Z", + "DateUTC": "2024-03-30", + "HourUTC": 12, + "Local": "2024-03-30T12:34:56.000+00:00", + "DateLocal": "2024-03-30", + "HourLocal": 12 + } + }, + { + "testDesc": "test id 51", + "input_srcTs": "2024-03-31T12:34:56Z", + "input_srcFormat": "2006-01-02T15:04:05Z", + "input_srcTz": "Europe/London", + "output_srcTz": "UTC", + "DateDim": { + "TimeZone": "UTC", + "TimeZoneOffset": "+00:00", + "YearMonth": 202403, + "YearWeek": 202413, + "YearIsoWeek": 202413, + "YearDay": 2024091, + "DateId": "Dates_20240331", + "DateKey": 20240331, + "DateTimeKey": 20240331113456000, + "HourId": "Hours_2024033111", + "HourKey": 2024033111, + "Millis": 1711884896000, + "RawValue": "2024-03-31T12:34:56Z", + "UTC": "2024-03-31T11:34:56.000Z", + "DateUTC": "2024-03-31", + "HourUTC": 11, + "Local": "2024-03-31T11:34:56.000+00:00", + "DateLocal": "2024-03-31", + "HourLocal": 11 + } + }, + { + "testDesc": "/* isoweek 53 - yearweek 0 test */", + "input_srcTs": "2021-01-03T12:13:14Z", + "input_srcFormat": "2006-01-02T15:04:05Z", + "input_srcTz": "Europe/London", + "output_srcTz": "Europe/London", + "DateDim": { + "TimeZone": "Europe/London", + "TimeZoneOffset": "+00:00", + "YearMonth": 202101, + "YearWeek": 202100, + "YearIsoWeek": 202053, + "YearDay": 2021003, + "DateId": "Dates_20210103", + "DateKey": 20210103, + "DateTimeKey": 20210103121314000, + "HourId": "Hours_2021010312", + "HourKey": 2021010312, + "Millis": 1609675994000, + "RawValue": "2021-01-03T12:13:14Z", + "UTC": "2021-01-03T12:13:14.000Z", + "DateUTC": "2021-01-03", + "HourUTC": 12, + "Local": "2021-01-03T12:13:14.000+00:00", + "DateLocal": "2021-01-03", + "HourLocal": 12 + } + }, + { + "testDesc": "/* isoweek 52 - yearweek 0 test */", + "input_srcTs": "2023-01-01T12:13:14Z", + "input_srcFormat": "2006-01-02T15:04:05Z", + "input_srcTz": "Europe/London", + "output_srcTz": "Europe/London", + "DateDim": { + "TimeZone": "Europe/London", + "TimeZoneOffset": "+00:00", + "YearMonth": 202301, + "YearWeek": 202300, + "YearIsoWeek": 202252, + "YearDay": 2023001, + "DateId": "Dates_20230101", + "DateKey": 20230101, + "DateTimeKey": 20230101121314000, + "HourId": "Hours_2023010112", + "HourKey": 2023010112, + "Millis": 1672575194000, + "RawValue": "2023-01-01T12:13:14Z", + "UTC": "2023-01-01T12:13:14.000Z", + "DateUTC": "2023-01-01", + "HourUTC": 12, + "Local": "2023-01-01T12:13:14.000+00:00", + "DateLocal": "2023-01-01", + "HourLocal": 12 + } + }, { "testDesc": "/* source has explicit input UTC, output UTC - this will convert XF official formatting */", "input_srcTs": "2023-08-06T00:23:41Z", @@ -9,7 +233,7 @@ "TimeZone": "UTC", "TimeZoneOffset": "+00:00", "YearMonth": 202308, - "YearWeek": 202330, + "YearWeek": 202331, "YearIsoWeek": 202331, "YearDay": 2023218, "DateId": "Dates_20230806", @@ -37,7 +261,7 @@ "TimeZone": "Europe/London", "TimeZoneOffset": "+01:00", "YearMonth": 202308, - "YearWeek": 202330, + "YearWeek": 202331, "YearIsoWeek": 202331, "YearDay": 2023218, "DateId": "Dates_20230806", @@ -65,7 +289,7 @@ "TimeZone": "Europe/London", "TimeZoneOffset": "+01:00", "YearMonth": 202308, - "YearWeek": 202330, + "YearWeek": 202331, "YearIsoWeek": 202331, "YearDay": 2023218, "DateId": "Dates_20230806", @@ -93,7 +317,7 @@ "TimeZone": "Europe/London", "TimeZoneOffset": "+01:00", "YearMonth": 202308, - "YearWeek": 202330, + "YearWeek": 202331, "YearIsoWeek": 202331, "YearDay": 2023218, "DateId": "Dates_20230806", @@ -149,7 +373,7 @@ "TimeZone": "Europe/London", "TimeZoneOffset": "+01:00", "YearMonth": 202308, - "YearWeek": 202330, + "YearWeek": 202331, "YearIsoWeek": 202331, "YearDay": 2023218, "DateId": "Dates_20230806", diff --git a/jlib/timeparse/timeparse.go b/jlib/timeparse/timeparse.go index 179f1da..821675d 100644 --- a/jlib/timeparse/timeparse.go +++ b/jlib/timeparse/timeparse.go @@ -5,6 +5,8 @@ import ( "strconv" "strings" "time" + + strtime "github.com/ncruces/go-strftime" ) // DateDim is the date dimension object returned from the timeparse function @@ -38,6 +40,7 @@ type DateDim struct { // TimeDateDimensions generates a JSON object dependent on input source timestamp, input source format and input source timezone // using golang time formats func TimeDateDimensions(inputSrcTs, inputSrcFormat, inputSrcTz, requiredTz string) (*DateDim, error) { + // first we create a time location based on the input source timezone location inputLocation, err := time.LoadLocation(inputSrcTz) if err != nil { return nil, err @@ -50,23 +53,29 @@ func TimeDateDimensions(inputSrcTs, inputSrcFormat, inputSrcTz, requiredTz strin return nil, err } + // then we create a time location based on the output timezone location outputLocation, err := time.LoadLocation(requiredTz) if err != nil { return nil, err } + // here we translate te input time into a local time based on the output location localTime := inputTime.In(outputLocation) // convert the parsed time into a UTC time for UTC calculations utcTime := localTime.UTC() - // UTC TIME values + // now we have inputTime (the time parsed from the input location) + // the local time which is the inputTime converted to the output location + // and the UTC time which is the local time converted into UTC time + // UTC TIME values utcAsYearMonthDay := utcTime.Format("2006-01-02") // Input time stamp TIME values (we confirmed there need to be a seperate set of UTC values) dateID := localTime.Format("20060102") + // here we get the year and week for golang standard library ISOWEEK for the local time year, week := localTime.ISOWeek() yearDay, err := strconv.Atoi(localTime.Format("2006") + localTime.Format("002")) @@ -76,26 +85,30 @@ func TimeDateDimensions(inputSrcTs, inputSrcFormat, inputSrcTz, requiredTz strin hourKeyStr := localTime.Format("2006010215") - mondayWeek, err := getWeekOfYearString(localTime) + yearMondayWeek, err := strconv.Atoi(strtime.Format(`%Y%W`, localTime)) if err != nil { return nil, err } + // the ISO yearweek yearIsoWeekInt, err := strconv.Atoi(fmt.Sprintf("%d%02d", year, week)) if err != nil { return nil, err } + // year month yearMonthInt, err := strconv.Atoi(localTime.Format("200601")) if err != nil { return nil, err } + // the date key dateKeyInt, err := strconv.Atoi(dateID) if err != nil { return nil, err } + // hack - golang time library doesn't handle millis correct unless you do the below dateTimeID := localTime.Format("20060102150405.000") dateTimeID = strings.ReplaceAll(dateTimeID, ".", "") @@ -105,6 +118,7 @@ func TimeDateDimensions(inputSrcTs, inputSrcFormat, inputSrcTz, requiredTz strin return nil, err } + // the hours hourKeyInt, err := strconv.Atoi(hourKeyStr) if err != nil { return nil, err @@ -116,7 +130,7 @@ func TimeDateDimensions(inputSrcTs, inputSrcFormat, inputSrcTz, requiredTz strin dateDim := &DateDim{ RawValue: inputSrcTs, TimeZoneOffset: offsetStr, - YearWeek: mondayWeek, + YearWeek: yearMondayWeek, YearDay: yearDay, YearIsoWeek: yearIsoWeekInt, YearMonth: yearMonthInt, @@ -137,13 +151,3 @@ func TimeDateDimensions(inputSrcTs, inputSrcFormat, inputSrcTz, requiredTz strin return dateDim, nil } - -func getWeekOfYearString(date time.Time) (int, error) { - _, week := date.ISOWeek() - - if date.Weekday() == time.Sunday { - week-- - } - - return strconv.Atoi(fmt.Sprintf("%04d%02d", date.Year(), week)) -} From c158b1f21487abbdf6e8a0b38a6269b5ad2cdc0b Mon Sep 17 00:00:00 2001 From: JamesXiatech Date: Wed, 27 Nov 2024 14:01:00 +0000 Subject: [PATCH 51/58] Updated objectsToDocument function to deal with array syntax --- jlib/new.go | 111 ++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 90 insertions(+), 21 deletions(-) diff --git a/jlib/new.go b/jlib/new.go index b4fadf2..11e84ad 100644 --- a/jlib/new.go +++ b/jlib/new.go @@ -1,11 +1,13 @@ package jlib import ( - "github.com/goccy/go-json" "errors" "fmt" "reflect" + "strconv" "strings" + + "github.com/goccy/go-json" ) // Unescape an escaped json string into JSON (once) @@ -231,25 +233,83 @@ func ObjMerge(i1, i2 interface{}) interface{} { return output } -// setValue sets the value in the obj map at the specified dot notation path. +// setValue handles setting values in a nested structure including array indices func setValue(obj map[string]interface{}, path string, value interface{}) { - paths := strings.Split(path, ".") // Split the path into parts - - // Iterate through path parts to navigate/create nested maps - for i := 0; i < len(paths)-1; i++ { - // If the key does not exist, create a new map at the key - _, ok := obj[paths[i]] - if !ok { - obj[paths[i]] = make(map[string]interface{}) + parts := strings.Split(path, ".") + current := obj + + for i := 0; i < len(parts)-1; i++ { + part := parts[i] + + // Check if this part contains an array index + arrayIndex := -1 + if idx := strings.Index(part, "["); idx != -1 { + // Extract the array index + if end := strings.Index(part, "]"); end != -1 { + indexStr := part[idx+1 : end] + if index, err := strconv.Atoi(indexStr); err == nil { + arrayIndex = index + part = part[:idx] // Remove the array notation from the part + } + } } - obj, ok = obj[paths[i]].(map[string]interface{}) - if !ok { - continue + // Handle array index if present + if arrayIndex != -1 { + // Ensure the current part exists and is an array + arr, exists := current[part].([]interface{}) + if !exists { + arr = make([]interface{}, 0) + current[part] = arr + } + + // Extend array if needed + for len(arr) <= arrayIndex { + arr = append(arr, make(map[string]interface{})) + } + current[part] = arr // Important: update the array in the map + + // Get or create map at array index + if arr[arrayIndex] == nil { + arr[arrayIndex] = make(map[string]interface{}) + } + + current = arr[arrayIndex].(map[string]interface{}) + } else { + // Normal object property + next, exists := current[part].(map[string]interface{}) + if !exists { + next = make(map[string]interface{}) + current[part] = next + } + current = next } } - obj[paths[len(paths)-1]] = value + // Handle the final part + lastPart := parts[len(parts)-1] + if idx := strings.Index(lastPart, "["); idx != -1 { + // Handle array index in the final part + if end := strings.Index(lastPart, "]"); end != -1 { + indexStr := lastPart[idx+1 : end] + if index, err := strconv.Atoi(indexStr); err == nil { + part := lastPart[:idx] + arr, exists := current[part].([]interface{}) + if !exists { + arr = make([]interface{}, 0) + } + // Extend array if needed + for len(arr) <= index { + arr = append(arr, nil) + } + arr[index] = value + current[part] = arr // Important: update the array in the map + return + } + } + } + // Set value for non-array final part + current[lastPart] = value } // objectsToDocument converts an array of Items to a nested map according to the Code paths. @@ -259,23 +319,32 @@ func ObjectsToDocument(input interface{}) (interface{}, error) { return nil, errors.New("$objectsToDocument input must be an array of objects") } - output := make(map[string]interface{}) // Initialize the output map - // Iterate through each item in the input + output := make(map[string]interface{}) for _, itemToInterface := range trueInput { item, ok := itemToInterface.(map[string]interface{}) if !ok { - return nil, errors.New("$objectsToDocument input must be an array of objects with Code and Value fields") + return nil, errors.New("$objectsToDocument input must be an array of objects with Code and Val/Value fields") } - // Call setValue for each item to set the value in the output map + code, ok := item["Code"].(string) if !ok { continue } - value := item["Value"] - setValue(output, code, value) + + // Try "Val" first, then fall back to "Value" + var value interface{} + if val, exists := item["Val"]; exists { + value = val + } else if val, exists := item["Value"]; exists { + value = val + } + + if value != nil { + setValue(output, code, value) + } } - return output, nil // Return the output map + return output, nil } // TransformRule defines a transformation rule with a search substring and a new name. From a868852027199c848d32abe0d7bad5ee30f3c728 Mon Sep 17 00:00:00 2001 From: tbal999 Date: Thu, 28 Nov 2024 11:37:34 +0000 Subject: [PATCH 52/58] chore: extended testing chassis --- callable.go | 2 +- example_eval_test.go | 2 +- extendedTestFiles/case1/input.json | 3 ++ extendedTestFiles/case1/input.jsonata | 3 ++ extendedTestFiles/case1/output.json | 3 ++ extended_test.go | 59 +++++++++++++++++++++++++++ jlib/string.go | 2 +- jsonata-server/main.go | 2 +- jsonata-test/main.go | 2 +- jsonata_test.go | 2 +- 10 files changed, 74 insertions(+), 6 deletions(-) create mode 100644 extendedTestFiles/case1/input.json create mode 100644 extendedTestFiles/case1/input.jsonata create mode 100644 extendedTestFiles/case1/output.json create mode 100644 extended_test.go diff --git a/callable.go b/callable.go index b54c82f..87700e6 100644 --- a/callable.go +++ b/callable.go @@ -5,8 +5,8 @@ package jsonata import ( - "github.com/goccy/go-json" "fmt" + "github.com/goccy/go-json" "reflect" "regexp" "strings" diff --git a/example_eval_test.go b/example_eval_test.go index d713791..65eb0b4 100644 --- a/example_eval_test.go +++ b/example_eval_test.go @@ -5,8 +5,8 @@ package jsonata_test import ( - "github.com/goccy/go-json" "fmt" + "github.com/goccy/go-json" "log" jsonata "github.com/xiatechs/jsonata-go" diff --git a/extendedTestFiles/case1/input.json b/extendedTestFiles/case1/input.json new file mode 100644 index 0000000..d02fab0 --- /dev/null +++ b/extendedTestFiles/case1/input.json @@ -0,0 +1,3 @@ +{ + "hello": "world" +} \ No newline at end of file diff --git a/extendedTestFiles/case1/input.jsonata b/extendedTestFiles/case1/input.jsonata new file mode 100644 index 0000000..3df85db --- /dev/null +++ b/extendedTestFiles/case1/input.jsonata @@ -0,0 +1,3 @@ +{ + "goodbye": "world" +} \ No newline at end of file diff --git a/extendedTestFiles/case1/output.json b/extendedTestFiles/case1/output.json new file mode 100644 index 0000000..3df85db --- /dev/null +++ b/extendedTestFiles/case1/output.json @@ -0,0 +1,3 @@ +{ + "goodbye": "world" +} \ No newline at end of file diff --git a/extended_test.go b/extended_test.go new file mode 100644 index 0000000..f59588e --- /dev/null +++ b/extended_test.go @@ -0,0 +1,59 @@ +package jsonata + +import ( + "encoding/json" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestChassis(t *testing.T) { + assert.Equal(t, nil, nil) + + tests := []struct{ + Name string + InputFile string + InputJsonataFile string + ExpectedOutputFile string + }{ + { + Name: "a simple test", + InputFile: "extendedTestFiles/case1/input.json", + InputJsonataFile: "extendedTestFiles/case1/input.jsonata", + ExpectedOutputFile: "extendedTestFiles/case1/output.json", + }, + } + + for index := range tests { + tt := tests[index] + + t.Run(tt.Name, func(t *testing.T) { + inputBytes, err := os.ReadFile(tt.InputFile) + require.NoError(t, err) + + jsonataBytes, err := os.ReadFile(tt.InputJsonataFile) + require.NoError(t, err) + + outputBytes, err := os.ReadFile(tt.ExpectedOutputFile) + require.NoError(t, err) + + expr, err := Compile(string(jsonataBytes)) + require.NoError(t, err) + + var input, output interface{} + + err = json.Unmarshal(inputBytes, &input) + require.NoError(t, err) + + result, err := expr.Eval(input) + require.NoError(t, err) + + err = json.Unmarshal(outputBytes, &output) + require.NoError(t, err) + + assert.Equal(t, result, output) + }) + } +} diff --git a/jlib/string.go b/jlib/string.go index 2f63deb..014acf8 100644 --- a/jlib/string.go +++ b/jlib/string.go @@ -7,8 +7,8 @@ package jlib import ( "bytes" "encoding/base64" - "github.com/goccy/go-json" "fmt" + "github.com/goccy/go-json" "math" "net/url" "reflect" diff --git a/jsonata-server/main.go b/jsonata-server/main.go index ad42c53..35bb843 100644 --- a/jsonata-server/main.go +++ b/jsonata-server/main.go @@ -6,9 +6,9 @@ package main import ( "bytes" - "github.com/goccy/go-json" "flag" "fmt" + "github.com/goccy/go-json" "log" "net/http" _ "net/http/pprof" diff --git a/jsonata-test/main.go b/jsonata-test/main.go index de768bf..14b3b0c 100644 --- a/jsonata-test/main.go +++ b/jsonata-test/main.go @@ -1,9 +1,9 @@ package main import ( - "github.com/goccy/go-json" "flag" "fmt" + "github.com/goccy/go-json" "io" "io/ioutil" "os" diff --git a/jsonata_test.go b/jsonata_test.go index 2965827..0adce16 100644 --- a/jsonata_test.go +++ b/jsonata_test.go @@ -5,9 +5,9 @@ package jsonata import ( - "github.com/goccy/go-json" "errors" "fmt" + "github.com/goccy/go-json" "io/ioutil" "math" "os" From 98d0360da8f44e7b8384369d5c49fdf11e44559b Mon Sep 17 00:00:00 2001 From: tbal999 Date: Thu, 28 Nov 2024 11:38:33 +0000 Subject: [PATCH 53/58] chore: switch to goccy --- extended_test.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/extended_test.go b/extended_test.go index f59588e..2b8cf58 100644 --- a/extended_test.go +++ b/extended_test.go @@ -1,10 +1,11 @@ package jsonata import ( - "encoding/json" "os" "testing" + "github.com/goccy/go-json" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -12,17 +13,17 @@ import ( func TestChassis(t *testing.T) { assert.Equal(t, nil, nil) - tests := []struct{ + tests := []struct { Name string InputFile string InputJsonataFile string ExpectedOutputFile string }{ { - Name: "a simple test", - InputFile: "extendedTestFiles/case1/input.json", - InputJsonataFile: "extendedTestFiles/case1/input.jsonata", - ExpectedOutputFile: "extendedTestFiles/case1/output.json", + Name: "a simple test", + InputFile: "extendedTestFiles/case1/input.json", + InputJsonataFile: "extendedTestFiles/case1/input.jsonata", + ExpectedOutputFile: "extendedTestFiles/case1/output.json", }, } From 88b5976e8f753ae54739bae105a1732c46b3dd99 Mon Sep 17 00:00:00 2001 From: tbal999 Date: Thu, 28 Nov 2024 11:40:21 +0000 Subject: [PATCH 54/58] chore: some comments --- extended_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/extended_test.go b/extended_test.go index 2b8cf58..053ca11 100644 --- a/extended_test.go +++ b/extended_test.go @@ -11,12 +11,14 @@ import ( ) func TestChassis(t *testing.T) { - assert.Equal(t, nil, nil) - tests := []struct { + // the name of the test Name string + // the input file (must be JSON) InputFile string + // the jsonata that is applied to the file InputJsonataFile string + // the expected output file that is the result of the transformations ExpectedOutputFile string }{ { From 7eebaae92e8b931029399db5d947061c208271ba Mon Sep 17 00:00:00 2001 From: tbal999 Date: Thu, 28 Nov 2024 12:51:33 +0000 Subject: [PATCH 55/58] chore: simplify chassis --- extendedTestFiles/readme.md | 13 ++++++ extended_test.go | 83 ++++++++++++++++++++----------------- 2 files changed, 59 insertions(+), 37 deletions(-) create mode 100644 extendedTestFiles/readme.md diff --git a/extendedTestFiles/readme.md b/extendedTestFiles/readme.md new file mode 100644 index 0000000..7cfb606 --- /dev/null +++ b/extendedTestFiles/readme.md @@ -0,0 +1,13 @@ +# add tests here + +## how? + +First, create a folder with a name i.e "exampleFunc" + +Second, add three files: input.json, input.jsonata and output.json + +Third, make sure the three files match context below: + +input.json - this is the test file you wish to assert against. +input.jsonata - this is the jsonata that will be applied to the input json +output.json - this is what you expect the output to be \ No newline at end of file diff --git a/extended_test.go b/extended_test.go index 053ca11..eeb39f1 100644 --- a/extended_test.go +++ b/extended_test.go @@ -1,7 +1,9 @@ package jsonata import ( + "log" "os" + "path/filepath" "testing" "github.com/goccy/go-json" @@ -10,53 +12,60 @@ import ( "github.com/stretchr/testify/require" ) +const testCasesPath = "extendedTestFiles" + +const ( + expectedInputFile = "input.json" + expectedOutputFile = "output.json" + expectedInputJsonata = "input.jsonata" +) + func TestChassis(t *testing.T) { - tests := []struct { - // the name of the test - Name string - // the input file (must be JSON) - InputFile string - // the jsonata that is applied to the file - InputJsonataFile string - // the expected output file that is the result of the transformations - ExpectedOutputFile string - }{ - { - Name: "a simple test", - InputFile: "extendedTestFiles/case1/input.json", - InputJsonataFile: "extendedTestFiles/case1/input.jsonata", - ExpectedOutputFile: "extendedTestFiles/case1/output.json", - }, + entries, err := os.ReadDir(testCasesPath) + if err != nil { + log.Fatalf("Failed to read directory: %v", err) } - for index := range tests { - tt := tests[index] + for _, entry := range entries { + if entry.IsDir() { + testCase := entry.Name() + + testCasePath := filepath.Join(testCasesPath, testCase) - t.Run(tt.Name, func(t *testing.T) { - inputBytes, err := os.ReadFile(tt.InputFile) - require.NoError(t, err) + t.Run(testCasePath, func(t *testing.T) { + runTest(t, + filepath.Join(testCasePath, expectedInputFile), + filepath.Join(testCasePath, expectedOutputFile), + filepath.Join(testCasePath, expectedInputJsonata), + ) + }) + } + } +} - jsonataBytes, err := os.ReadFile(tt.InputJsonataFile) - require.NoError(t, err) +func runTest(t *testing.T, inputfile, outputfile, jsonatafile string) { + inputBytes, err := os.ReadFile(inputfile) + require.NoError(t, err) - outputBytes, err := os.ReadFile(tt.ExpectedOutputFile) - require.NoError(t, err) + jsonataBytes, err := os.ReadFile(outputfile) + require.NoError(t, err) - expr, err := Compile(string(jsonataBytes)) - require.NoError(t, err) + outputBytes, err := os.ReadFile(jsonatafile) + require.NoError(t, err) - var input, output interface{} + expr, err := Compile(string(jsonataBytes)) + require.NoError(t, err) - err = json.Unmarshal(inputBytes, &input) - require.NoError(t, err) + var input, output interface{} - result, err := expr.Eval(input) - require.NoError(t, err) + err = json.Unmarshal(inputBytes, &input) + require.NoError(t, err) - err = json.Unmarshal(outputBytes, &output) - require.NoError(t, err) + result, err := expr.Eval(input) + require.NoError(t, err) - assert.Equal(t, result, output) - }) - } + err = json.Unmarshal(outputBytes, &output) + require.NoError(t, err) + + assert.Equal(t, result, output) } From 58374e256c2c0bc276667c337a96ad8cb709e8c6 Mon Sep 17 00:00:00 2001 From: tbal999 Date: Thu, 28 Nov 2024 12:53:10 +0000 Subject: [PATCH 56/58] chore: clean up readme --- extendedTestFiles/readme.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/extendedTestFiles/readme.md b/extendedTestFiles/readme.md index 7cfb606..6495eb1 100644 --- a/extendedTestFiles/readme.md +++ b/extendedTestFiles/readme.md @@ -8,6 +8,8 @@ Second, add three files: input.json, input.jsonata and output.json Third, make sure the three files match context below: +``` input.json - this is the test file you wish to assert against. input.jsonata - this is the jsonata that will be applied to the input json -output.json - this is what you expect the output to be \ No newline at end of file +output.json - this is what you expect the output to be +``` \ No newline at end of file From 629e86d6e175cc5f5da107bb20ef643b26f405d1 Mon Sep 17 00:00:00 2001 From: xiatech-ricky Date: Wed, 18 Dec 2024 20:02:13 +0000 Subject: [PATCH 57/58] chore(AS-3797): add test coverage fix nil pointer issue, add unit tests, add data driven tests, fix bug in data driven framework --- .../arrayIndexingComplex/input.json | 6 + .../arrayIndexingComplex/input.jsonata | 1 + .../arrayIndexingComplex/output.json | 29 ++++ .../arrayIndexingSimple/input.json | 4 + .../arrayIndexingSimple/input.jsonata | 1 + .../arrayIndexingSimple/output.json | 10 ++ extendedTestFiles/case1/input.json | 3 - extendedTestFiles/case1/input.jsonata | 3 - extendedTestFiles/case1/output.json | 3 - extendedTestFiles/noValNoValue/input.json | 4 + extendedTestFiles/noValNoValue/input.jsonata | 1 + extendedTestFiles/noValNoValue/output.json | 1 + extendedTestFiles/readme.md | 2 - .../valAndValueAtArrayIndex/input.json | 5 + .../valAndValueAtArrayIndex/input.jsonata | 1 + .../valAndValueAtArrayIndex/output.json | 7 + .../valIsNullUseValue/input.json | 3 + .../valIsNullUseValue/input.jsonata | 1 + .../valIsNullUseValue/output.json | 3 + extendedTestFiles/valPriority/input.json | 4 + extendedTestFiles/valPriority/input.jsonata | 1 + extendedTestFiles/valPriority/output.json | 6 + extended_test.go | 4 +- jlib/new.go | 7 +- jlib/new_test.go | 164 +++++++++++++++++- 25 files changed, 257 insertions(+), 17 deletions(-) create mode 100644 extendedTestFiles/arrayIndexingComplex/input.json create mode 100644 extendedTestFiles/arrayIndexingComplex/input.jsonata create mode 100644 extendedTestFiles/arrayIndexingComplex/output.json create mode 100644 extendedTestFiles/arrayIndexingSimple/input.json create mode 100644 extendedTestFiles/arrayIndexingSimple/input.jsonata create mode 100644 extendedTestFiles/arrayIndexingSimple/output.json delete mode 100644 extendedTestFiles/case1/input.json delete mode 100644 extendedTestFiles/case1/input.jsonata delete mode 100644 extendedTestFiles/case1/output.json create mode 100644 extendedTestFiles/noValNoValue/input.json create mode 100644 extendedTestFiles/noValNoValue/input.jsonata create mode 100644 extendedTestFiles/noValNoValue/output.json create mode 100644 extendedTestFiles/valAndValueAtArrayIndex/input.json create mode 100644 extendedTestFiles/valAndValueAtArrayIndex/input.jsonata create mode 100644 extendedTestFiles/valAndValueAtArrayIndex/output.json create mode 100644 extendedTestFiles/valIsNullUseValue/input.json create mode 100644 extendedTestFiles/valIsNullUseValue/input.jsonata create mode 100644 extendedTestFiles/valIsNullUseValue/output.json create mode 100644 extendedTestFiles/valPriority/input.json create mode 100644 extendedTestFiles/valPriority/input.jsonata create mode 100644 extendedTestFiles/valPriority/output.json diff --git a/extendedTestFiles/arrayIndexingComplex/input.json b/extendedTestFiles/arrayIndexingComplex/input.json new file mode 100644 index 0000000..4902504 --- /dev/null +++ b/extendedTestFiles/arrayIndexingComplex/input.json @@ -0,0 +1,6 @@ +[ + {"Code":"root.list[0].items[0].value","Val":"Item0-0"}, + {"Code":"root.list[0].items[1].value","Value":"Item0-1"}, + {"Code":"root.list[1].items[0].value","Val":"Item1-0"}, + {"Code":"root.list[2].info[2].details[1].desc","Value":"DeepNestedValue"} +] diff --git a/extendedTestFiles/arrayIndexingComplex/input.jsonata b/extendedTestFiles/arrayIndexingComplex/input.jsonata new file mode 100644 index 0000000..8f811b4 --- /dev/null +++ b/extendedTestFiles/arrayIndexingComplex/input.jsonata @@ -0,0 +1 @@ +$objectsToDocument($) \ No newline at end of file diff --git a/extendedTestFiles/arrayIndexingComplex/output.json b/extendedTestFiles/arrayIndexingComplex/output.json new file mode 100644 index 0000000..c96aca4 --- /dev/null +++ b/extendedTestFiles/arrayIndexingComplex/output.json @@ -0,0 +1,29 @@ +{ + "root": { + "list": [ + { + "items": [ + {"value":"Item0-0"}, + {"value":"Item0-1"} + ] + }, + { + "items": [ + {"value":"Item1-0"} + ] + }, + { + "info": [ + {}, + {}, + { + "details": [ + {}, + {"desc": "DeepNestedValue"} + ] + } + ] + } + ] + } +} diff --git a/extendedTestFiles/arrayIndexingSimple/input.json b/extendedTestFiles/arrayIndexingSimple/input.json new file mode 100644 index 0000000..256c35a --- /dev/null +++ b/extendedTestFiles/arrayIndexingSimple/input.json @@ -0,0 +1,4 @@ +[ + {"Code": "employees[0].name", "Val": "Alice"}, + {"Code": "employees[1].name", "Value": "Bob"} +] \ No newline at end of file diff --git a/extendedTestFiles/arrayIndexingSimple/input.jsonata b/extendedTestFiles/arrayIndexingSimple/input.jsonata new file mode 100644 index 0000000..8f811b4 --- /dev/null +++ b/extendedTestFiles/arrayIndexingSimple/input.jsonata @@ -0,0 +1 @@ +$objectsToDocument($) \ No newline at end of file diff --git a/extendedTestFiles/arrayIndexingSimple/output.json b/extendedTestFiles/arrayIndexingSimple/output.json new file mode 100644 index 0000000..028c5fe --- /dev/null +++ b/extendedTestFiles/arrayIndexingSimple/output.json @@ -0,0 +1,10 @@ +{ + "employees": [ + { + "name": "Alice" + }, + { + "name": "Bob" + } + ] +} \ No newline at end of file diff --git a/extendedTestFiles/case1/input.json b/extendedTestFiles/case1/input.json deleted file mode 100644 index d02fab0..0000000 --- a/extendedTestFiles/case1/input.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "hello": "world" -} \ No newline at end of file diff --git a/extendedTestFiles/case1/input.jsonata b/extendedTestFiles/case1/input.jsonata deleted file mode 100644 index 3df85db..0000000 --- a/extendedTestFiles/case1/input.jsonata +++ /dev/null @@ -1,3 +0,0 @@ -{ - "goodbye": "world" -} \ No newline at end of file diff --git a/extendedTestFiles/case1/output.json b/extendedTestFiles/case1/output.json deleted file mode 100644 index 3df85db..0000000 --- a/extendedTestFiles/case1/output.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "goodbye": "world" -} \ No newline at end of file diff --git a/extendedTestFiles/noValNoValue/input.json b/extendedTestFiles/noValNoValue/input.json new file mode 100644 index 0000000..abd3291 --- /dev/null +++ b/extendedTestFiles/noValNoValue/input.json @@ -0,0 +1,4 @@ +[ + {"Code":"topKey","OtherField":"none"}, + {"Code":"anotherKey"} +] diff --git a/extendedTestFiles/noValNoValue/input.jsonata b/extendedTestFiles/noValNoValue/input.jsonata new file mode 100644 index 0000000..8f811b4 --- /dev/null +++ b/extendedTestFiles/noValNoValue/input.jsonata @@ -0,0 +1 @@ +$objectsToDocument($) \ No newline at end of file diff --git a/extendedTestFiles/noValNoValue/output.json b/extendedTestFiles/noValNoValue/output.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/extendedTestFiles/noValNoValue/output.json @@ -0,0 +1 @@ +{} diff --git a/extendedTestFiles/readme.md b/extendedTestFiles/readme.md index 6495eb1..9eb13bc 100644 --- a/extendedTestFiles/readme.md +++ b/extendedTestFiles/readme.md @@ -1,5 +1,3 @@ -# add tests here - ## how? First, create a folder with a name i.e "exampleFunc" diff --git a/extendedTestFiles/valAndValueAtArrayIndex/input.json b/extendedTestFiles/valAndValueAtArrayIndex/input.json new file mode 100644 index 0000000..3f8aba0 --- /dev/null +++ b/extendedTestFiles/valAndValueAtArrayIndex/input.json @@ -0,0 +1,5 @@ +[ + {"Code":"data[2].entry","Val":"ThirdEntryVal","Value":"ThirdEntryValue"}, + {"Code":"data[1].entry","Value":"SecondEntryValue"}, + {"Code":"data[0].entry","Val":"FirstEntryVal"} +] diff --git a/extendedTestFiles/valAndValueAtArrayIndex/input.jsonata b/extendedTestFiles/valAndValueAtArrayIndex/input.jsonata new file mode 100644 index 0000000..8f811b4 --- /dev/null +++ b/extendedTestFiles/valAndValueAtArrayIndex/input.jsonata @@ -0,0 +1 @@ +$objectsToDocument($) \ No newline at end of file diff --git a/extendedTestFiles/valAndValueAtArrayIndex/output.json b/extendedTestFiles/valAndValueAtArrayIndex/output.json new file mode 100644 index 0000000..10f612d --- /dev/null +++ b/extendedTestFiles/valAndValueAtArrayIndex/output.json @@ -0,0 +1,7 @@ +{ + "data": [ + { "entry": "FirstEntryVal" }, + { "entry": "SecondEntryValue" }, + { "entry": "ThirdEntryVal" } + ] +} diff --git a/extendedTestFiles/valIsNullUseValue/input.json b/extendedTestFiles/valIsNullUseValue/input.json new file mode 100644 index 0000000..f86bba5 --- /dev/null +++ b/extendedTestFiles/valIsNullUseValue/input.json @@ -0,0 +1,3 @@ +[ + {"Code": "testKey", "Val": null, "Value": "RealValue"} +] \ No newline at end of file diff --git a/extendedTestFiles/valIsNullUseValue/input.jsonata b/extendedTestFiles/valIsNullUseValue/input.jsonata new file mode 100644 index 0000000..8f811b4 --- /dev/null +++ b/extendedTestFiles/valIsNullUseValue/input.jsonata @@ -0,0 +1 @@ +$objectsToDocument($) \ No newline at end of file diff --git a/extendedTestFiles/valIsNullUseValue/output.json b/extendedTestFiles/valIsNullUseValue/output.json new file mode 100644 index 0000000..37e89be --- /dev/null +++ b/extendedTestFiles/valIsNullUseValue/output.json @@ -0,0 +1,3 @@ +{ + "testKey": "RealValue" +} \ No newline at end of file diff --git a/extendedTestFiles/valPriority/input.json b/extendedTestFiles/valPriority/input.json new file mode 100644 index 0000000..fde9a14 --- /dev/null +++ b/extendedTestFiles/valPriority/input.json @@ -0,0 +1,4 @@ +[ + {"Code": "person.name", "Val": "Alice", "Value": "ShouldNotUse"}, + {"Code": "person.age", "Val": 30} +] \ No newline at end of file diff --git a/extendedTestFiles/valPriority/input.jsonata b/extendedTestFiles/valPriority/input.jsonata new file mode 100644 index 0000000..8f811b4 --- /dev/null +++ b/extendedTestFiles/valPriority/input.jsonata @@ -0,0 +1 @@ +$objectsToDocument($) \ No newline at end of file diff --git a/extendedTestFiles/valPriority/output.json b/extendedTestFiles/valPriority/output.json new file mode 100644 index 0000000..67420c7 --- /dev/null +++ b/extendedTestFiles/valPriority/output.json @@ -0,0 +1,6 @@ +{ + "person": { + "name": "Alice", + "age": 30 + } +} \ No newline at end of file diff --git a/extended_test.go b/extended_test.go index eeb39f1..4194f80 100644 --- a/extended_test.go +++ b/extended_test.go @@ -47,10 +47,10 @@ func runTest(t *testing.T, inputfile, outputfile, jsonatafile string) { inputBytes, err := os.ReadFile(inputfile) require.NoError(t, err) - jsonataBytes, err := os.ReadFile(outputfile) + outputBytes, err := os.ReadFile(outputfile) require.NoError(t, err) - outputBytes, err := os.ReadFile(jsonatafile) + jsonataBytes, err := os.ReadFile(jsonatafile) require.NoError(t, err) expr, err := Compile(string(jsonataBytes)) diff --git a/jlib/new.go b/jlib/new.go index 11e84ad..7d57488 100644 --- a/jlib/new.go +++ b/jlib/new.go @@ -331,11 +331,12 @@ func ObjectsToDocument(input interface{}) (interface{}, error) { continue } - // Try "Val" first, then fall back to "Value" var value interface{} - if val, exists := item["Val"]; exists { + if val, exists := item["Val"]; exists && val != nil { + // Use Val only if it's not nil value = val - } else if val, exists := item["Value"]; exists { + } else if val, exists := item["Value"]; exists && val != nil { + // Use Value if Val doesn't exist or was nil value = val } diff --git a/jlib/new_test.go b/jlib/new_test.go index 9c70e12..8eee88b 100644 --- a/jlib/new_test.go +++ b/jlib/new_test.go @@ -1,10 +1,11 @@ package jlib import ( - "github.com/goccy/go-json" "reflect" "testing" + "github.com/goccy/go-json" + "github.com/stretchr/testify/assert" ) @@ -120,3 +121,164 @@ func TestRenameKeys(t *testing.T) { }) } } + +func TestSetValue_ArrayIndexing(t *testing.T) { + tests := []struct { + description string + input map[string]interface{} + code string + value interface{} + expectedOutput map[string]interface{} + }{ + { + description: "Set simple value at array index", + input: map[string]interface{}{}, + code: "employees[0].name", + value: "Alice", + expectedOutput: map[string]interface{}{ + "employees": []interface{}{ + map[string]interface{}{"name": "Alice"}, + }, + }, + }, + { + description: "Extend array to meet required index", + input: map[string]interface{}{}, + code: "employees[2].name", + value: "Charlie", + expectedOutput: map[string]interface{}{ + "employees": []interface{}{ + map[string]interface{}{}, + map[string]interface{}{}, + map[string]interface{}{"name": "Charlie"}, + }, + }, + }, + { + description: "Nested arrays and objects", + input: map[string]interface{}{}, + code: "company.departments[1].staff[0].role", + value: "Manager", + expectedOutput: map[string]interface{}{ + "company": map[string]interface{}{ + "departments": []interface{}{ + map[string]interface{}{}, + map[string]interface{}{ + "staff": []interface{}{ + map[string]interface{}{"role": "Manager"}, + }, + }, + }, + }, + }, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.description, func(t *testing.T) { + setValue(tt.input, tt.code, tt.value) + assert.Equal(t, tt.expectedOutput, tt.input) + }) + } +} + +func TestObjectsToDocument_ValPriority(t *testing.T) { + tests := []struct { + description string + input string + expectedOutput string + }{ + { + description: "Use Val when present", + input: `[ + {"Code":"person.name","Val":"Alice","Value":"ShouldNotUse"}, + {"Code":"person.age","Val":30} + ]`, + expectedOutput: `{"person":{"age":30,"name":"Alice"}}`, + }, + { + description: "Use Value when Val not present", + input: `[ + {"Code":"person.name","Value":"Bob"}, + {"Code":"person.age","Val":25} + ]`, + // "person.name" should come from Value since Val is not present + expectedOutput: `{"person":{"age":25,"name":"Bob"}}`, + }, + { + description: "Array indexing in Code with Val", + input: `[ + {"Code":"people[0].name","Val":"Carol"}, + {"Code":"people[1].name","Value":"Dave"} + ]`, + expectedOutput: `{"people":[{"name":"Carol"},{"name":"Dave"}]}`, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.description, func(t *testing.T) { + var inputData interface{} + err := json.Unmarshal([]byte(tt.input), &inputData) + assert.NoError(t, err) + + output, err := ObjectsToDocument(inputData) + assert.NoError(t, err) + + bytes, err := json.Marshal(output) + assert.NoError(t, err) + assert.Equal(t, tt.expectedOutput, string(bytes)) + }) + } +} + +func TestObjectsToDocument_ComplexArrayPaths(t *testing.T) { + tests := []struct { + description string + input string + expectedOutput string + }{ + { + description: "Set multiple nested array values", + input: `[ + {"Code":"root.list[0].items[0].value","Val":"Item0-0"}, + {"Code":"root.list[0].items[1].value","Value":"Item0-1"}, + {"Code":"root.list[1].items[0].value","Val":"Item1-0"} + ]`, + expectedOutput: `{"root":{"list":[{"items":[{"value":"Item0-0"},{"value":"Item0-1"}]},{"items":[{"value":"Item1-0"}]}]}}`, + }, + { + description: "No Val or Value", + input: `[ + {"Code":"topKey","Whatever":"none"}, + {"Code":"anotherKey","Val":null} + ]`, + // No Val or Value means keys are not set + expectedOutput: `{}`, + }, + { + description: "Val is nil, use Value", + input: `[ + {"Code":"testKey","Val":null,"Value":"RealValue"} + ]`, + expectedOutput: `{"testKey":"RealValue"}`, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.description, func(t *testing.T) { + var inputData interface{} + err := json.Unmarshal([]byte(tt.input), &inputData) + assert.NoError(t, err) + + output, err := ObjectsToDocument(inputData) + assert.NoError(t, err) + + bytes, err := json.Marshal(output) + assert.NoError(t, err) + assert.Equal(t, tt.expectedOutput, string(bytes)) + }) + } +} From dd436d6283e52c7685475fea21712838913f7a1c Mon Sep 17 00:00:00 2001 From: xiatech-ricky Date: Wed, 18 Dec 2024 20:19:56 +0000 Subject: [PATCH 58/58] chore(as-3797): throw error if no Code found Changed `continue` to throw an Error if no Code is found in the array item list for $objectsToDocument --- jlib/new.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jlib/new.go b/jlib/new.go index 7d57488..e38b200 100644 --- a/jlib/new.go +++ b/jlib/new.go @@ -327,8 +327,8 @@ func ObjectsToDocument(input interface{}) (interface{}, error) { } code, ok := item["Code"].(string) - if !ok { - continue + if code == "" || !ok { + return nil, errors.New("$objectsToDocument input must contain a 'Code' field that is non-empty string") } var value interface{}