From 3f9bcdd8574c9cb709e0e943ac7c61d45354641c Mon Sep 17 00:00:00 2001 From: vizee Date: Sun, 23 Apr 2023 20:13:21 +0800 Subject: [PATCH] feat: transcode between json and protobuf --- go.mod | 5 + go.sum | 6 + helper.go | 12 + helper_test.go | 26 ++ json_builder.go | 49 ++++ json_builder_test.go | 25 ++ jsonlit/iter.go | 141 +++++++++++ jsonlit/iter_test.go | 21 ++ jsonlit/string.go | 78 ++++++ jsonlit/string_test.go | 24 ++ jtop.go | 524 +++++++++++++++++++++++++++++++++++++++++ jtop_test.go | 360 ++++++++++++++++++++++++++++ metadata.go | 152 ++++++++++++ metadata_test.go | 71 ++++++ proto/decode.go | 75 ++++++ proto/decode_test.go | 37 +++ proto/encode.go | 79 +++++++ proto/encode_test.go | 18 ++ ptoj.go | 412 ++++++++++++++++++++++++++++++++ ptoj_test.go | 343 +++++++++++++++++++++++++++ types_test.go | 41 ++++ 21 files changed, 2499 insertions(+) create mode 100644 go.mod create mode 100644 go.sum create mode 100644 helper.go create mode 100644 helper_test.go create mode 100644 json_builder.go create mode 100644 json_builder_test.go create mode 100644 jsonlit/iter.go create mode 100644 jsonlit/iter_test.go create mode 100644 jsonlit/string.go create mode 100644 jsonlit/string_test.go create mode 100644 jtop.go create mode 100644 jtop_test.go create mode 100644 metadata.go create mode 100644 metadata_test.go create mode 100644 proto/decode.go create mode 100644 proto/decode_test.go create mode 100644 proto/encode.go create mode 100644 proto/encode_test.go create mode 100644 ptoj.go create mode 100644 ptoj_test.go create mode 100644 types_test.go diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..c53632a --- /dev/null +++ b/go.mod @@ -0,0 +1,5 @@ +module github.com/vizee/jsonpb + +go 1.20 + +require google.golang.org/protobuf v1.30.0 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..37169cc --- /dev/null +++ b/go.sum @@ -0,0 +1,6 @@ +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= diff --git a/helper.go b/helper.go new file mode 100644 index 0000000..739a57c --- /dev/null +++ b/helper.go @@ -0,0 +1,12 @@ +package jsonpb + +import "unsafe" + +func noescape(p unsafe.Pointer) unsafe.Pointer { + x := uintptr(p) + return unsafe.Pointer(x ^ 0) +} + +func asString(s []byte) string { + return *(*string)(noescape(unsafe.Pointer(&s))) +} diff --git a/helper_test.go b/helper_test.go new file mode 100644 index 0000000..4d84945 --- /dev/null +++ b/helper_test.go @@ -0,0 +1,26 @@ +package jsonpb + +import ( + "testing" +) + +func Test_asString(t *testing.T) { + type args struct { + s []byte + } + tests := []struct { + name string + args args + want string + }{ + {name: "empty", args: args{s: []byte("")}, want: ""}, + {name: "abc", args: args{s: []byte("abc")}, want: "abc"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := asString(tt.args.s); got != tt.want { + t.Errorf("asString() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/json_builder.go b/json_builder.go new file mode 100644 index 0000000..bf485c7 --- /dev/null +++ b/json_builder.go @@ -0,0 +1,49 @@ +package jsonpb + +import "github.com/vizee/jsonpb/jsonlit" + +type JsonBuilder struct { + buf []byte +} + +func UnsafeJsonBuilder(buf []byte) *JsonBuilder { + return &JsonBuilder{buf: buf} +} + +func (b *JsonBuilder) Len() int { + return len(b.buf) +} + +func (b *JsonBuilder) Reserve(n int) { + if cap(b.buf)-len(b.buf) < n { + newbuf := make([]byte, len(b.buf), cap(b.buf)+n) + copy(newbuf, b.buf) + b.buf = newbuf + } +} + +func (b *JsonBuilder) String() string { + return asString(b.buf) +} + +func (b *JsonBuilder) IntoBytes() []byte { + buf := b.buf + b.buf = nil + return buf +} + +func (b *JsonBuilder) AppendBytes(s ...byte) { + b.buf = append(b.buf, s...) +} + +func (b *JsonBuilder) AppendString(s string) { + b.buf = append(b.buf, s...) +} + +func (b *JsonBuilder) AppendByte(c byte) { + b.buf = append(b.buf, c) +} + +func (b *JsonBuilder) AppendEscapedString(s string) { + b.buf = jsonlit.EscapeString(b.buf, s) +} diff --git a/json_builder_test.go b/json_builder_test.go new file mode 100644 index 0000000..fd0a88e --- /dev/null +++ b/json_builder_test.go @@ -0,0 +1,25 @@ +package jsonpb + +import ( + "testing" +) + +func TestJsonBuilder(t *testing.T) { + var b JsonBuilder + b.AppendByte('{') + b.AppendByte('"') + b.AppendEscapedString("b\tc") + b.AppendString(`":`) + b.AppendString("123") + b.AppendString("}") + if b.String() != `{"b\tc":123}` { + t.Fatal("b.String():", b.String()) + } + b1 := UnsafeJsonBuilder([]byte(`{"a":`)) + b1.Reserve(b.Len() + 1) + b1.AppendBytes(b.IntoBytes()...) + b1.AppendByte('}') + if b1.String() != `{"a":{"b\tc":123}}` { + t.Fatal("b1.String():", b1.String()) + } +} diff --git a/jsonlit/iter.go b/jsonlit/iter.go new file mode 100644 index 0000000..d58d998 --- /dev/null +++ b/jsonlit/iter.go @@ -0,0 +1,141 @@ +package jsonlit + +type Bytes interface { + ~string | []byte +} + +type Kind int + +const ( + Invalid Kind = iota + Null + Bool + Number + String + Opener + Object + ObjectClose + Array + ArrayClose + Comma + Colon + EOF +) + +type Iter[S Bytes] struct { + s S + p int +} + +func NewIter[S Bytes](s S) *Iter[S] { + return &Iter[S]{ + s: s, + } +} + +func (it *Iter[S]) Reset(data S) { + it.s = data + it.p = 0 +} + +func (it *Iter[S]) EOF() bool { + return it.p >= len(it.s) +} + +func (it *Iter[S]) nextString() (Kind, S) { + b := it.p + p := it.p + 1 + for p < len(it.s) { + if it.s[p] == '"' && it.s[p-1] != '\\' { + it.p = p + 1 + return String, it.s[b:it.p] + } + p++ + } + it.p = p + return Invalid, it.s[b:] +} + +func (it *Iter[S]) nextNumber() (Kind, S) { + b := it.p + p := it.p + 1 + for p < len(it.s) { + c := it.s[p] + if !isdigit(c) && c != '.' && c != '-' && c != 'e' && c != 'E' { + break + } + p++ + } + it.p = p + return Number, it.s[b:p] +} + +func (it *Iter[S]) consume(kind Kind) (Kind, S) { + p := it.p + it.p++ + return kind, it.s[p : p+1] +} + +func (it *Iter[S]) expect(expected string, kind Kind) (Kind, S) { + p := it.p + e := p + len(expected) + if e > len(it.s) { + e = len(it.s) + kind = Invalid + } else if string(it.s[p:e]) != expected { + // 如果 S 是 string，那么 `string(it.s[p:e])` 没有任何开销， + // 如果 S 是 []byte，根据已知 go 语言优化，也不会产生开销。 + // See: https://www.go101.org/article/string.html#conversion-optimizations + kind = Invalid + } + it.p = e + return kind, it.s[p:e] +} + +func (it *Iter[S]) Next() (Kind, S) { + p := it.p + for p < len(it.s) && iswhitespace(it.s[p]) { + p++ + } + if p >= len(it.s) { + return EOF, it.s[len(it.s):] + } + it.p = p + + c := it.s[p] + switch c { + case 'n': + return it.expect("null", Null) + case 't': + return it.expect("true", Bool) + case 'f': + return it.expect("false", Bool) + case '"': + return it.nextString() + case '{': + return it.consume(Object) + case '}': + return it.consume(ObjectClose) + case '[': + return it.consume(Array) + case ']': + return it.consume(ArrayClose) + case ',': + return it.consume(Comma) + case ':': + return it.consume(Colon) + default: + if isdigit(c) || c == '-' { + return it.nextNumber() + } + } + return it.consume(Invalid) +} + +func iswhitespace(c byte) bool { + return c == ' ' || c == '\n' || c == '\r' || c == '\t' +} + +func isdigit(c byte) bool { + return '0' <= c && c <= '9' +} diff --git a/jsonlit/iter_test.go b/jsonlit/iter_test.go new file mode 100644 index 0000000..d6a1c43 --- /dev/null +++ b/jsonlit/iter_test.go @@ -0,0 +1,21 @@ +package jsonlit + +import ( + "testing" +) + +func TestIter(t *testing.T) { + const input = "{\"animals\":{\"dog\":[{\"name\":\"Rufus\",\"age\":15,\"is_male\":true},{\"name\":\"Marty\",\"age\":null,\"is_male\":false}]}}" + it := NewIter(input) + for !it.EOF() { + k, s := it.Next() + if k == Invalid { + t.Fatal(k, string(s)) + } + t.Log(string(s)) + } + eof, _ := it.Next() + if eof != EOF { + t.Fatal(eof) + } +} diff --git a/jsonlit/string.go b/jsonlit/string.go new file mode 100644 index 0000000..2de40a6 --- /dev/null +++ b/jsonlit/string.go @@ -0,0 +1,78 @@ +package jsonlit + +import ( + "unicode/utf8" +) + +const ( + rawMark = '0' + escapeTable = "00000000btn0fr00000000000000000000\"000000000000/00000000000000000000000000000000000000000000\\" + unescapeTable = "0000000000000000000000000000000000\"000000000000/00000000000000000000000000000000000000000000\\00000\x08000\x0C0000000\n000\r0\tu" +) + +func EscapeString[S Bytes](dst []byte, s S) []byte { + begin := 0 + i := 0 + for i < len(s) { + c := s[i] + if int(c) >= len(escapeTable) || escapeTable[c] == rawMark { + i++ + continue + } + if begin < i { + dst = append(dst, s[begin:i]...) + } + dst = append(dst, '\\', escapeTable[c]) + i++ + begin = i + } + if begin < len(s) { + dst = append(dst, s[begin:]...) + } + return dst +} + +func UnescapeString[S Bytes](dst []byte, s S) ([]byte, bool) { + i := 0 + for i < len(s) { + c := s[i] + if c == '\\' { + i++ + if i >= len(s) { + return nil, false + } + c = s[i] + if int(c) >= len(unescapeTable) || unescapeTable[c] == rawMark { + return nil, false + } + if c == 'u' { + if i+4 >= len(s) { + return nil, false + } + uc := rune(0) + for k := 0; k < 4; k++ { + i++ + c = s[i] + if isdigit(c) { + uc = uc<<4 | rune(c-'0') + } else if 'A' <= c && c <= 'F' { + uc = uc<<4 | rune(c-'A'+10) + } else if 'a' <= c && c <= 'f' { + uc = uc<<4 | rune(c-'a'+10) + } else { + return nil, false + } + } + var u8 [6]byte + n := utf8.EncodeRune(u8[:], uc) + dst = append(dst, u8[:n]...) + } else { + dst = append(dst, unescapeTable[c]) + } + } else { + dst = append(dst, c) + } + i++ + } + return dst, true +} diff --git a/jsonlit/string_test.go b/jsonlit/string_test.go new file mode 100644 index 0000000..6ca95d0 --- /dev/null +++ b/jsonlit/string_test.go @@ -0,0 +1,24 @@ +package jsonlit + +import "testing" + +func TestEscapeString(t *testing.T) { + if s := EscapeString(nil, "\t"); string(s) != `\t` { + t.Fatal(string(s)) + } + if s := EscapeString(nil, "123\tabc"); string(s) != `123\tabc` { + t.Fatal(string(s)) + } +} + +func TestUnescapeString(t *testing.T) { + if s, ok := UnescapeString(nil, `\t`); ok && string(s) != "\t" { + t.Fatal(string(s)) + } + if s, ok := UnescapeString(nil, `123\tabc`); ok && string(s) != "123\tabc" { + t.Fatal(string(s)) + } + if s, ok := UnescapeString(nil, `\u4f60\u597d`); ok && string(s) != "你好" { + t.Fatal(string(s)) + } +} diff --git a/jtop.go b/jtop.go new file mode 100644 index 0000000..d028c7c --- /dev/null +++ b/jtop.go @@ -0,0 +1,524 @@ +package jsonpb + +import ( + "encoding/base64" + "errors" + "io" + "math" + "strconv" + + "github.com/vizee/jsonpb/jsonlit" + "github.com/vizee/jsonpb/proto" +) + +type JsonIter = jsonlit.Iter[[]byte] + +var ( + ErrUnexpectedToken = errors.New("unexpected token") + ErrTypeMismatch = errors.New("field type mismatch") +) + +func transJsonRepeatedMessage(p *proto.Encoder, j *JsonIter, field *Field) error { + var buf proto.Encoder + for !j.EOF() { + tok, _ := j.Next() + switch tok { + case jsonlit.ArrayClose: + return nil + case jsonlit.Comma: + case jsonlit.Object: + buf.Clear() + err := transJsonObject(&buf, j, field.Ref) + if err != nil { + return err + } + p.EmitBytes(field.Tag, buf.Bytes()) + case jsonlit.Null: + // null 会表达为一个空对象占位 + p.EmitBytes(field.Tag, nil) + default: + return ErrUnexpectedToken + } + } + return io.ErrUnexpectedEOF +} + +func walkJsonArray(j *JsonIter, expect jsonlit.Kind, f func([]byte) error) error { + for !j.EOF() { + tok, s := j.Next() + switch tok { + case jsonlit.ArrayClose: + return nil + case jsonlit.Comma: + case expect: + err := f(s) + if err != nil { + return err + } + default: + return ErrUnexpectedToken + } + } + return io.ErrUnexpectedEOF +} + +func transJsonArrayField(p *proto.Encoder, j *JsonIter, field *Field) error { + switch field.Kind { + case MessageKind: + return transJsonRepeatedMessage(p, j, field) + case BytesKind: + // 暂不允许 null 转到 bytes + err := walkJsonArray(j, jsonlit.String, func(s []byte) error { + return transJsonBytes(p, field.Tag, false, s) + }) + if err != nil { + return err + } + case StringKind: + err := walkJsonArray(j, jsonlit.String, func(s []byte) error { + return transJsonString(p, field.Tag, false, s) + }) + if err != nil { + return err + } + default: + var ( + packed proto.Encoder + err error + ) + switch field.Kind { + case DoubleKind: + err = walkJsonArray(j, jsonlit.Number, func(s []byte) error { + x, err := strconv.ParseFloat(asString(s), 64) + if err != nil { + return err + } + packed.WriteFixed64(math.Float64bits(x)) + return nil + }) + case FloatKind: + err = walkJsonArray(j, jsonlit.Number, func(s []byte) error { + x, err := strconv.ParseFloat(asString(s), 32) + if err != nil { + return err + } + packed.WriteFixed32(math.Float32bits(float32(x))) + return nil + }) + case Int32Kind: + err = walkJsonArray(j, jsonlit.Number, func(s []byte) error { + x, err := strconv.ParseInt(asString(s), 10, 32) + if err != nil { + return err + } + packed.WriteVarint(uint64(x)) + return nil + }) + case Int64Kind: + err = walkJsonArray(j, jsonlit.Number, func(s []byte) error { + x, err := strconv.ParseInt(asString(s), 10, 64) + if err != nil { + return err + } + packed.WriteVarint(uint64(x)) + return nil + }) + case Uint32Kind: + err = walkJsonArray(j, jsonlit.Number, func(s []byte) error { + x, err := strconv.ParseUint(asString(s), 10, 32) + if err != nil { + return err + } + packed.WriteVarint(x) + return nil + }) + case Uint64Kind: + err = walkJsonArray(j, jsonlit.Number, func(s []byte) error { + x, err := strconv.ParseUint(asString(s), 10, 64) + if err != nil { + return err + } + packed.WriteVarint(x) + return nil + }) + case Sint32Kind: + err = walkJsonArray(j, jsonlit.Number, func(s []byte) error { + x, err := strconv.ParseInt(asString(s), 10, 32) + if err != nil { + return err + } + packed.WriteZigzag(x) + return nil + }) + case Sint64Kind: + err = walkJsonArray(j, jsonlit.Number, func(s []byte) error { + x, err := strconv.ParseInt(asString(s), 10, 64) + if err != nil { + return err + } + packed.WriteZigzag(x) + return nil + }) + case Fixed32Kind: + err = walkJsonArray(j, jsonlit.Number, func(s []byte) error { + x, err := strconv.ParseUint(asString(s), 10, 32) + if err != nil { + return err + } + packed.WriteFixed32(uint32(x)) + return nil + }) + case Fixed64Kind: + err = walkJsonArray(j, jsonlit.Number, func(s []byte) error { + x, err := strconv.ParseUint(asString(s), 10, 64) + if err != nil { + return err + } + packed.WriteFixed64(x) + return nil + }) + case Sfixed32Kind: + err = walkJsonArray(j, jsonlit.Number, func(s []byte) error { + x, err := strconv.ParseInt(asString(s), 10, 32) + if err != nil { + return err + } + packed.WriteFixed32(uint32(x)) + return nil + }) + case Sfixed64Kind: + err = walkJsonArray(j, jsonlit.Number, func(s []byte) error { + x, err := strconv.ParseInt(asString(s), 10, 64) + if err != nil { + return err + } + packed.WriteFixed64(uint64(x)) + return nil + }) + case BoolKind: + err = walkJsonArray(j, jsonlit.Bool, func(s []byte) error { + var x uint64 + if len(s) == 4 { + x = 1 + } else { + x = 0 + } + packed.WriteVarint(x) + return nil + }) + default: + err = ErrTypeMismatch + } + if err != nil { + return err + } + if packed.Len() != 0 { + p.EmitBytes(field.Tag, packed.Bytes()) + } + } + return nil +} + +func transJsonToMap(p *proto.Encoder, j *JsonIter, tag uint32, entry *Message) error { + keyField, valueField := entry.FieldByTag(1), entry.FieldByTag(2) + // assert(keyField != nil && valueField != nil) + + var buf proto.Encoder + expectValue := false + for !j.EOF() { + lead, s := j.Next() + switch lead { + case jsonlit.ObjectClose: + if expectValue { + return ErrUnexpectedToken + } + return nil + case jsonlit.Comma, jsonlit.Colon: + // 忽略语法检查 + continue + default: + if expectValue { + // NOTE: transJsonField 会跳过 0 值字段，导致结果比 proto.Marshal 的结果字节数更少，但不影响反序列化结果 + err := transJsonField(&buf, j, valueField, lead, s) + if err != nil { + return err + } + if buf.Len() != 0 { + p.EmitBytes(tag, buf.Bytes()) + } + expectValue = false + } else if lead == jsonlit.String { + buf.Clear() + if keyField.Kind == StringKind { + err := transJsonString(&buf, 1, true, s) + if err != nil { + return err + } + } else if IsNumericKind(keyField.Kind) { + // 允许把 json key 转为将数值类型的 map key + err := transJsonNumeric(&buf, 1, keyField.Kind, s[1:len(s)-1]) + if err != nil { + return err + } + } else { + return ErrTypeMismatch + } + expectValue = true + } else { + return ErrUnexpectedToken + } + } + } + return io.ErrUnexpectedEOF +} + +func transJsonNumeric(p *proto.Encoder, tag uint32, kind Kind, s []byte) error { + if !IsNumericKind(kind) { + return ErrTypeMismatch + } + // 提前检查 0 值 + if len(s) == 1 && s[0] == '0' { + return nil + } + switch kind { + case DoubleKind: + x, err := strconv.ParseFloat(asString(s), 64) + if err != nil { + return err + } + p.EmitFixed64(tag, math.Float64bits(x)) + case FloatKind: + x, err := strconv.ParseFloat(asString(s), 32) + if err != nil { + return err + } + p.EmitFixed32(tag, math.Float32bits(float32(x))) + case Int32Kind: + x, err := strconv.ParseInt(asString(s), 10, 32) + if err != nil { + return err + } + p.EmitVarint(tag, uint64(x)) + case Int64Kind: + x, err := strconv.ParseInt(asString(s), 10, 64) + if err != nil { + return err + } + p.EmitVarint(tag, uint64(x)) + case Uint32Kind: + x, err := strconv.ParseUint(asString(s), 10, 32) + if err != nil { + return err + } + p.EmitVarint(tag, uint64(x)) + case Uint64Kind: + x, err := strconv.ParseUint(asString(s), 10, 64) + if err != nil { + return err + } + p.EmitVarint(tag, x) + case Sint32Kind: + x, err := strconv.ParseInt(asString(s), 10, 32) + if err != nil { + return err + } + p.EmitZigzag(tag, x) + case Sint64Kind: + x, err := strconv.ParseInt(asString(s), 10, 64) + if err != nil { + return err + } + p.EmitZigzag(tag, x) + case Fixed32Kind: + x, err := strconv.ParseUint(asString(s), 10, 32) + if err != nil { + return err + } + p.EmitFixed32(tag, uint32(x)) + case Fixed64Kind: + x, err := strconv.ParseUint(asString(s), 10, 64) + if err != nil { + return err + } + p.EmitFixed64(tag, x) + case Sfixed32Kind: + x, err := strconv.ParseInt(asString(s), 10, 32) + if err != nil { + return err + } + p.EmitFixed32(tag, uint32(x)) + case Sfixed64Kind: + x, err := strconv.ParseInt(asString(s), 10, 64) + if err != nil { + return err + } + p.EmitFixed64(tag, uint64(x)) + } + return nil +} + +func transJsonString(p *proto.Encoder, tag uint32, omitEmpty bool, s []byte) error { + if len(s) == 2 && omitEmpty { + return nil + } + z := make([]byte, 0, len(s)-2) + z, ok := jsonlit.UnescapeString(z, s[1:len(s)-1]) + if !ok { + return errors.New("unescape malformed string") + } + p.EmitBytes(tag, z) + return nil +} + +func transJsonBytes(p *proto.Encoder, tag uint32, omitEmpty bool, s []byte) error { + if len(s) == 2 && omitEmpty { + return nil + } + z := make([]byte, base64.StdEncoding.DecodedLen(len(s)-2)) + n, err := base64.StdEncoding.Decode(z, s[1:len(s)-1]) + if err != nil { + return err + } + p.EmitBytes(tag, z[:n]) + return nil +} + +func transJsonField(p *proto.Encoder, j *JsonIter, field *Field, lead jsonlit.Kind, s []byte) error { + switch lead { + case jsonlit.String: + switch field.Kind { + case BytesKind: + return transJsonBytes(p, field.Tag, true, s) + case StringKind: + return transJsonString(p, field.Tag, true, s) + default: + return ErrTypeMismatch + } + case jsonlit.Number: + return transJsonNumeric(p, field.Tag, field.Kind, s) + case jsonlit.Bool: + if field.Kind == BoolKind { + if len(s) == 4 { + p.EmitVarint(field.Tag, 1) + } + return nil + } else { + return ErrTypeMismatch + } + case jsonlit.Null: + // 忽略所有 null + return nil + case jsonlit.Object: + switch field.Kind { + case MessageKind: + var buf proto.Encoder + err := transJsonObject(&buf, j, field.Ref) + if err != nil { + return err + } + if buf.Len() != 0 { + p.EmitBytes(field.Tag, buf.Bytes()) + } + return nil + case MapKind: + return transJsonToMap(p, j, field.Tag, field.Ref) + default: + return ErrTypeMismatch + } + case jsonlit.Array: + if field.Repeated { + return transJsonArrayField(p, j, field) + } + return ErrTypeMismatch + } + return ErrUnexpectedToken +} + +func skipJsonValue(j *JsonIter, lead jsonlit.Kind) error { + switch lead { + case jsonlit.Null, jsonlit.Bool, jsonlit.Number, jsonlit.String: + return nil + case jsonlit.Object: + for !j.EOF() { + tok, _ := j.Next() + switch tok { + case jsonlit.ObjectClose: + return nil + case jsonlit.Comma, jsonlit.Colon: + default: + err := skipJsonValue(j, tok) + if err != nil { + return err + } + } + } + case jsonlit.Array: + for !j.EOF() { + tok, _ := j.Next() + switch tok { + case jsonlit.ArrayClose: + return nil + case jsonlit.Comma: + default: + err := skipJsonValue(j, tok) + if err != nil { + return err + } + } + } + return io.ErrUnexpectedEOF + } + return ErrUnexpectedToken +} + +func transJsonObject(p *proto.Encoder, j *JsonIter, msg *Message) error { + var key []byte + for !j.EOF() { + lead, s := j.Next() + switch lead { + case jsonlit.ObjectClose: + if len(key) == 0 { + return nil + } + return io.ErrUnexpectedEOF + case jsonlit.Comma, jsonlit.Colon: + // 忽略语法检查 + continue + default: + if len(key) != 0 { + // 暂不转义 key + field := msg.FieldByName(asString(key[1 : len(key)-1])) + if field != nil { + err := transJsonField(p, j, field, lead, s) + if err != nil { + return err + } + } else { + err := skipJsonValue(j, lead) + if err != nil { + return err + } + } + key = nil + } else if lead == jsonlit.String { + key = s + } else { + return ErrUnexpectedToken + } + } + } + return io.ErrUnexpectedEOF +} + +// TranscodeToProto 通过 JsonIter 解析 JSON，并且根据 msg 将 JSON 内容转译到 protobuf 二进制。 +// 注意，受限于 metadata 可表达的结构和一些取舍，对 JSON 的解析并不按照 JSON 标准。 +func TranscodeToProto(p *proto.Encoder, j *JsonIter, msg *Message) error { + tok, _ := j.Next() + switch tok { + case jsonlit.Object: + return transJsonObject(p, j, msg) + case jsonlit.EOF: + return io.ErrUnexpectedEOF + } + return ErrUnexpectedToken +} diff --git a/jtop_test.go b/jtop_test.go new file mode 100644 index 0000000..8040a52 --- /dev/null +++ b/jtop_test.go @@ -0,0 +1,360 @@ +package jsonpb + +import ( + "encoding/hex" + "fmt" + "testing" + + "github.com/vizee/jsonpb/jsonlit" + "github.com/vizee/jsonpb/proto" +) + +func skipJsonValueCase(j string) error { + it := jsonlit.NewIter([]byte(j)) + tok, _ := it.Next() + err := skipJsonValue(it, tok) + if err != nil { + return err + } + if !it.EOF() { + return fmt.Errorf("incomplete") + } + return nil +} + +func Test_skipJsonValue(t *testing.T) { + tests := []struct { + name string + arg string + wantErr bool + }{ + {name: "array", arg: `[1,"hello",false,{"k1":"v1","k2":null}]`, wantErr: false}, + {name: "object", arg: `{"a":1,"b":"hello","c":[1,"hello",false,{"k1":"v1","k2":"v2"}],"d":{"k1":"v1","k2":"v2"}}`, wantErr: false}, + {name: "ignore_syntax", arg: `{"a" 1 "b" "hello" "c":[1 "hello" false {"k1":"v1","k2":"v2"}],"d":{"k1":"v1","k2":"v2"}}`, wantErr: false}, + {name: "bad_token", arg: `:`, wantErr: true}, + {name: "unterminated", arg: `{"k1":1,"k2":2`, wantErr: true}, + {name: "unterminated_sub", arg: `{"k1":1,"k2":[`, wantErr: true}, + {name: "unexpected_token", arg: `{"k1":1,"k2":[}`, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := skipJsonValueCase(tt.arg); (err != nil) != tt.wantErr { + t.Errorf("skipJsonValueCase() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func transJsonBytesCase(tag uint32, omitEmpty bool, s []byte) (string, error) { + var buf proto.Encoder + err := transJsonBytes(&buf, tag, omitEmpty, s) + if err != nil { + return "", err + } + return hex.EncodeToString(buf.Bytes()), nil +} + +func Test_transJsonBytes(t *testing.T) { + type args struct { + tag uint32 + omitEmpty bool + s []byte + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + {name: "empty", args: args{tag: 1, s: []byte(`""`)}, want: "0a00"}, + {name: "omit_empty", args: args{tag: 1, omitEmpty: true, s: []byte(`""`)}, want: ""}, + {name: "simple", args: args{tag: 1, s: []byte(`"aGVsbG8gd29ybGQ="`)}, want: "0a0b68656c6c6f20776f726c64"}, + {name: "illegal_base64", args: args{tag: 1, s: []byte(`"aGVsbG8gd29ybGQ"`)}, want: "", wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := transJsonBytesCase(tt.args.tag, tt.args.omitEmpty, tt.args.s) + if (err != nil) != tt.wantErr { + t.Errorf("transJsonBytesCase() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("transJsonBytesCase() = %v, want %v", got, tt.want) + } + }) + } +} + +func transJsonStringCase(tag uint32, omitEmpty bool, s []byte) (string, error) { + var buf proto.Encoder + err := transJsonString(&buf, tag, omitEmpty, s) + if err != nil { + return "", err + } + return hex.EncodeToString(buf.Bytes()), nil +} + +func Test_transJsonStringCase(t *testing.T) { + type args struct { + tag uint32 + omitEmpty bool + s []byte + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + {name: "empty", args: args{tag: 1, s: []byte(`""`)}, want: "0a00"}, + {name: "omit_empty", args: args{tag: 1, omitEmpty: true, s: []byte(`""`)}, want: ""}, + {name: "simple", args: args{tag: 1, s: []byte(`"hello world"`)}, want: "0a0b68656c6c6f20776f726c64"}, + {name: "escape", args: args{tag: 1, s: []byte(`"\u4f60\u597d"`)}, want: "0a06e4bda0e5a5bd"}, + {name: "illegal_escape", args: args{tag: 1, s: []byte(`"\z"`)}, want: "", wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := transJsonStringCase(tt.args.tag, tt.args.omitEmpty, tt.args.s) + if (err != nil) != tt.wantErr { + t.Errorf("transJsonStringCase() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("transJsonStringCase() = %v, want %v", got, tt.want) + } + }) + } +} + +func transJsonNumericCase(tag uint32, kind Kind, s []byte) (string, error) { + var buf proto.Encoder + err := transJsonNumeric(&buf, tag, kind, s) + if err != nil { + return "", err + } + return hex.EncodeToString(buf.Bytes()), nil +} + +func Test_transJsonNumeric(t *testing.T) { + type args struct { + tag uint32 + kind Kind + s []byte + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + {name: "omit_zero", args: args{tag: 1, kind: Int32Kind, s: []byte(`0`)}, want: ""}, + {name: "double", args: args{tag: 1, kind: DoubleKind, s: []byte(`1`)}, want: "09000000000000f03f"}, + {name: "bad_double", args: args{tag: 1, kind: DoubleKind, s: []byte(`a`)}, wantErr: true}, + {name: "float", args: args{tag: 2, kind: FloatKind, s: []byte(`2`)}, want: "1500000040"}, + {name: "bad_float", args: args{tag: 2, kind: FloatKind, s: []byte(`a`)}, wantErr: true}, + {name: "int32", args: args{tag: 3, kind: Int32Kind, s: []byte(`3`)}, want: "1803"}, + {name: "bad_int32", args: args{tag: 3, kind: Int32Kind, s: []byte(`a`)}, wantErr: true}, + {name: "int64", args: args{tag: 4, kind: Int64Kind, s: []byte(`4`)}, want: "2004"}, + {name: "bad_int64", args: args{tag: 4, kind: Int64Kind, s: []byte(`a`)}, wantErr: true}, + {name: "uint32", args: args{tag: 5, kind: Uint32Kind, s: []byte(`5`)}, want: "2805"}, + {name: "bad_uint32", args: args{tag: 5, kind: Uint32Kind, s: []byte(`a`)}, wantErr: true}, + {name: "uint64", args: args{tag: 6, kind: Uint64Kind, s: []byte(`6`)}, want: "3006"}, + {name: "bad_uint64", args: args{tag: 6, kind: Uint64Kind, s: []byte(`a`)}, wantErr: true}, + {name: "sint32", args: args{tag: 7, kind: Sint32Kind, s: []byte(`7`)}, want: "380e"}, + {name: "bad_sint32", args: args{tag: 7, kind: Sint32Kind, s: []byte(`a`)}, wantErr: true}, + {name: "sint64", args: args{tag: 8, kind: Sint64Kind, s: []byte(`8`)}, want: "4010"}, + {name: "bad_sint64", args: args{tag: 8, kind: Sint64Kind, s: []byte(`a`)}, wantErr: true}, + {name: "fixed32", args: args{tag: 9, kind: Fixed32Kind, s: []byte(`9`)}, want: "4d09000000"}, + {name: "bad_fixed32", args: args{tag: 9, kind: Fixed32Kind, s: []byte(`a`)}, wantErr: true}, + {name: "fixed64", args: args{tag: 10, kind: Fixed64Kind, s: []byte(`10`)}, want: "510a00000000000000"}, + {name: "bad_fixed64", args: args{tag: 10, kind: Fixed64Kind, s: []byte(`a`)}, wantErr: true}, + {name: "sfixed32", args: args{tag: 11, kind: Sfixed32Kind, s: []byte(`11`)}, want: "5d0b000000"}, + {name: "bad_sfixed32", args: args{tag: 11, kind: Sfixed32Kind, s: []byte(`a`)}, wantErr: true}, + {name: "sfixed64", args: args{tag: 12, kind: Sfixed64Kind, s: []byte(`12`)}, want: "610c00000000000000"}, + {name: "bad_sfixed64", args: args{tag: 12, kind: Sfixed64Kind, s: []byte(`a`)}, wantErr: true}, + {name: "invalid_kind", args: args{tag: 1, kind: BoolKind, s: []byte(`1`)}, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := transJsonNumericCase(tt.args.tag, tt.args.kind, tt.args.s) + if (err != nil) != tt.wantErr { + t.Errorf("transJsonNumericCase() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("transJsonNumericCase() = %v, want %v", got, tt.want) + } + }) + } +} + +func transJsonArrayFieldCase(j string, field *Field) (string, error) { + var buf proto.Encoder + it := jsonlit.NewIter([]byte(j)) + it.Next() + err := transJsonArrayField(&buf, it, field) + if err != nil { + return "", err + } + return hex.EncodeToString(buf.Bytes()), nil +} + +func Test_transJsonArrayFieldCase(t *testing.T) { + type args struct { + j string + field *Field + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + {name: "empty", args: args{j: `[]`, field: &Field{Tag: 1, Kind: Int32Kind, Repeated: true}}, want: ""}, + {name: "strings", args: args{j: `["hello","中文","🚀"]`, field: &Field{Tag: 2, Kind: StringKind, Repeated: true}}, want: "120568656c6c6f1206e4b8ade696871204f09f9a80"}, + {name: "bytes", args: args{j: `["YWJj","aGVsbG8=","d29ybGQ="]`, field: &Field{Tag: 2, Kind: BytesKind, Repeated: true}}, want: "1203616263120568656c6c6f1205776f726c64"}, + {name: "packed_double", args: args{j: `[0,1,2]`, field: &Field{Tag: 2, Kind: DoubleKind, Repeated: true}}, want: "12180000000000000000000000000000f03f0000000000000040"}, + {name: "packed_float", args: args{j: `[0,1,2]`, field: &Field{Tag: 2, Kind: FloatKind, Repeated: true}}, want: "120c000000000000803f00000040"}, + {name: "packed_int32", args: args{j: `[0,1,2]`, field: &Field{Tag: 2, Kind: Int32Kind, Repeated: true}}, want: "1203000102"}, + {name: "packed_int64", args: args{j: `[0,1,2]`, field: &Field{Tag: 2, Kind: Int64Kind, Repeated: true}}, want: "1203000102"}, + {name: "packed_uint32", args: args{j: `[0,1,2]`, field: &Field{Tag: 2, Kind: Uint32Kind, Repeated: true}}, want: "1203000102"}, + {name: "packed_uint64", args: args{j: `[0,1,2]`, field: &Field{Tag: 2, Kind: Uint64Kind, Repeated: true}}, want: "1203000102"}, + {name: "packed_sint32", args: args{j: `[0,1,2]`, field: &Field{Tag: 2, Kind: Sint32Kind, Repeated: true}}, want: "1203000204"}, + {name: "packed_sint64", args: args{j: `[0,1,2]`, field: &Field{Tag: 2, Kind: Sint64Kind, Repeated: true}}, want: "1203000204"}, + {name: "packed_fixed32", args: args{j: `[0,1,2]`, field: &Field{Tag: 2, Kind: Fixed32Kind, Repeated: true}}, want: "120c000000000100000002000000"}, + {name: "packed_fixed64", args: args{j: `[0,1,2]`, field: &Field{Tag: 2, Kind: Fixed64Kind, Repeated: true}}, want: "1218000000000000000001000000000000000200000000000000"}, + {name: "packed_sfixed32", args: args{j: `[0,1,2]`, field: &Field{Tag: 2, Kind: Sfixed32Kind, Repeated: true}}, want: "120c000000000100000002000000"}, + {name: "packed_sfixed64", args: args{j: `[0,1,2]`, field: &Field{Tag: 2, Kind: Sfixed64Kind, Repeated: true}}, want: "1218000000000000000001000000000000000200000000000000"}, + {name: "packed_bool", args: args{j: `[false,true,false]`, field: &Field{Tag: 2, Kind: BoolKind, Repeated: true}}, want: "1203000100"}, + {name: "messages", args: args{j: `[{},null,{"name":"string","age":123},{"age":456}]`, field: &Field{Tag: 2, Kind: MessageKind, Ref: getTestSimpleMessage(), Repeated: true}}, want: "12001200120a0a06737472696e67107b120310c803"}, + {name: "unterminated", args: args{j: `[0,1,2`, field: &Field{Tag: 2, Kind: Int32Kind, Repeated: true}}, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := transJsonArrayFieldCase(tt.args.j, tt.args.field) + if (err != nil) != tt.wantErr { + t.Errorf("transJsonArrayFieldCase() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("transJsonArrayFieldCase() = %v, want %v", got, tt.want) + } + }) + } +} + +func transJsonToMapCase(j string, tag uint32, entry *Message) (string, error) { + var buf proto.Encoder + it := jsonlit.NewIter([]byte(j)) + it.Next() + err := transJsonToMap(&buf, it, tag, entry) + if err != nil { + return "", err + } + return hex.EncodeToString(buf.Bytes()), nil +} + +func Test_transJsonToMapCase(t *testing.T) { + type args struct { + j string + tag uint32 + entry *Message + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + {name: "empty", args: args{j: `{}`, tag: 2, entry: getTestMapEntry(StringKind, Int32Kind, nil)}, want: ""}, + {name: "string_key", args: args{j: `{"a":1,"b":2}`, tag: 2, entry: getTestMapEntry(StringKind, Int32Kind, nil)}, want: "12050a0161100112050a01621002"}, + {name: "numeric_key", args: args{j: `{"1":"a","2":"b"}`, tag: 2, entry: getTestMapEntry(Int32Kind, StringKind, nil)}, want: "1205080112016112050802120162"}, + {name: "message_value", args: args{j: `{"v":{"name":"ok"}}`, tag: 2, entry: getTestMapEntry(StringKind, MessageKind, getTestSimpleMessage())}, want: "12090a017612040a026f6b"}, + {name: "type_mismatched", args: args{j: `{"1":"a","2":"b"}`, tag: 2, entry: getTestMapEntry(BoolKind, StringKind, nil)}, wantErr: true}, + {name: "unexpected_key", args: args{j: `{null`, tag: 2, entry: getTestMapEntry(StringKind, Int32Kind, nil)}, wantErr: true}, + {name: "unexpected_termination", args: args{j: `{"key":}`, tag: 2, entry: getTestMapEntry(StringKind, Int32Kind, nil)}, wantErr: true}, + {name: "eof", args: args{j: `{`, tag: 2, entry: getTestMapEntry(StringKind, Int32Kind, nil)}, wantErr: true}, + {name: "zero_value", args: args{j: `{"v":0}`, tag: 3, entry: getTestMapEntry(StringKind, Int32Kind, nil)}, want: "1a030a0176"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := transJsonToMapCase(tt.args.j, tt.args.tag, tt.args.entry) + if (err != nil) != tt.wantErr { + t.Errorf("transJsonToMapCase() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("transJsonToMapCase() = %v, want %v", got, tt.want) + } + }) + } +} + +func transJsonObjectCase(j string, msg *Message) (string, error) { + var buf proto.Encoder + it := jsonlit.NewIter([]byte(j)) + it.Next() + err := transJsonObject(&buf, it, msg) + if err != nil { + return "", err + } + return hex.EncodeToString(buf.Bytes()), nil +} + +func Test_transJsonObjectCase(t *testing.T) { + const ( + complexJson = `{"noexisted":null,"fdouble":123,"ffloat":123,"fint32":123,"fint64":123,"fuint32":123,"fuint64":123,"fsint32":123,"fsint64":123,"ffixed32":123,"ffixed64":123,"fsfixed32":123,"fsfixed64":123,"fbool":true,"fstring":"okk","fbytes":"AQID","fmap1":{"k":1},"fmap2":{"u":{"name":"abc","age":23,"male":true},"v":null},"fsubmsg":{"name":"efg","age":23,"male":true},"fint32s":[1,2,3],"fitems":[{"name":"abc","age":12,"male":true},null,{"name":"efg","age":23}]}` + complexWant = `090000000000c05e40150000f642187b207b287b307b38f60140f6014d7b000000517b000000000000005d7b000000617b00000000000000680172036f6b6b7a030102038201050a016b10018a010e0a017512090a03616263101718018a01030a01769201090a03656667101718019a0103010203a201090a03616263100c1801a20100a201070a036566671017` + ) + type args struct { + j string + msg *Message + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + {name: "empty", args: args{j: `{}`, msg: getTestSimpleMessage()}, want: ""}, + {name: "simple", args: args{j: `{"name":"string","age":123,"male":true}`, msg: getTestSimpleMessage()}, want: "0a06737472696e67107b1801"}, + {name: "complex", args: args{j: complexJson, msg: getTestComplexMessage()}, want: complexWant}, + {name: "eof", args: args{j: `{`, msg: getTestSimpleMessage()}, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := transJsonObjectCase(tt.args.j, tt.args.msg) + if (err != nil) != tt.wantErr { + t.Errorf("transJsonObjectCase() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("transJsonObjectCase() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestTranscodeToProto(t *testing.T) { + type args struct { + p *proto.Encoder + j *JsonIter + msg *Message + } + tests := []struct { + name string + args args + wantErr bool + }{ + {name: "simple", args: args{p: proto.NewEncoder(nil), j: jsonlit.NewIter([]byte(`{}`)), msg: getTestSimpleMessage()}}, + {name: "not_object", args: args{p: proto.NewEncoder(nil), j: jsonlit.NewIter([]byte(`[]`)), msg: getTestSimpleMessage()}, wantErr: true}, + {name: "eof", args: args{p: proto.NewEncoder(nil), j: jsonlit.NewIter([]byte(``)), msg: getTestSimpleMessage()}, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := TranscodeToProto(tt.args.p, tt.args.j, tt.args.msg); (err != nil) != tt.wantErr { + t.Errorf("TranscodeToProto() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/metadata.go b/metadata.go new file mode 100644 index 0000000..bd92336 --- /dev/null +++ b/metadata.go @@ -0,0 +1,152 @@ +package jsonpb + +import "sort" + +type Kind uint8 + +const ( + DoubleKind Kind = iota + FloatKind + Int32Kind + Int64Kind + Uint32Kind + Uint64Kind + Sint32Kind + Sint64Kind + Fixed32Kind + Fixed64Kind + Sfixed32Kind + Sfixed64Kind + BoolKind + StringKind + BytesKind + MapKind + MessageKind +) + +func IsNumericKind(k Kind) bool { + return DoubleKind <= k && k <= Sfixed64Kind +} + +type Message struct { + Name string + Fields []Field + + tagIdx []int + nameIdx map[string]int +} + +func NewMessage(name string, fields []Field, indexTag bool, indexName bool) *Message { + msg := &Message{ + Name: name, + Fields: fields, + } + if indexTag { + msg.BakeTagIndex() + } + if indexName { + msg.BakeNameIndex() + } + return msg +} + +func (m *Message) BakeTagIndex() { + fields := m.Fields + maxTag := uint32(0) + for i := range fields { + if fields[i].Tag > maxTag { + maxTag = fields[i].Tag + } + } + var tagIdx []int + if int(maxTag) < len(fields)+len(fields)/4+3 { + tagIdx = make([]int, maxTag+1) + for i := range tagIdx { + tagIdx[i] = -1 + } + for i := range fields { + tagIdx[fields[i].Tag] = i + } + } else { + // sparse-index + tagIdx = make([]int, len(fields)) + for i := range tagIdx { + tagIdx[i] = i + } + sort.Slice(tagIdx, func(i, j int) bool { + return fields[tagIdx[i]].Tag < fields[tagIdx[j]].Tag + }) + } + m.tagIdx = tagIdx +} + +func (m *Message) FieldIndexByTag(tag uint32) int { + if len(m.tagIdx) == len(m.Fields) { + l, r := 0, len(m.tagIdx)-1 + for l <= r { + mid := (l + r) / 2 + i := m.tagIdx[mid] + x := m.Fields[i].Tag + if x == tag { + return i + } else if x > tag { + r = mid - 1 + } else { + l = mid + 1 + } + } + } else if int(tag) < len(m.tagIdx) { + idx := m.tagIdx[tag] + if idx >= 0 { + return idx + } + } else { + for i := 0; i < len(m.Fields); i++ { + if m.Fields[i].Tag == tag { + return i + } + } + } + return -1 +} + +func (m *Message) FieldByTag(tag uint32) *Field { + idx := m.FieldIndexByTag(tag) + if idx >= 0 { + return &m.Fields[idx] + } + return nil +} + +func (m *Message) BakeNameIndex() { + names := make(map[string]int, len(m.Fields)) + for i := range m.Fields { + names[m.Fields[i].Name] = i + } + m.nameIdx = names +} + +func (m *Message) FieldByName(name string) *Field { + if m.nameIdx != nil { + idx, ok := m.nameIdx[name] + if ok { + return &m.Fields[idx] + } + } else { + for i := 0; i < len(m.Fields); i++ { + if m.Fields[i].Name == name { + return &m.Fields[i] + } + } + } + return nil +} + +type Field struct { + Name string + Kind Kind + Ref *Message + Tag uint32 + Repeated bool + OmitEmpty bool +} diff --git a/metadata_test.go b/metadata_test.go new file mode 100644 index 0000000..a4036b9 --- /dev/null +++ b/metadata_test.go @@ -0,0 +1,71 @@ +package jsonpb + +import ( + "testing" +) + +func TestMessage_noIndex(t *testing.T) { + m := &Message{ + Fields: []Field{ + {Name: "a", Tag: 1}, + {Name: "b", Tag: 10}, + {Name: "c", Tag: 11}, + {Name: "d", Tag: 20}, + }, + } + type args struct { + tag uint32 + name string + } + tests := []struct { + name string + args args + }{ + {name: "a", args: args{tag: 1, name: "a"}}, + {name: "b", args: args{tag: 10, name: "b"}}, + {name: "c", args: args{tag: 11, name: "c"}}, + {name: "d", args: args{tag: 20, name: "d"}}, + {name: "not_found", args: args{tag: 12, name: "e"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if byTag, byName := m.FieldByTag(tt.args.tag), m.FieldByName(tt.args.name); byTag != byName { + t.Errorf("byTag = %p, byName = %p", byTag, byName) + } + }) + } +} + +func TestMessage_sparseTagIndex(t *testing.T) { + m := &Message{ + Fields: []Field{ + {Name: "a", Tag: 1}, + {Name: "b", Tag: 10}, + {Name: "c", Tag: 11}, + {Name: "d", Tag: 20}, + }, + } + m.BakeTagIndex() + + type args struct { + tag uint32 + } + tests := []struct { + name string + args args + want int + }{ + {name: "a", args: args{tag: 1}, want: 0}, + {name: "b", args: args{tag: 10}, want: 1}, + {name: "c", args: args{tag: 11}, want: 2}, + {name: "d", args: args{tag: 20}, want: 3}, + {name: "not_found", args: args{tag: 12}, want: -1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := m.FieldIndexByTag(tt.args.tag); got != tt.want { + t.Errorf("Message.FieldIndexByTag() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/proto/decode.go b/proto/decode.go new file mode 100644 index 0000000..a9a790d --- /dev/null +++ b/proto/decode.go @@ -0,0 +1,75 @@ +package proto + +import "google.golang.org/protobuf/encoding/protowire" + +type Decoder struct { + buf []byte + i int +} + +func (d *Decoder) EOF() bool { + return d.i >= len(d.buf) +} + +func (d *Decoder) ReadVarint() (uint64, int) { + v, n := protowire.ConsumeVarint(d.buf[d.i:]) + if n < 0 { + return 0, n + } + d.i += n + return v, 0 +} + +func (d *Decoder) ReadZigzag() (int64, int) { + v, e := d.ReadVarint() + return protowire.DecodeZigZag(v), e +} + +func (d *Decoder) PeekTag() (uint32, protowire.Type, int) { + v, n := protowire.ConsumeVarint(d.buf[d.i:]) + if n < 0 { + return 0, 0, n + } + tag, wire := protowire.DecodeTag(v) + return uint32(tag), wire, 0 +} + +func (d *Decoder) ReadTag() (uint32, protowire.Type, int) { + v, e := d.ReadVarint() + if e < 0 { + return 0, 0, e + } + tag, wire := protowire.DecodeTag(v) + return uint32(tag), wire, 0 +} + +func (d *Decoder) ReadFixed32() (uint32, int) { + v, n := protowire.ConsumeFixed32(d.buf[d.i:]) + if n < 0 { + return 0, n + } + d.i += n + return v, 0 +} + +func (d *Decoder) ReadFixed64() (uint64, int) { + v, n := protowire.ConsumeFixed64(d.buf[d.i:]) + if n < 0 { + return 0, n + } + d.i += n + return v, 0 +} + +func (d *Decoder) ReadBytes() ([]byte, int) { + v, n := protowire.ConsumeBytes(d.buf[d.i:]) + if n < 0 { + return nil, n + } + d.i += n + return v, 0 +} + +func NewDecoder(buf []byte) *Decoder { + return &Decoder{buf: buf} +} diff --git a/proto/decode_test.go b/proto/decode_test.go new file mode 100644 index 0000000..fe42187 --- /dev/null +++ b/proto/decode_test.go @@ -0,0 +1,37 @@ +package proto + +import ( + "testing" + + "google.golang.org/protobuf/encoding/protowire" +) + +func assert2[A, B comparable](t *testing.T, f func() (A, B), a2 A, b2 B) { + a1, b1 := f() + if a1 != a2 { + t.Fatal("assert", a1, a2) + } + if b1 != b2 { + t.Fatal("assert", b1, b2) + } +} + +func TestDecode(t *testing.T) { + raw := []byte{8, 233, 1, 18, 4, 116, 101, 115, 116, 29, 219, 3, 0, 0, 32, 209, 3} + dec := NewDecoder(raw) + readTag := func() (uint32, protowire.Type) { + a, b, _ := dec.ReadTag() + return a, b + } + assert2(t, readTag, 1, protowire.VarintType) + assert2(t, dec.ReadVarint, 233, 0) + assert2(t, readTag, 2, protowire.BytesType) + data, n := dec.ReadBytes() + if n < 0 || string(data) != "test" { + t.Fatal("ReadBytes", string(data)) + } + assert2(t, readTag, 3, protowire.Fixed32Type) + assert2(t, dec.ReadFixed32, 987, 0) + assert2(t, readTag, 4, protowire.VarintType) + assert2(t, dec.ReadZigzag, -233, 0) +} diff --git a/proto/encode.go b/proto/encode.go new file mode 100644 index 0000000..32cf326 --- /dev/null +++ b/proto/encode.go @@ -0,0 +1,79 @@ +package proto + +import ( + "google.golang.org/protobuf/encoding/protowire" +) + +type Encoder struct { + buf []byte +} + +func (e *Encoder) Len() int { + return len(e.buf) +} + +func (e *Encoder) Clear() { + e.buf = e.buf[:0] +} + +func (e *Encoder) Bytes() []byte { + return e.buf +} + +func (e *Encoder) WriteBytes(s []byte) { + e.buf = append(e.buf, s...) +} + +func (e *Encoder) WriteVarint(v uint64) { + e.buf = protowire.AppendVarint(e.buf, v) +} + +func (e *Encoder) WriteZigzag(x int64) { + e.buf = protowire.AppendVarint(e.buf, protowire.EncodeZigZag(x)) +} + +func (e *Encoder) WriteFixed32(v uint32) { + e.buf = protowire.AppendFixed32(e.buf, v) +} + +func (e *Encoder) WriteFixed64(v uint64) { + e.buf = protowire.AppendFixed64(e.buf, v) +} + +func (e *Encoder) EmitVarint(tag uint32, v uint64) { + e.buf = protowire.AppendVarint(e.buf, protowire.EncodeTag(protowire.Number(tag), protowire.VarintType)) + e.buf = protowire.AppendVarint(e.buf, v) +} + +func (e *Encoder) EmitZigzag(tag uint32, x int64) { + e.buf = protowire.AppendVarint(e.buf, protowire.EncodeTag(protowire.Number(tag), protowire.VarintType)) + e.buf = protowire.AppendVarint(e.buf, protowire.EncodeZigZag(x)) +} + +func (e *Encoder) EmitFixed32(tag uint32, v uint32) { + e.buf = protowire.AppendVarint(e.buf, protowire.EncodeTag(protowire.Number(tag), protowire.Fixed32Type)) + e.buf = protowire.AppendFixed32(e.buf, v) +} + +func (e *Encoder) EmitFixed64(tag uint32, v uint64) { + e.buf = protowire.AppendVarint(e.buf, protowire.EncodeTag(protowire.Number(tag), protowire.Fixed64Type)) + e.buf = protowire.AppendFixed64(e.buf, v) +} + +func (e *Encoder) EmitBytes(tag uint32, s []byte) { + e.buf = protowire.AppendVarint(e.buf, protowire.EncodeTag(protowire.Number(tag), protowire.BytesType)) + e.buf = protowire.AppendVarint(e.buf, uint64(len(s))) + e.buf = append(e.buf, s...) +} + +func (e *Encoder) EmitString(tag uint32, s string) { + e.buf = protowire.AppendVarint(e.buf, protowire.EncodeTag(protowire.Number(tag), protowire.BytesType)) + e.buf = protowire.AppendVarint(e.buf, uint64(len(s))) + e.buf = append(e.buf, s...) +} + +func NewEncoder(buf []byte) *Encoder { + return &Encoder{ + buf: buf, + } +} diff --git a/proto/encode_test.go b/proto/encode_test.go new file mode 100644 index 0000000..c7c4c72 --- /dev/null +++ b/proto/encode_test.go @@ -0,0 +1,18 @@ +package proto + +import ( + "bytes" + "testing" +) + +func TestEncode(t *testing.T) { + e := NewEncoder(nil) + e.EmitVarint(1, 233) + e.EmitBytes(2, []byte(`test`)) + e.EmitFixed32(3, 987) + e.EmitZigzag(4, -233) + want := []byte{8, 233, 1, 18, 4, 116, 101, 115, 116, 29, 219, 3, 0, 0, 32, 209, 3} + if !bytes.Equal(e.Bytes(), want) { + t.Fail() + } +} diff --git a/ptoj.go b/ptoj.go new file mode 100644 index 0000000..51668bb --- /dev/null +++ b/ptoj.go @@ -0,0 +1,412 @@ +package jsonpb + +import ( + "encoding/base64" + "errors" + "math" + "strconv" + + "github.com/vizee/jsonpb/proto" + "google.golang.org/protobuf/encoding/protowire" +) + +var ( + ErrInvalidWireType = errors.New("invalid wire type") +) + +type protoValue struct { + x uint64 + s []byte +} + +func readProtoValue(p *proto.Decoder, wire protowire.Type) (val protoValue, e int) { + switch wire { + case protowire.VarintType: + val.x, e = p.ReadVarint() + case protowire.Fixed32Type: + var t uint32 + t, e = p.ReadFixed32() + val.x = uint64(t) + case protowire.Fixed64Type: + val.x, e = p.ReadFixed64() + case protowire.BytesType: + val.s, e = p.ReadBytes() + default: + e = -100 + } + return +} + +var wireTypeOfKind = [...]protowire.Type{ + DoubleKind: protowire.Fixed64Type, + FloatKind: protowire.Fixed32Type, + Int32Kind: protowire.VarintType, + Int64Kind: protowire.VarintType, + Uint32Kind: protowire.VarintType, + Uint64Kind: protowire.VarintType, + Sint32Kind: protowire.VarintType, + Sint64Kind: protowire.VarintType, + Fixed32Kind: protowire.Fixed32Type, + Fixed64Kind: protowire.Fixed64Type, + Sfixed32Kind: protowire.Fixed32Type, + Sfixed64Kind: protowire.Fixed64Type, + BoolKind: protowire.VarintType, + // StringKind: protowire.BytesType, + // BytesKind: protowire.BytesType, + // MapKind: protowire.BytesType, + // MessageKind: protowire.BytesType, +} + +func getFieldWireType(kind Kind, repeated bool) protowire.Type { + // 如果字段设置 repeated，那么值应该是 packed/string/bytes/message，所以 wire 一定是 BytesType + if !repeated && int(kind) < len(wireTypeOfKind) { + return wireTypeOfKind[kind] + } + return protowire.BytesType +} + +var defaultValues = [...]string{ + DoubleKind: `0`, + FloatKind: `0`, + Int32Kind: `0`, + Int64Kind: `0`, + Uint32Kind: `0`, + Uint64Kind: `0`, + Sint32Kind: `0`, + Sint64Kind: `0`, + Fixed32Kind: `0`, + Fixed64Kind: `0`, + Sfixed32Kind: `0`, + Sfixed64Kind: `0`, + BoolKind: `false`, + StringKind: `""`, + BytesKind: `""`, + MapKind: `{}`, + MessageKind: `{}`, +} + +func writeDefaultValue(j *JsonBuilder, repeated bool, kind Kind) { + if repeated { + j.AppendString("[]") + } else { + j.AppendString(defaultValues[kind]) + } +} + +func transProtoMap(j *JsonBuilder, p *proto.Decoder, tag uint32, entry *Message, s []byte) error { + j.AppendByte('{') + + keyField, valueField := entry.FieldByTag(1), entry.FieldByTag(2) + // assert(keyField != nil && valueField != nil) + keyWire := getFieldWireType(keyField.Kind, keyField.Repeated) + valueWire := getFieldWireType(valueField.Kind, valueField.Repeated) + // 暂不检查 keyField.Kind + + more := false + for { + if !more { + more = true + } else { + j.AppendByte(',') + } + + // 上下文比较复杂，直接嵌套逻辑读取 KV + + var values [2]protoValue + assigned := 0 + dec := proto.NewDecoder(s) + for !dec.EOF() && assigned != 3 { + tag, wire, e := dec.ReadTag() + if e < 0 { + return protowire.ParseError(e) + } + val, e := readProtoValue(dec, wire) + if e < 0 { + return protowire.ParseError(e) + } + switch tag { + case 1: + if wire != keyWire { + return ErrInvalidWireType + } + values[0] = val + assigned |= 1 + case 2: + if wire != valueWire { + return ErrInvalidWireType + } + values[1] = val + assigned |= 2 + } + } + + if assigned&1 != 0 { + if keyField.Kind == StringKind { + transProtoString(j, values[0].s) + } else { + j.AppendByte('"') + transProtoSimpleValue(j, keyField.Kind, values[0].x) + j.AppendByte('"') + } + } else { + j.AppendString(`""`) + } + + j.AppendByte(':') + + if assigned&2 != 0 { + switch valueField.Kind { + case StringKind: + transProtoString(j, values[1].s) + case BytesKind: + transProtoBytes(j, values[1].s) + case MessageKind: + err := transProtoMessage(j, proto.NewDecoder(values[1].s), valueField.Ref) + if err != nil { + return err + } + default: + transProtoSimpleValue(j, valueField.Kind, values[1].x) + } + } else { + writeDefaultValue(j, valueField.Repeated, valueField.Kind) + } + + if p.EOF() { + break + } + nextTag, wire, e := p.PeekTag() + if e < 0 { + return protowire.ParseError(e) + } + if nextTag != tag { + break + } + if wire != protowire.BytesType { + return ErrInvalidWireType + } + p.ReadVarint() // consume tag + s, e = p.ReadBytes() + if e < 0 { + return protowire.ParseError(e) + } + } + + j.AppendByte('}') + return nil +} + +func transProtoRepeatedBytes(j *JsonBuilder, p *proto.Decoder, field *Field, s []byte) error { + j.AppendByte('[') + + more := false + for { + if !more { + more = true + } else { + j.AppendByte(',') + } + + switch field.Kind { + case StringKind: + transProtoString(j, s) + case BytesKind: + transProtoBytes(j, s) + case MessageKind: + err := transProtoMessage(j, proto.NewDecoder(s), field.Ref) + if err != nil { + return err + } + } + + if p.EOF() { + break + } + tag, wire, e := p.PeekTag() + if e < 0 { + return protowire.ParseError(e) + } + if tag != field.Tag { + break + } + if wire != protowire.BytesType { + return ErrInvalidWireType + } + p.ReadVarint() // consume tag + s, e = p.ReadBytes() + if e < 0 { + return protowire.ParseError(e) + } + } + + j.AppendByte(']') + return nil +} + +func transProtoPackedArray(j *JsonBuilder, s []byte, field *Field) error { + p := proto.NewDecoder(s) + + j.AppendByte('[') + + wire := getFieldWireType(field.Kind, false) + more := false + for !p.EOF() { + if !more { + more = true + } else { + j.AppendByte(',') + } + val, e := readProtoValue(p, wire) + if e < 0 { + return protowire.ParseError(e) + } + transProtoSimpleValue(j, field.Kind, val.x) + } + + j.AppendByte(']') + return nil +} + +func transProtoBytes(j *JsonBuilder, s []byte) { + j.AppendByte('"') + n := base64.StdEncoding.EncodedLen(len(s)) + j.Reserve(n) + m := len(j.buf) + d := j.buf[m : m+n] + base64.StdEncoding.Encode(d, s) + j.buf = j.buf[:m+n] + j.AppendByte('"') +} + +func transProtoString(j *JsonBuilder, s []byte) { + j.AppendByte('"') + j.AppendEscapedString(asString(s)) + j.AppendByte('"') +} + +func transProtoSimpleValue(j *JsonBuilder, kind Kind, x uint64) { + switch kind { + case DoubleKind: + j.buf = strconv.AppendFloat(j.buf, math.Float64frombits(x), 'f', -1, 64) + case FloatKind: + j.buf = strconv.AppendFloat(j.buf, float64(math.Float32frombits(uint32(x))), 'f', -1, 32) + case Int32Kind, Int64Kind, Sfixed64Kind: + j.buf = strconv.AppendInt(j.buf, int64(x), 10) + case Uint32Kind, Uint64Kind, Fixed32Kind, Fixed64Kind: + j.buf = strconv.AppendUint(j.buf, x, 10) + case Sint32Kind, Sint64Kind: + j.buf = strconv.AppendInt(j.buf, protowire.DecodeZigZag(x), 10) + case Sfixed32Kind: + j.buf = strconv.AppendInt(j.buf, int64(int32(x)), 10) + case BoolKind: + if x != 0 { + j.AppendString("true") + } else { + j.AppendString("false") + } + } +} + +func transProtoMessage(j *JsonBuilder, p *proto.Decoder, msg *Message) error { + j.AppendByte('{') + + const preAllocSize = 16 + var ( + preAlloc [preAllocSize]bool + emitted []bool + ) + if len(msg.Fields) <= preAllocSize { + emitted = preAlloc[:] + } else { + emitted = make([]bool, len(msg.Fields)) + } + + more := false + for !p.EOF() { + tag, wire, e := p.ReadTag() + if e < 0 { + return protowire.ParseError(e) + } + + val, e := readProtoValue(p, wire) + if e < 0 { + return protowire.ParseError(e) + } + + fieldIdx := msg.FieldIndexByTag(tag) + if fieldIdx < 0 { + continue + } + field := &msg.Fields[fieldIdx] + expectedWire := getFieldWireType(field.Kind, field.Repeated) + if expectedWire != wire { + return ErrInvalidWireType + } + + if emitted[fieldIdx] { + continue + } + + if !more { + more = true + } else { + j.AppendByte(',') + } + j.AppendByte('"') + j.AppendString(field.Name) + j.AppendByte('"') + j.AppendByte(':') + + var err error + if field.Repeated { + switch field.Kind { + case StringKind, BytesKind, MessageKind: + err = transProtoRepeatedBytes(j, p, field, val.s) + default: + err = transProtoPackedArray(j, val.s, field) + } + } else if field.Kind == MapKind { + err = transProtoMap(j, p, field.Tag, field.Ref, val.s) + } else { + switch field.Kind { + case StringKind: + transProtoString(j, val.s) + case BytesKind: + transProtoBytes(j, val.s) + case MessageKind: + err = transProtoMessage(j, proto.NewDecoder(val.s), field.Ref) + default: + transProtoSimpleValue(j, field.Kind, val.x) + } + } + if err != nil { + return err + } + + emitted[fieldIdx] = true + } + + for i := range msg.Fields { + field := &msg.Fields[i] + if emitted[i] || field.OmitEmpty { + continue + } + if !more { + more = true + } else { + j.AppendByte(',') + } + j.AppendByte('"') + j.AppendString(field.Name) + j.AppendByte('"') + j.AppendByte(':') + writeDefaultValue(j, field.Repeated, field.Kind) + } + + j.AppendByte('}') + return nil +} + +func TranscodeToJson(j *JsonBuilder, p *proto.Decoder, msg *Message) error { + return transProtoMessage(j, p, msg) +} diff --git a/ptoj_test.go b/ptoj_test.go new file mode 100644 index 0000000..79debac --- /dev/null +++ b/ptoj_test.go @@ -0,0 +1,343 @@ +package jsonpb + +import ( + "encoding/hex" + "reflect" + "testing" + + "github.com/vizee/jsonpb/proto" + "google.golang.org/protobuf/encoding/protowire" +) + +func decodeBytes(s string) []byte { + b, err := hex.DecodeString(s) + if err != nil { + panic(err) + } + return b +} + +func readProtoValueCase(s string, wire protowire.Type) (protoValue, int) { + return readProtoValue(proto.NewDecoder(decodeBytes(s)), wire) +} + +func Test_readProtoValueCase(t *testing.T) { + type args struct { + s string + wire protowire.Type + } + tests := []struct { + name string + args args + want protoValue + want1 int + }{ + {name: "varint", args: args{s: "7b", wire: protowire.VarintType}, want: protoValue{x: 123}}, + {name: "fixed32", args: args{s: "7b000000", wire: protowire.Fixed32Type}, want: protoValue{x: 123}}, + {name: "fixed64", args: args{s: "7b00000000000000", wire: protowire.Fixed64Type}, want: protoValue{x: 123}}, + {name: "bytes", args: args{s: "036f6b6b", wire: protowire.BytesType}, want: protoValue{s: []byte("okk")}}, + {name: "bad_wire", args: args{s: "", wire: protowire.StartGroupType}, want: protoValue{}, want1: -100}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1 := readProtoValueCase(tt.args.s, tt.args.wire) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("readProtoValueCase() got = %v, want %v", got, tt.want) + } + if got1 != tt.want1 { + t.Errorf("readProtoValueCase() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} + +func transProtoBytesCase(s string) string { + var j JsonBuilder + transProtoBytes(&j, decodeBytes(s)) + return j.String() +} + +func Test_transProtoBytesCase(t *testing.T) { + type args struct { + s string + } + tests := []struct { + name string + args args + want string + }{ + {name: "empty", args: args{s: ""}, want: `""`}, + {name: "hello", args: args{s: "68656c6c6f"}, want: `"aGVsbG8="`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := transProtoBytesCase(tt.args.s) + if got != tt.want { + t.Errorf("transProtoBytesCase() = %v, want %v", got, tt.want) + } + }) + } +} + +func transProtoStringCase(s string) string { + var j JsonBuilder + transProtoString(&j, decodeBytes(s)) + return j.String() +} + +func Test_transProtoStringCase(t *testing.T) { + type args struct { + s string + } + tests := []struct { + name string + args args + want string + }{ + {name: "empty", args: args{s: ""}, want: `""`}, + {name: "hello", args: args{s: "68656c6c6f"}, want: `"hello"`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := transProtoStringCase(tt.args.s) + if got != tt.want { + t.Errorf("transProtoStringCase() = %v, want %v", got, tt.want) + } + }) + } +} + +func transProtoSimpleValueCase(kind Kind, s string) string { + pv, _ := readProtoValue(proto.NewDecoder(decodeBytes(s)), getFieldWireType(kind, false)) + var j JsonBuilder + transProtoSimpleValue(&j, kind, pv.x) + return j.String() +} + +func Test_transProtoSimpleValueCase(t *testing.T) { + type args struct { + kind Kind + s string + } + tests := []struct { + name string + args args + want string + }{ + {name: "double", args: args{kind: DoubleKind, s: "ae47e17a14aef33f"}, want: "1.23"}, + {name: "float", args: args{kind: FloatKind, s: "a4709d3f"}, want: "1.23"}, + {name: "int32", args: args{kind: Int32Kind, s: "7b"}, want: "123"}, + {name: "int64", args: args{kind: Int64Kind, s: "7b"}, want: "123"}, + {name: "uint32", args: args{kind: Uint32Kind, s: "7b"}, want: "123"}, + {name: "uint64", args: args{kind: Uint64Kind, s: "7b"}, want: "123"}, + {name: "sint32", args: args{kind: Sint32Kind, s: "f501"}, want: "-123"}, + {name: "sint64", args: args{kind: Sint64Kind, s: "f501"}, want: "-123"}, + {name: "fixed32", args: args{kind: Fixed32Kind, s: "7b000000"}, want: "123"}, + {name: "fixed64", args: args{kind: Fixed64Kind, s: "7b00000000000000"}, want: "123"}, + {name: "sfixed32", args: args{kind: Sfixed32Kind, s: "85ffffff"}, want: "-123"}, + {name: "sfixed64", args: args{kind: Sfixed64Kind, s: "85ffffffffffffff"}, want: "-123"}, + {name: "bool_true", args: args{kind: BoolKind, s: "01"}, want: "true"}, + {name: "bool_false", args: args{kind: BoolKind, s: "00"}, want: "false"}, + {name: "unexpected_kind", args: args{kind: StringKind, s: "00"}, want: ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := transProtoSimpleValueCase(tt.args.kind, tt.args.s); got != tt.want { + t.Errorf("transProtoSimpleValueCase() = %v, want %v", got, tt.want) + } + }) + } +} + +func transProtoRepeatedBytesCase(p string, field *Field, s string) (string, error) { + var j JsonBuilder + err := transProtoRepeatedBytes(&j, proto.NewDecoder(decodeBytes(p)), field, decodeBytes(s)) + if err != nil { + return "", err + } + return j.String(), nil +} + +func Test_transProtoRepeatedBytesCase(t *testing.T) { + type args struct { + p string + field *Field + s string + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + {name: "one_string", args: args{p: "", field: &Field{Tag: 1, Kind: StringKind}, s: "616263"}, want: `["abc"]`}, + {name: "more_strings", args: args{p: "0a0568656c6c6f0a05776f726c64", field: &Field{Tag: 1, Kind: StringKind}, s: "616263"}, want: `["abc","hello","world"]`}, + {name: "more_bytes", args: args{p: "0a0568656c6c6f0a05776f726c64", field: &Field{Tag: 1, Kind: BytesKind}, s: "616263"}, want: `["YWJj","aGVsbG8=","d29ybGQ="]`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := transProtoRepeatedBytesCase(tt.args.p, tt.args.field, tt.args.s) + if (err != nil) != tt.wantErr { + t.Errorf("transProtoRepeatedBytesCase() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("transProtoRepeatedBytesCase() = %v, want %v", got, tt.want) + } + }) + } +} + +func transProtoPackedArrayCase(p string, field *Field) (string, error) { + var j JsonBuilder + err := transProtoPackedArray(&j, decodeBytes(p), field) + if err != nil { + return "", err + } + return j.String(), nil +} + +func Test_transProtoPackedArrayCase(t *testing.T) { + type args struct { + p string + field *Field + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + {name: "empty", args: args{p: "", field: &Field{Tag: 1, Kind: Int32Kind}}, want: `[]`}, + {name: "int32s", args: args{p: "7bc8039506", field: &Field{Tag: 1, Kind: Int32Kind}}, want: `[123,456,789]`}, + {name: "doubles", args: args{p: "ae47e17a14aef33f3d0ad7a3703d12408fc2f5285c8f1f40", field: &Field{Tag: 1, Kind: DoubleKind}}, want: `[1.23,4.56,7.89]`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := transProtoPackedArrayCase(tt.args.p, tt.args.field) + if (err != nil) != tt.wantErr { + t.Errorf("transProtoPackedArrayCase() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("transProtoPackedArrayCase() = %v, want %v", got, tt.want) + } + }) + } +} + +func transProtoMapCase(p string, tag uint32, entry *Message, s string) (string, error) { + var j JsonBuilder + err := transProtoMap(&j, proto.NewDecoder(decodeBytes(p)), tag, entry, decodeBytes(s)) + if err != nil { + return "", err + } + return j.String(), nil +} + +func Test_transProtoMapCase(t *testing.T) { + type args struct { + p string + tag uint32 + entry *Message + s string + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + {name: "empty", args: args{p: "", tag: 1, entry: getTestMapEntry(StringKind, Int32Kind, nil), s: ""}, want: `{"":0}`}, + {name: "simple", args: args{p: "8201050a01621002", tag: 16, entry: getTestMapEntry(StringKind, Int32Kind, nil), s: "0a01611001"}, want: `{"a":1,"b":2}`}, + {name: "stop", args: args{p: "8201050a01621002", tag: 17, entry: getTestMapEntry(StringKind, Int32Kind, nil), s: "0a01611001"}, want: `{"a":1}`}, + {name: "int_key", args: args{p: "0a0608c803120162", tag: 1, entry: getTestMapEntry(Int32Kind, StringKind, nil), s: "087b120161"}, want: `{"123":"a","456":"b"}`}, + {name: "bytes_value", args: args{p: "", tag: 1, entry: getTestMapEntry(StringKind, BytesKind, nil), s: "0a0568656c6c6f1205776f726c64"}, want: `{"hello":"d29ybGQ="}`}, + {name: "message_value", args: args{p: "", tag: 1, entry: getTestMapEntry(StringKind, MessageKind, getTestSimpleMessage()), s: "0a0361626312090a03626f6210171801"}, want: `{"abc":{"name":"bob","age":23,"male":true}}`}, + {name: "default_key", args: args{p: "", tag: 1, entry: getTestMapEntry(StringKind, Int32Kind, nil), s: "107b"}, want: `{"":123}`}, + {name: "default_int32_value", args: args{p: "", tag: 1, entry: getTestMapEntry(StringKind, Int32Kind, nil), s: "0a0161"}, want: `{"a":0}`}, + {name: "default_string_value", args: args{p: "", tag: 1, entry: getTestMapEntry(StringKind, StringKind, nil), s: "0a0161"}, want: `{"a":""}`}, + {name: "default_message_value", args: args{p: "", tag: 1, entry: getTestMapEntry(StringKind, MessageKind, getTestSimpleMessage()), s: "0a0161"}, want: `{"a":{}}`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := transProtoMapCase(tt.args.p, tt.args.tag, tt.args.entry, tt.args.s) + if (err != nil) != tt.wantErr { + t.Errorf("transProtoMapCase() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("transProtoMapCase() = %v, want %v", got, tt.want) + } + }) + } +} + +func transProtoMessageCase(p string, msg *Message) (string, error) { + var j JsonBuilder + err := transProtoMessage(&j, proto.NewDecoder(decodeBytes(p)), msg) + if err != nil { + return "", err + } + return j.String(), nil +} + +func Test_transProtoMessageCase(t *testing.T) { + const ( + complexProto = `090000000000c05e40150000f642187b207b287b307b38f60140f6014d7b000000517b000000000000005d7b000000617b00000000000000680172036f6b6b7a030102038201050a016b10018a010e0a017512090a03616263101718018a01050a017612009201090a03656667101718019a0103010203a201090a03616263100c1801a20100a201070a036566671017` + complexWant = `{"fdouble":123,"ffloat":123,"fint32":123,"fint64":123,"fuint32":123,"fuint64":123,"fsint32":123,"fsint64":123,"ffixed32":123,"ffixed64":123,"fsfixed32":123,"fsfixed64":123,"fbool":true,"fstring":"okk","fbytes":"AQID","fmap1":{"k":1},"fmap2":{"u":{"name":"abc","age":23,"male":true},"v":{}},"fsubmsg":{"name":"efg","age":23,"male":true},"fint32s":[1,2,3],"fitems":[{"name":"abc","age":12,"male":true},{},{"name":"efg","age":23}]}` + complexDefaultWant = `{"fdouble":0,"ffloat":0,"fint32":0,"fint64":0,"fuint32":0,"fuint64":0,"fsint32":0,"fsint64":0,"ffixed32":0,"ffixed64":0,"fsfixed32":0,"fsfixed64":0,"fbool":false,"fstring":"","fbytes":"","fmap1":{},"fmap2":{},"fsubmsg":{},"fint32s":[],"fitems":[]}` + ) + + type args struct { + p string + msg *Message + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + {name: "empty", args: args{p: "", msg: getTestSimpleMessage()}, want: `{}`}, + {name: "simple", args: args{p: "0a03626f6210171801", msg: getTestSimpleMessage()}, want: `{"name":"bob","age":23,"male":true}`}, + {name: "emitted", args: args{p: "0a03626f6210170a03626f621801", msg: getTestSimpleMessage()}, want: `{"name":"bob","age":23,"male":true}`}, + {name: "complex", args: args{p: complexProto, msg: getTestComplexMessage()}, want: complexWant}, + {name: "default", args: args{p: "", msg: getTestComplexMessage()}, want: complexDefaultWant}, + {name: "eof", args: args{p: "0a", msg: getTestComplexMessage()}, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := transProtoMessageCase(tt.args.p, tt.args.msg) + if (err != nil) != tt.wantErr { + t.Errorf("transProtoMessageCase() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("transProtoMessageCase() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestTranscodeToJson(t *testing.T) { + type args struct { + j *JsonBuilder + p *proto.Decoder + msg *Message + } + tests := []struct { + name string + args args + wantErr bool + }{ + {name: "simple", args: args{j: &JsonBuilder{}, p: proto.NewDecoder(decodeBytes("")), msg: getTestSimpleMessage()}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := TranscodeToJson(tt.args.j, tt.args.p, tt.args.msg); (err != nil) != tt.wantErr { + t.Errorf("TranscodeToJson() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/types_test.go b/types_test.go new file mode 100644 index 0000000..71ac34a --- /dev/null +++ b/types_test.go @@ -0,0 +1,41 @@ +package jsonpb + +func getTestSimpleMessage() *Message { + return NewMessage("Simple", []Field{ + {Name: "name", Tag: 1, Kind: StringKind, OmitEmpty: true}, + {Name: "age", Tag: 2, Kind: Int32Kind, OmitEmpty: true}, + {Name: "male", Tag: 3, Kind: BoolKind, OmitEmpty: true}, + }, true, true) +} + +func getTestMapEntry(keyKind Kind, valueKind Kind, valueRef *Message) *Message { + return NewMessage("", []Field{ + 0: {Tag: 1, Kind: keyKind}, + 1: {Tag: 2, Kind: valueKind, Ref: valueRef}, + }, true, true) +} + +func getTestComplexMessage() *Message { + return NewMessage("Complex", []Field{ + {Name: "fdouble", Kind: DoubleKind, Tag: 1}, + {Name: "ffloat", Kind: FloatKind, Tag: 2}, + {Name: "fint32", Kind: Int32Kind, Tag: 3}, + {Name: "fint64", Kind: Int64Kind, Tag: 4}, + {Name: "fuint32", Kind: Uint32Kind, Tag: 5}, + {Name: "fuint64", Kind: Uint64Kind, Tag: 6}, + {Name: "fsint32", Kind: Sint32Kind, Tag: 7}, + {Name: "fsint64", Kind: Sint64Kind, Tag: 8}, + {Name: "ffixed32", Kind: Fixed32Kind, Tag: 9}, + {Name: "ffixed64", Kind: Fixed64Kind, Tag: 10}, + {Name: "fsfixed32", Kind: Sfixed32Kind, Tag: 11}, + {Name: "fsfixed64", Kind: Sfixed64Kind, Tag: 12}, + {Name: "fbool", Kind: BoolKind, Tag: 13}, + {Name: "fstring", Kind: StringKind, Tag: 14}, + {Name: "fbytes", Kind: BytesKind, Tag: 15}, + {Name: "fmap1", Kind: MapKind, Tag: 16, Ref: getTestMapEntry(StringKind, Int32Kind, nil)}, + {Name: "fmap2", Kind: MapKind, Tag: 17, Ref: getTestMapEntry(StringKind, MessageKind, getTestSimpleMessage())}, + {Name: "fsubmsg", Kind: MessageKind, Tag: 18, Ref: getTestSimpleMessage()}, + {Name: "fint32s", Kind: Int32Kind, Tag: 19, Repeated: true}, + {Name: "fitems", Kind: MessageKind, Tag: 20, Repeated: true, Ref: getTestSimpleMessage()}, + }, true, true) +}