diff --git a/internal/protogen/exporter.go b/internal/protogen/exporter.go index 4f6753db..18b7b918 100644 --- a/internal/protogen/exporter.go +++ b/internal/protogen/exporter.go @@ -45,7 +45,7 @@ func newBookExporter(protoPackage string, protoFileOptions map[string]string, ou } func (x *bookExporter) GetProtoFilePath() string { - return genProtoFilePath(x.wb.Name, x.FilenameSuffix) + return genProtoFilePath(x.wb.GetName(), x.FilenameSuffix) } func (x *bookExporter) export(checkProtoFileConflicts bool) error { @@ -203,11 +203,17 @@ func (x *sheetExporter) exportStruct() error { opts := &tableaupb.StructOptions{Name: x.ws.GetOptions().GetName(), Note: x.ws.Note} x.p.P(" option (tableau.struct) = {", marshalToText(opts), "};") x.p.P("") + + md := x.parseMessagerFromGeneratedProtos(x.ws.Name) + x.addNumberToFields(x.ws.Fields, md) // generate the fields depth := 1 - for i, field := range x.ws.Fields { - tagid := i + 1 - if err := x.exportField(depth, tagid, field, x.ws.Name); err != nil { + for _, field := range x.ws.Fields { + var fd protoreflect.FieldDescriptor + if md != nil { + fd = md.Fields().ByNumber(protoreflect.FieldNumber(field.GetNumber())) + } + if err := x.exportField(depth, field, x.ws.Name, fd); err != nil { return err } } @@ -273,16 +279,16 @@ func (x *sheetExporter) exportUnion() error { x.p.P(" message ", typ, " {") // generate the fields depth := 2 - tagid := 1 + tagid := int32(1) for _, field := range msgField.Fields { - if err := x.exportField(depth, tagid, field, msgField.Name); err != nil { + field.Number = tagid + cross := max(field.GetOptions().GetProp().GetCross(), 1) + tagid += cross + } + for _, field := range msgField.Fields { + if err := x.exportField(depth, field, msgField.Name, nil); err != nil { return err } - cross := int(field.GetOptions().GetProp().GetCross()) - if cross < 1 { - cross = 1 - } - tagid += cross } x.p.P(" }") } @@ -294,6 +300,54 @@ func (x *sheetExporter) exportUnion() error { return nil } +func (x *sheetExporter) parseMessagerFromGeneratedProtos(name string) protoreflect.MessageDescriptor { + if !x.be.gen.OutputOpt.PreserveFieldNumbers { + return nil + } + relPath := x.be.GetProtoFilePath() + fd, err := x.be.gen.GeneratedProtoRegistryFiles.FindFileByPath(relPath) + if err != nil { + return nil + } + return fd.Messages().ByName(protoreflect.Name(name)) +} + +func (*sheetExporter) addNumberToFields(fields []*internalpb.Field, md protoreflect.MessageDescriptor) { + if md == nil { + tagid := int32(1) + for _, field := range fields { + field.Number = tagid + tagid++ + } + return + } + fieldNameNumberMap := make(map[string]int32) + fieldNumbers := make(map[int32]bool) + for _, field := range fields { + if fd := md.Fields().ByName(protoreflect.Name(field.Name)); fd != nil { + number := int32(fd.Number()) + fieldNameNumberMap[field.Name] = number + fieldNumbers[number] = true + } + } + var missingNumbers []int32 + for i := int32(1); len(fieldNumbers)+len(missingNumbers) < len(fields); i++ { + if fieldNumbers[i] { + continue + } + missingNumbers = append(missingNumbers, i) + } + missingNumberIndex := 0 + for _, field := range fields { + if number, ok := fieldNameNumberMap[field.Name]; ok { + field.Number = number + } else { + field.Number = missingNumbers[missingNumberIndex] + missingNumberIndex++ + } + } +} + func (x *sheetExporter) exportMessager() error { // log.Debugf("workbook: %s", x.ws.String()) if x.be.messagerPatternRegexp != nil && !x.be.messagerPatternRegexp.MatchString(x.ws.Name) { @@ -302,11 +356,17 @@ func (x *sheetExporter) exportMessager() error { x.p.P("message ", x.ws.Name, " {") x.p.P(" option (tableau.worksheet) = {", marshalToText(x.ws.Options), "};") x.p.P("") + + md := x.parseMessagerFromGeneratedProtos(x.ws.Name) + x.addNumberToFields(x.ws.Fields, md) // generate the fields depth := 1 - for i, field := range x.ws.Fields { - tagid := i + 1 - if err := x.exportField(depth, tagid, field, x.ws.Name); err != nil { + for _, field := range x.ws.Fields { + var fd protoreflect.FieldDescriptor + if md != nil { + fd = md.Fields().ByNumber(protoreflect.FieldNumber(field.GetNumber())) + } + if err := x.exportField(depth, field, x.ws.Name, fd); err != nil { return err } } @@ -317,7 +377,7 @@ func (x *sheetExporter) exportMessager() error { return nil } -func (x *sheetExporter) exportField(depth int, tagid int, field *internalpb.Field, prefix string) error { +func (x *sheetExporter) exportField(depth int, field *internalpb.Field, prefix string, fd protoreflect.FieldDescriptor) error { label := "" if x.ws.GetOptions().GetFieldPresence() && types.IsScalarType(field.FullType) && @@ -328,8 +388,12 @@ func (x *sheetExporter) exportField(depth int, tagid int, field *internalpb.Fiel if field.Note != "" { note = " // " + field.Note } - x.p.P(printer.Indent(depth), label, field.FullType, " ", field.Name, " = ", tagid, " ", genFieldOptionsString(field.Options), ";", note) + x.p.P(printer.Indent(depth), label, field.FullType, " ", field.Name, " = ", field.Number, " ", genFieldOptionsString(field.Options), ";", note) + var md protoreflect.MessageDescriptor + if fd != nil { + md = fd.Message() + } typeName := field.Type fullTypeName := field.FullType if field.ListEntry != nil { @@ -339,6 +403,11 @@ func (x *sheetExporter) exportField(depth int, tagid int, field *internalpb.Fiel if field.MapEntry != nil { typeName = field.MapEntry.ValueType fullTypeName = field.MapEntry.ValueFullType + if fd != nil { + if v := fd.MapValue(); v != nil { + md = v.Message() + } + } } if types.IsWellKnownMessage(fullTypeName) { @@ -377,9 +446,14 @@ func (x *sheetExporter) exportField(depth int, tagid int, field *internalpb.Fiel // x.g.P("") x.p.P(printer.Indent(depth), "message ", typeName, " {") - for i, f := range field.Fields { - tagid := i + 1 - if err := x.exportField(depth+1, tagid, f, nestedMsgName); err != nil { + + x.addNumberToFields(field.Fields, md) + for _, f := range field.Fields { + var nextFd protoreflect.FieldDescriptor + if md != nil { + nextFd = md.Fields().ByNumber(protoreflect.FieldNumber(field.GetNumber())) + } + if err := x.exportField(depth+1, f, nestedMsgName, nextFd); err != nil { return err } } diff --git a/internal/protogen/exporter_test.go b/internal/protogen/exporter_test.go index f106506c..96ede7f0 100644 --- a/internal/protogen/exporter_test.go +++ b/internal/protogen/exporter_test.go @@ -8,10 +8,13 @@ import ( "github.com/stretchr/testify/assert" "github.com/tableauio/tableau/internal/printer" "github.com/tableauio/tableau/internal/x/xproto" + "github.com/tableauio/tableau/options" "github.com/tableauio/tableau/proto/tableaupb" "github.com/tableauio/tableau/proto/tableaupb/internalpb" + _ "github.com/tableauio/tableau/proto/tableaupb/unittestpb" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/reflect/protoregistry" ) func Test_genFieldOptionsString(t *testing.T) { @@ -351,7 +354,12 @@ func Test_sheetExporter_exportStruct(t *testing.T) { {Name: "fruit_type", Type: "FruitType", FullType: "protoconf.FruitType", Predefined: true, Options: &tableaupb.FieldOptions{Name: "FruitType"}}, }, }, - p: printer.New(), + p: printer.New(), + be: &bookExporter{ + gen: &Generator{ + OutputOpt: &options.ProtoOutputOption{}, + }, + }, typeInfos: &xproto.TypeInfos{}, nestedMessages: make(map[string]*internalpb.Field), }, @@ -363,6 +371,46 @@ func Test_sheetExporter_exportStruct(t *testing.T) { protoconf.FruitType fruit_type = 3 [(tableau.field) = {name:"FruitType"}]; } +`, + wantErr: false, + }, + { + name: "field-number-compatibility", + x: &sheetExporter{ + ws: &internalpb.Worksheet{ + Name: "Item", + Options: &tableaupb.WorksheetOptions{ + Name: "StructItem", + }, + Fields: []*internalpb.Field{ + {Name: "id", Type: "uint32", FullType: "uint32", Options: &tableaupb.FieldOptions{Name: "ID"}}, + {Name: "fruit_type", Type: "FruitType", FullType: "protoconf.FruitType", Predefined: true, Options: &tableaupb.FieldOptions{Name: "FruitType"}}, + {Name: "num", Type: "int32", FullType: "int32", Options: &tableaupb.FieldOptions{Name: "Num"}}, + }, + }, + p: printer.New(), + be: &bookExporter{ + wb: &internalpb.Workbook{ + Name: "tableau/protobuf/unittest/common", + }, + gen: &Generator{ + OutputOpt: &options.ProtoOutputOption{ + PreserveFieldNumbers: true, + }, + GeneratedProtoRegistryFiles: protoregistry.GlobalFiles, + }, + }, + typeInfos: &xproto.TypeInfos{}, + nestedMessages: make(map[string]*internalpb.Field), + }, + want: `message Item { + option (tableau.struct) = {name:"StructItem"}; + + uint32 id = 1 [(tableau.field) = {name:"ID"}]; + protoconf.FruitType fruit_type = 3 [(tableau.field) = {name:"FruitType"}]; + int32 num = 2 [(tableau.field) = {name:"Num"}]; +} + `, wantErr: false, }, @@ -468,7 +516,9 @@ func Test_sheetExporter_exportMessager(t *testing.T) { }, p: printer.New(), be: &bookExporter{ - gen: &Generator{}, + gen: &Generator{ + OutputOpt: &options.ProtoOutputOption{}, + }, messagerPatternRegexp: regexp.MustCompile(`Conf$`), }, typeInfos: &xproto.TypeInfos{}, @@ -494,7 +544,9 @@ func Test_sheetExporter_exportMessager(t *testing.T) { }, p: printer.New(), be: &bookExporter{ - gen: &Generator{}, + gen: &Generator{ + OutputOpt: &options.ProtoOutputOption{}, + }, messagerPatternRegexp: regexp.MustCompile(`Data$`), }, typeInfos: &xproto.TypeInfos{}, @@ -502,6 +554,125 @@ func Test_sheetExporter_exportMessager(t *testing.T) { }, wantErr: true, }, + { + name: "field-number-compatibility", + x: &sheetExporter{ + ws: &internalpb.Worksheet{ + Name: "YamlScalarConf", + Options: &tableaupb.WorksheetOptions{ + Name: "YamlScalarConf", + }, + Fields: []*internalpb.Field{ + {Name: "id", Type: "uint32", FullType: "uint32", Options: &tableaupb.FieldOptions{Name: "ID"}}, + {Name: "num", Type: "int32", FullType: "int32", Options: &tableaupb.FieldOptions{Name: "Num"}}, + {Name: "value", Type: "uint64", FullType: "uint64", Options: &tableaupb.FieldOptions{Name: "Value"}}, + {Name: "inserted_field", Type: "int32", FullType: "int32", Options: &tableaupb.FieldOptions{Name: "InsertedField"}}, + {Name: "weight", Type: "int64", FullType: "int64", Options: &tableaupb.FieldOptions{Name: "Weight"}}, + {Name: "percentage", Type: "float", FullType: "float", Options: &tableaupb.FieldOptions{Name: "Percentage"}}, + {Name: "ratio", Type: "double", FullType: "double", Options: &tableaupb.FieldOptions{Name: "Ratio"}}, + {Name: "another_inserted_field", Type: "int32", FullType: "int32", Options: &tableaupb.FieldOptions{Name: "AnotherInsertedField"}}, + {Name: "name", Type: "string", FullType: "string", Options: &tableaupb.FieldOptions{Name: "Name"}}, + {Name: "blob", Type: "bytes", FullType: "bytes", Options: &tableaupb.FieldOptions{Name: "Blob"}}, + {Name: "ok", Type: "bool", FullType: "bool", Options: &tableaupb.FieldOptions{Name: "OK"}}, + }, + }, + p: printer.New(), + be: &bookExporter{ + wb: &internalpb.Workbook{ + Name: "tableau/protobuf/unittest/unittest", + }, + gen: &Generator{ + OutputOpt: &options.ProtoOutputOption{ + PreserveFieldNumbers: true, + }, + GeneratedProtoRegistryFiles: protoregistry.GlobalFiles, + }, + }, + typeInfos: &xproto.TypeInfos{}, + nestedMessages: make(map[string]*internalpb.Field), + }, + want: `message YamlScalarConf { + option (tableau.worksheet) = {name:"YamlScalarConf"}; + + uint32 id = 1 [(tableau.field) = {name:"ID"}]; + int32 num = 2 [(tableau.field) = {name:"Num"}]; + uint64 value = 3 [(tableau.field) = {name:"Value"}]; + int32 inserted_field = 10 [(tableau.field) = {name:"InsertedField"}]; + int64 weight = 4 [(tableau.field) = {name:"Weight"}]; + float percentage = 5 [(tableau.field) = {name:"Percentage"}]; + double ratio = 6 [(tableau.field) = {name:"Ratio"}]; + int32 another_inserted_field = 11 [(tableau.field) = {name:"AnotherInsertedField"}]; + string name = 7 [(tableau.field) = {name:"Name"}]; + bytes blob = 8 [(tableau.field) = {name:"Blob"}]; + bool ok = 9 [(tableau.field) = {name:"OK"}]; +} + +`, + wantErr: false, + }, + { + name: "field-number-compatibility-in-sub-structs", + x: &sheetExporter{ + ws: &internalpb.Worksheet{ + Name: "RewardConf", + Options: &tableaupb.WorksheetOptions{ + Name: "RewardConf", + }, + Fields: []*internalpb.Field{ + { + Name: "reward_map", + Type: "map", + FullType: "map", + MapEntry: &internalpb.Field_MapEntry{ + KeyType: "uint32", + ValueType: "Reward", + ValueFullType: "Reward", + }, + Options: &tableaupb.FieldOptions{ + Key: "RewardID", + Layout: tableaupb.Layout_LAYOUT_VERTICAL, + }, + Fields: []*internalpb.Field{ + {Name: "reward_id", Type: "uint32", FullType: "uint32", Options: &tableaupb.FieldOptions{Name: "RewardID"}}, + {Name: "reward_name", Type: "string", FullType: "string", Options: &tableaupb.FieldOptions{Name: "RewardName"}}, + { + Name: "item_map", Type: "map", FullType: "map", Predefined: true, + MapEntry: &internalpb.Field_MapEntry{KeyType: "uint32", ValueType: "Item", ValueFullType: "unittest.Item"}, + Options: &tableaupb.FieldOptions{Name: "Item", Key: "ID", Layout: tableaupb.Layout_LAYOUT_HORIZONTAL}, + }, + }, + }, + }, + }, + p: printer.New(), + be: &bookExporter{ + wb: &internalpb.Workbook{ + Name: "tableau/protobuf/unittest/unittest", + }, + gen: &Generator{ + OutputOpt: &options.ProtoOutputOption{ + PreserveFieldNumbers: true, + }, + GeneratedProtoRegistryFiles: protoregistry.GlobalFiles, + }, + }, + typeInfos: &xproto.TypeInfos{}, + nestedMessages: make(map[string]*internalpb.Field), + }, + want: `message RewardConf { + option (tableau.worksheet) = {name:"RewardConf"}; + + map reward_map = 1 [(tableau.field) = {key:"RewardID" layout:LAYOUT_VERTICAL}]; + message Reward { + uint32 reward_id = 1 [(tableau.field) = {name:"RewardID"}]; + string reward_name = 3 [(tableau.field) = {name:"RewardName"}]; + map item_map = 2 [(tableau.field) = {name:"Item" key:"ID" layout:LAYOUT_HORIZONTAL}]; + } +} + +`, + wantErr: false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/internal/protogen/protogen.go b/internal/protogen/protogen.go index 1d4e2efa..7dbe1185 100644 --- a/internal/protogen/protogen.go +++ b/internal/protogen/protogen.go @@ -27,6 +27,7 @@ import ( "golang.org/x/sync/errgroup" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/reflect/protoregistry" ) type Generator struct { @@ -39,6 +40,8 @@ type Generator struct { InputOpt *options.ProtoInputOption OutputOpt *options.ProtoOutputOption + GeneratedProtoRegistryFiles *protoregistry.Files + // internal typeInfos *xproto.TypeInfos // predefined type infos @@ -68,11 +71,14 @@ func NewGeneratorWithOptions(protoPackage, indir, outdir string, opts *options.O cachedImporters: make(map[string]importer.Importer), } + if opts.Proto.Output.PreserveFieldNumbers { + gen.GeneratedProtoRegistryFiles, _ = gen.parseProtoRegistryFiles(true) + } return gen } -func (gen *Generator) preprocess(useGeneratedProtos, delExisted bool) error { +func (gen *Generator) parseProtoRegistryFiles(useGeneratedProtos bool) (*protoregistry.Files, error) { outdir := filepath.Join(gen.OutputDir, gen.OutputOpt.Subdir) var protoFiles []string protoFiles = append(protoFiles, gen.InputOpt.ProtoFiles...) @@ -84,9 +90,15 @@ func (gen *Generator) preprocess(useGeneratedProtos, delExisted bool) error { protoFiles = append(protoFiles, xfs.CleanSlashPath(filepath.Join(outdir, "*.proto"))) } // parse custom imported proto files - protoRegistryFiles, err := protoc.NewFiles( + return protoc.NewFiles( gen.InputOpt.ProtoPaths, protoFiles) +} + +func (gen *Generator) preprocess(useGeneratedProtos, delExisted bool) error { + outdir := filepath.Join(gen.OutputDir, gen.OutputOpt.Subdir) + // parse custom imported proto files + protoRegistryFiles, err := gen.parseProtoRegistryFiles(useGeneratedProtos) if err != nil { return err } diff --git a/options/options.go b/options/options.go index 9cfdb050..5d6f6be9 100644 --- a/options/options.go +++ b/options/options.go @@ -203,6 +203,28 @@ type ProtoOutputOption struct { // // Default: false. EnumValueWithPrefix bool `yaml:"enumValueWithPrefix"` + + // Whether converter will preserve field numbers for fields that already + // exists in current protos. For example, if you already have the message + // below in your proto file: + // + // message Item { + // int32 id = 1 [(tableau.field) = { name: "ID" }]; + // int32 count = 2 [(tableau.field) = { name: "Count" }]; + // } + // + // Then you try to add a string field "Name" between "ID" and "Count", its + // field number will be 3. "ID" and "Count" will preserve their previous + // field number 1 and 2. + // + // message Item { + // int32 id = 1 [(tableau.field) = { name: "ID" }]; + // string name = 3 [(tableau.field) = { name: "Name" }]; + // int32 count = 2 [(tableau.field) = { name: "Count" }]; + // } + // + // Default: false. + PreserveFieldNumbers bool `yaml:"preserveFieldNumbers"` } // Options for generating conf files. Only for confgen.