From e650ff6fe6b1acfe86c99bf76ded62fa34177dd2 Mon Sep 17 00:00:00 2001 From: walon Date: Wed, 8 Dec 2021 20:39:58 +0800 Subject: [PATCH] =?UTF-8?q?=E3=80=90=E7=89=B9=E6=80=A7=E3=80=91=E6=96=B0?= =?UTF-8?q?=E5=A2=9Ecode=5Fprotobuf3=E5=8F=8Adata=5Fprotobuf3=5Fjson?= =?UTF-8?q?=E6=94=AF=E6=8C=81=20=E3=80=90=E4=BF=AE=E5=A4=8D=E3=80=91?= =?UTF-8?q?=E4=BF=AE=E5=A4=8DFlatBuffers=20json=E5=AF=BC=E5=87=BA=E6=A0=BC?= =?UTF-8?q?=E5=BC=8F=E4=B8=ADmap=E6=95=B0=E6=8D=AE=E7=9A=84=E6=A0=BC?= =?UTF-8?q?=E5=BC=8F=E9=94=99=E8=AF=AF=E3=80=82=E6=9C=AC=E5=BA=94=E8=AF=A5?= =?UTF-8?q?=E6=98=AF{"K":v}=E5=8D=B4=E5=BA=8F=E5=88=97=E5=8C=96=E6=88=90[[?= =?UTF-8?q?k1,v1]]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../DataExporters/FlatBuffersJsonExportor.cs | 15 ++++ ...obufExportor.cs => ProtobufBinExportor.cs} | 4 +- .../DataExporters/ProtobufJsonExportor.cs | 88 +++++++++++++++++++ src/Luban.Job.Cfg/Source/GenArgs.cs | 2 +- .../Source/Generate/DataScatterRender.cs | 3 +- .../Source/Generate/Protobuf2SchemaRender.cs | 2 + .../Source/Generate/Protobuf3SchemaRender.cs | 4 +- .../Source/Utils/DataExporterUtil.cs | 17 +++- .../Source/Defs/TTypeTemplateCommonExtends.cs | 23 +++++ src/Luban.Server/Luban.Server.csproj | 17 +++- .../common/{protobuf => protobuf2}/enum.tpl | 2 +- .../Templates/common/protobuf3/enum.tpl | 15 ++++ .../Templates/config/protobuf3/all.tpl | 36 ++++++++ .../Templates/config/protobuf3/bean.tpl | 20 +++++ .../Templates/config/protobuf3/table.tpl | 9 ++ .../Templates/config/protobuf3/tables.tpl | 5 ++ 16 files changed, 253 insertions(+), 9 deletions(-) rename src/Luban.Job.Cfg/Source/DataExporters/{ProtobufExportor.cs => ProtobufBinExportor.cs} (98%) create mode 100644 src/Luban.Job.Cfg/Source/DataExporters/ProtobufJsonExportor.cs rename src/Luban.Server/Templates/common/{protobuf => protobuf2}/enum.tpl (92%) create mode 100644 src/Luban.Server/Templates/common/protobuf3/enum.tpl create mode 100644 src/Luban.Server/Templates/config/protobuf3/all.tpl create mode 100644 src/Luban.Server/Templates/config/protobuf3/bean.tpl create mode 100644 src/Luban.Server/Templates/config/protobuf3/table.tpl create mode 100644 src/Luban.Server/Templates/config/protobuf3/tables.tpl diff --git a/src/Luban.Job.Cfg/Source/DataExporters/FlatBuffersJsonExportor.cs b/src/Luban.Job.Cfg/Source/DataExporters/FlatBuffersJsonExportor.cs index 1806b3e..cd7ec68 100644 --- a/src/Luban.Job.Cfg/Source/DataExporters/FlatBuffersJsonExportor.cs +++ b/src/Luban.Job.Cfg/Source/DataExporters/FlatBuffersJsonExportor.cs @@ -78,5 +78,20 @@ namespace Luban.Job.Cfg.DataExporters } x.WriteEndObject(); } + + public override void Accept(DMap type, Utf8JsonWriter x) + { + x.WriteStartArray(); + foreach (var d in type.Datas) + { + x.WriteStartObject(); + x.WritePropertyName("key"); + d.Key.Apply(this, x); + x.WritePropertyName("value"); + d.Value.Apply(this, x); + x.WriteEndObject(); + } + x.WriteEndArray(); + } } } diff --git a/src/Luban.Job.Cfg/Source/DataExporters/ProtobufExportor.cs b/src/Luban.Job.Cfg/Source/DataExporters/ProtobufBinExportor.cs similarity index 98% rename from src/Luban.Job.Cfg/Source/DataExporters/ProtobufExportor.cs rename to src/Luban.Job.Cfg/Source/DataExporters/ProtobufBinExportor.cs index 6826867..58b6f0e 100644 --- a/src/Luban.Job.Cfg/Source/DataExporters/ProtobufExportor.cs +++ b/src/Luban.Job.Cfg/Source/DataExporters/ProtobufBinExportor.cs @@ -15,9 +15,9 @@ using System.Threading.Tasks; namespace Luban.Job.Cfg.DataExporters { - class ProtobufExportor : IDataActionVisitor + class ProtobufBinExportor : IDataActionVisitor { - public static ProtobufExportor Ins { get; } = new(); + public static ProtobufBinExportor Ins { get; } = new(); public void WriteList(DefTable table, List datas, MemoryStream x) { diff --git a/src/Luban.Job.Cfg/Source/DataExporters/ProtobufJsonExportor.cs b/src/Luban.Job.Cfg/Source/DataExporters/ProtobufJsonExportor.cs new file mode 100644 index 0000000..cdf3d68 --- /dev/null +++ b/src/Luban.Job.Cfg/Source/DataExporters/ProtobufJsonExportor.cs @@ -0,0 +1,88 @@ +using Luban.Job.Cfg.Datas; +using Luban.Job.Cfg.DataSources; +using Luban.Job.Cfg.DataVisitors; +using Luban.Job.Cfg.Defs; +using Luban.Job.Common.Types; +using Luban.Job.Common.TypeVisitors; +using System; +using System.Collections.Generic; +using System.Text.Json; + +namespace Luban.Job.Cfg.DataExporters +{ + class ProtobufJsonExportor : JsonExportor + { + public static new FlatBuffersJsonExportor Ins { get; } = new(); + + public void WriteAsTable(List datas, Utf8JsonWriter x) + { + x.WriteStartObject(); + // 如果修改了这个名字，请同时修改table.tpl + x.WritePropertyName("data_list"); + x.WriteStartArray(); + foreach (var d in datas) + { + d.Data.Apply(this, x); + } + x.WriteEndArray(); + x.WriteEndObject(); + } + + public override void Accept(DText type, Utf8JsonWriter x) + { + // 不支持本地化。只能简单起见这么做了 + //x.WriteStartObject(); + //x.WritePropertyName(DText.KEY_NAME); + //x.WriteStringValue(type.Key); + //x.WritePropertyName(DText.TEXT_NAME); + x.WriteStringValue(type.TextOfCurrentAssembly); + //x.WriteEndObject(); + } + + public override void Accept(DBean type, Utf8JsonWriter x) + { + x.WriteStartObject(); + + if (type.Type.IsAbstractType) + { + // protobuf oneof 用 @type来识别类型 + x.WritePropertyName("@type"); + x.WriteStringValue(TBean.Create(false, type.ImplType, null).Apply(ProtobufTypeNameVisitor.Ins)); + } + + var defFields = type.ImplType.HierarchyFields; + int index = 0; + foreach (var d in type.Fields) + { + var defField = (DefField)defFields[index++]; + + // 特殊处理 bean 多态类型 + // 另外，不生成 xxx:null 这样 + if (d == null || !defField.NeedExport) + { + //x.WriteNullValue(); + } + else + { + x.WritePropertyName(defField.Name); + d.Apply(this, x); + } + } + x.WriteEndObject(); + } + + + public override void Accept(DMap type, Utf8JsonWriter x) + { + x.WriteStartArray(); + foreach (var d in type.Datas) + { + x.WriteStartArray(); + x.WriteStringValue(d.Key.Apply(ToJsonLiteralVisitor.Ins)); + d.Value.Apply(this, x); + x.WriteEndArray(); + } + x.WriteEndArray(); + } + } +} diff --git a/src/Luban.Job.Cfg/Source/GenArgs.cs b/src/Luban.Job.Cfg/Source/GenArgs.cs index 036e79a..18d2c62 100644 --- a/src/Luban.Job.Cfg/Source/GenArgs.cs +++ b/src/Luban.Job.Cfg/Source/GenArgs.cs @@ -8,7 +8,7 @@ namespace Luban.Job.Cfg [Option('s', "service", Required = true, HelpText = "service")] public string Service { get; set; } - [Option("gen_types", Required = true, HelpText = "code_cs_bin,code_cs_json,code_cs_unity_json,code_lua_bin,code_java_bin,code_java_json,code_go_bin,code_go_json,code_cpp_bin,code_python3_json,code_typescript_bin,code_typescript_json,code_rust_json,code_protobuf,code_template,code_flatbuffers,data_bin,data_lua,data_json,data_json2,data_json_monolithic,data_resources,data_template,data_protobuf,data_flatbuffers_json,convert_json,convert_lua,convert_xlsx . can be multi")] + [Option("gen_types", Required = true, HelpText = "code_cs_bin,code_cs_json,code_cs_unity_json,code_lua_bin,code_java_bin,code_java_json,code_go_bin,code_go_json,code_cpp_bin,code_python3_json,code_typescript_bin,code_typescript_json,code_rust_json,code_protobuf,code_template,code_flatbuffers,data_bin,data_lua,data_json,data_json2,data_json_monolithic,data_resources,data_template,data_protobuf_bin,data_protobuf_json,data_flatbuffers_json,convert_json,convert_lua,convert_xlsx . can be multi")] public string GenType { get; set; } [Option("input_data_dir", Required = true, HelpText = "input data dir")] diff --git a/src/Luban.Job.Cfg/Source/Generate/DataScatterRender.cs b/src/Luban.Job.Cfg/Source/Generate/DataScatterRender.cs index 87293b7..0d70c40 100644 --- a/src/Luban.Job.Cfg/Source/Generate/DataScatterRender.cs +++ b/src/Luban.Job.Cfg/Source/Generate/DataScatterRender.cs @@ -13,7 +13,8 @@ namespace Luban.Job.Cfg.Generate [Render("data_lua")] [Render("data_xml")] [Render("data_yaml")] - [Render("data_protobuf")] + [Render("data_protobuf_bin")] + [Render("data_protobuf_json")] [Render("data_msgpack")] [Render("data_flatbuffers_json")] class DataScatterRender : DataRenderBase diff --git a/src/Luban.Job.Cfg/Source/Generate/Protobuf2SchemaRender.cs b/src/Luban.Job.Cfg/Source/Generate/Protobuf2SchemaRender.cs index 292b948..6825c9f 100644 --- a/src/Luban.Job.Cfg/Source/Generate/Protobuf2SchemaRender.cs +++ b/src/Luban.Job.Cfg/Source/Generate/Protobuf2SchemaRender.cs @@ -15,6 +15,8 @@ namespace Luban.Job.Cfg.Generate [Render("code_protobuf2")] class Protobuf2SchemaRender : ProtobufSchemaRenderBase { + protected override string CommonRenderTemplateDir => "protobuf2"; + protected override string RenderTemplateDir => "protobuf2"; } } diff --git a/src/Luban.Job.Cfg/Source/Generate/Protobuf3SchemaRender.cs b/src/Luban.Job.Cfg/Source/Generate/Protobuf3SchemaRender.cs index 83eb9b7..01fda52 100644 --- a/src/Luban.Job.Cfg/Source/Generate/Protobuf3SchemaRender.cs +++ b/src/Luban.Job.Cfg/Source/Generate/Protobuf3SchemaRender.cs @@ -10,6 +10,8 @@ namespace Luban.Job.Cfg.Generate [Render("code_protobuf3")] class Protobuf3SchemaRender : ProtobufSchemaRenderBase { - protected override string RenderTemplateDir => "protobuf"; + protected override string CommonRenderTemplateDir => "protobuf3"; + + protected override string RenderTemplateDir => "protobuf3"; } } diff --git a/src/Luban.Job.Cfg/Source/Utils/DataExporterUtil.cs b/src/Luban.Job.Cfg/Source/Utils/DataExporterUtil.cs index f7d7b91..47387e7 100644 --- a/src/Luban.Job.Cfg/Source/Utils/DataExporterUtil.cs +++ b/src/Luban.Job.Cfg/Source/Utils/DataExporterUtil.cs @@ -93,12 +93,25 @@ namespace Luban.Job.Cfg.Utils } return string.Join('\n', content); } - case "data_protobuf": + case "data_protobuf_bin": { var ms = new MemoryStream(); - ProtobufExportor.Ins.WriteList(table, records, ms); + ProtobufBinExportor.Ins.WriteList(table, records, ms); return DataUtil.StreamToBytes(ms); } + case "data_protobuf_json": + { + var ss = new MemoryStream(); + var jsonWriter = new Utf8JsonWriter(ss, new JsonWriterOptions() + { + Indented = !table.Assembly.OutputCompactJson, + SkipValidation = false, + Encoder = System.Text.Encodings.Web.JavaScriptEncoder.Create(System.Text.Unicode.UnicodeRanges.All), + }); + ProtobufJsonExportor.Ins.WriteAsTable(records, jsonWriter); + jsonWriter.Flush(); + return DataUtil.StreamToBytes(ss); + } case "data_msgpack": { var ms = new System.Buffers.ArrayBufferWriter(); diff --git a/src/Luban.Job.Common/Source/Defs/TTypeTemplateCommonExtends.cs b/src/Luban.Job.Common/Source/Defs/TTypeTemplateCommonExtends.cs index 0db7ab6..cb7dd8b 100644 --- a/src/Luban.Job.Common/Source/Defs/TTypeTemplateCommonExtends.cs +++ b/src/Luban.Job.Common/Source/Defs/TTypeTemplateCommonExtends.cs @@ -245,6 +245,29 @@ namespace Luban.Job.Common.Defs } } + public static string Protobuf3PreDecorator(TType type) + { + if (type.IsNullable) + { + return "optional"; + } + else if (type.IsCollection) + { + if (type is TMap) + { + return ""; + } + else + { + return "repeated"; + } + } + else + { + return ""; + } + } + public static string FlatBuffersDefineType(TType type) { return type.Apply(FlatBuffersTypeNameVisitor.Ins); diff --git a/src/Luban.Server/Luban.Server.csproj b/src/Luban.Server/Luban.Server.csproj index eefed7c..fe486b9 100644 --- a/src/Luban.Server/Luban.Server.csproj +++ b/src/Luban.Server/Luban.Server.csproj @@ -64,7 +64,10 @@ PreserveNewest - + + PreserveNewest + + PreserveNewest @@ -187,6 +190,18 @@ PreserveNewest + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + PreserveNewest diff --git a/src/Luban.Server/Templates/common/protobuf/enum.tpl b/src/Luban.Server/Templates/common/protobuf2/enum.tpl similarity index 92% rename from src/Luban.Server/Templates/common/protobuf/enum.tpl rename to src/Luban.Server/Templates/common/protobuf2/enum.tpl index 39467e9..33e8046 100644 --- a/src/Luban.Server/Templates/common/protobuf/enum.tpl +++ b/src/Luban.Server/Templates/common/protobuf2/enum.tpl @@ -9,7 +9,7 @@ enum {{x.pb_full_name}} { {{~for item in items ~}} {{x.pb_full_name}}_{{item.name}} = {{item.int_value}}; {{~end~}} - {{~if items.empty?~}} + {{~if items.empty?}} {{x.pb_full_name}}_EMPTY_PLACEHOLDER = 0; {{~end~}} } \ No newline at end of file diff --git a/src/Luban.Server/Templates/common/protobuf3/enum.tpl b/src/Luban.Server/Templates/common/protobuf3/enum.tpl new file mode 100644 index 0000000..167ef97 --- /dev/null +++ b/src/Luban.Server/Templates/common/protobuf3/enum.tpl @@ -0,0 +1,15 @@ +{{~ + name = x.name + namespace_with_top_module = x.namespace_with_top_module + comment = x.comment + items = x.items +~}} + +enum {{x.pb_full_name}} { + {{~if items.empty? || items[0].int_value != 0~}} + {{x.pb_full_name}}_EMPTY_PLACEHOLDER = 0; + {{~end~}} + {{~for item in items ~}} + {{x.pb_full_name}}_{{item.name}} = {{item.int_value}}; + {{~end~}} +} \ No newline at end of file diff --git a/src/Luban.Server/Templates/config/protobuf3/all.tpl b/src/Luban.Server/Templates/config/protobuf3/all.tpl new file mode 100644 index 0000000..4581f78 --- /dev/null +++ b/src/Luban.Server/Templates/config/protobuf3/all.tpl @@ -0,0 +1,36 @@ +syntax = "proto3"; + +package {{x.namespace}}; + +// luban internal types begin +message Vector2 { + float x = 1; + float y = 2; +} + +message Vector3 { + float x = 1; + float y = 2; + float z = 3; +} + +message Vector4 { + float x = 1; + float y = 2; + float z = 3; + float w = 4; +} +// luban internal types end + +{{~for enum in x.enums ~}} +{{enum}} +{{~end~}} + +{{~for bean in x.beans~}} +{{bean}} +{{~end~}} + +{{~for table in x.tables~}} +{{table}} +{{~end~}} + diff --git a/src/Luban.Server/Templates/config/protobuf3/bean.tpl b/src/Luban.Server/Templates/config/protobuf3/bean.tpl new file mode 100644 index 0000000..309f18e --- /dev/null +++ b/src/Luban.Server/Templates/config/protobuf3/bean.tpl @@ -0,0 +1,20 @@ +{{~ + name = x.name + parent_def_type = x.parent_def_type + export_fields = x.export_fields + hierarchy_export_fields = x.hierarchy_export_fields +~}} + +message {{x.pb_full_name}} { +{{~if x.is_abstract_type ~}} + oneof value { + {{~for c in x.hierarchy_not_abstract_children~}} + {{c.pb_full_name}} {{c.name}} = {{c.auto_id}}; + {{~end~}} + } +{{~else~}} + {{~for f in hierarchy_export_fields ~}} + {{protobuf3_pre_decorator f.ctype}} {{protobuf_define_type f.ctype}} {{f.name}} = {{f.auto_id}} {{protobuf_suffix_options f.ctype}}; + {{~end~}} +{{~end~}} +} \ No newline at end of file diff --git a/src/Luban.Server/Templates/config/protobuf3/table.tpl b/src/Luban.Server/Templates/config/protobuf3/table.tpl new file mode 100644 index 0000000..36d1699 --- /dev/null +++ b/src/Luban.Server/Templates/config/protobuf3/table.tpl @@ -0,0 +1,9 @@ +{{~ + name = x.name + key_type = x.key_ttype + value_type = x.value_ttype +~}} + +message {{x.pb_full_name}} { + repeated {{protobuf_define_type value_type}} data_list = 1 [packed = false]; +} \ No newline at end of file diff --git a/src/Luban.Server/Templates/config/protobuf3/tables.tpl b/src/Luban.Server/Templates/config/protobuf3/tables.tpl new file mode 100644 index 0000000..827db8e --- /dev/null +++ b/src/Luban.Server/Templates/config/protobuf3/tables.tpl @@ -0,0 +1,5 @@ +{{~ + name = x.name + namespace = x.namespace + tables = x.tables +~}} \ No newline at end of file