From 665b9a4cea85a1c13e1cb05563566af8b62243d5 Mon Sep 17 00:00:00 2001 From: walon Date: Tue, 19 Jul 2022 11:42:49 +0800 Subject: [PATCH] =?UTF-8?q?[new]=20=E6=96=B0=E5=A2=9Ebidx=EF=BC=8C?= =?UTF-8?q?=E5=8D=B3binary=E6=A0=BC=E5=BC=8F=E7=9A=84=E7=B4=A2=E5=BC=95?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E8=BE=93=E5=87=BA=E3=80=82=E6=96=B9=E4=BE=BF?= =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=8C=89=E9=9C=80=E5=8A=A0=E8=BD=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../DataExporters/BinaryIndexExportor.cs | 40 +++++++++++++++++++ src/Luban.Job.Cfg/Source/GenArgs.cs | 2 +- .../Source/Generate/DataScatterRender.cs | 1 + .../Source/Utils/DataExporterUtil.cs | 8 ++++ .../Source/Utils/RenderFileUtil.cs | 3 +- 5 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 src/Luban.Job.Cfg/Source/DataExporters/BinaryIndexExportor.cs diff --git a/src/Luban.Job.Cfg/Source/DataExporters/BinaryIndexExportor.cs b/src/Luban.Job.Cfg/Source/DataExporters/BinaryIndexExportor.cs new file mode 100644 index 0000000..f0812b1 --- /dev/null +++ b/src/Luban.Job.Cfg/Source/DataExporters/BinaryIndexExportor.cs @@ -0,0 +1,40 @@ +using Bright.Serialization; +using Luban.Job.Cfg.Datas; +using Luban.Job.Cfg.DataSources; +using Luban.Job.Cfg.DataVisitors; +using Luban.Job.Cfg.Defs; +using Luban.Job.Cfg.RawDefs; +using System; +using System.Collections.Generic; + +namespace Luban.Job.Cfg.DataExporters +{ + class BinaryIndexExportor + { + public static BinaryIndexExportor Ins { get; } = new BinaryIndexExportor(); + + public void WriteList(DefTable table, List datas, ByteBuf x) + { + x.WriteSize(datas.Count); + var tableDataBuf = new ByteBuf(10 * 1024); + tableDataBuf.WriteSize(datas.Count); + + foreach (var d in datas) + { + int offset = tableDataBuf.Size; + d.Data.Apply(BinaryExportor.Ins, tableDataBuf); + + string keyStr = ""; + foreach (IndexInfo index in table.IndexList) + { + DType key = d.Data.Fields[index.IndexFieldIdIndex]; + key.Apply(BinaryExportor.Ins, x); + keyStr += key.ToString() + ","; + } + x.WriteSize(offset); + Console.WriteLine($"table:{table.Name} key:{keyStr} offset:{offset}"); + } + + } + } +} diff --git a/src/Luban.Job.Cfg/Source/GenArgs.cs b/src/Luban.Job.Cfg/Source/GenArgs.cs index b09c089..1eda829 100644 --- a/src/Luban.Job.Cfg/Source/GenArgs.cs +++ b/src/Luban.Job.Cfg/Source/GenArgs.cs @@ -8,7 +8,7 @@ namespace Luban.Job.Cfg [Option('s', "service", Required = true, HelpText = "service")] public string Service { get; set; } - [Option("gen_types", Required = true, HelpText = "code_cs_bin,code_cs_dotnet_json,code_cs_unity_json,code_cs_unity_editor_json,code_lua_bin,code_java_bin,code_java_json,code_go_bin,code_go_json,code_cpp_bin,code_cpp_ue_editor_json,code_python2_json,code_python3_json,code_typescript_bin,code_typescript_json,code_rust_json,code_protobuf,code_template,code_flatbuffers,code_gdscript_json,data_bin,data_lua,data_json,data_json2,data_json_monolithic,data_yaml,data_xml,data_resources,data_template,data_protobuf_bin,data_protobuf_json,data_flatbuffers_json,convert_json,convert_lua,convert_xlsx . can be multi")] + [Option("gen_types", Required = true, HelpText = "code_cs_bin,code_cs_dotnet_json,code_cs_unity_json,code_cs_unity_editor_json,code_lua_bin,code_java_bin,code_java_json,code_go_bin,code_go_json,code_cpp_bin,code_cpp_ue_editor_json,code_python2_json,code_python3_json,code_typescript_bin,code_typescript_json,code_rust_json,code_protobuf,code_template,code_flatbuffers,code_gdscript_json,data_bin,data_bidx,data_lua,data_json,data_json2,data_json_monolithic,data_bson,data_yaml,data_xml,data_resources,data_template,data_protobuf_bin,data_protobuf_json,data_flatbuffers_json,convert_json,convert_lua,convert_xlsx . can be multi")] public string GenType { get; set; } [Option("input_data_dir", Required = true, HelpText = "input data dir")] diff --git a/src/Luban.Job.Cfg/Source/Generate/DataScatterRender.cs b/src/Luban.Job.Cfg/Source/Generate/DataScatterRender.cs index 3c07f7b..b6c03e6 100644 --- a/src/Luban.Job.Cfg/Source/Generate/DataScatterRender.cs +++ b/src/Luban.Job.Cfg/Source/Generate/DataScatterRender.cs @@ -8,6 +8,7 @@ using System.Threading.Tasks; namespace Luban.Job.Cfg.Generate { [Render("data_bin")] + [Render("data_bidx")] [Render("data_json")] [Render("data_json2")] [Render("data_bson")] diff --git a/src/Luban.Job.Cfg/Source/Utils/DataExporterUtil.cs b/src/Luban.Job.Cfg/Source/Utils/DataExporterUtil.cs index ea4fc5e..8e46a7b 100644 --- a/src/Luban.Job.Cfg/Source/Utils/DataExporterUtil.cs +++ b/src/Luban.Job.Cfg/Source/Utils/DataExporterUtil.cs @@ -44,6 +44,14 @@ namespace Luban.Job.Cfg.Utils ThreadLocalTemporalByteBufPool.Free(buf); return bytes; } + case "data_bidx": + { + var buf = ThreadLocalTemporalByteBufPool.Alloc(1024 * 1024); + BinaryIndexExportor.Ins.WriteList(table, records, buf); + var bytes = buf.CopyData(); + ThreadLocalTemporalByteBufPool.Free(buf); + return bytes; + } case "data_json": case "data_json2": { diff --git a/src/Luban.Job.Common/Source/Utils/RenderFileUtil.cs b/src/Luban.Job.Common/Source/Utils/RenderFileUtil.cs index ea229ab..6e46e3e 100644 --- a/src/Luban.Job.Common/Source/Utils/RenderFileUtil.cs +++ b/src/Luban.Job.Common/Source/Utils/RenderFileUtil.cs @@ -111,9 +111,10 @@ namespace Luban.Job.Common.Utils private static readonly Dictionary s_name2Suxxifx = new() { { "json", "json" }, - {"bson", "bson" }, + { "bson", "bson" }, { "lua", "lua" }, { "bin", "bytes" }, + { "bidx", "bytes" }, { "xml", "xml" }, { "yaml", "yml" }, { "yml", "yml" },