diff --git a/README.md b/README.md index 39278fa..f1341a4 100644 --- a/README.md +++ b/README.md @@ -53,8 +53,8 @@ Luban适合有以下需求的开发者： - 支持res资源标记。可以一键导出配置中引用的所有资源列表(icon,ui,assetbundle等等) - 生成代码良好模块化。 - **支持文本静态本地化。导出时所有text类型数据正确替换为最终的本地化字符串。** +- **支持main + braches 多地区版本数据。对于需要针对不同地区有部分差异配置的海外项目非常有用。 - **[TODO] 支持文本动态本地化。运行时动态切换所有text类型数据为目标本地化字符串。** -- **[TODO] 支持main + braches 多地区版本数据。对于需要针对不同地区有部分差异配置的海外项目非常有用。** - 支持主流的游戏开发语言 - c++ (11+) - c# (.net framework 4+. dotnet core 3+) diff --git a/config/Datas/l10n/cn/test/full_type_cn1.xlsx b/config/Datas/l10n/cn/test/full_type_cn1.xlsx new file mode 100644 index 0000000..8d3025f Binary files /dev/null and b/config/Datas/l10n/cn/test/full_type_cn1.xlsx differ diff --git a/config/Datas/l10n/cn/test/full_type_cn2.xlsx b/config/Datas/l10n/cn/test/full_type_cn2.xlsx new file mode 100644 index 0000000..f525910 Binary files /dev/null and b/config/Datas/l10n/cn/test/full_type_cn2.xlsx differ diff --git a/config/Datas/l10n/en/texx/full_type_en.xlsx b/config/Datas/l10n/en/texx/full_type_en.xlsx new file mode 100644 index 0000000..bb5374a Binary files /dev/null and b/config/Datas/l10n/en/texx/full_type_en.xlsx differ diff --git a/config/Defines/__root__.xml b/config/Defines/__root__.xml index b2b2af7..f3b7df4 100644 --- a/config/Defines/__root__.xml +++ b/config/Defines/__root__.xml @@ -6,6 +6,10 @@ editor分组 + + + + diff --git a/config/Defines/test.xml b/config/Defines/test.xml index 2d32cbe..44a65bc 100644 --- a/config/Defines/test.xml +++ b/config/Defines/test.xml @@ -79,7 +79,7 @@ - 最常见的普通 key-value表 +
最常见的普通 key-value表 diff --git a/config/branch生成.bat b/config/branch生成.bat new file mode 100644 index 0000000..d79687a --- /dev/null +++ b/config/branch生成.bat @@ -0,0 +1,14 @@ +..\src\Luban.Client\bin\Debug\net5.0\Luban.Client.exe ^ + -h %LUBAN_SERVER_IP% ^ + -j cfg ^ + -- ^ + -d Defines/__root__.xml ^ + --input_data_dir Datas ^ + --output_data_dir output_lua ^ + -s client ^ + --gen_types data_lua ^ + --export_test_data ^ + --branch cn ^ + --branch_input_data_dir Datas/l10n/cn + +pause \ No newline at end of file diff --git a/src/Luban.Job.Cfg/Source/Datas/Record.cs b/src/Luban.Job.Cfg/Source/Datas/Record.cs index 67d0450..cfe442a 100644 --- a/src/Luban.Job.Cfg/Source/Datas/Record.cs +++ b/src/Luban.Job.Cfg/Source/Datas/Record.cs @@ -12,6 +12,8 @@ namespace Luban.Job.Cfg.Datas public string Source { get; } + public int Index { get; set; } + public Record(DBean data, string source) { Data = data; diff --git a/src/Luban.Job.Cfg/Source/Defs/CfgDefLoader.cs b/src/Luban.Job.Cfg/Source/Defs/CfgDefLoader.cs index f8f2f61..b15695d 100644 --- a/src/Luban.Job.Cfg/Source/Defs/CfgDefLoader.cs +++ b/src/Luban.Job.Cfg/Source/Defs/CfgDefLoader.cs @@ -1,5 +1,5 @@ using Luban.Common.Utils; -using Luban.Config.Common.RawDefs; +using Luban.Job.Cfg.RawDefs; using Luban.Job.Common.Defs; using Luban.Job.Common.RawDefs; using Luban.Server.Common; @@ -14,6 +14,8 @@ namespace Luban.Job.Cfg.Defs { private static readonly NLog.Logger s_logger = NLog.LogManager.GetCurrentClassLogger(); + private readonly List _branches = new(); + private readonly List
_cfgTables = new List
(); private readonly List _cfgServices = new List(); @@ -24,11 +26,13 @@ namespace Luban.Job.Cfg.Defs public CfgDefLoader(RemoteAgent agent) : base(agent) { + RegisterRootDefineHandler("branch", AddBranch); RegisterRootDefineHandler("service", AddService); RegisterRootDefineHandler("group", AddGroup); RegisterModuleDefineHandler("table", AddTable); + IsBeanFieldMustDefineId = false; } @@ -37,6 +41,7 @@ namespace Luban.Job.Cfg.Defs return new Defines() { TopModule = TopModule, + Branches = _branches, Consts = this._consts, Enums = _enums, Beans = _beans, @@ -47,6 +52,21 @@ namespace Luban.Job.Cfg.Defs } + private static readonly List _branchRequireAttrs = new List { "name" }; + private void AddBranch(XElement e) + { + var branchName = e.Attribute("name").Value; + if (string.IsNullOrWhiteSpace(branchName)) + { + throw new Exception("branch 属性name不能为空"); + } + if (this._branches.Any(b => b.Name == branchName)) + { + throw new Exception($"branch {branchName} 重复"); + } + _branches.Add(new Branch(branchName)); + } + private static readonly List _groupOptionalAttrs = new List { "default" }; private static readonly List _groupRequireAttrs = new List { "name" }; @@ -132,7 +152,7 @@ namespace Luban.Job.Cfg.Defs _cfgServices.Add(new Service() { Name = name, Manager = manager, Groups = groups, Refs = refs }); } - private readonly List _tableOptionalAttrs = new List { "index", "mode", "group" }; + private readonly List _tableOptionalAttrs = new List { "index", "mode", "group", "branch_input" }; private readonly List _tableRequireAttrs = new List { "name", "value", "input" }; @@ -236,6 +256,24 @@ namespace Luban.Job.Cfg.Defs } p.InputFiles.AddRange(XmlUtil.GetRequiredAttribute(e, "input").Split(',')); + var branchInputAttr = e.Attribute("branch_input"); + if (branchInputAttr != null) + { + foreach (var subBranchStr in branchInputAttr.Value.Split('|').Select(s => s.Trim()).Where(s => !string.IsNullOrWhiteSpace(s))) + { + var nameAndDirs = subBranchStr.Split(':'); + if (nameAndDirs.Length != 2) + { + throw new Exception($"定义文件:{CurImportFile} table:{p.Name} branch_input:{subBranchStr} 定义不合法"); + } + var branchDirs = nameAndDirs[1].Split(',', ';').ToList(); + if (!p.BranchInputFiles.TryAdd(nameAndDirs[0], branchDirs)) + { + throw new Exception($"定义文件:{CurImportFile} table:{p.Name} branch_input:{subBranchStr} 子branch:{nameAndDirs[0]} 重复"); + } + } + } + if (!_name2CfgTable.TryAdd(p.Name, p)) { var exist = _name2CfgTable[p.Name]; diff --git a/src/Luban.Job.Cfg/Source/Defs/DefAssembly.cs b/src/Luban.Job.Cfg/Source/Defs/DefAssembly.cs index 33d3330..d2d4888 100644 --- a/src/Luban.Job.Cfg/Source/Defs/DefAssembly.cs +++ b/src/Luban.Job.Cfg/Source/Defs/DefAssembly.cs @@ -1,6 +1,6 @@ -using Luban.Config.Common.RawDefs; using Luban.Job.Cfg.Datas; using Luban.Job.Cfg.l10n; +using Luban.Job.Cfg.RawDefs; using Luban.Job.Cfg.TypeVisitors; using Luban.Job.Common.Defs; using Luban.Server.Common; @@ -11,12 +11,31 @@ using System.Linq; namespace Luban.Job.Cfg.Defs { + public class TableDataInfo + { + public List MainRecords { get; } + + public List BranchRecords { get; } + + public List FinalRecords { get; set; } + + public Dictionary FinalRecordMap { get; set; } + + public TableDataInfo(List mainRecords, List branchRecords) + { + MainRecords = mainRecords; + BranchRecords = branchRecords; + } + } + public class DefAssembly : DefAssemblyBase { private static readonly NLog.Logger s_logger = NLog.LogManager.GetCurrentClassLogger(); public Service CfgTargetService { get; private set; } + public Branch TargetBranch { get; private set; } + public TimeZoneInfo TimeZone { get; } public DefAssembly(TimeZoneInfo timezone) @@ -33,10 +52,11 @@ namespace Luban.Job.Cfg.Defs return groups.Any(g => CfgTargetService.Groups.Contains(g)); } + private readonly List _branches = new List(); + private readonly List _cfgServices = new List(); - private readonly ConcurrentDictionary> _recordsByTables = new(); - private readonly ConcurrentDictionary> _recordsMapByTables = new(); + private readonly ConcurrentDictionary _recordsByTables = new(); public Dictionary CfgTables { get; } = new Dictionary(); @@ -52,6 +72,11 @@ namespace Luban.Job.Cfg.Defs NotConvertTextSet = new NotConvertTextSet(); } + public Branch GetBranch(string name) + { + return _branches.Find(b => b.Name == name); + } + public void AddCfgTable(DefTable table) { if (!CfgTables.TryAdd(table.FullName, table)) @@ -65,24 +90,24 @@ namespace Luban.Job.Cfg.Defs return CfgTables.TryGetValue(name, out var t) ? t : null; } - public void AddDataTable(DefTable table, List records) + public void AddDataTable(DefTable table, List mainRecords, List branchRecords) { - _recordsByTables[table.FullName] = records; + _recordsByTables[table.FullName] = new TableDataInfo(mainRecords, branchRecords); } - public void SetDataTableMap(DefTable table, Dictionary recordMap) - { - _recordsMapByTables[table.FullName] = recordMap; - } + //public void SetDataTableMap(DefTable table, Dictionary recordMap) + //{ + // _recordsByTables[table.FullName].FinalRecordMap = recordMap; + //} public List GetTableDataList(DefTable table) { - return _recordsByTables[table.FullName]; + return _recordsByTables[table.FullName].FinalRecords; } - public Dictionary GetTableDataMap(DefTable table) + public TableDataInfo GetTableDataInfo(DefTable table) { - return _recordsMapByTables[table.FullName]; + return _recordsByTables[table.FullName]; } public List GetExportTables() @@ -122,7 +147,7 @@ namespace Luban.Job.Cfg.Defs return refTypes.Values.ToList(); } - public void Load(string outputService, Defines defines, RemoteAgent agent) + public void Load(string outputService, string branchName, Defines defines, RemoteAgent agent) { this.Agent = agent; SupportDatetimeType = true; @@ -136,6 +161,17 @@ namespace Luban.Job.Cfg.Defs throw new ArgumentException($"service:{outputService} not exists"); } + if (!string.IsNullOrWhiteSpace(branchName)) + { + TargetBranch = defines.Branches.Find(b => b.Name == branchName); + if (TargetBranch == null) + { + throw new Exception($"branch {branchName} not in valid branch set"); + } + } + + this._branches.AddRange(defines.Branches); + foreach (var c in defines.Consts) { AddType(new DefConst(c)); diff --git a/src/Luban.Job.Cfg/Source/Defs/DefBean.cs b/src/Luban.Job.Cfg/Source/Defs/DefBean.cs index b93cd1a..1167453 100644 --- a/src/Luban.Job.Cfg/Source/Defs/DefBean.cs +++ b/src/Luban.Job.Cfg/Source/Defs/DefBean.cs @@ -1,5 +1,5 @@ using Luban.Common.Utils; -using Luban.Config.Common.RawDefs; +using Luban.Job.Cfg.RawDefs; using Luban.Job.Cfg.TypeVisitors; using Luban.Job.Common.Defs; using System; diff --git a/src/Luban.Job.Cfg/Source/Defs/DefField.cs b/src/Luban.Job.Cfg/Source/Defs/DefField.cs index 7ebda3a..8a64bdd 100644 --- a/src/Luban.Job.Cfg/Source/Defs/DefField.cs +++ b/src/Luban.Job.Cfg/Source/Defs/DefField.cs @@ -1,5 +1,5 @@ using Luban.Common.Utils; -using Luban.Config.Common.RawDefs; +using Luban.Job.Cfg.RawDefs; using Luban.Job.Cfg.Validators; using Luban.Job.Common.Defs; using Luban.Job.Common.Types; diff --git a/src/Luban.Job.Cfg/Source/Defs/DefTable.cs b/src/Luban.Job.Cfg/Source/Defs/DefTable.cs index 5acd608..58ebba3 100644 --- a/src/Luban.Job.Cfg/Source/Defs/DefTable.cs +++ b/src/Luban.Job.Cfg/Source/Defs/DefTable.cs @@ -1,4 +1,4 @@ -using Luban.Config.Common.RawDefs; +using Luban.Job.Cfg.RawDefs; using Luban.Job.Common.Types; using System; using System.Collections.Generic; @@ -19,6 +19,7 @@ namespace Luban.Job.Cfg.Defs Mode = b.Mode; InputFiles = b.InputFiles; Groups = b.Groups; + _branchInputFiles = b.BranchInputFiles; } @@ -36,6 +37,8 @@ namespace Luban.Job.Cfg.Defs public List InputFiles { get; } + private readonly Dictionary> _branchInputFiles; + public List Groups { get; } public TType KeyTType { get; private set; } @@ -62,11 +65,24 @@ namespace Luban.Job.Cfg.Defs public string JsonOutputDataFile => $"{FullName}.json"; + public List GetBranchInputFiles(string branchName) + { + return _branchInputFiles.GetValueOrDefault(branchName); + } + public override void Compile() { - var pass = Assembly; + var ass = Assembly; - if ((ValueTType = (TBean)pass.CreateType(Namespace, ValueType)) == null) + foreach (var branchName in _branchInputFiles.Keys) + { + if (ass.GetBranch(branchName) == null) + { + throw new Exception($"table:{FullName} branch_input branch:{branchName} 不存在"); + } + } + + if ((ValueTType = (TBean)ass.CreateType(Namespace, ValueType)) == null) { throw new Exception($"table:{FullName} 的 value类型:{ValueType} 不存在"); } diff --git a/src/Luban.Job.Cfg/Source/JobController.cs b/src/Luban.Job.Cfg/Source/JobController.cs index 5a0cb80..0c49b61 100644 --- a/src/Luban.Job.Cfg/Source/JobController.cs +++ b/src/Luban.Job.Cfg/Source/JobController.cs @@ -58,7 +58,7 @@ namespace Luban.Job.Cfg [Option("export_test_data", Required = false, HelpText = "export test data")] public bool ExportTestData { get; set; } = false; - [Option('t', "i10n_timezone", Required = false, HelpText = "timezone")] + [Option('t', "l10n_timezone", Required = false, HelpText = "timezone")] public string TimeZone { get; set; } [Option("input_l10n_text_files", Required = false, HelpText = "input l10n text table files. can be multi, sep by ','")] @@ -66,6 +66,12 @@ namespace Luban.Job.Cfg [Option("output_l10n_not_converted_text_file", Required = false, HelpText = "the file save not converted l10n texts.")] public string OutputNotConvertTextFile { get; set; } + + [Option("branch", Required = false, HelpText = "branch name")] + public string BranchName { get; set; } + + [Option("branch_input_data_dir", Required = false, HelpText = "branch input data root dir")] + public string BranchInputDataDir { get; set; } } private ICodeRender CreateCodeRender(string genType) @@ -95,7 +101,7 @@ namespace Luban.Job.Cfg } - private static bool TryParseArg(List args, out GenArgs result, out string errMsg) + private static bool TryParseArg(List args, out GenArgs options, out string errMsg) { var helpWriter = new StringWriter(); var parser = new Parser(ps => @@ -106,19 +112,19 @@ namespace Luban.Job.Cfg if (parseResult.Tag == ParserResultType.NotParsed) { errMsg = helpWriter.ToString(); - result = null; + options = null; return false; } else { - result = (parseResult as Parsed).Value; + options = (parseResult as Parsed).Value; errMsg = null; - string inputDataDir = result.InputDataDir; - string outputCodeDir = result.OutputCodeDir; - string outputDataDir = result.OutputDataDir; + string inputDataDir = options.InputDataDir; + string outputCodeDir = options.OutputCodeDir; + string outputDataDir = options.OutputDataDir; - var genTypes = result.GenType.Split(',').Select(s => s.Trim()).ToList(); + var genTypes = options.GenType.Split(',').Select(s => s.Trim()).ToList(); if (genTypes.Any(t => t.StartsWith("code_", StringComparison.Ordinal)) && string.IsNullOrWhiteSpace(outputCodeDir)) { @@ -137,24 +143,30 @@ namespace Luban.Job.Cfg errMsg = "--outputdatadir missing"; return false; } - if (genTypes.Contains("data_resources") && string.IsNullOrWhiteSpace(result.OutputDataResourceListFile)) + if (genTypes.Contains("data_resources") && string.IsNullOrWhiteSpace(options.OutputDataResourceListFile)) { errMsg = "--output_data_resource_list_file missing"; return false; } - if (genTypes.Contains("data_json_monolithic") && string.IsNullOrWhiteSpace(result.OutputDataJsonMonolithicFile)) + if (genTypes.Contains("data_json_monolithic") && string.IsNullOrWhiteSpace(options.OutputDataJsonMonolithicFile)) { errMsg = "--output_data_json_monolithic_file missing"; return false; } - if (string.IsNullOrWhiteSpace(result.InputTextTableFiles) ^ string.IsNullOrWhiteSpace(result.OutputNotConvertTextFile)) + if (string.IsNullOrWhiteSpace(options.InputTextTableFiles) ^ string.IsNullOrWhiteSpace(options.OutputNotConvertTextFile)) { errMsg = "--input_l10n_text_files must be provided with --output_l10n_not_converted_text_file"; return false; } } + if (string.IsNullOrWhiteSpace(options.BranchName) ^ string.IsNullOrWhiteSpace(options.BranchInputDataDir)) + { + errMsg = "--branch must be provided with --branch_input_data_dir"; + return false; + } + return true; } } @@ -197,7 +209,7 @@ namespace Luban.Job.Cfg var ass = new DefAssembly(timeZoneInfo); - ass.Load(args.Service, rawDefines, agent); + ass.Load(args.Service, args.BranchName, rawDefines, agent); var targetService = ass.CfgTargetService; @@ -216,7 +228,7 @@ namespace Luban.Job.Cfg hasLoadCfgData = true; var timer = new ProfileTimer(); timer.StartPhase("load config data"); - await DataLoaderUtil.LoadCfgDataAsync(agent, ass, args.InputDataDir, args.ExportTestData); + await DataLoaderUtil.LoadCfgDataAsync(agent, ass, args.InputDataDir, args.BranchName, args.BranchInputDataDir, args.ExportTestData); timer.EndPhaseAndLog(); if (needL10NTextConvert) diff --git a/src/Luban.Job.Cfg/Source/RawDefs/Branch.cs b/src/Luban.Job.Cfg/Source/RawDefs/Branch.cs new file mode 100644 index 0000000..9cfc04c --- /dev/null +++ b/src/Luban.Job.Cfg/Source/RawDefs/Branch.cs @@ -0,0 +1,18 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Luban.Job.Cfg.RawDefs +{ + public class Branch + { + public string Name { get; } + + public Branch(string name) + { + Name = name; + } + } +} diff --git a/src/Luban.Job.Cfg/Source/RawDefs/CfgBean.cs b/src/Luban.Job.Cfg/Source/RawDefs/CfgBean.cs index 9790d7f..ab6b176 100644 --- a/src/Luban.Job.Cfg/Source/RawDefs/CfgBean.cs +++ b/src/Luban.Job.Cfg/Source/RawDefs/CfgBean.cs @@ -1,6 +1,6 @@ using Luban.Job.Common.RawDefs; -namespace Luban.Config.Common.RawDefs +namespace Luban.Job.Cfg.RawDefs { public class CfgBean : Bean { diff --git a/src/Luban.Job.Cfg/Source/RawDefs/CfgField.cs b/src/Luban.Job.Cfg/Source/RawDefs/CfgField.cs index a6014e3..4237334 100644 --- a/src/Luban.Job.Cfg/Source/RawDefs/CfgField.cs +++ b/src/Luban.Job.Cfg/Source/RawDefs/CfgField.cs @@ -1,7 +1,7 @@ using Luban.Job.Common.RawDefs; using System.Collections.Generic; -namespace Luban.Config.Common.RawDefs +namespace Luban.Job.Cfg.RawDefs { public class Validator diff --git a/src/Luban.Job.Cfg/Source/RawDefs/Defines.cs b/src/Luban.Job.Cfg/Source/RawDefs/Defines.cs index 4b4ed22..4760f1c 100644 --- a/src/Luban.Job.Cfg/Source/RawDefs/Defines.cs +++ b/src/Luban.Job.Cfg/Source/RawDefs/Defines.cs @@ -1,12 +1,15 @@ +using Luban.Job.Cfg.RawDefs; using Luban.Job.Common.RawDefs; using System.Collections.Generic; -namespace Luban.Config.Common.RawDefs +namespace Luban.Job.Cfg.RawDefs { public class Defines { public string TopModule { get; set; } = ""; + public List Branches { get; set; } = new List(); + public List Beans { get; set; } = new List(); public List Consts { get; set; } = new List(); diff --git a/src/Luban.Job.Cfg/Source/RawDefs/Group.cs b/src/Luban.Job.Cfg/Source/RawDefs/Group.cs index bd63340..30ceb44 100644 --- a/src/Luban.Job.Cfg/Source/RawDefs/Group.cs +++ b/src/Luban.Job.Cfg/Source/RawDefs/Group.cs @@ -1,6 +1,6 @@ using System.Collections.Generic; -namespace Luban.Config.Common.RawDefs +namespace Luban.Job.Cfg.RawDefs { public class Group { diff --git a/src/Luban.Job.Cfg/Source/RawDefs/Service.cs b/src/Luban.Job.Cfg/Source/RawDefs/Service.cs index 59bd7c9..3054710 100644 --- a/src/Luban.Job.Cfg/Source/RawDefs/Service.cs +++ b/src/Luban.Job.Cfg/Source/RawDefs/Service.cs @@ -1,6 +1,6 @@ using System.Collections.Generic; -namespace Luban.Config.Common.RawDefs +namespace Luban.Job.Cfg.RawDefs { public class Service { diff --git a/src/Luban.Job.Cfg/Source/RawDefs/Table.cs b/src/Luban.Job.Cfg/Source/RawDefs/Table.cs index 038547e..fe7ef88 100644 --- a/src/Luban.Job.Cfg/Source/RawDefs/Table.cs +++ b/src/Luban.Job.Cfg/Source/RawDefs/Table.cs @@ -1,6 +1,6 @@ using System.Collections.Generic; -namespace Luban.Config.Common.RawDefs +namespace Luban.Job.Cfg.RawDefs { public enum ETableMode { @@ -35,5 +35,7 @@ namespace Luban.Config.Common.RawDefs public List Groups { get; set; } = new List(); public List InputFiles { get; set; } = new List(); + + public Dictionary> BranchInputFiles { get; set; } = new Dictionary>(); } } diff --git a/src/Luban.Job.Cfg/Source/TypeVisitors/DeepCompareTypeDefine.cs b/src/Luban.Job.Cfg/Source/TypeVisitors/DeepCompareTypeDefine.cs index 0c49151..561ea8d 100644 --- a/src/Luban.Job.Cfg/Source/TypeVisitors/DeepCompareTypeDefine.cs +++ b/src/Luban.Job.Cfg/Source/TypeVisitors/DeepCompareTypeDefine.cs @@ -1,5 +1,5 @@ -using Luban.Config.Common.RawDefs; using Luban.Job.Cfg.Defs; +using Luban.Job.Cfg.RawDefs; using Luban.Job.Common.Defs; using Luban.Job.Common.Types; using Luban.Job.Common.TypeVisitors; diff --git a/src/Luban.Job.Cfg/Source/Utils/DataExporterUtil.cs b/src/Luban.Job.Cfg/Source/Utils/DataExporterUtil.cs index a14efbf..3b91cb5 100644 --- a/src/Luban.Job.Cfg/Source/Utils/DataExporterUtil.cs +++ b/src/Luban.Job.Cfg/Source/Utils/DataExporterUtil.cs @@ -1,5 +1,4 @@ using Bright.Serialization; -using Luban.Config.Common.RawDefs; using Luban.Job.Cfg.Datas; using Luban.Job.Cfg.DataVisitors; using Luban.Job.Cfg.Defs; diff --git a/src/Luban.Job.Cfg/Source/Utils/DataLoaderUtil.cs b/src/Luban.Job.Cfg/Source/Utils/DataLoaderUtil.cs index ea72aa6..14b1fff 100644 --- a/src/Luban.Job.Cfg/Source/Utils/DataLoaderUtil.cs +++ b/src/Luban.Job.Cfg/Source/Utils/DataLoaderUtil.cs @@ -71,16 +71,14 @@ namespace Luban.Job.Cfg.Utils // return CollectInputFilesAsync(agent, table.InputFiles, dataDir) //} - public static async Task LoadTableAsync(RemoteAgent agent, DefTable table, string dataDir, bool exportTestData) + public static async Task GenerateLoadRecordFromFileTasksAsync(RemoteAgent agent, DefTable table, string dataDir, List inputFiles2, bool exportTestData, List>> tasks) { - var tasks = new List>>(); - - var inputFiles = await CollectInputFilesAsync(agent, table.InputFiles, dataDir); + var inputFileInfos = await CollectInputFilesAsync(agent, inputFiles2, dataDir); // check cache (table, exporttestdata) -> (list, List) // (md5, sheetName,exportTestData) -> (value_type, List) - foreach (var file in inputFiles) + foreach (var file in inputFileInfos) { var actualFile = file.ActualFile; //s_logger.Info("== get input file:{file} actualFile:{actual}", file, actualFile); @@ -103,21 +101,52 @@ namespace Luban.Job.Cfg.Utils return res; })); } - - var records = new List(tasks.Count); - foreach (var task in tasks) - { - records.AddRange(await task); - } - - s_logger.Trace("== load recors. count:{count}", records.Count); - - table.Assembly.AddDataTable(table, records); - - s_logger.Trace("table:{name} record num:{num}", table.FullName, records.Count); } - public static async Task LoadCfgDataAsync(RemoteAgent agent, DefAssembly ass, string dataDir, bool exportTestData) + public static async Task LoadTableAsync(RemoteAgent agent, DefTable table, string dataDir, string branchName, string branchDataDir, bool exportTestData) + { + var mainLoadTasks = new List>>(); + var mainGenerateTask = GenerateLoadRecordFromFileTasksAsync(agent, table, dataDir, table.InputFiles, exportTestData, mainLoadTasks); + + var branchLoadTasks = new List>>(); + + Task branchGenerateTask = null; + if (!string.IsNullOrWhiteSpace(branchName)) + { + var branchInputFiles = table.GetBranchInputFiles(branchName); + if (branchInputFiles != null) + { + branchGenerateTask = GenerateLoadRecordFromFileTasksAsync(agent, table, branchDataDir, branchInputFiles, exportTestData, branchLoadTasks); + } + } + + await mainGenerateTask; + + var mainRecords = new List(256); + foreach (var task in mainLoadTasks) + { + mainRecords.AddRange(await task); + } + s_logger.Trace("== load main records. count:{count}", mainRecords.Count); + + List branchRecords = null; + if (branchGenerateTask != null) + { + branchRecords = new List(64); + await branchGenerateTask; + foreach (var task in branchLoadTasks) + { + branchRecords.AddRange(await task); + } + s_logger.Trace("== load branch records. count:{count}", branchRecords.Count); + } + + table.Assembly.AddDataTable(table, mainRecords, branchRecords); + + s_logger.Trace("table:{name} record num:{num}", table.FullName, mainRecords.Count); + } + + public static async Task LoadCfgDataAsync(RemoteAgent agent, DefAssembly ass, string dataDir, string branchName, string branchDataDir, bool exportTestData) { var ctx = agent; List exportTables = ass.Types.Values.Where(t => t is DefTable ct && ct.NeedExport).Select(t => (DefTable)t).ToList(); @@ -130,7 +159,7 @@ namespace Luban.Job.Cfg.Utils genDataTasks.Add(Task.Run(async () => { long beginTime = TimeUtil.NowMillis; - await LoadTableAsync(agent, c, dataDir, exportTestData); + await LoadTableAsync(agent, c, dataDir, branchName, branchDataDir, exportTestData); long endTime = TimeUtil.NowMillis; if (endTime - beginTime > 100) { diff --git a/src/Luban.Job.Cfg/Source/ValidatorContext.cs b/src/Luban.Job.Cfg/Source/ValidatorContext.cs index ba6152a..55add5d 100644 --- a/src/Luban.Job.Cfg/Source/ValidatorContext.cs +++ b/src/Luban.Job.Cfg/Source/ValidatorContext.cs @@ -1,8 +1,8 @@ using Luban.Common.Utils; -using Luban.Config.Common.RawDefs; using Luban.Job.Cfg.Datas; using Luban.Job.Cfg.DataVisitors; using Luban.Job.Cfg.Defs; +using Luban.Job.Cfg.RawDefs; using Luban.Job.Cfg.Utils; using Luban.Job.Cfg.Validators; using System; @@ -27,6 +27,8 @@ namespace Luban.Job.Cfg public class ValidatorContext { + private static readonly NLog.Logger s_logger = NLog.LogManager.GetCurrentClassLogger(); + [ThreadStatic] private static ValidatorVisitor t_visitor; @@ -67,8 +69,7 @@ namespace Luban.Job.Cfg { tasks.Add(Task.Run(() => { - var records = t.Assembly.GetTableDataList(t); - ValidateTableModeIndex(t, records); + ValidateTableModeIndex(t); })); } await Task.WhenAll(tasks); @@ -171,58 +172,134 @@ namespace Luban.Job.Cfg } } - private void ValidateTableModeIndex(DefTable table, List records) + private void ValidateTableModeIndex(DefTable table) { - var recordMap = new Dictionary(); + var tableDataInfo = Assembly.GetTableDataInfo(table); + + List mainRecords = tableDataInfo.MainRecords; + List branchRecords = tableDataInfo.BranchRecords; + + // 这么大费周张是为了保证被覆盖的id仍然保持原来的顺序，而不是出现在最后 + int index = 0; + foreach (var r in mainRecords) + { + r.Index = index++; + } + if (branchRecords != null) + { + foreach (var r in branchRecords) + { + r.Index = index++; + } + } + + var mainRecordMap = new Dictionary(); switch (table.Mode) { case ETableMode.ONE: { - if (records.Count != 1) + if (mainRecords.Count != 1) { - throw new Exception($"配置表 {table.FullName} 是单值表 mode=one,但数据个数:{records.Count} != 1"); + throw new Exception($"配置表 {table.FullName} 是单值表 mode=one,但主文件数据个数:{mainRecords.Count} != 1"); + } + if (branchRecords != null && branchRecords.Count != 1) + { + throw new Exception($"配置表 {table.FullName} 是单值表 mode=one,但分支文件数据个数:{branchRecords.Count} != 1"); + } + if (branchRecords != null) + { + mainRecords[0] = branchRecords[0]; } break; } case ETableMode.MAP: { - foreach (Record r in records) + foreach (Record r in mainRecords) { DType key = r.Data.Fields[table.IndexFieldIdIndex]; - if (!recordMap.TryAdd(key, r)) + if (!mainRecordMap.TryAdd(key, r)) { - throw new Exception($@"配置表 {table.FullName} 主键字段:{table.Index} 主键值:{key} 重复. + throw new Exception($@"配置表 {table.FullName} 主文件 主键字段:{table.Index} 主键值:{key} 重复. 记录1 来自文件:{r.Source} - 记录2 来自文件:{recordMap[key].Source} + 记录2 来自文件:{mainRecordMap[key].Source} "); } - + } + if (branchRecords != null) + { + var branchRecordMap = new Dictionary(); + foreach (Record r in branchRecords) + { + DType key = r.Data.Fields[table.IndexFieldIdIndex]; + if (!branchRecordMap.TryAdd(key, r)) + { + throw new Exception($@"配置表 {table.FullName} 分支文件 主键字段:{table.Index} 主键值:{key} 重复. + 记录1 来自文件:{r.Source} + 记录2 来自文件:{branchRecordMap[key].Source} +"); + } + if (mainRecordMap.TryGetValue(key, out var old)) + { + s_logger.Debug("配置表 {} 分支文件 主键:{} 覆盖 主文件记录", table.FullName, key); + mainRecords[old.Index] = r; + } + mainRecordMap[key] = r; + } } break; } case ETableMode.BMAP: { - var twoKeyMap = new Dictionary<(DType, DType), Record>(); - foreach (Record r in records) + var mainTwoKeyMap = new Dictionary<(DType, DType), Record>(); + foreach (Record r in mainRecords) { DType key1 = r.Data.Fields[table.IndexFieldIdIndex1]; DType key2 = r.Data.Fields[table.IndexFieldIdIndex2]; - if (!twoKeyMap.TryAdd((key1, key2), r)) + if (!mainTwoKeyMap.TryAdd((key1, key2), r)) { - throw new Exception($@"配置表 {table.FullName} 主键字段:{table.Index} 主键值:({key1},{key2})重复. + throw new Exception($@"配置表 {table.FullName} 主文件 主键字段:{table.Index} 主键值:({key1},{key2})重复. 记录1 来自文件:{r.Source} - 记录2 来自文件:{twoKeyMap[(key1, key2)].Source} + 记录2 来自文件:{mainTwoKeyMap[(key1, key2)].Source} "); } // 目前不支持 双key索引检查,但支持主key索引检查. // 所以至少塞入一个,让ref检查能通过 - recordMap[key1] = r; + mainRecordMap[key1] = r; } + + if (branchRecords != null) + { + var branchTwoKeyMap = new Dictionary<(DType, DType), Record>(); + foreach (Record r in branchRecords) + { + DType key1 = r.Data.Fields[table.IndexFieldIdIndex1]; + DType key2 = r.Data.Fields[table.IndexFieldIdIndex2]; + if (!branchTwoKeyMap.TryAdd((key1, key2), r)) + { + throw new Exception($@"配置表 {table.FullName} 分支文件 主键字段:{table.Index} 主键值:({key1},{key2})重复. + 记录1 来自文件:{r.Source} + 记录2 来自文件:{branchTwoKeyMap[(key1, key2)].Source} +"); + } + if (mainTwoKeyMap.TryGetValue((key1, key2), out var old)) + { + s_logger.Debug("配置表 {} 分支文件 主键:({},{}) 覆盖 主文件记录", table.FullName, key1, key2); + mainRecords[old.Index] = r; + } + mainTwoKeyMap[(key1, key2)] = r; + // 目前不支持 双key索引检查,但支持主key索引检查. + // 所以至少塞入一个,让ref检查能通过 + mainRecordMap[key1] = r; + } + } + + break; } } - table.Assembly.SetDataTableMap(table, recordMap); + tableDataInfo.FinalRecords = mainRecords; + tableDataInfo.FinalRecordMap = mainRecordMap; } } } diff --git a/src/Luban.Job.Cfg/Source/Validators/RefValidator.cs b/src/Luban.Job.Cfg/Source/Validators/RefValidator.cs index 2a8b307..eacc928 100644 --- a/src/Luban.Job.Cfg/Source/Validators/RefValidator.cs +++ b/src/Luban.Job.Cfg/Source/Validators/RefValidator.cs @@ -53,7 +53,7 @@ namespace Luban.Job.Cfg.Validators return; } DefTable ct = assembly.GetCfgTable(actualTable); - var recordMap = assembly.GetTableDataMap(ct); + var recordMap = assembly.GetTableDataInfo(ct).FinalRecordMap; if (/*recordMap != null &&*/ recordMap.ContainsKey(key)) { return; diff --git a/src/Luban.Job.Cfg/Source/Validators/ValidatorFactory.cs b/src/Luban.Job.Cfg/Source/Validators/ValidatorFactory.cs index 4efe0cc..e6a13fc 100644 --- a/src/Luban.Job.Cfg/Source/Validators/ValidatorFactory.cs +++ b/src/Luban.Job.Cfg/Source/Validators/ValidatorFactory.cs @@ -1,4 +1,4 @@ -using Luban.Config.Common.RawDefs; +using Luban.Job.Cfg.RawDefs; using System; using System.Collections.Generic; using System.Linq; diff --git a/src/Luban.Job.Cfg/Source/l10n/TextTable.cs b/src/Luban.Job.Cfg/Source/l10n/TextTable.cs index 65ee6b9..ca8cde8 100644 --- a/src/Luban.Job.Cfg/Source/l10n/TextTable.cs +++ b/src/Luban.Job.Cfg/Source/l10n/TextTable.cs @@ -1,7 +1,7 @@ -using Luban.Config.Common.RawDefs; -using Luban.Job.Cfg.Datas; +using Luban.Job.Cfg.Datas; using Luban.Job.Cfg.DataVisitors; using Luban.Job.Cfg.Defs; +using Luban.Job.Cfg.RawDefs; using Luban.Job.Cfg.Utils; using Luban.Job.Common.Types; using System;