diff --git a/README.md b/README.md index 5de5ac9..2a32094 100644 --- a/README.md +++ b/README.md @@ -221,9 +221,9 @@ array与list类型都能表示列表,它们区别在于array生成的代码为 - 表级别分组 - 字段级别分组(任意bean字段粒度,而不仅限于顶层字段) -### 数据标签 +### 数据标签过滤 -开发期有时候希望临时注释掉一些记录,另外开发期经常会制作一些仅供开发使用的配置,比如测试道具,比如自动化测试使用的配置,在正式上线时不导出这些数据。 luban支持数据标签及数据注释及测试数据过滤导出,导出正式配置时不需要手动将测试数据删掉。 +开发期经常会制作一些仅供开发使用的配置,比如测试道具,比如自动化测试使用的配置,希望在正式发布时不导出这些数据。 ![pipeline](docs/images/examples/c_11.jpg) diff --git a/docs/images/examples/c_11.jpg b/docs/images/examples/c_11.jpg index 5de9c5f..442919e 100644 Binary files a/docs/images/examples/c_11.jpg and b/docs/images/examples/c_11.jpg differ diff --git a/src/Luban.Job.Cfg/Source/DataSources/AbstractDataSource.cs b/src/Luban.Job.Cfg/Source/DataSources/AbstractDataSource.cs index 54e9307..ef1761d 100644 --- a/src/Luban.Job.Cfg/Source/DataSources/AbstractDataSource.cs +++ b/src/Luban.Job.Cfg/Source/DataSources/AbstractDataSource.cs @@ -15,6 +15,6 @@ namespace Luban.Job.Cfg.DataSources public abstract List ReadMulti(TBean type); - public abstract void Load(string rawUrl, string sheetName, Stream stream, bool exportDebugData); + public abstract void Load(string rawUrl, string sheetName, Stream stream); } } diff --git a/src/Luban.Job.Cfg/Source/DataSources/Binary/BinaryDataSource.cs b/src/Luban.Job.Cfg/Source/DataSources/Binary/BinaryDataSource.cs index 2e3407a..6cdfe6c 100644 --- a/src/Luban.Job.Cfg/Source/DataSources/Binary/BinaryDataSource.cs +++ b/src/Luban.Job.Cfg/Source/DataSources/Binary/BinaryDataSource.cs @@ -8,7 +8,7 @@ namespace Luban.Job.Cfg.DataSources.Binary { class BinaryDataSource : AbstractDataSource { - public override void Load(string rawUrl, string sheetName, Stream stream, bool exportDebugData) + public override void Load(string rawUrl, string sheetName, Stream stream) { throw new NotImplementedException(); } diff --git a/src/Luban.Job.Cfg/Source/DataSources/DataSourceFactory.cs b/src/Luban.Job.Cfg/Source/DataSources/DataSourceFactory.cs index e9f6c4a..bbe881a 100644 --- a/src/Luban.Job.Cfg/Source/DataSources/DataSourceFactory.cs +++ b/src/Luban.Job.Cfg/Source/DataSources/DataSourceFactory.cs @@ -18,7 +18,7 @@ namespace Luban.Job.Cfg.DataSources ".bin", }; - public static AbstractDataSource Create(string url, string sheetName, Stream stream, bool exportTestData) + public static AbstractDataSource Create(string url, string sheetName, Stream stream) { try { @@ -36,7 +36,7 @@ namespace Luban.Job.Cfg.DataSources case "yml": source = new Yaml.YamlDataSource(); break; default: throw new Exception($"不支持的文件类型:{url}"); } - source.Load(url, sheetName, stream, exportTestData); + source.Load(url, sheetName, stream); return source; } catch (DataCreateException) diff --git a/src/Luban.Job.Cfg/Source/DataSources/Excel/ExcelDataSource.cs b/src/Luban.Job.Cfg/Source/DataSources/Excel/ExcelDataSource.cs index 3272bb1..6d38dda 100644 --- a/src/Luban.Job.Cfg/Source/DataSources/Excel/ExcelDataSource.cs +++ b/src/Luban.Job.Cfg/Source/DataSources/Excel/ExcelDataSource.cs @@ -33,7 +33,7 @@ namespace Luban.Job.Cfg.DataSources.Excel } } - public override void Load(string rawUrl, string sheetName, Stream stream, bool exportTestData) + public override void Load(string rawUrl, string sheetName, Stream stream) { s_logger.Trace("{filename} {sheet}", rawUrl, sheetName); RawUrl = rawUrl; diff --git a/src/Luban.Job.Cfg/Source/DataSources/Json/JsonDataSource.cs b/src/Luban.Job.Cfg/Source/DataSources/Json/JsonDataSource.cs index 9d09fd7..70ceaf7 100644 --- a/src/Luban.Job.Cfg/Source/DataSources/Json/JsonDataSource.cs +++ b/src/Luban.Job.Cfg/Source/DataSources/Json/JsonDataSource.cs @@ -14,7 +14,7 @@ namespace Luban.Job.Cfg.DataSources.Json { JsonElement _data; - public override void Load(string rawUrl, string sheetName, Stream stream, bool exportDebugData) + public override void Load(string rawUrl, string sheetName, Stream stream) { RawUrl = rawUrl; this._data = JsonDocument.Parse(stream).RootElement; diff --git a/src/Luban.Job.Cfg/Source/DataSources/Lua/LuaDataSource.cs b/src/Luban.Job.Cfg/Source/DataSources/Lua/LuaDataSource.cs index f59030d..ea05fc4 100644 --- a/src/Luban.Job.Cfg/Source/DataSources/Lua/LuaDataSource.cs +++ b/src/Luban.Job.Cfg/Source/DataSources/Lua/LuaDataSource.cs @@ -17,7 +17,7 @@ namespace Luban.Job.Cfg.DataSources.Lua private LuaGlobal _env; private LuaTable _dataTable; - public override void Load(string rawUrl, string sheetName, Stream stream, bool exportDebugData) + public override void Load(string rawUrl, string sheetName, Stream stream) { RawUrl = rawUrl; _env = LuaManager.CreateEnvironment(); diff --git a/src/Luban.Job.Cfg/Source/DataSources/Xml/XmlDataSource.cs b/src/Luban.Job.Cfg/Source/DataSources/Xml/XmlDataSource.cs index 534b559..007c221 100644 --- a/src/Luban.Job.Cfg/Source/DataSources/Xml/XmlDataSource.cs +++ b/src/Luban.Job.Cfg/Source/DataSources/Xml/XmlDataSource.cs @@ -14,7 +14,7 @@ namespace Luban.Job.Cfg.DataSources.Xml { private XElement _doc; - public override void Load(string rawUrl, string sheetName, Stream stream, bool exportDebugData) + public override void Load(string rawUrl, string sheetName, Stream stream) { RawUrl = rawUrl; _doc = XElement.Load(stream); diff --git a/src/Luban.Job.Cfg/Source/DataSources/Yaml/YamlDataSource.cs b/src/Luban.Job.Cfg/Source/DataSources/Yaml/YamlDataSource.cs index 5e94b55..e7bf986 100644 --- a/src/Luban.Job.Cfg/Source/DataSources/Yaml/YamlDataSource.cs +++ b/src/Luban.Job.Cfg/Source/DataSources/Yaml/YamlDataSource.cs @@ -13,7 +13,7 @@ namespace Luban.Job.Cfg.DataSources.Yaml class YamlDataSource : AbstractDataSource { private YamlMappingNode _root; - public override void Load(string rawUrl, string sheetName, Stream stream, bool exportDebugData) + public override void Load(string rawUrl, string sheetName, Stream stream) { var ys = new YamlStream(); ys.Load(new StreamReader(stream)); diff --git a/src/Luban.Job.Cfg/Source/Datas/Record.cs b/src/Luban.Job.Cfg/Source/Datas/Record.cs index ac023de..ef4b84f 100644 --- a/src/Luban.Job.Cfg/Source/Datas/Record.cs +++ b/src/Luban.Job.Cfg/Source/Datas/Record.cs @@ -13,7 +13,14 @@ namespace Luban.Job.Cfg.Datas public int Index { get; set; } - public bool IsTest => Tags != null && DataUtil.IsTestTag(Tags); + public bool IsNotFiltered(List excludeTags) + { + if (Tags == null) + { + return true; + } + return Tags.TrueForAll(t => !excludeTags.Contains(t)); + } public Record(DBean data, string source, List tags) { diff --git a/src/Luban.Job.Cfg/Source/Defs/CfgDefLoader.cs b/src/Luban.Job.Cfg/Source/Defs/CfgDefLoader.cs index 0545f17..f35962d 100644 --- a/src/Luban.Job.Cfg/Source/Defs/CfgDefLoader.cs +++ b/src/Luban.Job.Cfg/Source/Defs/CfgDefLoader.cs @@ -478,7 +478,7 @@ namespace Luban.Job.Cfg.Defs } }) { - AssemblyBase = new DefAssembly("", null, true, Agent), + AssemblyBase = new DefAssembly("", null, new List(), Agent), }; defTableRecordType.PreCompile(); defTableRecordType.Compile(); @@ -489,7 +489,7 @@ namespace Luban.Job.Cfg.Defs { var source = new ExcelDataSource(); var bytes = await this.Agent.GetFromCacheOrReadAllBytesAsync(file.ActualFile, file.MD5); - var records = DataLoaderUtil.LoadCfgRecords(tableRecordType, file.OriginFile, null, bytes, true, false); + var records = DataLoaderUtil.LoadCfgRecords(tableRecordType, file.OriginFile, null, bytes, true); foreach (var r in records) { DBean data = r.Data; @@ -544,7 +544,7 @@ namespace Luban.Job.Cfg.Defs } }) { - AssemblyBase = new DefAssembly("", null, true, Agent), + AssemblyBase = new DefAssembly("", null, new List(), Agent), }; defTableRecordType.PreCompile(); defTableRecordType.Compile(); @@ -555,7 +555,7 @@ namespace Luban.Job.Cfg.Defs { var source = new ExcelDataSource(); var bytes = await this.Agent.GetFromCacheOrReadAllBytesAsync(file.ActualFile, file.MD5); - var records = DataLoaderUtil.LoadCfgRecords(tableRecordType, file.OriginFile, null, bytes, true, false); + var records = DataLoaderUtil.LoadCfgRecords(tableRecordType, file.OriginFile, null, bytes, true); PEnum curEnum = null; foreach (var r in records) @@ -599,7 +599,7 @@ namespace Luban.Job.Cfg.Defs var inputFileInfos = await DataLoaderUtil.CollectInputFilesAsync(this.Agent, this._importExcelBeanFiles, dataDir); - var ass = new DefAssembly("", null, true, Agent); + var ass = new DefAssembly("", null, new List(), Agent); var defBeanFieldType = new DefBean(new CfgBean() { @@ -669,7 +669,7 @@ namespace Luban.Job.Cfg.Defs { var source = new ExcelDataSource(); var bytes = await this.Agent.GetFromCacheOrReadAllBytesAsync(file.ActualFile, file.MD5); - var records = DataLoaderUtil.LoadCfgRecords(tableRecordType, file.OriginFile, null, bytes, true, false); + var records = DataLoaderUtil.LoadCfgRecords(tableRecordType, file.OriginFile, null, bytes, true); foreach (var r in records) { diff --git a/src/Luban.Job.Cfg/Source/Defs/DefAssembly.cs b/src/Luban.Job.Cfg/Source/Defs/DefAssembly.cs index 66b120b..9e73521 100644 --- a/src/Luban.Job.Cfg/Source/Defs/DefAssembly.cs +++ b/src/Luban.Job.Cfg/Source/Defs/DefAssembly.cs @@ -19,19 +19,6 @@ namespace Luban.Job.Cfg.Defs public List FinalRecords { get; set; } - private List _notTestRecords; - public List NotTestRecords - { - get - { - if (_notTestRecords == null) - { - _notTestRecords = FinalRecords.Where(r => !r.IsTest).ToList(); - } - return _notTestRecords; - } - } - public Dictionary FinalRecordMap { get; set; } public TableDataInfo(List mainRecords, List patchRecords) @@ -50,17 +37,17 @@ namespace Luban.Job.Cfg.Defs public Service CfgTargetService { get; private set; } private readonly string _patchName; - private readonly bool _exportTestData; + private readonly List _excludeTags; public Patch TargetPatch { get; private set; } public TimeZoneInfo TimeZone { get; } - public DefAssembly(string patchName, TimeZoneInfo timezone, bool exportTestData, RemoteAgent agent) + public DefAssembly(string patchName, TimeZoneInfo timezone, List excludeTags, RemoteAgent agent) { this._patchName = patchName; this.TimeZone = timezone; - this._exportTestData = exportTestData; + this._excludeTags = excludeTags; this.Agent = agent; } @@ -126,7 +113,19 @@ namespace Luban.Job.Cfg.Defs public List GetTableExportDataList(DefTable table) { var tableDataInfo = _recordsByTables[table.FullName]; - return _exportTestData ? tableDataInfo.FinalRecords : tableDataInfo.NotTestRecords; + if (_excludeTags.Count == 0) + { + return tableDataInfo.FinalRecords; + } + else + { + var finalRecords = tableDataInfo.FinalRecords.Where(r => r.IsNotFiltered(_excludeTags)).ToList(); + if (table.IsOneValueTable && finalRecords.Count != 1) + { + throw new Exception($"配置表 {table.FullName} 是单值表 mode=one,但数据个数:{finalRecords.Count} != 1"); + } + return finalRecords; + } } public TableDataInfo GetTableDataInfo(DefTable table) diff --git a/src/Luban.Job.Cfg/Source/GenArgs.cs b/src/Luban.Job.Cfg/Source/GenArgs.cs index 5256ea7..62269d8 100644 --- a/src/Luban.Job.Cfg/Source/GenArgs.cs +++ b/src/Luban.Job.Cfg/Source/GenArgs.cs @@ -32,11 +32,8 @@ namespace Luban.Job.Cfg [Option('s', "service", Required = true, HelpText = "service")] public string Service { get; set; } - [Option("export_test_data", Required = false, HelpText = "export test data")] - public bool ExportTestData { get; set; } = false; - - [Option("export_tag_filter", Required = false, HelpText = "export tag filter")] - public string ExportTagFilter { get; set; } = ""; + [Option("export_exclude_tags", Required = false, HelpText = "export exclude tags. default export all tags")] + public string ExportExcludeTags { get; set; } = ""; [Option('t', "l10n_timezone", Required = false, HelpText = "timezone")] public string TimeZone { get; set; } diff --git a/src/Luban.Job.Cfg/Source/JobController.cs b/src/Luban.Job.Cfg/Source/JobController.cs index 842f85c..c9ad06b 100644 --- a/src/Luban.Job.Cfg/Source/JobController.cs +++ b/src/Luban.Job.Cfg/Source/JobController.cs @@ -140,7 +140,8 @@ namespace Luban.Job.Cfg TimeZoneInfo timeZoneInfo = string.IsNullOrEmpty(args.TimeZone) ? null : TimeZoneInfo.FindSystemTimeZoneById(args.TimeZone); - var ass = new DefAssembly(args.PatchName, timeZoneInfo, args.ExportTestData, agent) + var excludeTags = args.ExportExcludeTags.Split(',').Select(t => t.Trim().ToLowerInvariant()).Where(t => !string.IsNullOrEmpty(t)).ToList(); + var ass = new DefAssembly(args.PatchName, timeZoneInfo, excludeTags, agent) { UseUnityVectors = args.UseUnityVectors }; @@ -166,7 +167,7 @@ namespace Luban.Job.Cfg hasLoadCfgData = true; var timer = new ProfileTimer(); timer.StartPhase("load config data"); - await DataLoaderUtil.LoadCfgDataAsync(agent, ass, args.InputDataDir, args.PatchName, args.PatchInputDataDir, args.ExportTestData); + await DataLoaderUtil.LoadCfgDataAsync(agent, ass, args.InputDataDir, args.PatchName, args.PatchInputDataDir); timer.EndPhaseAndLog(); if (needL10NTextConvert) diff --git a/src/Luban.Job.Cfg/Source/Utils/DataLoaderUtil.cs b/src/Luban.Job.Cfg/Source/Utils/DataLoaderUtil.cs index c7ff457..f7660a9 100644 --- a/src/Luban.Job.Cfg/Source/Utils/DataLoaderUtil.cs +++ b/src/Luban.Job.Cfg/Source/Utils/DataLoaderUtil.cs @@ -71,7 +71,7 @@ namespace Luban.Job.Cfg.Utils // return CollectInputFilesAsync(agent, table.InputFiles, dataDir) //} - public static async Task GenerateLoadRecordFromFileTasksAsync(RemoteAgent agent, DefTable table, string dataDir, List inputFiles2, bool exportTestData, List>> tasks) + public static async Task GenerateLoadRecordFromFileTasksAsync(RemoteAgent agent, DefTable table, string dataDir, List inputFiles2, List>> tasks) { var inputFileInfos = await CollectInputFilesAsync(agent, inputFiles2, dataDir); @@ -93,8 +93,7 @@ namespace Luban.Job.Cfg.Utils file.OriginFile, file.SheetName, await agent.GetFromCacheOrReadAllBytesAsync(file.ActualFile, file.MD5), - RenderFileUtil.IsExcelFile(file.ActualFile), - exportTestData); + RenderFileUtil.IsExcelFile(file.ActualFile)); FileRecordCacheManager.Ins.AddCacheLoadedRecords(table, file.MD5, file.SheetName, res); @@ -103,10 +102,10 @@ namespace Luban.Job.Cfg.Utils } } - public static async Task LoadTableAsync(RemoteAgent agent, DefTable table, string dataDir, string patchName, string patchDataDir, bool exportTestData) + public static async Task LoadTableAsync(RemoteAgent agent, DefTable table, string dataDir, string patchName, string patchDataDir) { var mainLoadTasks = new List>>(); - var mainGenerateTask = GenerateLoadRecordFromFileTasksAsync(agent, table, dataDir, table.InputFiles, exportTestData, mainLoadTasks); + var mainGenerateTask = GenerateLoadRecordFromFileTasksAsync(agent, table, dataDir, table.InputFiles, mainLoadTasks); var patchLoadTasks = new List>>(); @@ -116,7 +115,7 @@ namespace Luban.Job.Cfg.Utils var patchInputFiles = table.GetPatchInputFiles(patchName); if (patchInputFiles != null) { - patchGenerateTask = GenerateLoadRecordFromFileTasksAsync(agent, table, patchDataDir, patchInputFiles, exportTestData, patchLoadTasks); + patchGenerateTask = GenerateLoadRecordFromFileTasksAsync(agent, table, patchDataDir, patchInputFiles, patchLoadTasks); } } @@ -146,7 +145,7 @@ namespace Luban.Job.Cfg.Utils s_logger.Trace("table:{name} record num:{num}", table.FullName, mainRecords.Count); } - public static async Task LoadCfgDataAsync(RemoteAgent agent, DefAssembly ass, string dataDir, string patchName, string patchDataDir, bool exportTestData) + public static async Task LoadCfgDataAsync(RemoteAgent agent, DefAssembly ass, string dataDir, string patchName, string patchDataDir) { var ctx = agent; List exportTables = ass.Types.Values.Where(t => t is DefTable ct && ct.NeedExport).Select(t => (DefTable)t).ToList(); @@ -160,7 +159,7 @@ namespace Luban.Job.Cfg.Utils genDataTasks.Add(Task.Run(async () => { long beginTime = TimeUtil.NowMillis; - await LoadTableAsync(agent, table, dataDir, patchName, patchDataDir, exportTestData); + await LoadTableAsync(agent, table, dataDir, patchName, patchDataDir); long endTime = TimeUtil.NowMillis; if (endTime - beginTime > 100) { @@ -171,10 +170,10 @@ namespace Luban.Job.Cfg.Utils await Task.WhenAll(genDataTasks.ToArray()); } - public static List LoadCfgRecords(TBean recordType, string originFile, string sheetName, byte[] content, bool multiRecord, bool exportTestData) + public static List LoadCfgRecords(TBean recordType, string originFile, string sheetName, byte[] content, bool multiRecord) { // (md5,sheet,multiRecord,exportTestData) -> (valuetype, List<(datas)>) - var dataSource = DataSourceFactory.Create(originFile, sheetName, new MemoryStream(content), exportTestData); + var dataSource = DataSourceFactory.Create(originFile, sheetName, new MemoryStream(content)); try { if (multiRecord) diff --git a/src/Luban.Job.Cfg/Source/Utils/DataUtil.cs b/src/Luban.Job.Cfg/Source/Utils/DataUtil.cs index 7abbfd1..9d983d3 100644 --- a/src/Luban.Job.Cfg/Source/Utils/DataUtil.cs +++ b/src/Luban.Job.Cfg/Source/Utils/DataUtil.cs @@ -117,33 +117,18 @@ namespace Luban.Job.Cfg.Utils { return !string.IsNullOrEmpty(tagName) && ( - tagName.Equals("false", System.StringComparison.OrdinalIgnoreCase) - || tagName.Equals("no", System.StringComparison.OrdinalIgnoreCase) + tagName.Equals("no", System.StringComparison.OrdinalIgnoreCase) || tagName.Equals("##", System.StringComparison.Ordinal) - //|| tagName.Equals("", System.StringComparison.Ordinal) ); } - private static bool IsTestTag(string tagName) - { - return !string.IsNullOrEmpty(tagName) && - (tagName.Equals("test", System.StringComparison.OrdinalIgnoreCase) - || tagName.Equals("", System.StringComparison.Ordinal) - ); - } - - public static bool IsTestTag(List tagNames) - { - return tagNames.Any(IsTestTag); - } - public static List ParseTags(string rawTagStr) { if (string.IsNullOrWhiteSpace(rawTagStr)) { return null; } - var tags = new List(rawTagStr.Split(',').Select(t => t.Trim()).Where(t => !string.IsNullOrEmpty(t))); + var tags = new List(rawTagStr.Split(',').Select(t => t.Trim().ToLower()).Where(t => !string.IsNullOrEmpty(t))); return tags.Count > 0 ? tags : null; } diff --git a/src/Luban.Job.Cfg/Source/ValidatorContext.cs b/src/Luban.Job.Cfg/Source/ValidatorContext.cs index f0dc71a..a9bd996 100644 --- a/src/Luban.Job.Cfg/Source/ValidatorContext.cs +++ b/src/Luban.Job.Cfg/Source/ValidatorContext.cs @@ -201,17 +201,17 @@ namespace Luban.Job.Cfg { case ETableMode.ONE: { - if (mainRecords.Count != 1) - { - throw new Exception($"配置表 {table.FullName} 是单值表 mode=one,但主文件数据个数:{mainRecords.Count} != 1"); - } - if (patchRecords != null && patchRecords.Count != 1) - { - throw new Exception($"配置表 {table.FullName} 是单值表 mode=one,但分支文件数据个数:{patchRecords.Count} != 1"); - } + //if (mainRecords.Count != 1) + //{ + // throw new Exception($"配置表 {table.FullName} 是单值表 mode=one,但主文件数据个数:{mainRecords.Count} != 1"); + //} + //if (patchRecords != null && patchRecords.Count != 1) + //{ + // throw new Exception($"配置表 {table.FullName} 是单值表 mode=one,但分支文件数据个数:{patchRecords.Count} != 1"); + //} if (patchRecords != null) { - mainRecords[0] = patchRecords[0]; + mainRecords = patchRecords; } break; } diff --git a/src/Luban.Job.Cfg/Source/l10n/TextTable.cs b/src/Luban.Job.Cfg/Source/l10n/TextTable.cs index b7ce122..75f7390 100644 --- a/src/Luban.Job.Cfg/Source/l10n/TextTable.cs +++ b/src/Luban.Job.Cfg/Source/l10n/TextTable.cs @@ -66,7 +66,7 @@ namespace Luban.Job.Cfg.l10n public void LoadFromFile(string fileName, byte[] bytes) { - var records = DataLoaderUtil.LoadCfgRecords(_textRowType, fileName, null, bytes, true, false); + var records = DataLoaderUtil.LoadCfgRecords(_textRowType, fileName, null, bytes, true); foreach (var r in records) { //s_logger.Info("== read text:{}", r.Data);