【删除】移除 cfg --export_test_data 选项，如果想不导出测试数据，使用 --export_exclude_tags test

【特性】新增 cfg --export_exclude_tags选项，支持过滤指定tags列表的数据

README.md

@@ -221,9 +221,9 @@ array与list类型都能表示列表，它们区别在于array生成的代码为
 - 表级别分组
 - 字段级别分组(任意bean字段粒度，而不仅限于顶层字段)
 
-### 数据标签
+### 数据标签过滤
 
-开发期有时候希望临时注释掉一些记录，另外开发期经常会制作一些仅供开发使用的配置，比如测试道具，比如自动化测试使用的配置，在正式上线时不导出这些数据。 luban支持数据标签及数据注释及测试数据过滤导出，导出正式配置时不需要手动将测试数据删掉。
+开发期经常会制作一些仅供开发使用的配置，比如测试道具，比如自动化测试使用的配置，希望在正式发布时不导出这些数据。 
 
 ![pipeline](docs/images/examples/c_11.jpg)
 

src/Luban.Job.Cfg/Source/DataSources/AbstractDataSource.cs

@@ -15,6 +15,6 @@ namespace Luban.Job.Cfg.DataSources
 
         public abstract List<Record> ReadMulti(TBean type);
 
-        public abstract void Load(string rawUrl, string sheetName, Stream stream, bool exportDebugData);
+        public abstract void Load(string rawUrl, string sheetName, Stream stream);
     }
 }

src/Luban.Job.Cfg/Source/DataSources/Binary/BinaryDataSource.cs

@@ -8,7 +8,7 @@ namespace Luban.Job.Cfg.DataSources.Binary
 {
     class BinaryDataSource : AbstractDataSource
     {
-        public override void Load(string rawUrl, string sheetName, Stream stream, bool exportDebugData)
+        public override void Load(string rawUrl, string sheetName, Stream stream)
         {
             throw new NotImplementedException();
         }

src/Luban.Job.Cfg/Source/DataSources/DataSourceFactory.cs

@@ -18,7 +18,7 @@ namespace Luban.Job.Cfg.DataSources
             ".bin",
         };
 
-        public static AbstractDataSource Create(string url, string sheetName, Stream stream, bool exportTestData)
+        public static AbstractDataSource Create(string url, string sheetName, Stream stream)
         {
             try
             {
@@ -36,7 +36,7 @@ namespace Luban.Job.Cfg.DataSources
                     case "yml": source = new Yaml.YamlDataSource(); break;
                     default: throw new Exception($"不支持的文件类型:{url}");
                 }
-                source.Load(url, sheetName, stream, exportTestData);
+                source.Load(url, sheetName, stream);
                 return source;
             }
             catch (DataCreateException)

src/Luban.Job.Cfg/Source/DataSources/Excel/ExcelDataSource.cs

@@ -33,7 +33,7 @@ namespace Luban.Job.Cfg.DataSources.Excel
             }
         }
 
-        public override void Load(string rawUrl, string sheetName, Stream stream, bool exportTestData)
+        public override void Load(string rawUrl, string sheetName, Stream stream)
         {
             s_logger.Trace("{filename} {sheet}", rawUrl, sheetName);
             RawUrl = rawUrl;

src/Luban.Job.Cfg/Source/DataSources/Json/JsonDataSource.cs

@@ -14,7 +14,7 @@ namespace Luban.Job.Cfg.DataSources.Json
     {
         JsonElement _data;
 
-        public override void Load(string rawUrl, string sheetName, Stream stream, bool exportDebugData)
+        public override void Load(string rawUrl, string sheetName, Stream stream)
         {
             RawUrl = rawUrl;
             this._data = JsonDocument.Parse(stream).RootElement;

src/Luban.Job.Cfg/Source/DataSources/Lua/LuaDataSource.cs

@@ -17,7 +17,7 @@ namespace Luban.Job.Cfg.DataSources.Lua
         private LuaGlobal _env;
         private LuaTable _dataTable;
 
-        public override void Load(string rawUrl, string sheetName, Stream stream, bool exportDebugData)
+        public override void Load(string rawUrl, string sheetName, Stream stream)
         {
             RawUrl = rawUrl;
             _env = LuaManager.CreateEnvironment();

src/Luban.Job.Cfg/Source/DataSources/Xml/XmlDataSource.cs

@@ -14,7 +14,7 @@ namespace Luban.Job.Cfg.DataSources.Xml
     {
         private XElement _doc;
 
-        public override void Load(string rawUrl, string sheetName, Stream stream, bool exportDebugData)
+        public override void Load(string rawUrl, string sheetName, Stream stream)
         {
             RawUrl = rawUrl;
             _doc = XElement.Load(stream);

src/Luban.Job.Cfg/Source/DataSources/Yaml/YamlDataSource.cs

@@ -13,7 +13,7 @@ namespace Luban.Job.Cfg.DataSources.Yaml
     class YamlDataSource : AbstractDataSource
     {
         private YamlMappingNode _root;
-        public override void Load(string rawUrl, string sheetName, Stream stream, bool exportDebugData)
+        public override void Load(string rawUrl, string sheetName, Stream stream)
         {
             var ys = new YamlStream();
             ys.Load(new StreamReader(stream));

src/Luban.Job.Cfg/Source/Datas/Record.cs

@@ -13,7 +13,14 @@ namespace Luban.Job.Cfg.Datas
 
         public int Index { get; set; }
 
-        public bool IsTest => Tags != null && DataUtil.IsTestTag(Tags);
+        public bool IsNotFiltered(List<string> excludeTags)
+        {
+            if (Tags == null)
+            {
+                return true;
+            }
+            return Tags.TrueForAll(t => !excludeTags.Contains(t));
+        }
 
         public Record(DBean data, string source, List<string> tags)
         {

src/Luban.Job.Cfg/Source/Defs/CfgDefLoader.cs

@@ -478,7 +478,7 @@ namespace Luban.Job.Cfg.Defs
                 }
             })
             {
-                AssemblyBase = new DefAssembly("", null, true, Agent),
+                AssemblyBase = new DefAssembly("", null, new List<string>(), Agent),
             };
             defTableRecordType.PreCompile();
             defTableRecordType.Compile();
@@ -489,7 +489,7 @@ namespace Luban.Job.Cfg.Defs
             {
                 var source = new ExcelDataSource();
                 var bytes = await this.Agent.GetFromCacheOrReadAllBytesAsync(file.ActualFile, file.MD5);
-                var records = DataLoaderUtil.LoadCfgRecords(tableRecordType, file.OriginFile, null, bytes, true, false);
+                var records = DataLoaderUtil.LoadCfgRecords(tableRecordType, file.OriginFile, null, bytes, true);
                 foreach (var r in records)
                 {
                     DBean data = r.Data;
@@ -544,7 +544,7 @@ namespace Luban.Job.Cfg.Defs
                 }
             })
             {
-                AssemblyBase = new DefAssembly("", null, true, Agent),
+                AssemblyBase = new DefAssembly("", null, new List<string>(), Agent),
             };
             defTableRecordType.PreCompile();
             defTableRecordType.Compile();
@@ -555,7 +555,7 @@ namespace Luban.Job.Cfg.Defs
             {
                 var source = new ExcelDataSource();
                 var bytes = await this.Agent.GetFromCacheOrReadAllBytesAsync(file.ActualFile, file.MD5);
-                var records = DataLoaderUtil.LoadCfgRecords(tableRecordType, file.OriginFile, null, bytes, true, false);
+                var records = DataLoaderUtil.LoadCfgRecords(tableRecordType, file.OriginFile, null, bytes, true);
 
                 PEnum curEnum = null;
                 foreach (var r in records)
@@ -599,7 +599,7 @@ namespace Luban.Job.Cfg.Defs
             var inputFileInfos = await DataLoaderUtil.CollectInputFilesAsync(this.Agent, this._importExcelBeanFiles, dataDir);
 
 
-            var ass = new DefAssembly("", null, true, Agent);
+            var ass = new DefAssembly("", null, new List<string>(), Agent);
 
             var defBeanFieldType = new DefBean(new CfgBean()
             {
@@ -669,7 +669,7 @@ namespace Luban.Job.Cfg.Defs
             {
                 var source = new ExcelDataSource();
                 var bytes = await this.Agent.GetFromCacheOrReadAllBytesAsync(file.ActualFile, file.MD5);
-                var records = DataLoaderUtil.LoadCfgRecords(tableRecordType, file.OriginFile, null, bytes, true, false);
+                var records = DataLoaderUtil.LoadCfgRecords(tableRecordType, file.OriginFile, null, bytes, true);
 
                 foreach (var r in records)
                 {

src/Luban.Job.Cfg/Source/Defs/DefAssembly.cs

@@ -19,19 +19,6 @@ namespace Luban.Job.Cfg.Defs
 
         public List<Record> FinalRecords { get; set; }
 
-        private List<Record> _notTestRecords;
-        public List<Record> NotTestRecords
-        {
-            get
-            {
-                if (_notTestRecords == null)
-                {
-                    _notTestRecords = FinalRecords.Where(r => !r.IsTest).ToList();
-                }
-                return _notTestRecords;
-            }
-        }
-
         public Dictionary<DType, Record> FinalRecordMap { get; set; }
 
         public TableDataInfo(List<Record> mainRecords, List<Record> patchRecords)
@@ -50,17 +37,17 @@ namespace Luban.Job.Cfg.Defs
         public Service CfgTargetService { get; private set; }
 
         private readonly string _patchName;
-        private readonly bool _exportTestData;
+        private readonly List<string> _excludeTags;
 
         public Patch TargetPatch { get; private set; }
 
         public TimeZoneInfo TimeZone { get; }
 
-        public DefAssembly(string patchName, TimeZoneInfo timezone, bool exportTestData, RemoteAgent agent)
+        public DefAssembly(string patchName, TimeZoneInfo timezone, List<string> excludeTags, RemoteAgent agent)
         {
             this._patchName = patchName;
             this.TimeZone = timezone;
-            this._exportTestData = exportTestData;
+            this._excludeTags = excludeTags;
             this.Agent = agent;
         }
 
@@ -126,7 +113,19 @@ namespace Luban.Job.Cfg.Defs
         public List<Record> GetTableExportDataList(DefTable table)
         {
             var tableDataInfo = _recordsByTables[table.FullName];
-            return _exportTestData ? tableDataInfo.FinalRecords : tableDataInfo.NotTestRecords;
+            if (_excludeTags.Count == 0)
+            {
+                return tableDataInfo.FinalRecords;
+            }
+            else
+            {
+                var finalRecords = tableDataInfo.FinalRecords.Where(r => r.IsNotFiltered(_excludeTags)).ToList();
+                if (table.IsOneValueTable && finalRecords.Count != 1)
+                {
+                    throw new Exception($"配置表 {table.FullName} 是单值表 mode=one,但数据个数:{finalRecords.Count} != 1");
+                }
+                return finalRecords;
+            }
         }
 
         public TableDataInfo GetTableDataInfo(DefTable table)

src/Luban.Job.Cfg/Source/GenArgs.cs

@@ -32,11 +32,8 @@ namespace Luban.Job.Cfg
         [Option('s', "service", Required = true, HelpText = "service")]
         public string Service { get; set; }
 
-        [Option("export_test_data", Required = false, HelpText = "export test data")]
+        [Option("export_exclude_tags", Required = false, HelpText = "export exclude tags. default export all tags")]
-        public bool ExportTestData { get; set; } = false;
+        public string ExportExcludeTags { get; set; } = "";
-
-        [Option("export_tag_filter", Required = false, HelpText = "export tag filter")]
-        public string ExportTagFilter { get; set; } = "";
 
         [Option('t', "l10n_timezone", Required = false, HelpText = "timezone")]
         public string TimeZone { get; set; }

src/Luban.Job.Cfg/Source/JobController.cs

@@ -140,7 +140,8 @@ namespace Luban.Job.Cfg
 
                 TimeZoneInfo timeZoneInfo = string.IsNullOrEmpty(args.TimeZone) ? null : TimeZoneInfo.FindSystemTimeZoneById(args.TimeZone);
 
-                var ass = new DefAssembly(args.PatchName, timeZoneInfo, args.ExportTestData, agent)
+                var excludeTags = args.ExportExcludeTags.Split(',').Select(t => t.Trim().ToLowerInvariant()).Where(t => !string.IsNullOrEmpty(t)).ToList();
+                var ass = new DefAssembly(args.PatchName, timeZoneInfo, excludeTags, agent)
                 {
                     UseUnityVectors = args.UseUnityVectors
                 };
@@ -166,7 +167,7 @@ namespace Luban.Job.Cfg
                         hasLoadCfgData = true;
                         var timer = new ProfileTimer();
                         timer.StartPhase("load config data");
-                        await DataLoaderUtil.LoadCfgDataAsync(agent, ass, args.InputDataDir, args.PatchName, args.PatchInputDataDir, args.ExportTestData);
+                        await DataLoaderUtil.LoadCfgDataAsync(agent, ass, args.InputDataDir, args.PatchName, args.PatchInputDataDir);
                         timer.EndPhaseAndLog();
 
                         if (needL10NTextConvert)

src/Luban.Job.Cfg/Source/Utils/DataLoaderUtil.cs

@@ -71,7 +71,7 @@ namespace Luban.Job.Cfg.Utils
         //   return CollectInputFilesAsync(agent, table.InputFiles, dataDir)
         //}
 
-        public static async Task GenerateLoadRecordFromFileTasksAsync(RemoteAgent agent, DefTable table, string dataDir, List<string> inputFiles2, bool exportTestData, List<Task<List<Record>>> tasks)
+        public static async Task GenerateLoadRecordFromFileTasksAsync(RemoteAgent agent, DefTable table, string dataDir, List<string> inputFiles2, List<Task<List<Record>>> tasks)
         {
             var inputFileInfos = await CollectInputFilesAsync(agent, inputFiles2, dataDir);
 
@@ -93,8 +93,7 @@ namespace Luban.Job.Cfg.Utils
                         file.OriginFile,
                         file.SheetName,
                         await agent.GetFromCacheOrReadAllBytesAsync(file.ActualFile, file.MD5),
-                        RenderFileUtil.IsExcelFile(file.ActualFile),
+                        RenderFileUtil.IsExcelFile(file.ActualFile));
-                        exportTestData);
 
                     FileRecordCacheManager.Ins.AddCacheLoadedRecords(table, file.MD5, file.SheetName, res);
 
@@ -103,10 +102,10 @@ namespace Luban.Job.Cfg.Utils
             }
         }
 
-        public static async Task LoadTableAsync(RemoteAgent agent, DefTable table, string dataDir, string patchName, string patchDataDir, bool exportTestData)
+        public static async Task LoadTableAsync(RemoteAgent agent, DefTable table, string dataDir, string patchName, string patchDataDir)
         {
             var mainLoadTasks = new List<Task<List<Record>>>();
-            var mainGenerateTask = GenerateLoadRecordFromFileTasksAsync(agent, table, dataDir, table.InputFiles, exportTestData, mainLoadTasks);
+            var mainGenerateTask = GenerateLoadRecordFromFileTasksAsync(agent, table, dataDir, table.InputFiles, mainLoadTasks);
 
             var patchLoadTasks = new List<Task<List<Record>>>();
 
@@ -116,7 +115,7 @@ namespace Luban.Job.Cfg.Utils
                 var patchInputFiles = table.GetPatchInputFiles(patchName);
                 if (patchInputFiles != null)
                 {
-                    patchGenerateTask = GenerateLoadRecordFromFileTasksAsync(agent, table, patchDataDir, patchInputFiles, exportTestData, patchLoadTasks);
+                    patchGenerateTask = GenerateLoadRecordFromFileTasksAsync(agent, table, patchDataDir, patchInputFiles, patchLoadTasks);
                 }
             }
 
@@ -146,7 +145,7 @@ namespace Luban.Job.Cfg.Utils
             s_logger.Trace("table:{name} record num:{num}", table.FullName, mainRecords.Count);
         }
 
-        public static async Task LoadCfgDataAsync(RemoteAgent agent, DefAssembly ass, string dataDir, string patchName, string patchDataDir, bool exportTestData)
+        public static async Task LoadCfgDataAsync(RemoteAgent agent, DefAssembly ass, string dataDir, string patchName, string patchDataDir)
         {
             var ctx = agent;
             List<DefTable> exportTables = ass.Types.Values.Where(t => t is DefTable ct && ct.NeedExport).Select(t => (DefTable)t).ToList();
@@ -160,7 +159,7 @@ namespace Luban.Job.Cfg.Utils
                 genDataTasks.Add(Task.Run(async () =>
                 {
                     long beginTime = TimeUtil.NowMillis;
-                    await LoadTableAsync(agent, table, dataDir, patchName, patchDataDir, exportTestData);
+                    await LoadTableAsync(agent, table, dataDir, patchName, patchDataDir);
                     long endTime = TimeUtil.NowMillis;
                     if (endTime - beginTime > 100)
                     {
@@ -171,10 +170,10 @@ namespace Luban.Job.Cfg.Utils
             await Task.WhenAll(genDataTasks.ToArray());
         }
 
-        public static List<Record> LoadCfgRecords(TBean recordType, string originFile, string sheetName, byte[] content, bool multiRecord, bool exportTestData)
+        public static List<Record> LoadCfgRecords(TBean recordType, string originFile, string sheetName, byte[] content, bool multiRecord)
         {
             // (md5,sheet,multiRecord,exportTestData) -> (valuetype, List<(datas)>)
-            var dataSource = DataSourceFactory.Create(originFile, sheetName, new MemoryStream(content), exportTestData);
+            var dataSource = DataSourceFactory.Create(originFile, sheetName, new MemoryStream(content));
             try
             {
                 if (multiRecord)

src/Luban.Job.Cfg/Source/Utils/DataUtil.cs

@@ -117,33 +117,18 @@ namespace Luban.Job.Cfg.Utils
         {
             return !string.IsNullOrEmpty(tagName) &&
                 (
-                   tagName.Equals("false", System.StringComparison.OrdinalIgnoreCase)
+                tagName.Equals("no", System.StringComparison.OrdinalIgnoreCase)
-                || tagName.Equals("no", System.StringComparison.OrdinalIgnoreCase)
                 || tagName.Equals("##", System.StringComparison.Ordinal)
-                //|| tagName.Equals("·ñ", System.StringComparison.Ordinal)
                 );
         }
 
-        private static bool IsTestTag(string tagName)
-        {
-            return !string.IsNullOrEmpty(tagName) &&
-                (tagName.Equals("test", System.StringComparison.OrdinalIgnoreCase)
-                || tagName.Equals("²âÊÔ", System.StringComparison.Ordinal)
-                );
-        }
-
-        public static bool IsTestTag(List<string> tagNames)
-        {
-            return tagNames.Any(IsTestTag);
-        }
-
         public static List<string> ParseTags(string rawTagStr)
         {
             if (string.IsNullOrWhiteSpace(rawTagStr))
             {
                 return null;
             }
-            var tags = new List<string>(rawTagStr.Split(',').Select(t => t.Trim()).Where(t => !string.IsNullOrEmpty(t)));
+            var tags = new List<string>(rawTagStr.Split(',').Select(t => t.Trim().ToLower()).Where(t => !string.IsNullOrEmpty(t)));
             return tags.Count > 0 ? tags : null;
         }
 

src/Luban.Job.Cfg/Source/ValidatorContext.cs

@@ -201,17 +201,17 @@ namespace Luban.Job.Cfg
             {
                 case ETableMode.ONE:
                 {
-                    if (mainRecords.Count != 1)
+                    //if (mainRecords.Count != 1)
-                    {
+                    //{
-                        throw new Exception($"配置表 {table.FullName} 是单值表 mode=one,但主文件数据个数:{mainRecords.Count} != 1");
+                    //    throw new Exception($"配置表 {table.FullName} 是单值表 mode=one,但主文件数据个数:{mainRecords.Count} != 1");
-                    }
+                    //}
-                    if (patchRecords != null && patchRecords.Count != 1)
+                    //if (patchRecords != null && patchRecords.Count != 1)
-                    {
+                    //{
-                        throw new Exception($"配置表 {table.FullName} 是单值表 mode=one,但分支文件数据个数:{patchRecords.Count} != 1");
+                    //    throw new Exception($"配置表 {table.FullName} 是单值表 mode=one,但分支文件数据个数:{patchRecords.Count} != 1");
-                    }
+                    //}
                     if (patchRecords != null)
                     {
-                        mainRecords[0] = patchRecords[0];
+                        mainRecords = patchRecords;
                     }
                     break;
                 }

src/Luban.Job.Cfg/Source/l10n/TextTable.cs

@@ -66,7 +66,7 @@ namespace Luban.Job.Cfg.l10n
 
         public void LoadFromFile(string fileName, byte[] bytes)
         {
-            var records = DataLoaderUtil.LoadCfgRecords(_textRowType, fileName, null, bytes, true, false);
+            var records = DataLoaderUtil.LoadCfgRecords(_textRowType, fileName, null, bytes, true);
             foreach (var r in records)
             {
                 //s_logger.Info("== read text:{}", r.Data);