xiang.li
/
luban
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

【删除】移除 cfg --export_test_data 选项，如果想不导出测试数据，使用 --export_exclude_tags test 

【特性】新增 cfg --export_exclude_tags选项，支持过滤指定tags列表的数据
main
walon 2021-10-11 15:43:51 +08:00
parent dd5bae123f
commit 68676cf8ab
19 changed files with 67 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@ -221,9 +221,9 @@ array与list类型都能表示列表，它们区别在于array生成的代码为
- 表级别分组
- 字段级别分组(任意bean字段粒度，而不仅限于顶层字段)
### 数据标签
### 数据标签过滤
开发期有时候希望临时注释掉一些记录，另外开发期经常会制作一些仅供开发使用的配置，比如测试道具，比如自动化测试使用的配置，在正式上线时不导出这些数据。 luban支持数据标签及数据注释及测试数据过滤导出，导出正式配置时不需要手动将测试数据删掉。
开发期经常会制作一些仅供开发使用的配置，比如测试道具，比如自动化测试使用的配置，希望在正式发布时不导出这些数据。
![pipeline](docs/images/examples/c_11.jpg)

BIN
docs/images/examples/c_11.jpg
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 32 KiB

After

Width:  |  Height:  |  Size: 37 KiB

2
src/Luban.Job.Cfg/Source/DataSources/AbstractDataSource.cs
View File

@ -15,6 +15,6 @@ namespace Luban.Job.Cfg.DataSources
public abstract List<Record> ReadMulti(TBean type);
public abstract void Load(string rawUrl, string sheetName, Stream stream, bool exportDebugData);
public abstract void Load(string rawUrl, string sheetName, Stream stream);
}
}

2
src/Luban.Job.Cfg/Source/DataSources/Binary/BinaryDataSource.cs
View File

@ -8,7 +8,7 @@ namespace Luban.Job.Cfg.DataSources.Binary
{
class BinaryDataSource : AbstractDataSource
{
public override void Load(string rawUrl, string sheetName, Stream stream, bool exportDebugData)
public override void Load(string rawUrl, string sheetName, Stream stream)
{
throw new NotImplementedException();
}

4
src/Luban.Job.Cfg/Source/DataSources/DataSourceFactory.cs
View File

@ -18,7 +18,7 @@ namespace Luban.Job.Cfg.DataSources
".bin",
};
public static AbstractDataSource Create(string url, string sheetName, Stream stream, bool exportTestData)
public static AbstractDataSource Create(string url, string sheetName, Stream stream)
{
try
{
@ -36,7 +36,7 @@ namespace Luban.Job.Cfg.DataSources
case "yml": source = new Yaml.YamlDataSource(); break;
default: throw new Exception($"不支持的文件类型:{url}");
}
source.Load(url, sheetName, stream, exportTestData);
source.Load(url, sheetName, stream);
return source;
}
catch (DataCreateException)

2
src/Luban.Job.Cfg/Source/DataSources/Excel/ExcelDataSource.cs
View File

@ -33,7 +33,7 @@ namespace Luban.Job.Cfg.DataSources.Excel
}
}
public override void Load(string rawUrl, string sheetName, Stream stream, bool exportTestData)
public override void Load(string rawUrl, string sheetName, Stream stream)
{
s_logger.Trace("{filename} {sheet}", rawUrl, sheetName);
RawUrl = rawUrl;

2
src/Luban.Job.Cfg/Source/DataSources/Json/JsonDataSource.cs
View File

@ -14,7 +14,7 @@ namespace Luban.Job.Cfg.DataSources.Json
{
JsonElement _data;
public override void Load(string rawUrl, string sheetName, Stream stream, bool exportDebugData)
public override void Load(string rawUrl, string sheetName, Stream stream)
{
RawUrl = rawUrl;
this._data = JsonDocument.Parse(stream).RootElement;

2
src/Luban.Job.Cfg/Source/DataSources/Lua/LuaDataSource.cs
View File

@ -17,7 +17,7 @@ namespace Luban.Job.Cfg.DataSources.Lua
private LuaGlobal _env;
private LuaTable _dataTable;
public override void Load(string rawUrl, string sheetName, Stream stream, bool exportDebugData)
public override void Load(string rawUrl, string sheetName, Stream stream)
{
RawUrl = rawUrl;
_env = LuaManager.CreateEnvironment();

2
src/Luban.Job.Cfg/Source/DataSources/Xml/XmlDataSource.cs
View File

@ -14,7 +14,7 @@ namespace Luban.Job.Cfg.DataSources.Xml
{
private XElement _doc;
public override void Load(string rawUrl, string sheetName, Stream stream, bool exportDebugData)
public override void Load(string rawUrl, string sheetName, Stream stream)
{
RawUrl = rawUrl;
_doc = XElement.Load(stream);

2
src/Luban.Job.Cfg/Source/DataSources/Yaml/YamlDataSource.cs
View File

@ -13,7 +13,7 @@ namespace Luban.Job.Cfg.DataSources.Yaml
class YamlDataSource : AbstractDataSource
{
private YamlMappingNode _root;
public override void Load(string rawUrl, string sheetName, Stream stream, bool exportDebugData)
public override void Load(string rawUrl, string sheetName, Stream stream)
{
var ys = new YamlStream();
ys.Load(new StreamReader(stream));

9
src/Luban.Job.Cfg/Source/Datas/Record.cs
View File

@ -13,7 +13,14 @@ namespace Luban.Job.Cfg.Datas
public int Index { get; set; }
public bool IsTest => Tags != null && DataUtil.IsTestTag(Tags);
public bool IsNotFiltered(List<string> excludeTags)
{
if (Tags == null)
{
return true;
}
return Tags.TrueForAll(t => !excludeTags.Contains(t));
}
public Record(DBean data, string source, List<string> tags)
{

12
src/Luban.Job.Cfg/Source/Defs/CfgDefLoader.cs
View File

@ -478,7 +478,7 @@ namespace Luban.Job.Cfg.Defs
}
})
{
AssemblyBase = new DefAssembly("", null, true, Agent),
AssemblyBase = new DefAssembly("", null, new List<string>(), Agent),
};
defTableRecordType.PreCompile();
defTableRecordType.Compile();
@ -489,7 +489,7 @@ namespace Luban.Job.Cfg.Defs
{
var source = new ExcelDataSource();
var bytes = await this.Agent.GetFromCacheOrReadAllBytesAsync(file.ActualFile, file.MD5);
var records = DataLoaderUtil.LoadCfgRecords(tableRecordType, file.OriginFile, null, bytes, true, false);
var records = DataLoaderUtil.LoadCfgRecords(tableRecordType, file.OriginFile, null, bytes, true);
foreach (var r in records)
{
DBean data = r.Data;
@ -544,7 +544,7 @@ namespace Luban.Job.Cfg.Defs
}
})
{
AssemblyBase = new DefAssembly("", null, true, Agent),
AssemblyBase = new DefAssembly("", null, new List<string>(), Agent),
};
defTableRecordType.PreCompile();
defTableRecordType.Compile();
@ -555,7 +555,7 @@ namespace Luban.Job.Cfg.Defs
{
var source = new ExcelDataSource();
var bytes = await this.Agent.GetFromCacheOrReadAllBytesAsync(file.ActualFile, file.MD5);
var records = DataLoaderUtil.LoadCfgRecords(tableRecordType, file.OriginFile, null, bytes, true, false);
var records = DataLoaderUtil.LoadCfgRecords(tableRecordType, file.OriginFile, null, bytes, true);
PEnum curEnum = null;
foreach (var r in records)
@ -599,7 +599,7 @@ namespace Luban.Job.Cfg.Defs
var inputFileInfos = await DataLoaderUtil.CollectInputFilesAsync(this.Agent, this._importExcelBeanFiles, dataDir);
var ass = new DefAssembly("", null, true, Agent);
var ass = new DefAssembly("", null, new List<string>(), Agent);
var defBeanFieldType = new DefBean(new CfgBean()
{
@ -669,7 +669,7 @@ namespace Luban.Job.Cfg.Defs
{
var source = new ExcelDataSource();
var bytes = await this.Agent.GetFromCacheOrReadAllBytesAsync(file.ActualFile, file.MD5);
var records = DataLoaderUtil.LoadCfgRecords(tableRecordType, file.OriginFile, null, bytes, true, false);
var records = DataLoaderUtil.LoadCfgRecords(tableRecordType, file.OriginFile, null, bytes, true);
foreach (var r in records)
{

33
src/Luban.Job.Cfg/Source/Defs/DefAssembly.cs
View File

@ -19,19 +19,6 @@ namespace Luban.Job.Cfg.Defs
public List<Record> FinalRecords { get; set; }
private List<Record> _notTestRecords;
public List<Record> NotTestRecords
{
get
{
if (_notTestRecords == null)
{
_notTestRecords = FinalRecords.Where(r => !r.IsTest).ToList();
}
return _notTestRecords;
}
}
public Dictionary<DType, Record> FinalRecordMap { get; set; }
public TableDataInfo(List<Record> mainRecords, List<Record> patchRecords)
@ -50,17 +37,17 @@ namespace Luban.Job.Cfg.Defs
public Service CfgTargetService { get; private set; }
private readonly string _patchName;
private readonly bool _exportTestData;
private readonly List<string> _excludeTags;
public Patch TargetPatch { get; private set; }
public TimeZoneInfo TimeZone { get; }
public DefAssembly(string patchName, TimeZoneInfo timezone, bool exportTestData, RemoteAgent agent)
public DefAssembly(string patchName, TimeZoneInfo timezone, List<string> excludeTags, RemoteAgent agent)
{
this._patchName = patchName;
this.TimeZone = timezone;
this._exportTestData = exportTestData;
this._excludeTags = excludeTags;
this.Agent = agent;
}
@ -126,7 +113,19 @@ namespace Luban.Job.Cfg.Defs
public List<Record> GetTableExportDataList(DefTable table)
{
var tableDataInfo = _recordsByTables[table.FullName];
return _exportTestData ? tableDataInfo.FinalRecords : tableDataInfo.NotTestRecords;
if (_excludeTags.Count == 0)
{
return tableDataInfo.FinalRecords;
}
else
{
var finalRecords = tableDataInfo.FinalRecords.Where(r => r.IsNotFiltered(_excludeTags)).ToList();
if (table.IsOneValueTable && finalRecords.Count != 1)
{
throw new Exception($"配置表 {table.FullName} 是单值表 mode=one,但数据个数:{finalRecords.Count} != 1");
}
return finalRecords;
}
}
public TableDataInfo GetTableDataInfo(DefTable table)

7
src/Luban.Job.Cfg/Source/GenArgs.cs
View File

@ -32,11 +32,8 @@ namespace Luban.Job.Cfg
[Option('s', "service", Required = true, HelpText = "service")]
public string Service { get; set; }
[Option("export_test_data", Required = false, HelpText = "export test data")]
public bool ExportTestData { get; set; } = false;
[Option("export_tag_filter", Required = false, HelpText = "export tag filter")]
public string ExportTagFilter { get; set; } = "";
[Option("export_exclude_tags", Required = false, HelpText = "export exclude tags. default export all tags")]
public string ExportExcludeTags { get; set; } = "";
[Option('t', "l10n_timezone", Required = false, HelpText = "timezone")]
public string TimeZone { get; set; }

5
src/Luban.Job.Cfg/Source/JobController.cs
View File

@ -140,7 +140,8 @@ namespace Luban.Job.Cfg
TimeZoneInfo timeZoneInfo = string.IsNullOrEmpty(args.TimeZone) ? null : TimeZoneInfo.FindSystemTimeZoneById(args.TimeZone);
var ass = new DefAssembly(args.PatchName, timeZoneInfo, args.ExportTestData, agent)
var excludeTags = args.ExportExcludeTags.Split(',').Select(t => t.Trim().ToLowerInvariant()).Where(t => !string.IsNullOrEmpty(t)).ToList();
var ass = new DefAssembly(args.PatchName, timeZoneInfo, excludeTags, agent)
{
UseUnityVectors = args.UseUnityVectors
};
@ -166,7 +167,7 @@ namespace Luban.Job.Cfg
hasLoadCfgData = true;
var timer = new ProfileTimer();
timer.StartPhase("load config data");
await DataLoaderUtil.LoadCfgDataAsync(agent, ass, args.InputDataDir, args.PatchName, args.PatchInputDataDir, args.ExportTestData);
await DataLoaderUtil.LoadCfgDataAsync(agent, ass, args.InputDataDir, args.PatchName, args.PatchInputDataDir);
timer.EndPhaseAndLog();
if (needL10NTextConvert)

19
src/Luban.Job.Cfg/Source/Utils/DataLoaderUtil.cs
View File

@ -71,7 +71,7 @@ namespace Luban.Job.Cfg.Utils
// return CollectInputFilesAsync(agent, table.InputFiles, dataDir)
//}
public static async Task GenerateLoadRecordFromFileTasksAsync(RemoteAgent agent, DefTable table, string dataDir, List<string> inputFiles2, bool exportTestData, List<Task<List<Record>>> tasks)
public static async Task GenerateLoadRecordFromFileTasksAsync(RemoteAgent agent, DefTable table, string dataDir, List<string> inputFiles2, List<Task<List<Record>>> tasks)
{
var inputFileInfos = await CollectInputFilesAsync(agent, inputFiles2, dataDir);
@ -93,8 +93,7 @@ namespace Luban.Job.Cfg.Utils
file.OriginFile,
file.SheetName,
await agent.GetFromCacheOrReadAllBytesAsync(file.ActualFile, file.MD5),
RenderFileUtil.IsExcelFile(file.ActualFile),
exportTestData);
RenderFileUtil.IsExcelFile(file.ActualFile));
FileRecordCacheManager.Ins.AddCacheLoadedRecords(table, file.MD5, file.SheetName, res);
@ -103,10 +102,10 @@ namespace Luban.Job.Cfg.Utils
}
}
public static async Task LoadTableAsync(RemoteAgent agent, DefTable table, string dataDir, string patchName, string patchDataDir, bool exportTestData)
public static async Task LoadTableAsync(RemoteAgent agent, DefTable table, string dataDir, string patchName, string patchDataDir)
{
var mainLoadTasks = new List<Task<List<Record>>>();
var mainGenerateTask = GenerateLoadRecordFromFileTasksAsync(agent, table, dataDir, table.InputFiles, exportTestData, mainLoadTasks);
var mainGenerateTask = GenerateLoadRecordFromFileTasksAsync(agent, table, dataDir, table.InputFiles, mainLoadTasks);
var patchLoadTasks = new List<Task<List<Record>>>();
@ -116,7 +115,7 @@ namespace Luban.Job.Cfg.Utils
var patchInputFiles = table.GetPatchInputFiles(patchName);
if (patchInputFiles != null)
{
patchGenerateTask = GenerateLoadRecordFromFileTasksAsync(agent, table, patchDataDir, patchInputFiles, exportTestData, patchLoadTasks);
patchGenerateTask = GenerateLoadRecordFromFileTasksAsync(agent, table, patchDataDir, patchInputFiles, patchLoadTasks);
}
}
@ -146,7 +145,7 @@ namespace Luban.Job.Cfg.Utils
s_logger.Trace("table:{name} record num:{num}", table.FullName, mainRecords.Count);
}
public static async Task LoadCfgDataAsync(RemoteAgent agent, DefAssembly ass, string dataDir, string patchName, string patchDataDir, bool exportTestData)
public static async Task LoadCfgDataAsync(RemoteAgent agent, DefAssembly ass, string dataDir, string patchName, string patchDataDir)
{
var ctx = agent;
List<DefTable> exportTables = ass.Types.Values.Where(t => t is DefTable ct && ct.NeedExport).Select(t => (DefTable)t).ToList();
@ -160,7 +159,7 @@ namespace Luban.Job.Cfg.Utils
genDataTasks.Add(Task.Run(async () =>
{
long beginTime = TimeUtil.NowMillis;
await LoadTableAsync(agent, table, dataDir, patchName, patchDataDir, exportTestData);
await LoadTableAsync(agent, table, dataDir, patchName, patchDataDir);
long endTime = TimeUtil.NowMillis;
if (endTime - beginTime > 100)
{
@ -171,10 +170,10 @@ namespace Luban.Job.Cfg.Utils
await Task.WhenAll(genDataTasks.ToArray());
}
public static List<Record> LoadCfgRecords(TBean recordType, string originFile, string sheetName, byte[] content, bool multiRecord, bool exportTestData)
public static List<Record> LoadCfgRecords(TBean recordType, string originFile, string sheetName, byte[] content, bool multiRecord)
{
// (md5,sheet,multiRecord,exportTestData) -> (valuetype, List<(datas)>)
var dataSource = DataSourceFactory.Create(originFile, sheetName, new MemoryStream(content), exportTestData);
var dataSource = DataSourceFactory.Create(originFile, sheetName, new MemoryStream(content));
try
{
if (multiRecord)

19
src/Luban.Job.Cfg/Source/Utils/DataUtil.cs
View File

@ -117,33 +117,18 @@ namespace Luban.Job.Cfg.Utils
{
return !string.IsNullOrEmpty(tagName) &&
(
tagName.Equals("false", System.StringComparison.OrdinalIgnoreCase)
|| tagName.Equals("no", System.StringComparison.OrdinalIgnoreCase)
tagName.Equals("no", System.StringComparison.OrdinalIgnoreCase)
|| tagName.Equals("##", System.StringComparison.Ordinal)
//|| tagName.Equals("·ñ", System.StringComparison.Ordinal)
);
}
private static bool IsTestTag(string tagName)
{
return !string.IsNullOrEmpty(tagName) &&
(tagName.Equals("test", System.StringComparison.OrdinalIgnoreCase)
|| tagName.Equals("²âÊÔ", System.StringComparison.Ordinal)
);
}
public static bool IsTestTag(List<string> tagNames)
{
return tagNames.Any(IsTestTag);
}
public static List<string> ParseTags(string rawTagStr)
{
if (string.IsNullOrWhiteSpace(rawTagStr))
{
return null;
}
var tags = new List<string>(rawTagStr.Split(',').Select(t => t.Trim()).Where(t => !string.IsNullOrEmpty(t)));
var tags = new List<string>(rawTagStr.Split(',').Select(t => t.Trim().ToLower()).Where(t => !string.IsNullOrEmpty(t)));
return tags.Count > 0 ? tags : null;
}

18
src/Luban.Job.Cfg/Source/ValidatorContext.cs
View File

@ -201,17 +201,17 @@ namespace Luban.Job.Cfg
{
case ETableMode.ONE:
{
if (mainRecords.Count != 1)
{
throw new Exception($"配置表 {table.FullName} 是单值表 mode=one,但主文件数据个数:{mainRecords.Count} != 1");
}
if (patchRecords != null && patchRecords.Count != 1)
{
throw new Exception($"配置表 {table.FullName} 是单值表 mode=one,但分支文件数据个数:{patchRecords.Count} != 1");
}
//if (mainRecords.Count != 1)
//{
// throw new Exception($"配置表 {table.FullName} 是单值表 mode=one,但主文件数据个数:{mainRecords.Count} != 1");
//}
//if (patchRecords != null && patchRecords.Count != 1)
//{
// throw new Exception($"配置表 {table.FullName} 是单值表 mode=one,但分支文件数据个数:{patchRecords.Count} != 1");
//}
if (patchRecords != null)
{
mainRecords[0] = patchRecords[0];
mainRecords = patchRecords;
}
break;
}

2
src/Luban.Job.Cfg/Source/l10n/TextTable.cs
View File

@ -66,7 +66,7 @@ namespace Luban.Job.Cfg.l10n
public void LoadFromFile(string fileName, byte[] bytes)
{
var records = DataLoaderUtil.LoadCfgRecords(_textRowType, fileName, null, bytes, true, false);
var records = DataLoaderUtil.LoadCfgRecords(_textRowType, fileName, null, bytes, true);
foreach (var r in records)
{
//s_logger.Info("== read text:{}", r.Data);