diff --git a/.gitignore b/.gitignore index 0336fe7..2dd2119 100644 --- a/.gitignore +++ b/.gitignore @@ -268,3 +268,4 @@ __pycache__/ /config/output_code /config/output_data /config/output_lua +/config/output_lua_without_test diff --git a/config/Datas/test/table_one.xlsx b/config/Datas/test/table_one.xlsx index 7b5a98e..8b9c6d8 100644 Binary files a/config/Datas/test/table_one.xlsx and b/config/Datas/test/table_one.xlsx differ diff --git a/config/Datas/test/tag_datas/j_not_tag.json b/config/Datas/test/tag_datas/j_not_tag.json new file mode 100644 index 0000000..24bff63 --- /dev/null +++ b/config/Datas/test/tag_datas/j_not_tag.json @@ -0,0 +1,4 @@ +{ + "id":2001, + "value": "导出" +} \ No newline at end of file diff --git a/config/Datas/test/tag_datas/j_tag_any.json b/config/Datas/test/tag_datas/j_tag_any.json new file mode 100644 index 0000000..7a3c6e9 --- /dev/null +++ b/config/Datas/test/tag_datas/j_tag_any.json @@ -0,0 +1,5 @@ +{ + "__tag__":"any", + "id":2004, + "value": "导出" +} \ No newline at end of file diff --git a/config/Datas/test/tag_datas/j_tag_ignore.json b/config/Datas/test/tag_datas/j_tag_ignore.json new file mode 100644 index 0000000..a7415db --- /dev/null +++ b/config/Datas/test/tag_datas/j_tag_ignore.json @@ -0,0 +1,5 @@ +{ + "__tag__":"no", + "id":2002, + "value": "忽略" +} \ No newline at end of file diff --git a/config/Datas/test/tag_datas/j_tag_test.json b/config/Datas/test/tag_datas/j_tag_test.json new file mode 100644 index 0000000..bfa461e --- /dev/null +++ b/config/Datas/test/tag_datas/j_tag_test.json @@ -0,0 +1,5 @@ +{ + "__tag__":"测试", + "id":2003, + "value": "导出" +} \ No newline at end of file diff --git a/config/Datas/test/tag_datas/not_tag.lua b/config/Datas/test/tag_datas/not_tag.lua new file mode 100644 index 0000000..76b5d3c --- /dev/null +++ b/config/Datas/test/tag_datas/not_tag.lua @@ -0,0 +1,5 @@ + +return { + id = 100, + value = "导出", +} \ No newline at end of file diff --git a/config/Datas/test/tag_datas/tag.xlsx b/config/Datas/test/tag_datas/tag.xlsx new file mode 100644 index 0000000..9a15a80 Binary files /dev/null and b/config/Datas/test/tag_datas/tag.xlsx differ diff --git a/config/Datas/test/tag_datas/tag_any.lua b/config/Datas/test/tag_datas/tag_any.lua new file mode 100644 index 0000000..1ebb530 --- /dev/null +++ b/config/Datas/test/tag_datas/tag_any.lua @@ -0,0 +1,5 @@ +return { + __tag__ = "随便", + id = 104, + value="导出", +} \ No newline at end of file diff --git a/config/Datas/test/tag_datas/tag_not_export.lua b/config/Datas/test/tag_datas/tag_not_export.lua new file mode 100644 index 0000000..6e0e3ae --- /dev/null +++ b/config/Datas/test/tag_datas/tag_not_export.lua @@ -0,0 +1,5 @@ +return { + __tag__ = "no", + id = 101, + value="不导出", +} \ No newline at end of file diff --git a/config/Datas/test/tag_datas/tag_test.lua b/config/Datas/test/tag_datas/tag_test.lua new file mode 100644 index 0000000..b853e1c --- /dev/null +++ b/config/Datas/test/tag_datas/tag_test.lua @@ -0,0 +1,5 @@ +return { + __tag__ = "test", + id = 102, + value="测试", +} \ No newline at end of file diff --git a/config/Datas/test/tag_datas/x_not_tag.xml b/config/Datas/test/tag_datas/x_not_tag.xml new file mode 100644 index 0000000..9cb7ac1 --- /dev/null +++ b/config/Datas/test/tag_datas/x_not_tag.xml @@ -0,0 +1,4 @@ + + 3001 + export + \ No newline at end of file diff --git a/config/Datas/test/tag_datas/x_tag_any.xml b/config/Datas/test/tag_datas/x_tag_any.xml new file mode 100644 index 0000000..4a1d24c --- /dev/null +++ b/config/Datas/test/tag_datas/x_tag_any.xml @@ -0,0 +1,5 @@ + + <__tag__>any + 3004 + 其他 + \ No newline at end of file diff --git a/config/Datas/test/tag_datas/x_tag_no.xml b/config/Datas/test/tag_datas/x_tag_no.xml new file mode 100644 index 0000000..5251b04 --- /dev/null +++ b/config/Datas/test/tag_datas/x_tag_no.xml @@ -0,0 +1,5 @@ + + <__tag__>no + 3002 + 不导出 + \ No newline at end of file diff --git a/config/Datas/test/tag_datas/x_tag_test.xml b/config/Datas/test/tag_datas/x_tag_test.xml new file mode 100644 index 0000000..9a49e72 --- /dev/null +++ b/config/Datas/test/tag_datas/x_tag_test.xml @@ -0,0 +1,5 @@ + + <__tag__>test + 3003 + 测试 + \ No newline at end of file diff --git a/config/Defines/test.xml b/config/Defines/test.xml index 44a65bc..e77324e 100644 --- a/config/Defines/test.xml +++ b/config/Defines/test.xml @@ -196,4 +196,10 @@ + + + + + + \ No newline at end of file diff --git a/config/生成lua数据_不包含测试数据.bat b/config/生成lua数据_不包含测试数据.bat new file mode 100644 index 0000000..ee10521 --- /dev/null +++ b/config/生成lua数据_不包含测试数据.bat @@ -0,0 +1,11 @@ +..\src\Luban.Client\bin\Debug\net5.0\Luban.Client.exe ^ + -h %LUBAN_SERVER_IP% ^ + -j cfg ^ + -- ^ + -d Defines/__root__.xml ^ + --input_data_dir Datas ^ + --output_data_dir output_lua_without_test ^ + -s client ^ + --gen_types data_lua + +pause \ No newline at end of file diff --git a/config/生成lua数据.bat b/config/生成lua数据_包含测试数据.bat similarity index 100% rename from config/生成lua数据.bat rename to config/生成lua数据_包含测试数据.bat diff --git a/src/Luban.Job.Cfg/Source/Cache/FileRecordCacheManager.cs b/src/Luban.Job.Cfg/Source/Cache/FileRecordCacheManager.cs index 85d25d6..bfe2314 100644 --- a/src/Luban.Job.Cfg/Source/Cache/FileRecordCacheManager.cs +++ b/src/Luban.Job.Cfg/Source/Cache/FileRecordCacheManager.cs @@ -35,15 +35,15 @@ namespace Luban.Job.Cfg.Cache } } - private readonly ConcurrentDictionary<(string, string, string, bool), FileRecordCache> _caches = new ConcurrentDictionary<(string, string, string, bool), FileRecordCache>(); + private readonly ConcurrentDictionary<(string MD5, string SheetName), FileRecordCache> _caches = new(); private readonly object _shrinkLocker = new object(); - public bool TryGetCacheLoadedRecords(DefTable table, string md5, string originFile, string sheetName, bool exportTestData, out List cacheRecords) + public bool TryGetCacheLoadedRecords(DefTable table, string md5, string originFile, string sheetName, out List cacheRecords) { // TODO text localization check cacheRecords = null; - if (!_caches.TryGetValue((table.Assembly.TimeZone.Id, md5, sheetName, exportTestData), out var r)) + if (!_caches.TryGetValue((md5, sheetName), out var r)) { return false; } @@ -60,11 +60,11 @@ namespace Luban.Job.Cfg.Cache } } - public void AddCacheLoadedRecords(DefTable table, string md5, string sheetName, bool exportTestData, List cacheRecords) + public void AddCacheLoadedRecords(DefTable table, string md5, string sheetName, List cacheRecords) { lock (_shrinkLocker) { - _caches[(table.Assembly.TimeZone.Id, md5, sheetName, exportTestData)] = new FileRecordCache(table, cacheRecords); + _caches[(md5, sheetName)] = new FileRecordCache(table, cacheRecords); if (_caches.Count > CACHE_FILE_HIGH_WATER_MARK) { s_logger.Info("ShrinkCaches. cache count > high CACHE_FILE_HIGH_WATER_MARK:{}", CACHE_FILE_HIGH_WATER_MARK); diff --git a/src/Luban.Job.Cfg/Source/DataSources/AbstractDataSource.cs b/src/Luban.Job.Cfg/Source/DataSources/AbstractDataSource.cs index f6d53a0..132beb9 100644 --- a/src/Luban.Job.Cfg/Source/DataSources/AbstractDataSource.cs +++ b/src/Luban.Job.Cfg/Source/DataSources/AbstractDataSource.cs @@ -7,9 +7,32 @@ namespace Luban.Job.Cfg.DataSources { abstract class AbstractDataSource { - public abstract DType ReadOne(TBean type); + public const string TAG_KEY = "__tag__"; - public abstract List ReadMulti(TBean type); + public static bool IsIgnoreTag(string tagName) + { + return !string.IsNullOrWhiteSpace(tagName) && + ( + tagName.Equals("false", System.StringComparison.OrdinalIgnoreCase) + || tagName.Equals("no", System.StringComparison.OrdinalIgnoreCase) + || tagName.Equals("##", System.StringComparison.Ordinal) + || tagName.Equals("", System.StringComparison.Ordinal) + ); + } + + public static bool IsTestTag(string tagName) + { + return !string.IsNullOrWhiteSpace(tagName) && + (tagName.Equals("test", System.StringComparison.OrdinalIgnoreCase) + || tagName.Equals("", System.StringComparison.Ordinal) + ); + } + + public string RawUrl { get; protected set; } + + public abstract Record ReadOne(TBean type); + + public abstract List ReadMulti(TBean type); public abstract void Load(string rawUrl, string sheetName, Stream stream, bool exportDebugData); } diff --git a/src/Luban.Job.Cfg/Source/DataSources/Binary/BinaryDataSource.cs b/src/Luban.Job.Cfg/Source/DataSources/Binary/BinaryDataSource.cs index 43ecb78..2e3407a 100644 --- a/src/Luban.Job.Cfg/Source/DataSources/Binary/BinaryDataSource.cs +++ b/src/Luban.Job.Cfg/Source/DataSources/Binary/BinaryDataSource.cs @@ -13,12 +13,12 @@ namespace Luban.Job.Cfg.DataSources.Binary throw new NotImplementedException(); } - public override List ReadMulti(TBean type) + public override List ReadMulti(TBean type) { throw new NotImplementedException(); } - public override DType ReadOne(TBean type) + public override Record ReadOne(TBean type) { throw new NotImplementedException(); } diff --git a/src/Luban.Job.Cfg/Source/DataSources/Excel/ExcelDataSource.cs b/src/Luban.Job.Cfg/Source/DataSources/Excel/ExcelDataSource.cs index 37303b2..20099fa 100644 --- a/src/Luban.Job.Cfg/Source/DataSources/Excel/ExcelDataSource.cs +++ b/src/Luban.Job.Cfg/Source/DataSources/Excel/ExcelDataSource.cs @@ -21,6 +21,7 @@ namespace Luban.Job.Cfg.DataSources.Excel public override void Load(string rawUrl, string sheetName, Stream stream, bool exportTestData) { s_logger.Trace("{filename} {sheet}", rawUrl, sheetName); + RawUrl = rawUrl; string ext = Path.GetExtension(rawUrl); using (var reader = ext != ".csv" ? ExcelReaderFactory.CreateReader(stream) : ExcelReaderFactory.CreateCsvReader(stream)) { @@ -30,7 +31,7 @@ namespace Luban.Job.Cfg.DataSources.Excel { try { - var sheet = ReadSheet(reader, exportTestData); + var sheet = ReadSheet(rawUrl, reader); if (sheet != null) { _sheets.Add(sheet); @@ -50,15 +51,15 @@ namespace Luban.Job.Cfg.DataSources.Excel } } - private Sheet ReadSheet(IExcelDataReader reader, bool exportTestData) + private Sheet ReadSheet(string url, IExcelDataReader reader) { - var sheet = new Sheet(reader.Name ?? "", exportTestData); + var sheet = new Sheet(url, reader.Name ?? ""); return sheet.Load(reader) ? sheet : null; } - public override List ReadMulti(TBean type) + public override List ReadMulti(TBean type) { - var datas = new List(); + var datas = new List(); foreach (var sheet in _sheets) { try @@ -73,7 +74,7 @@ namespace Luban.Job.Cfg.DataSources.Excel return datas; } - public override DType ReadOne(TBean type) + public override Record ReadOne(TBean type) { var datas = ReadMulti(type); switch (datas.Count) diff --git a/src/Luban.Job.Cfg/Source/DataSources/Excel/Sheet.cs b/src/Luban.Job.Cfg/Source/DataSources/Excel/Sheet.cs index 7c55073..c62bc25 100644 --- a/src/Luban.Job.Cfg/Source/DataSources/Excel/Sheet.cs +++ b/src/Luban.Job.Cfg/Source/DataSources/Excel/Sheet.cs @@ -16,14 +16,14 @@ namespace Luban.Job.Cfg.DataSources.Excel private int TitleRows { get; set; } = 3; // 默认有三行是标题行. 第一行是字段名，第二行是中文描述，第三行是注释 + public string RawUrl { get; } + public string Name { get; } private List> _rowColumns; private Title _rootTitle; - private bool ExportTestData { get; } - public class Title { public int FromIndex { get; set; } @@ -206,10 +206,10 @@ namespace Luban.Job.Cfg.DataSources.Excel } } - public Sheet(string name, bool exportTestData) + public Sheet(string rawUrl, string name) { + this.RawUrl = rawUrl; this.Name = name; - this.ExportTestData = exportTestData; } public bool Load(IExcelDataReader reader) @@ -292,35 +292,17 @@ namespace Luban.Job.Cfg.DataSources.Excel return true; } - private bool NotExport(List row) + private string GetRowTag(List row) { if (row.Count == 0) { - return true; + return null; } if (row[0].Value == null) { - return false; - } - - string exportFlag = row[0].Value.ToString().Trim().ToLower(); - switch (exportFlag) - { - case "false": - case "否": return true; - case "true": - case "是": return false; - case "test": - case "测试": - { - if (!ExportTestData) - { - s_logger.Debug("忽略测试数据. row:{row}", row); - } - return !ExportTestData; - } - default: throw new Exception($"不支持的excel 导出标记: {exportFlag}"); + return null; } + return row[0].Value.ToString().Trim(); } private void InitSubTitles(Title parentTitle, List> rows, CellRange[] mergeCells, int maxDepth, int depth, int fromColumn, int toColumn) @@ -519,15 +501,16 @@ namespace Luban.Job.Cfg.DataSources.Excel throw new Exception($"没有定义任何有效 列"); } _rootTitle.SortSubTitles(); - foreach (var title in _rootTitle.SubTitleList) - { - // s_logger.Info("============ sheet:{sheet} title:{title}", Name, title); - } + //foreach (var title in _rootTitle.SubTitleList) + //{ + // // s_logger.Info("============ sheet:{sheet} title:{title}", Name, title); + //} // 删除标题行 this._rowColumns.RemoveRange(0, Math.Min(TitleRows + titleRowNum - 1, this._rowColumns.Count)); - this._rowColumns.RemoveAll(row => NotExport(row)); + // 删除忽略的记录行 + this._rowColumns.RemoveAll(row => AbstractDataSource.IsIgnoreTag(GetRowTag(row))); } @@ -604,11 +587,11 @@ namespace Luban.Job.Cfg.DataSources.Excel - public List ReadMulti(TBean type, bool enableMultiRowRecord) + public List ReadMulti(TBean type, bool enableMultiRowRecord) { - var datas = new List(); + var datas = new List(); - for (DType data; (data = ReadOne(type, enableMultiRowRecord)) != null;) + for (Record data; (data = ReadOne(type, enableMultiRowRecord)) != null;) { datas.Add(data); } @@ -616,7 +599,7 @@ namespace Luban.Job.Cfg.DataSources.Excel } private int curReadIndex = 0; - public DType ReadOne(TBean type, bool enableMultiRowRecord) + public Record ReadOne(TBean type, bool enableMultiRowRecord) { if (!enableMultiRowRecord) { @@ -625,7 +608,9 @@ namespace Luban.Job.Cfg.DataSources.Excel { return null; } - return ExcelNamedRowDataCreator.Ins.ReadExcel(new NamedRow(_rootTitle, row), type); + bool isTest = AbstractDataSource.IsTestTag(GetRowTag(row)); + var data = (DBean)ExcelNamedRowDataCreator.Ins.ReadExcel(new NamedRow(_rootTitle, row), type); + return new Record(data, RawUrl, isTest); } else { @@ -634,7 +619,9 @@ namespace Luban.Job.Cfg.DataSources.Excel { return null; } - return ExcelNamedRowDataCreator.Ins.ReadExcel(new NamedRow(_rootTitle, rows), type); + bool isTest = AbstractDataSource.IsTestTag(GetRowTag(rows[0])); + var data = (DBean)ExcelNamedRowDataCreator.Ins.ReadExcel(new NamedRow(_rootTitle, rows), type); + return new Record(data, RawUrl, isTest); } } } diff --git a/src/Luban.Job.Cfg/Source/DataSources/Json/JsonDataSource.cs b/src/Luban.Job.Cfg/Source/DataSources/Json/JsonDataSource.cs index 5c83b33..f0ca351 100644 --- a/src/Luban.Job.Cfg/Source/DataSources/Json/JsonDataSource.cs +++ b/src/Luban.Job.Cfg/Source/DataSources/Json/JsonDataSource.cs @@ -15,17 +15,30 @@ namespace Luban.Job.Cfg.DataSources.Json public override void Load(string rawUrl, string sheetName, Stream stream, bool exportDebugData) { + RawUrl = rawUrl; this._data = JsonDocument.Parse(stream).RootElement; } - public override List ReadMulti(TBean type) + public override List ReadMulti(TBean type) { throw new NotImplementedException(); } - public override DType ReadOne(TBean type) + public override Record ReadOne(TBean type) { - return type.Apply(JsonDataCreator.Ins, _data, (DefAssembly)type.Bean.AssemblyBase); + bool isTest = false; + if (_data.TryGetProperty(TAG_KEY, out var tagEle)) + { + var tagName = tagEle.GetString(); + if (IsIgnoreTag(tagName)) + { + return null; + } + isTest = IsTestTag(tagName); + } + + var data = (DBean)type.Apply(JsonDataCreator.Ins, _data, (DefAssembly)type.Bean.AssemblyBase); + return new Record(data, RawUrl, isTest); } } } diff --git a/src/Luban.Job.Cfg/Source/DataSources/Lua/LuaDataSource.cs b/src/Luban.Job.Cfg/Source/DataSources/Lua/LuaDataSource.cs index e6a74c5..4b293a6 100644 --- a/src/Luban.Job.Cfg/Source/DataSources/Lua/LuaDataSource.cs +++ b/src/Luban.Job.Cfg/Source/DataSources/Lua/LuaDataSource.cs @@ -18,25 +18,42 @@ namespace Luban.Job.Cfg.DataSources.Lua public override void Load(string rawUrl, string sheetName, Stream stream, bool exportDebugData) { + RawUrl = rawUrl; _env = LuaManager.CreateEnvironment(); _dataTable = (LuaTable)_env.DoChunk(new StreamReader(stream, Encoding.UTF8), rawUrl)[0]; } - public override List ReadMulti(TBean type) + public override List ReadMulti(TBean type) { - var records = new List(); + var records = new List(); foreach (LuaTable t in _dataTable.Values.Values) { - records.Add(type.Apply(LuaDataCreator.Ins, t, (DefAssembly)type.Bean.AssemblyBase)); + Record r = ReadRecord(t, type); + if (r != null) + { + records.Add(r); + } } return records; } - public override DType ReadOne(TBean type) + public override Record ReadOne(TBean type) { - return type.Apply(LuaDataCreator.Ins, _dataTable, (DefAssembly)type.Bean.AssemblyBase); + return ReadRecord(_dataTable, type); + } + + protected Record ReadRecord(LuaTable table, TBean type) + { + string tagName = table.GetValue(TAG_KEY)?.ToString(); + if (IsIgnoreTag(tagName)) + { + return null; + } + var data = (DBean)type.Apply(LuaDataCreator.Ins, table, (DefAssembly)type.Bean.AssemblyBase); + var isTest = IsTestTag(tagName); + return new Record(data, RawUrl, isTest); } } } diff --git a/src/Luban.Job.Cfg/Source/DataSources/Xml/XmlDataSource.cs b/src/Luban.Job.Cfg/Source/DataSources/Xml/XmlDataSource.cs index d3736e2..0abf98c 100644 --- a/src/Luban.Job.Cfg/Source/DataSources/Xml/XmlDataSource.cs +++ b/src/Luban.Job.Cfg/Source/DataSources/Xml/XmlDataSource.cs @@ -12,19 +12,28 @@ namespace Luban.Job.Cfg.DataSources.Xml class XmlDataSource : AbstractDataSource { private XElement _doc; + public override void Load(string rawUrl, string sheetName, Stream stream, bool exportDebugData) { + RawUrl = rawUrl; _doc = XElement.Load(stream); } - public override List ReadMulti(TBean type) + public override List ReadMulti(TBean type) { throw new NotSupportedException(); } - public override DType ReadOne(TBean type) + public override Record ReadOne(TBean type) { - return type.Apply(XmlDataCreator.Ins, _doc, (DefAssembly)type.Bean.AssemblyBase); + string tagName = _doc.Element(TAG_KEY)?.Value; + if (IsIgnoreTag(tagName)) + { + return null; + } + var data = (DBean)type.Apply(XmlDataCreator.Ins, _doc, (DefAssembly)type.Bean.AssemblyBase); + bool isTest = IsTestTag(tagName); + return new Record(data, RawUrl, isTest); } } } diff --git a/src/Luban.Job.Cfg/Source/DataVisitors/BinaryExportor.cs b/src/Luban.Job.Cfg/Source/DataVisitors/BinaryExportor.cs index 671c865..4927196 100644 --- a/src/Luban.Job.Cfg/Source/DataVisitors/BinaryExportor.cs +++ b/src/Luban.Job.Cfg/Source/DataVisitors/BinaryExportor.cs @@ -183,7 +183,7 @@ namespace Luban.Job.Cfg.DataVisitors public void Accept(DDateTime type, DefAssembly ass, ByteBuf x) { - x.WriteInt(type.UnixTime); + x.WriteInt(type.GetUnixTime(ass.TimeZone)); } } } diff --git a/src/Luban.Job.Cfg/Source/DataVisitors/IsDefaultValue.cs b/src/Luban.Job.Cfg/Source/DataVisitors/IsDefaultValue.cs index c36d4bf..c138d4f 100644 --- a/src/Luban.Job.Cfg/Source/DataVisitors/IsDefaultValue.cs +++ b/src/Luban.Job.Cfg/Source/DataVisitors/IsDefaultValue.cs @@ -70,7 +70,7 @@ namespace Luban.Job.Cfg.DataVisitors public bool Accept(DBytes type) { - throw new NotImplementedException(); + throw new NotSupportedException(); } public bool Accept(DText type) @@ -120,7 +120,7 @@ namespace Luban.Job.Cfg.DataVisitors public bool Accept(DDateTime type) { - return type.UnixTime == 0; + throw new NotSupportedException(); } } } diff --git a/src/Luban.Job.Cfg/Source/DataVisitors/JsonExportor.cs b/src/Luban.Job.Cfg/Source/DataVisitors/JsonExportor.cs index a3f2ce2..cb58a3c 100644 --- a/src/Luban.Job.Cfg/Source/DataVisitors/JsonExportor.cs +++ b/src/Luban.Job.Cfg/Source/DataVisitors/JsonExportor.cs @@ -191,7 +191,7 @@ namespace Luban.Job.Cfg.DataVisitors public void Accept(DDateTime type, DefAssembly ass, Utf8JsonWriter x) { - x.WriteNumberValue(type.UnixTime); + x.WriteNumberValue(type.GetUnixTime(ass.TimeZone)); } } } diff --git a/src/Luban.Job.Cfg/Source/DataVisitors/LuaExportor.cs b/src/Luban.Job.Cfg/Source/DataVisitors/LuaExportor.cs index 1d25f4a..dedab22 100644 --- a/src/Luban.Job.Cfg/Source/DataVisitors/LuaExportor.cs +++ b/src/Luban.Job.Cfg/Source/DataVisitors/LuaExportor.cs @@ -247,7 +247,7 @@ namespace Luban.Job.Cfg.DataVisitors public void Accept(DDateTime type, DefAssembly ass, StringBuilder line) { - line.Append(type.UnixTime); + line.Append(type.GetUnixTime(ass.TimeZone)); } } } diff --git a/src/Luban.Job.Cfg/Source/Datas/DDateTime.cs b/src/Luban.Job.Cfg/Source/Datas/DDateTime.cs index a1c658c..30d931b 100644 --- a/src/Luban.Job.Cfg/Source/Datas/DDateTime.cs +++ b/src/Luban.Job.Cfg/Source/Datas/DDateTime.cs @@ -7,12 +7,28 @@ namespace Luban.Job.Cfg.Datas { public DateTime Time { get; } - public int UnixTime { get; } + //public int UnixTime { get; } + private readonly int _localTime; public DDateTime(DateTime time) { this.Time = time; - this.UnixTime = (int)new DateTimeOffset(time).ToUnixTimeSeconds(); + // time.Kind == DateTimeKind.Unspecified + // DateTimeOffsetLocal + this._localTime = (int)new DateTimeOffset(time).ToUnixTimeSeconds(); + } + + public int GetUnixTime(TimeZoneInfo asTimeZone) + { + if (asTimeZone == null || asTimeZone == TimeZoneInfo.Local) + { + return this._localTime; + } + else + { + var destDateTime = TimeZoneInfo.ConvertTime(Time, asTimeZone, TimeZoneInfo.Utc); + return (int)new DateTimeOffset(destDateTime).ToUnixTimeSeconds(); + } } public override void Apply(IDataActionVisitor visitor, T x) diff --git a/src/Luban.Job.Cfg/Source/Datas/Record.cs b/src/Luban.Job.Cfg/Source/Datas/Record.cs index cfe442a..5f5d6af 100644 --- a/src/Luban.Job.Cfg/Source/Datas/Record.cs +++ b/src/Luban.Job.Cfg/Source/Datas/Record.cs @@ -14,10 +14,13 @@ namespace Luban.Job.Cfg.Datas public int Index { get; set; } - public Record(DBean data, string source) + public bool IsTest { get; } + + public Record(DBean data, string source, bool isTest) { Data = data; Source = source; + IsTest = isTest; } } } diff --git a/src/Luban.Job.Cfg/Source/Defs/DefAssembly.cs b/src/Luban.Job.Cfg/Source/Defs/DefAssembly.cs index c943363..54234ae 100644 --- a/src/Luban.Job.Cfg/Source/Defs/DefAssembly.cs +++ b/src/Luban.Job.Cfg/Source/Defs/DefAssembly.cs @@ -19,6 +19,19 @@ namespace Luban.Job.Cfg.Defs public List FinalRecords { get; set; } + private List _notTestRecords; + public List NotTestRecords + { + get + { + if (_notTestRecords == null) + { + _notTestRecords = FinalRecords.Where(r => !r.IsTest).ToList(); + } + return _notTestRecords; + } + } + public Dictionary FinalRecordMap { get; set; } public TableDataInfo(List mainRecords, List branchRecords) @@ -34,13 +47,19 @@ namespace Luban.Job.Cfg.Defs public Service CfgTargetService { get; private set; } + private readonly string _branchName; + private readonly bool _exportTestData; + public Branch TargetBranch { get; private set; } public TimeZoneInfo TimeZone { get; } - public DefAssembly(TimeZoneInfo timezone) + public DefAssembly(string branchName, TimeZoneInfo timezone, bool exportTestData, RemoteAgent agent) { + this._branchName = branchName; this.TimeZone = timezone; + this._exportTestData = exportTestData; + this.Agent = agent; } public bool NeedExport(List groups) @@ -100,11 +119,17 @@ namespace Luban.Job.Cfg.Defs // _recordsByTables[table.FullName].FinalRecordMap = recordMap; //} - public List GetTableDataList(DefTable table) + public List GetTableAllDataList(DefTable table) { return _recordsByTables[table.FullName].FinalRecords; } + public List GetTableExportDataList(DefTable table) + { + var tableDataInfo = _recordsByTables[table.FullName]; + return _exportTestData ? tableDataInfo.FinalRecords : tableDataInfo.NotTestRecords; + } + public TableDataInfo GetTableDataInfo(DefTable table) { return _recordsByTables[table.FullName]; @@ -147,9 +172,8 @@ namespace Luban.Job.Cfg.Defs return refTypes.Values.ToList(); } - public void Load(string outputService, string branchName, Defines defines, RemoteAgent agent) + public void Load(string outputService, Defines defines) { - this.Agent = agent; SupportDatetimeType = true; TopModule = defines.TopModule; @@ -161,12 +185,12 @@ namespace Luban.Job.Cfg.Defs throw new ArgumentException($"service:{outputService} not exists"); } - if (!string.IsNullOrWhiteSpace(branchName)) + if (!string.IsNullOrWhiteSpace(_branchName)) { - TargetBranch = defines.Branches.Find(b => b.Name == branchName); + TargetBranch = defines.Branches.Find(b => b.Name == _branchName); if (TargetBranch == null) { - throw new Exception($"branch {branchName} not in valid branch set"); + throw new Exception($"branch {_branchName} not in valid branch set"); } } @@ -211,7 +235,7 @@ namespace Luban.Job.Cfg.Defs } catch (Exception) { - agent.Error("precompile type:{0} error", type.FullName); + this.Agent.Error("precompile type:{0} error", type.FullName); throw; } } @@ -225,7 +249,7 @@ namespace Luban.Job.Cfg.Defs } catch (Exception) { - agent.Error("compile type:{0} error", type.FullName); + this.Agent.Error("compile type:{0} error", type.FullName); s_logger.Error("compile type:{0} error", type.FullName); throw; } @@ -240,7 +264,7 @@ namespace Luban.Job.Cfg.Defs } catch (Exception) { - agent.Error("post compile type:{0} error", type.FullName); + this.Agent.Error("post compile type:{0} error", type.FullName); s_logger.Error("post compile type:{0} error", type.FullName); throw; } diff --git a/src/Luban.Job.Cfg/Source/JobController.cs b/src/Luban.Job.Cfg/Source/JobController.cs index cae212b..2fc980d 100644 --- a/src/Luban.Job.Cfg/Source/JobController.cs +++ b/src/Luban.Job.Cfg/Source/JobController.cs @@ -208,11 +208,11 @@ namespace Luban.Job.Cfg var rawDefines = loader.BuildDefines(); - TimeZoneInfo timeZoneInfo = string.IsNullOrEmpty(args.TimeZone) ? TimeZoneInfo.Local : TimeZoneInfo.FindSystemTimeZoneById(args.TimeZone); + TimeZoneInfo timeZoneInfo = string.IsNullOrEmpty(args.TimeZone) ? null : TimeZoneInfo.FindSystemTimeZoneById(args.TimeZone); - var ass = new DefAssembly(timeZoneInfo); + var ass = new DefAssembly(args.BranchName, timeZoneInfo, args.ExportTestData, agent); - ass.Load(args.Service, args.BranchName, rawDefines, agent); + ass.Load(args.Service, rawDefines); var targetService = ass.CfgTargetService; @@ -948,7 +948,7 @@ class Vector4: { tasks.Add(Task.Run(() => { - var content = DataExporterUtil.ToOutputData(c, ass.GetTableDataList(c), genType); + var content = DataExporterUtil.ToOutputData(c, ass.GetTableExportDataList(c), genType); var file = genType.EndsWith("json") ? c.JsonOutputDataFile : c.OutputDataFile; var md5 = FileUtil.CalcMD5(content); CacheManager.Ins.AddCache(file, md5, content); @@ -965,7 +965,7 @@ class Vector4: { allJsonTask.Add(Task.Run(() => { - return DataExporterUtil.ToOutputData(c, ass.GetTableDataList(c), "data_json"); + return DataExporterUtil.ToOutputData(c, ass.GetTableExportDataList(c), "data_json"); })); } await Task.WhenAll(allJsonTask); @@ -1010,7 +1010,7 @@ class Vector4: { tasks.Add(Task.Run(() => { - var content = DataExporterUtil.ToOutputData(c, ass.GetTableDataList(c), genType); + var content = DataExporterUtil.ToOutputData(c, ass.GetTableExportDataList(c), genType); var file = $"{c.Name}.lua"; var md5 = FileUtil.CalcMD5(content); CacheManager.Ins.AddCache(file, md5, content); @@ -1027,7 +1027,7 @@ class Vector4: { genDataTasks.Add(Task.Run(() => { - return DataExporterUtil.ExportResourceList(ass.GetTableDataList(c)); + return DataExporterUtil.ExportResourceList(ass.GetTableExportDataList(c)); })); } diff --git a/src/Luban.Job.Cfg/Source/TypeVisitors/ExcelDataCreator.cs b/src/Luban.Job.Cfg/Source/TypeVisitors/ExcelDataCreator.cs index 19a95b4..681cee2 100644 --- a/src/Luban.Job.Cfg/Source/TypeVisitors/ExcelDataCreator.cs +++ b/src/Luban.Job.Cfg/Source/TypeVisitors/ExcelDataCreator.cs @@ -436,7 +436,7 @@ namespace Luban.Job.Cfg.TypeVisitors { return null; } - return DataUtil.CreateDateTime(d.ToString(), ass.TimeZone); + return DataUtil.CreateDateTime(d.ToString()); } } } diff --git a/src/Luban.Job.Cfg/Source/TypeVisitors/JsonDataCreator.cs b/src/Luban.Job.Cfg/Source/TypeVisitors/JsonDataCreator.cs index c817711..ce6e9b4 100644 --- a/src/Luban.Job.Cfg/Source/TypeVisitors/JsonDataCreator.cs +++ b/src/Luban.Job.Cfg/Source/TypeVisitors/JsonDataCreator.cs @@ -197,7 +197,7 @@ namespace Luban.Job.Cfg.TypeVisitors public DType Accept(TDateTime type, JsonElement x, DefAssembly ass) { - return DataUtil.CreateDateTime(x.GetString(), ass.TimeZone); + return DataUtil.CreateDateTime(x.GetString()); } public DType Accept(TVector2 type, JsonElement x, DefAssembly ass) diff --git a/src/Luban.Job.Cfg/Source/TypeVisitors/LuaDataCreator.cs b/src/Luban.Job.Cfg/Source/TypeVisitors/LuaDataCreator.cs index c5d65ac..73cd942 100644 --- a/src/Luban.Job.Cfg/Source/TypeVisitors/LuaDataCreator.cs +++ b/src/Luban.Job.Cfg/Source/TypeVisitors/LuaDataCreator.cs @@ -257,7 +257,7 @@ namespace Luban.Job.Cfg.TypeVisitors public DType Accept(TDateTime type, object x, DefAssembly ass) { - return DataUtil.CreateDateTime(x.ToString(), ass.TimeZone); + return DataUtil.CreateDateTime(x.ToString()); } } } diff --git a/src/Luban.Job.Cfg/Source/TypeVisitors/XmlDataCreator.cs b/src/Luban.Job.Cfg/Source/TypeVisitors/XmlDataCreator.cs index 3f287c0..706a6f9 100644 --- a/src/Luban.Job.Cfg/Source/TypeVisitors/XmlDataCreator.cs +++ b/src/Luban.Job.Cfg/Source/TypeVisitors/XmlDataCreator.cs @@ -193,7 +193,7 @@ namespace Luban.Job.Cfg.TypeVisitors public DType Accept(TDateTime type, XElement x, DefAssembly ass) { - return DataUtil.CreateDateTime(x.Value, ass.TimeZone); + return DataUtil.CreateDateTime(x.Value); } } } diff --git a/src/Luban.Job.Cfg/Source/Utils/DataLoaderUtil.cs b/src/Luban.Job.Cfg/Source/Utils/DataLoaderUtil.cs index 14b1fff..dae266c 100644 --- a/src/Luban.Job.Cfg/Source/Utils/DataLoaderUtil.cs +++ b/src/Luban.Job.Cfg/Source/Utils/DataLoaderUtil.cs @@ -85,7 +85,7 @@ namespace Luban.Job.Cfg.Utils tasks.Add(Task.Run(async () => { - if (FileRecordCacheManager.Ins.TryGetCacheLoadedRecords(table, file.MD5, actualFile, file.SheetName, exportTestData, out var cacheRecords)) + if (FileRecordCacheManager.Ins.TryGetCacheLoadedRecords(table, file.MD5, actualFile, file.SheetName, out var cacheRecords)) { return cacheRecords; } @@ -96,7 +96,7 @@ namespace Luban.Job.Cfg.Utils RenderFileUtil.IsExcelFile(file.ActualFile), exportTestData); - FileRecordCacheManager.Ins.AddCacheLoadedRecords(table, file.MD5, file.SheetName, exportTestData, res); + FileRecordCacheManager.Ins.AddCacheLoadedRecords(table, file.MD5, file.SheetName, res); return res; })); @@ -176,21 +176,15 @@ namespace Luban.Job.Cfg.Utils var dataSource = DataSourceFactory.Create(originFile, sheetName, new MemoryStream(content), exportTestData); try { - List datas; if (multiRecord) { - datas = dataSource.ReadMulti(recordType); + return dataSource.ReadMulti(recordType); } else { - datas = new List { dataSource.ReadOne(recordType) }; + Record record = dataSource.ReadOne(recordType); + return record != null ? new List { record } : new List(); } - var records = new List(datas.Count); - foreach (var data in datas) - { - records.Add(new Record((DBean)data, originFile)); - } - return records; } catch (Exception e) { diff --git a/src/Luban.Job.Cfg/Source/Utils/DataUtil.cs b/src/Luban.Job.Cfg/Source/Utils/DataUtil.cs index fb21d79..1af1222 100644 --- a/src/Luban.Job.Cfg/Source/Utils/DataUtil.cs +++ b/src/Luban.Job.Cfg/Source/Utils/DataUtil.cs @@ -39,16 +39,25 @@ namespace Luban.Job.Cfg.Utils return new DVector4(new System.Numerics.Vector4(float.Parse(values[0]), float.Parse(values[1]), float.Parse(values[2]), float.Parse(values[3]))); } - public static DDateTime CreateDateTime(string x, TimeZoneInfo timeZoneInfo) - { + //public static DDateTime CreateDateTime(string x, TimeZoneInfo timeZoneInfo) + //{ - DateTime dateTime = DateTime.ParseExact(x, - new string[] { + // DateTime dateTime = DateTime.ParseExact(x, + // new string[] { + // "yyyy-MM-dd HH:mm:ss", "yyyy-MM-dd HH:mm", "yyyy-MM-dd HH", "yyyy-MM-dd", + // //"yyyy/MM/dd HH:mm:ss", "yyyy/MM/dd HH:mm", "yyyy/MM/dd HH", "yyyy/MM/dd", + // }, + // System.Globalization.CultureInfo.InvariantCulture); + // return new DDateTime(TimeZoneInfo.ConvertTimeToUtc(dateTime, timeZoneInfo)); + //} + private static readonly string[] dateTimeFormats = new string[] { "yyyy-MM-dd HH:mm:ss", "yyyy-MM-dd HH:mm", "yyyy-MM-dd HH", "yyyy-MM-dd", //"yyyy/MM/dd HH:mm:ss", "yyyy/MM/dd HH:mm", "yyyy/MM/dd HH", "yyyy/MM/dd", - }, - System.Globalization.CultureInfo.InvariantCulture); - return new DDateTime(TimeZoneInfo.ConvertTimeToUtc(dateTime, timeZoneInfo)); + }; + public static DDateTime CreateDateTime(string x) + { + DateTime dateTime = DateTime.ParseExact(x, dateTimeFormats, System.Globalization.CultureInfo.InvariantCulture); + return new DDateTime(dateTime); } public static byte[] StreamToBytes(Stream stream) diff --git a/src/Luban.Job.Cfg/Source/ValidatorContext.cs b/src/Luban.Job.Cfg/Source/ValidatorContext.cs index 55add5d..047ff06 100644 --- a/src/Luban.Job.Cfg/Source/ValidatorContext.cs +++ b/src/Luban.Job.Cfg/Source/ValidatorContext.cs @@ -81,7 +81,7 @@ namespace Luban.Job.Cfg { tasks.Add(Task.Run(() => { - var records = t.Assembly.GetTableDataList(t); + var records = t.Assembly.GetTableAllDataList(t); var visitor = new ValidatorVisitor(this); try {