【特性】excel、lua、xml、json 支持标签过滤

【调整】datetime 类型数据不再在创建时确定utc时间，改为导出时根据时区再确定（跟text的方式保持一致，即所有原始数据加载时保持不变，只有导出时才做转换）

.gitignore

@@ -268,3 +268,4 @@ __pycache__/
 /config/output_code
 /config/output_data
 /config/output_lua
+/config/output_lua_without_test

config/Datas/test/tag_datas/j_not_tag.json

@@ -0,0 +1,4 @@
+{
+	"id":2001,
+	"value": "导出"
+}

config/Datas/test/tag_datas/j_tag_any.json

@@ -0,0 +1,5 @@
+{
+	"__tag__":"any",
+	"id":2004,
+	"value": "导出"
+}

config/Datas/test/tag_datas/j_tag_ignore.json

@@ -0,0 +1,5 @@
+{
+	"__tag__":"no",
+	"id":2002,
+	"value": "忽略"
+}

config/Datas/test/tag_datas/j_tag_test.json

@@ -0,0 +1,5 @@
+{
+	"__tag__":"测试",
+	"id":2003,
+	"value": "导出"
+}

config/Datas/test/tag_datas/not_tag.lua

@@ -0,0 +1,5 @@
+
+return {
+	id = 100,
+	value = "导出",
+}

config/Datas/test/tag_datas/tag_any.lua

@@ -0,0 +1,5 @@
+return {
+	__tag__ = "随便",
+	id = 104,
+	value="导出",
+}

config/Datas/test/tag_datas/tag_not_export.lua

@@ -0,0 +1,5 @@
+return {
+	__tag__ = "no",
+	id = 101,
+	value="不导出",
+}

config/Datas/test/tag_datas/tag_test.lua

@@ -0,0 +1,5 @@
+return {
+	__tag__ = "test",
+	id = 102,
+	value="测试",
+}

config/Datas/test/tag_datas/x_not_tag.xml

@@ -0,0 +1,4 @@
+<data>
+	<id>3001</id>
+	<value>export</value>
+</data>

config/Datas/test/tag_datas/x_tag_any.xml

@@ -0,0 +1,5 @@
+<data>
+	<__tag__>any</__tag__>
+	<id>3004</id>
+	<value>其他</value>
+</data>

config/Datas/test/tag_datas/x_tag_no.xml

@@ -0,0 +1,5 @@
+<data>
+	<__tag__>no</__tag__>
+	<id>3002</id>
+	<value>不导出</value>
+</data>

config/Datas/test/tag_datas/x_tag_test.xml

@@ -0,0 +1,5 @@
+<data>
+	<__tag__>test</__tag__>
+	<id>3003</id>
+	<value>测试</value>
+</data>

config/Defines/test.xml

@@ -196,4 +196,10 @@
 			<var name="role_id" type="long"/>
 		</bean>
 	</module>
+	
+	<bean name="TestTag">
+		<var name="id" type="int"/>
+		<var name="value" type="string"/>
+	</bean>
+	<table name="TbTestTag" value="TestTag" input="test/tag_datas"/>
 </module>

config/生成lua数据_不包含测试数据.bat

@@ -0,0 +1,11 @@
+..\src\Luban.Client\bin\Debug\net5.0\Luban.Client.exe ^
+	-h %LUBAN_SERVER_IP% ^
+	-j cfg ^
+	-- ^
+	-d Defines/__root__.xml ^
+	--input_data_dir Datas ^
+	--output_data_dir output_lua_without_test ^
+	-s client ^
+	--gen_types data_lua 
+
+pause

src/Luban.Job.Cfg/Source/Cache/FileRecordCacheManager.cs

@@ -35,15 +35,15 @@ namespace Luban.Job.Cfg.Cache
             }
         }
 
-        private readonly ConcurrentDictionary<(string, string, string, bool), FileRecordCache> _caches = new ConcurrentDictionary<(string, string, string, bool), FileRecordCache>();
+        private readonly ConcurrentDictionary<(string MD5, string SheetName), FileRecordCache> _caches = new();
 
         private readonly object _shrinkLocker = new object();
 
-        public bool TryGetCacheLoadedRecords(DefTable table, string md5, string originFile, string sheetName, bool exportTestData, out List<Record> cacheRecords)
+        public bool TryGetCacheLoadedRecords(DefTable table, string md5, string originFile, string sheetName, out List<Record> cacheRecords)
         {
             // TODO text localization check
             cacheRecords = null;
-            if (!_caches.TryGetValue((table.Assembly.TimeZone.Id, md5, sheetName, exportTestData), out var r))
+            if (!_caches.TryGetValue((md5, sheetName), out var r))
             {
                 return false;
             }
@@ -60,11 +60,11 @@ namespace Luban.Job.Cfg.Cache
             }
         }
 
-        public void AddCacheLoadedRecords(DefTable table, string md5, string sheetName, bool exportTestData, List<Record> cacheRecords)
+        public void AddCacheLoadedRecords(DefTable table, string md5, string sheetName, List<Record> cacheRecords)
         {
             lock (_shrinkLocker)
             {
-                _caches[(table.Assembly.TimeZone.Id, md5, sheetName, exportTestData)] = new FileRecordCache(table, cacheRecords);
+                _caches[(md5, sheetName)] = new FileRecordCache(table, cacheRecords);
                 if (_caches.Count > CACHE_FILE_HIGH_WATER_MARK)
                 {
                     s_logger.Info("ShrinkCaches. cache count > high CACHE_FILE_HIGH_WATER_MARK:{}", CACHE_FILE_HIGH_WATER_MARK);

src/Luban.Job.Cfg/Source/DataSources/AbstractDataSource.cs

@@ -7,9 +7,32 @@ namespace Luban.Job.Cfg.DataSources
 {
     abstract class AbstractDataSource
     {
-        public abstract DType ReadOne(TBean type);
+        public const string TAG_KEY = "__tag__";
 
-        public abstract List<DType> ReadMulti(TBean type);
+        public static bool IsIgnoreTag(string tagName)
+        {
+            return !string.IsNullOrWhiteSpace(tagName) &&
+                (
+                   tagName.Equals("false", System.StringComparison.OrdinalIgnoreCase)
+                || tagName.Equals("no", System.StringComparison.OrdinalIgnoreCase)
+                || tagName.Equals("##", System.StringComparison.Ordinal)
+                || tagName.Equals("·ñ", System.StringComparison.Ordinal)
+                );
+        }
+
+        public static bool IsTestTag(string tagName)
+        {
+            return !string.IsNullOrWhiteSpace(tagName) &&
+                (tagName.Equals("test", System.StringComparison.OrdinalIgnoreCase)
+                || tagName.Equals("²âÊÔ", System.StringComparison.Ordinal)
+                );
+        }
+
+        public string RawUrl { get; protected set; }
+
+        public abstract Record ReadOne(TBean type);
+
+        public abstract List<Record> ReadMulti(TBean type);
 
         public abstract void Load(string rawUrl, string sheetName, Stream stream, bool exportDebugData);
     }

src/Luban.Job.Cfg/Source/DataSources/Binary/BinaryDataSource.cs

@@ -13,12 +13,12 @@ namespace Luban.Job.Cfg.DataSources.Binary
             throw new NotImplementedException();
         }
 
-        public override List<DType> ReadMulti(TBean type)
+        public override List<Record> ReadMulti(TBean type)
         {
             throw new NotImplementedException();
         }
 
-        public override DType ReadOne(TBean type)
+        public override Record ReadOne(TBean type)
         {
             throw new NotImplementedException();
         }

src/Luban.Job.Cfg/Source/DataSources/Excel/ExcelDataSource.cs

@@ -21,6 +21,7 @@ namespace Luban.Job.Cfg.DataSources.Excel
         public override void Load(string rawUrl, string sheetName, Stream stream, bool exportTestData)
         {
             s_logger.Trace("{filename} {sheet}", rawUrl, sheetName);
+            RawUrl = rawUrl;
             string ext = Path.GetExtension(rawUrl);
             using (var reader = ext != ".csv" ? ExcelReaderFactory.CreateReader(stream) : ExcelReaderFactory.CreateCsvReader(stream))
             {
@@ -30,7 +31,7 @@ namespace Luban.Job.Cfg.DataSources.Excel
                     {
                         try
                         {
-                            var sheet = ReadSheet(reader, exportTestData);
+                            var sheet = ReadSheet(rawUrl, reader);
                             if (sheet != null)
                             {
                                 _sheets.Add(sheet);
@@ -50,15 +51,15 @@ namespace Luban.Job.Cfg.DataSources.Excel
             }
         }
 
-        private Sheet ReadSheet(IExcelDataReader reader, bool exportTestData)
+        private Sheet ReadSheet(string url, IExcelDataReader reader)
         {
-            var sheet = new Sheet(reader.Name ?? "", exportTestData);
+            var sheet = new Sheet(url, reader.Name ?? "");
             return sheet.Load(reader) ? sheet : null;
         }
 
-        public override List<DType> ReadMulti(TBean type)
+        public override List<Record> ReadMulti(TBean type)
         {
-            var datas = new List<DType>();
+            var datas = new List<Record>();
             foreach (var sheet in _sheets)
             {
                 try
@@ -73,7 +74,7 @@ namespace Luban.Job.Cfg.DataSources.Excel
             return datas;
         }
 
-        public override DType ReadOne(TBean type)
+        public override Record ReadOne(TBean type)
         {
             var datas = ReadMulti(type);
             switch (datas.Count)

src/Luban.Job.Cfg/Source/DataSources/Excel/Sheet.cs

@@ -16,14 +16,14 @@ namespace Luban.Job.Cfg.DataSources.Excel
 
         private int TitleRows { get; set; } = 3; // 默认有三行是标题行. 第一行是字段名，第二行是中文描述，第三行是注释
 
+        public string RawUrl { get; }
+
         public string Name { get; }
 
         private List<List<Cell>> _rowColumns;
 
         private Title _rootTitle;
 
-        private bool ExportTestData { get; }
-
         public class Title
         {
             public int FromIndex { get; set; }
@@ -206,10 +206,10 @@ namespace Luban.Job.Cfg.DataSources.Excel
             }
         }
 
-        public Sheet(string name, bool exportTestData)
+        public Sheet(string rawUrl, string name)
         {
+            this.RawUrl = rawUrl;
             this.Name = name;
-            this.ExportTestData = exportTestData;
         }
 
         public bool Load(IExcelDataReader reader)
@@ -292,35 +292,17 @@ namespace Luban.Job.Cfg.DataSources.Excel
             return true;
         }
 
-        private bool NotExport(List<Cell> row)
+        private string GetRowTag(List<Cell> row)
         {
             if (row.Count == 0)
             {
-                return true;
+                return null;
             }
             if (row[0].Value == null)
             {
-                return false;
+                return null;
-            }
-
-            string exportFlag = row[0].Value.ToString().Trim().ToLower();
-            switch (exportFlag)
-            {
-                case "false":
-                case "否": return true;
-                case "true":
-                case "是": return false;
-                case "test":
-                case "测试":
-                {
-                    if (!ExportTestData)
-                    {
-                        s_logger.Debug("忽略测试数据. row:{row}", row);
-                    }
-                    return !ExportTestData;
-                }
-                default: throw new Exception($"不支持的excel 导出标记: {exportFlag}");
             }
+            return row[0].Value.ToString().Trim();
         }
 
         private void InitSubTitles(Title parentTitle, List<List<Cell>> rows, CellRange[] mergeCells, int maxDepth, int depth, int fromColumn, int toColumn)
@@ -519,15 +501,16 @@ namespace Luban.Job.Cfg.DataSources.Excel
                 throw new Exception($"没有定义任何有效 列");
             }
             _rootTitle.SortSubTitles();
-            foreach (var title in _rootTitle.SubTitleList)
+            //foreach (var title in _rootTitle.SubTitleList)
-            {
+            //{
-                // s_logger.Info("============ sheet:{sheet} title:{title}", Name, title);
+            //    // s_logger.Info("============ sheet:{sheet} title:{title}", Name, title);
-            }
+            //}
 
             // 删除标题行
             this._rowColumns.RemoveRange(0, Math.Min(TitleRows + titleRowNum - 1, this._rowColumns.Count));
 
-            this._rowColumns.RemoveAll(row => NotExport(row));
+            // 删除忽略的记录行
+            this._rowColumns.RemoveAll(row => AbstractDataSource.IsIgnoreTag(GetRowTag(row)));
         }
 
 
@@ -604,11 +587,11 @@ namespace Luban.Job.Cfg.DataSources.Excel
 
 
 
-        public List<DType> ReadMulti(TBean type, bool enableMultiRowRecord)
+        public List<Record> ReadMulti(TBean type, bool enableMultiRowRecord)
         {
-            var datas = new List<DType>();
+            var datas = new List<Record>();
 
-            for (DType data; (data = ReadOne(type, enableMultiRowRecord)) != null;)
+            for (Record data; (data = ReadOne(type, enableMultiRowRecord)) != null;)
             {
                 datas.Add(data);
             }
@@ -616,7 +599,7 @@ namespace Luban.Job.Cfg.DataSources.Excel
         }
 
         private int curReadIndex = 0;
-        public DType ReadOne(TBean type, bool enableMultiRowRecord)
+        public Record ReadOne(TBean type, bool enableMultiRowRecord)
         {
             if (!enableMultiRowRecord)
             {
@@ -625,7 +608,9 @@ namespace Luban.Job.Cfg.DataSources.Excel
                 {
                     return null;
                 }
-                return ExcelNamedRowDataCreator.Ins.ReadExcel(new NamedRow(_rootTitle, row), type);
+                bool isTest = AbstractDataSource.IsTestTag(GetRowTag(row));
+                var data = (DBean)ExcelNamedRowDataCreator.Ins.ReadExcel(new NamedRow(_rootTitle, row), type);
+                return new Record(data, RawUrl, isTest);
             }
             else
             {
@@ -634,7 +619,9 @@ namespace Luban.Job.Cfg.DataSources.Excel
                 {
                     return null;
                 }
-                return ExcelNamedRowDataCreator.Ins.ReadExcel(new NamedRow(_rootTitle, rows), type);
+                bool isTest = AbstractDataSource.IsTestTag(GetRowTag(rows[0]));
+                var data = (DBean)ExcelNamedRowDataCreator.Ins.ReadExcel(new NamedRow(_rootTitle, rows), type);
+                return new Record(data, RawUrl, isTest);
             }
         }
     }

src/Luban.Job.Cfg/Source/DataSources/Json/JsonDataSource.cs

@@ -15,17 +15,30 @@ namespace Luban.Job.Cfg.DataSources.Json
 
         public override void Load(string rawUrl, string sheetName, Stream stream, bool exportDebugData)
         {
+            RawUrl = rawUrl;
             this._data = JsonDocument.Parse(stream).RootElement;
         }
 
-        public override List<DType> ReadMulti(TBean type)
+        public override List<Record> ReadMulti(TBean type)
         {
             throw new NotImplementedException();
         }
 
-        public override DType ReadOne(TBean type)
+        public override Record ReadOne(TBean type)
         {
-            return type.Apply(JsonDataCreator.Ins, _data, (DefAssembly)type.Bean.AssemblyBase);
+            bool isTest = false;
+            if (_data.TryGetProperty(TAG_KEY, out var tagEle))
+            {
+                var tagName = tagEle.GetString();
+                if (IsIgnoreTag(tagName))
+                {
+                    return null;
+                }
+                isTest = IsTestTag(tagName);
+            }
+
+            var data = (DBean)type.Apply(JsonDataCreator.Ins, _data, (DefAssembly)type.Bean.AssemblyBase);
+            return new Record(data, RawUrl, isTest);
         }
     }
 }

src/Luban.Job.Cfg/Source/DataSources/Lua/LuaDataSource.cs

@@ -18,25 +18,42 @@ namespace Luban.Job.Cfg.DataSources.Lua
 
         public override void Load(string rawUrl, string sheetName, Stream stream, bool exportDebugData)
         {
+            RawUrl = rawUrl;
             _env = LuaManager.CreateEnvironment();
             _dataTable = (LuaTable)_env.DoChunk(new StreamReader(stream, Encoding.UTF8), rawUrl)[0];
         }
 
-        public override List<DType> ReadMulti(TBean type)
+        public override List<Record> ReadMulti(TBean type)
         {
-            var records = new List<DType>();
+            var records = new List<Record>();
 
             foreach (LuaTable t in _dataTable.Values.Values)
             {
-                records.Add(type.Apply(LuaDataCreator.Ins, t, (DefAssembly)type.Bean.AssemblyBase));
+                Record r = ReadRecord(t, type);
+                if (r != null)
+                {
+                    records.Add(r);
+                }
             }
 
             return records;
         }
 
-        public override DType ReadOne(TBean type)
+        public override Record ReadOne(TBean type)
         {
-            return type.Apply(LuaDataCreator.Ins, _dataTable, (DefAssembly)type.Bean.AssemblyBase);
+            return ReadRecord(_dataTable, type);
+        }
+
+        protected Record ReadRecord(LuaTable table, TBean type)
+        {
+            string tagName = table.GetValue(TAG_KEY)?.ToString();
+            if (IsIgnoreTag(tagName))
+            {
+                return null;
+            }
+            var data = (DBean)type.Apply(LuaDataCreator.Ins, table, (DefAssembly)type.Bean.AssemblyBase);
+            var isTest = IsTestTag(tagName);
+            return new Record(data, RawUrl, isTest);
         }
     }
 }

src/Luban.Job.Cfg/Source/DataSources/Xml/XmlDataSource.cs

@@ -12,19 +12,28 @@ namespace Luban.Job.Cfg.DataSources.Xml
     class XmlDataSource : AbstractDataSource
     {
         private XElement _doc;
+
         public override void Load(string rawUrl, string sheetName, Stream stream, bool exportDebugData)
         {
+            RawUrl = rawUrl;
             _doc = XElement.Load(stream);
         }
 
-        public override List<DType> ReadMulti(TBean type)
+        public override List<Record> ReadMulti(TBean type)
         {
             throw new NotSupportedException();
         }
 
-        public override DType ReadOne(TBean type)
+        public override Record ReadOne(TBean type)
         {
-            return type.Apply(XmlDataCreator.Ins, _doc, (DefAssembly)type.Bean.AssemblyBase);
+            string tagName = _doc.Element(TAG_KEY)?.Value;
+            if (IsIgnoreTag(tagName))
+            {
+                return null;
+            }
+            var data = (DBean)type.Apply(XmlDataCreator.Ins, _doc, (DefAssembly)type.Bean.AssemblyBase);
+            bool isTest = IsTestTag(tagName);
+            return new Record(data, RawUrl, isTest);
         }
     }
 }

src/Luban.Job.Cfg/Source/DataVisitors/BinaryExportor.cs

@@ -183,7 +183,7 @@ namespace Luban.Job.Cfg.DataVisitors
 
         public void Accept(DDateTime type, DefAssembly ass, ByteBuf x)
         {
-            x.WriteInt(type.UnixTime);
+            x.WriteInt(type.GetUnixTime(ass.TimeZone));
         }
     }
 }

src/Luban.Job.Cfg/Source/DataVisitors/IsDefaultValue.cs

@@ -70,7 +70,7 @@ namespace Luban.Job.Cfg.DataVisitors
 
         public bool Accept(DBytes type)
         {
-            throw new NotImplementedException();
+            throw new NotSupportedException();
         }
 
         public bool Accept(DText type)
@@ -120,7 +120,7 @@ namespace Luban.Job.Cfg.DataVisitors
 
         public bool Accept(DDateTime type)
         {
-            return type.UnixTime == 0;
+            throw new NotSupportedException();
         }
     }
 }

src/Luban.Job.Cfg/Source/DataVisitors/JsonExportor.cs

@@ -191,7 +191,7 @@ namespace Luban.Job.Cfg.DataVisitors
 
         public void Accept(DDateTime type, DefAssembly ass, Utf8JsonWriter x)
         {
-            x.WriteNumberValue(type.UnixTime);
+            x.WriteNumberValue(type.GetUnixTime(ass.TimeZone));
         }
     }
 }

src/Luban.Job.Cfg/Source/DataVisitors/LuaExportor.cs

@@ -247,7 +247,7 @@ namespace Luban.Job.Cfg.DataVisitors
 
         public void Accept(DDateTime type, DefAssembly ass, StringBuilder line)
         {
-            line.Append(type.UnixTime);
+            line.Append(type.GetUnixTime(ass.TimeZone));
         }
     }
 }

src/Luban.Job.Cfg/Source/Datas/DDateTime.cs

@@ -7,12 +7,28 @@ namespace Luban.Job.Cfg.Datas
     {
         public DateTime Time { get; }
 
-        public int UnixTime { get; }
+        //public int UnixTime { get; }
+        private readonly int _localTime;
 
         public DDateTime(DateTime time)
         {
             this.Time = time;
-            this.UnixTime = (int)new DateTimeOffset(time).ToUnixTimeSeconds();
+            // time.Kind == DateTimeKind.Unspecified
+            // DateTimeOffset°ÑËüµ±×÷Local´¦Àí
+            this._localTime = (int)new DateTimeOffset(time).ToUnixTimeSeconds();
+        }
+
+        public int GetUnixTime(TimeZoneInfo asTimeZone)
+        {
+            if (asTimeZone == null || asTimeZone == TimeZoneInfo.Local)
+            {
+                return this._localTime;
+            }
+            else
+            {
+                var destDateTime = TimeZoneInfo.ConvertTime(Time, asTimeZone, TimeZoneInfo.Utc);
+                return (int)new DateTimeOffset(destDateTime).ToUnixTimeSeconds();
+            }
         }
 
         public override void Apply<T>(IDataActionVisitor<T> visitor, T x)

src/Luban.Job.Cfg/Source/Datas/Record.cs

@@ -14,10 +14,13 @@ namespace Luban.Job.Cfg.Datas
 
         public int Index { get; set; }
 
-        public Record(DBean data, string source)
+        public bool IsTest { get; }
+
+        public Record(DBean data, string source, bool isTest)
         {
             Data = data;
             Source = source;
+            IsTest = isTest;
         }
     }
 }

src/Luban.Job.Cfg/Source/Defs/DefAssembly.cs

@@ -19,6 +19,19 @@ namespace Luban.Job.Cfg.Defs
 
         public List<Record> FinalRecords { get; set; }
 
+        private List<Record> _notTestRecords;
+        public List<Record> NotTestRecords
+        {
+            get
+            {
+                if (_notTestRecords == null)
+                {
+                    _notTestRecords = FinalRecords.Where(r => !r.IsTest).ToList();
+                }
+                return _notTestRecords;
+            }
+        }
+
         public Dictionary<DType, Record> FinalRecordMap { get; set; }
 
         public TableDataInfo(List<Record> mainRecords, List<Record> branchRecords)
@@ -34,13 +47,19 @@ namespace Luban.Job.Cfg.Defs
 
         public Service CfgTargetService { get; private set; }
 
+        private readonly string _branchName;
+        private readonly bool _exportTestData;
+
         public Branch TargetBranch { get; private set; }
 
         public TimeZoneInfo TimeZone { get; }
 
-        public DefAssembly(TimeZoneInfo timezone)
+        public DefAssembly(string branchName, TimeZoneInfo timezone, bool exportTestData, RemoteAgent agent)
         {
+            this._branchName = branchName;
             this.TimeZone = timezone;
+            this._exportTestData = exportTestData;
+            this.Agent = agent;
         }
 
         public bool NeedExport(List<string> groups)
@@ -100,11 +119,17 @@ namespace Luban.Job.Cfg.Defs
         //    _recordsByTables[table.FullName].FinalRecordMap = recordMap;
         //}
 
-        public List<Record> GetTableDataList(DefTable table)
+        public List<Record> GetTableAllDataList(DefTable table)
         {
             return _recordsByTables[table.FullName].FinalRecords;
         }
 
+        public List<Record> GetTableExportDataList(DefTable table)
+        {
+            var tableDataInfo = _recordsByTables[table.FullName];
+            return _exportTestData ? tableDataInfo.FinalRecords : tableDataInfo.NotTestRecords;
+        }
+
         public TableDataInfo GetTableDataInfo(DefTable table)
         {
             return _recordsByTables[table.FullName];
@@ -147,9 +172,8 @@ namespace Luban.Job.Cfg.Defs
             return refTypes.Values.ToList();
         }
 
-        public void Load(string outputService, string branchName, Defines defines, RemoteAgent agent)
+        public void Load(string outputService, Defines defines)
         {
-            this.Agent = agent;
             SupportDatetimeType = true;
 
             TopModule = defines.TopModule;
@@ -161,12 +185,12 @@ namespace Luban.Job.Cfg.Defs
                 throw new ArgumentException($"service:{outputService} not exists");
             }
 
-            if (!string.IsNullOrWhiteSpace(branchName))
+            if (!string.IsNullOrWhiteSpace(_branchName))
             {
-                TargetBranch = defines.Branches.Find(b => b.Name == branchName);
+                TargetBranch = defines.Branches.Find(b => b.Name == _branchName);
                 if (TargetBranch == null)
                 {
-                    throw new Exception($"branch {branchName} not in valid branch set");
+                    throw new Exception($"branch {_branchName} not in valid branch set");
                 }
             }
 
@@ -211,7 +235,7 @@ namespace Luban.Job.Cfg.Defs
                 }
                 catch (Exception)
                 {
-                    agent.Error("precompile type:{0} error", type.FullName);
+                    this.Agent.Error("precompile type:{0} error", type.FullName);
                     throw;
                 }
             }
@@ -225,7 +249,7 @@ namespace Luban.Job.Cfg.Defs
                 }
                 catch (Exception)
                 {
-                    agent.Error("compile type:{0} error", type.FullName);
+                    this.Agent.Error("compile type:{0} error", type.FullName);
                     s_logger.Error("compile type:{0} error", type.FullName);
                     throw;
                 }
@@ -240,7 +264,7 @@ namespace Luban.Job.Cfg.Defs
                 }
                 catch (Exception)
                 {
-                    agent.Error("post compile type:{0} error", type.FullName);
+                    this.Agent.Error("post compile type:{0} error", type.FullName);
                     s_logger.Error("post compile type:{0} error", type.FullName);
                     throw;
                 }

src/Luban.Job.Cfg/Source/JobController.cs

@@ -208,11 +208,11 @@ namespace Luban.Job.Cfg
 
                 var rawDefines = loader.BuildDefines();
 
-                TimeZoneInfo timeZoneInfo = string.IsNullOrEmpty(args.TimeZone) ? TimeZoneInfo.Local : TimeZoneInfo.FindSystemTimeZoneById(args.TimeZone);
+                TimeZoneInfo timeZoneInfo = string.IsNullOrEmpty(args.TimeZone) ? null : TimeZoneInfo.FindSystemTimeZoneById(args.TimeZone);
 
-                var ass = new DefAssembly(timeZoneInfo);
+                var ass = new DefAssembly(args.BranchName, timeZoneInfo, args.ExportTestData, agent);
 
-                ass.Load(args.Service, args.BranchName, rawDefines, agent);
+                ass.Load(args.Service, rawDefines);
 
                 var targetService = ass.CfgTargetService;
 
@@ -948,7 +948,7 @@ class Vector4:
                             {
                                 tasks.Add(Task.Run(() =>
                                 {
-                                    var content = DataExporterUtil.ToOutputData(c, ass.GetTableDataList(c), genType);
+                                    var content = DataExporterUtil.ToOutputData(c, ass.GetTableExportDataList(c), genType);
                                     var file = genType.EndsWith("json") ? c.JsonOutputDataFile : c.OutputDataFile;
                                     var md5 = FileUtil.CalcMD5(content);
                                     CacheManager.Ins.AddCache(file, md5, content);
@@ -965,7 +965,7 @@ class Vector4:
                             {
                                 allJsonTask.Add(Task.Run(() =>
                                 {
-                                    return DataExporterUtil.ToOutputData(c, ass.GetTableDataList(c), "data_json");
+                                    return DataExporterUtil.ToOutputData(c, ass.GetTableExportDataList(c), "data_json");
                                 }));
                             }
                             await Task.WhenAll(allJsonTask);
@@ -1010,7 +1010,7 @@ class Vector4:
                             {
                                 tasks.Add(Task.Run(() =>
                                 {
-                                    var content = DataExporterUtil.ToOutputData(c, ass.GetTableDataList(c), genType);
+                                    var content = DataExporterUtil.ToOutputData(c, ass.GetTableExportDataList(c), genType);
                                     var file = $"{c.Name}.lua";
                                     var md5 = FileUtil.CalcMD5(content);
                                     CacheManager.Ins.AddCache(file, md5, content);
@@ -1027,7 +1027,7 @@ class Vector4:
                             {
                                 genDataTasks.Add(Task.Run(() =>
                                 {
-                                    return DataExporterUtil.ExportResourceList(ass.GetTableDataList(c));
+                                    return DataExporterUtil.ExportResourceList(ass.GetTableExportDataList(c));
                                 }));
                             }
 

src/Luban.Job.Cfg/Source/TypeVisitors/ExcelDataCreator.cs

@@ -436,7 +436,7 @@ namespace Luban.Job.Cfg.TypeVisitors
             {
                 return null;
             }
-            return DataUtil.CreateDateTime(d.ToString(), ass.TimeZone);
+            return DataUtil.CreateDateTime(d.ToString());
         }
     }
 }

src/Luban.Job.Cfg/Source/TypeVisitors/JsonDataCreator.cs

@@ -197,7 +197,7 @@ namespace Luban.Job.Cfg.TypeVisitors
 
         public DType Accept(TDateTime type, JsonElement x, DefAssembly ass)
         {
-            return DataUtil.CreateDateTime(x.GetString(), ass.TimeZone);
+            return DataUtil.CreateDateTime(x.GetString());
         }
 
         public DType Accept(TVector2 type, JsonElement x, DefAssembly ass)

src/Luban.Job.Cfg/Source/TypeVisitors/LuaDataCreator.cs

@@ -257,7 +257,7 @@ namespace Luban.Job.Cfg.TypeVisitors
 
         public DType Accept(TDateTime type, object x, DefAssembly ass)
         {
-            return DataUtil.CreateDateTime(x.ToString(), ass.TimeZone);
+            return DataUtil.CreateDateTime(x.ToString());
         }
     }
 }

src/Luban.Job.Cfg/Source/TypeVisitors/XmlDataCreator.cs

@@ -193,7 +193,7 @@ namespace Luban.Job.Cfg.TypeVisitors
 
         public DType Accept(TDateTime type, XElement x, DefAssembly ass)
         {
-            return DataUtil.CreateDateTime(x.Value, ass.TimeZone);
+            return DataUtil.CreateDateTime(x.Value);
         }
     }
 }

src/Luban.Job.Cfg/Source/Utils/DataLoaderUtil.cs

@@ -85,7 +85,7 @@ namespace Luban.Job.Cfg.Utils
 
                 tasks.Add(Task.Run(async () =>
                 {
-                    if (FileRecordCacheManager.Ins.TryGetCacheLoadedRecords(table, file.MD5, actualFile, file.SheetName, exportTestData, out var cacheRecords))
+                    if (FileRecordCacheManager.Ins.TryGetCacheLoadedRecords(table, file.MD5, actualFile, file.SheetName, out var cacheRecords))
                     {
                         return cacheRecords;
                     }
@@ -96,7 +96,7 @@ namespace Luban.Job.Cfg.Utils
                         RenderFileUtil.IsExcelFile(file.ActualFile),
                         exportTestData);
 
-                    FileRecordCacheManager.Ins.AddCacheLoadedRecords(table, file.MD5, file.SheetName, exportTestData, res);
+                    FileRecordCacheManager.Ins.AddCacheLoadedRecords(table, file.MD5, file.SheetName, res);
 
                     return res;
                 }));
@@ -176,21 +176,15 @@ namespace Luban.Job.Cfg.Utils
             var dataSource = DataSourceFactory.Create(originFile, sheetName, new MemoryStream(content), exportTestData);
             try
             {
-                List<DType> datas;
                 if (multiRecord)
                 {
-                    datas = dataSource.ReadMulti(recordType);
+                    return dataSource.ReadMulti(recordType);
                 }
                 else
                 {
-                    datas = new List<DType> { dataSource.ReadOne(recordType) };
+                    Record record = dataSource.ReadOne(recordType);
+                    return record != null ? new List<Record> { record } : new List<Record>();
                 }
-                var records = new List<Record>(datas.Count);
-                foreach (var data in datas)
-                {
-                    records.Add(new Record((DBean)data, originFile));
-                }
-                return records;
             }
             catch (Exception e)
             {

src/Luban.Job.Cfg/Source/Utils/DataUtil.cs

@@ -39,16 +39,25 @@ namespace Luban.Job.Cfg.Utils
             return new DVector4(new System.Numerics.Vector4(float.Parse(values[0]), float.Parse(values[1]), float.Parse(values[2]), float.Parse(values[3])));
         }
 
-        public static DDateTime CreateDateTime(string x, TimeZoneInfo timeZoneInfo)
+        //public static DDateTime CreateDateTime(string x, TimeZoneInfo timeZoneInfo)
-        {
+        //{
 
-            DateTime dateTime = DateTime.ParseExact(x,
+        //    DateTime dateTime = DateTime.ParseExact(x,
-                new string[] {
+        //        new string[] {
+        //            "yyyy-MM-dd HH:mm:ss", "yyyy-MM-dd HH:mm", "yyyy-MM-dd HH", "yyyy-MM-dd",
+        //            //"yyyy/MM/dd HH:mm:ss", "yyyy/MM/dd HH:mm", "yyyy/MM/dd HH", "yyyy/MM/dd",
+        //        },
+        //        System.Globalization.CultureInfo.InvariantCulture);
+        //    return new DDateTime(TimeZoneInfo.ConvertTimeToUtc(dateTime, timeZoneInfo));
+        //}
+        private static readonly string[] dateTimeFormats = new string[] {
                     "yyyy-MM-dd HH:mm:ss", "yyyy-MM-dd HH:mm", "yyyy-MM-dd HH", "yyyy-MM-dd",
                     //"yyyy/MM/dd HH:mm:ss", "yyyy/MM/dd HH:mm", "yyyy/MM/dd HH", "yyyy/MM/dd",
-                },
+                };
-                System.Globalization.CultureInfo.InvariantCulture);
+        public static DDateTime CreateDateTime(string x)
-            return new DDateTime(TimeZoneInfo.ConvertTimeToUtc(dateTime, timeZoneInfo));
+        {
+            DateTime dateTime = DateTime.ParseExact(x, dateTimeFormats, System.Globalization.CultureInfo.InvariantCulture);
+            return new DDateTime(dateTime);
         }
 
         public static byte[] StreamToBytes(Stream stream)

src/Luban.Job.Cfg/Source/ValidatorContext.cs

@@ -81,7 +81,7 @@ namespace Luban.Job.Cfg
                 {
                     tasks.Add(Task.Run(() =>
                     {
-                        var records = t.Assembly.GetTableDataList(t);
+                        var records = t.Assembly.GetTableAllDataList(t);
                         var visitor = new ValidatorVisitor(this);
                         try
                         {