From cd6f0f1282837c87ffbfa0b5a6185f29cd989fb0 Mon Sep 17 00:00:00 2001 From: walon Date: Mon, 2 Aug 2021 13:46:59 +0800 Subject: [PATCH] =?UTF-8?q?=E3=80=90=E4=BF=AE=E5=A4=8D=E3=80=91=E8=A7=A3?= =?UTF-8?q?=E5=86=B3=E4=B8=8D=E5=90=8C=E5=B9=B3=E5=8F=B0=E4=B8=8B=E7=94=9F?= =?UTF-8?q?=E6=88=90json=E5=92=8Clua=E6=95=B0=E6=8D=AE=E6=96=87=E4=BB=B6?= =?UTF-8?q?=EF=BC=88=E6=8D=A2=E8=A1=8C=E7=AC=A6=EF=BC=89=E4=B8=8D=E5=90=8C?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Source/DataSources/AbstractDataSource.cs | 19 --------------- .../Source/DataSources/Excel/Sheet.cs | 7 +++--- .../Source/DataSources/Json/JsonDataSource.cs | 5 ++-- .../Source/DataSources/Lua/LuaDataSource.cs | 5 ++-- .../Source/DataSources/Xml/XmlDataSource.cs | 5 ++-- src/Luban.Job.Cfg/Source/JobController.cs | 24 +++++++++---------- .../Source/Utils/DataExporterUtil.cs | 6 ++--- src/Luban.Job.Cfg/Source/Utils/DataUtil.cs | 20 ++++++++++++++++ .../Source/Utils/CacheFileUtil.cs | 10 ++++++++ 9 files changed, 57 insertions(+), 44 deletions(-) diff --git a/src/Luban.Job.Cfg/Source/DataSources/AbstractDataSource.cs b/src/Luban.Job.Cfg/Source/DataSources/AbstractDataSource.cs index 132beb9..54e9307 100644 --- a/src/Luban.Job.Cfg/Source/DataSources/AbstractDataSource.cs +++ b/src/Luban.Job.Cfg/Source/DataSources/AbstractDataSource.cs @@ -9,25 +9,6 @@ namespace Luban.Job.Cfg.DataSources { public const string TAG_KEY = "__tag__"; - public static bool IsIgnoreTag(string tagName) - { - return !string.IsNullOrWhiteSpace(tagName) && - ( - tagName.Equals("false", System.StringComparison.OrdinalIgnoreCase) - || tagName.Equals("no", System.StringComparison.OrdinalIgnoreCase) - || tagName.Equals("##", System.StringComparison.Ordinal) - || tagName.Equals("", System.StringComparison.Ordinal) - ); - } - - public static bool IsTestTag(string tagName) - { - return !string.IsNullOrWhiteSpace(tagName) && - (tagName.Equals("test", System.StringComparison.OrdinalIgnoreCase) - || tagName.Equals("", System.StringComparison.Ordinal) - ); - } - public string RawUrl { get; protected set; } public abstract Record ReadOne(TBean type); diff --git a/src/Luban.Job.Cfg/Source/DataSources/Excel/Sheet.cs b/src/Luban.Job.Cfg/Source/DataSources/Excel/Sheet.cs index 0b2374d..be7be61 100644 --- a/src/Luban.Job.Cfg/Source/DataSources/Excel/Sheet.cs +++ b/src/Luban.Job.Cfg/Source/DataSources/Excel/Sheet.cs @@ -2,6 +2,7 @@ using ExcelDataReader; using Luban.Job.Cfg.DataCreators; using Luban.Job.Cfg.Datas; using Luban.Job.Cfg.TypeVisitors; +using Luban.Job.Cfg.Utils; using Luban.Job.Common.Types; using System; using System.Collections.Generic; @@ -529,7 +530,7 @@ namespace Luban.Job.Cfg.DataSources.Excel // 删除所有标题行，包含字段名行、属性行、标题、描述等等非有效数据行 this._rowColumns.RemoveRange(0, Math.Min(TitleRows + titleRowNum - 1, this._rowColumns.Count)); // 删除忽略的记录行 - this._rowColumns.RemoveAll(row => AbstractDataSource.IsIgnoreTag(GetRowTag(row))); + this._rowColumns.RemoveAll(row => DataUtil.IsIgnoreTag(GetRowTag(row))); } } @@ -629,7 +630,7 @@ namespace Luban.Job.Cfg.DataSources.Excel { return null; } - bool isTest = AbstractDataSource.IsTestTag(GetRowTag(row)); + bool isTest = DataUtil.IsTestTag(GetRowTag(row)); var data = (DBean)ExcelNamedRowDataCreator.Ins.ReadExcel(new NamedRow(_rootTitle, row), type); return new Record(data, RawUrl, isTest); } @@ -640,7 +641,7 @@ namespace Luban.Job.Cfg.DataSources.Excel { return null; } - bool isTest = AbstractDataSource.IsTestTag(GetRowTag(rows[0])); + bool isTest = DataUtil.IsTestTag(GetRowTag(rows[0])); var data = (DBean)ExcelNamedRowDataCreator.Ins.ReadExcel(new NamedRow(_rootTitle, rows), type); return new Record(data, RawUrl, isTest); } diff --git a/src/Luban.Job.Cfg/Source/DataSources/Json/JsonDataSource.cs b/src/Luban.Job.Cfg/Source/DataSources/Json/JsonDataSource.cs index 550e7ba..8f4ff54 100644 --- a/src/Luban.Job.Cfg/Source/DataSources/Json/JsonDataSource.cs +++ b/src/Luban.Job.Cfg/Source/DataSources/Json/JsonDataSource.cs @@ -2,6 +2,7 @@ using Luban.Job.Cfg.DataCreators; using Luban.Job.Cfg.Datas; using Luban.Job.Cfg.Defs; using Luban.Job.Cfg.TypeVisitors; +using Luban.Job.Cfg.Utils; using Luban.Job.Common.Types; using System; using System.Collections.Generic; @@ -31,11 +32,11 @@ namespace Luban.Job.Cfg.DataSources.Json if (_data.TryGetProperty(TAG_KEY, out var tagEle)) { var tagName = tagEle.GetString(); - if (IsIgnoreTag(tagName)) + if (DataUtil.IsIgnoreTag(tagName)) { return null; } - isTest = IsTestTag(tagName); + isTest = DataUtil.IsTestTag(tagName); } var data = (DBean)type.Apply(JsonDataCreator.Ins, _data, (DefAssembly)type.Bean.AssemblyBase); diff --git a/src/Luban.Job.Cfg/Source/DataSources/Lua/LuaDataSource.cs b/src/Luban.Job.Cfg/Source/DataSources/Lua/LuaDataSource.cs index a60c394..6d3ee6a 100644 --- a/src/Luban.Job.Cfg/Source/DataSources/Lua/LuaDataSource.cs +++ b/src/Luban.Job.Cfg/Source/DataSources/Lua/LuaDataSource.cs @@ -2,6 +2,7 @@ using Luban.Job.Cfg.DataCreators; using Luban.Job.Cfg.Datas; using Luban.Job.Cfg.Defs; using Luban.Job.Cfg.TypeVisitors; +using Luban.Job.Cfg.Utils; using Luban.Job.Common.Types; using Neo.IronLua; using System.Collections.Generic; @@ -48,12 +49,12 @@ namespace Luban.Job.Cfg.DataSources.Lua protected Record ReadRecord(LuaTable table, TBean type) { string tagName = table.GetValue(TAG_KEY)?.ToString(); - if (IsIgnoreTag(tagName)) + if (DataUtil.IsIgnoreTag(tagName)) { return null; } var data = (DBean)type.Apply(LuaDataCreator.Ins, table, (DefAssembly)type.Bean.AssemblyBase); - var isTest = IsTestTag(tagName); + var isTest = DataUtil.IsTestTag(tagName); return new Record(data, RawUrl, isTest); } } diff --git a/src/Luban.Job.Cfg/Source/DataSources/Xml/XmlDataSource.cs b/src/Luban.Job.Cfg/Source/DataSources/Xml/XmlDataSource.cs index bcd1cee..5049acb 100644 --- a/src/Luban.Job.Cfg/Source/DataSources/Xml/XmlDataSource.cs +++ b/src/Luban.Job.Cfg/Source/DataSources/Xml/XmlDataSource.cs @@ -2,6 +2,7 @@ using Luban.Job.Cfg.DataCreators; using Luban.Job.Cfg.Datas; using Luban.Job.Cfg.Defs; using Luban.Job.Cfg.TypeVisitors; +using Luban.Job.Cfg.Utils; using Luban.Job.Common.Types; using System; using System.Collections.Generic; @@ -28,12 +29,12 @@ namespace Luban.Job.Cfg.DataSources.Xml public override Record ReadOne(TBean type) { string tagName = _doc.Element(TAG_KEY)?.Value; - if (IsIgnoreTag(tagName)) + if (DataUtil.IsIgnoreTag(tagName)) { return null; } var data = (DBean)type.Apply(XmlDataCreator.Ins, _doc, (DefAssembly)type.Bean.AssemblyBase); - bool isTest = IsTestTag(tagName); + bool isTest = DataUtil.IsTestTag(tagName); return new Record(data, RawUrl, isTest); } } diff --git a/src/Luban.Job.Cfg/Source/JobController.cs b/src/Luban.Job.Cfg/Source/JobController.cs index 1717536..f044c51 100644 --- a/src/Luban.Job.Cfg/Source/JobController.cs +++ b/src/Luban.Job.Cfg/Source/JobController.cs @@ -770,7 +770,7 @@ namespace {ctx.TopModule} { var content = DataExporterUtil.ToOutputData(c, ctx.Assembly.GetTableExportDataList(c), genType); var file = GetOutputFileName(genType, c.OutputDataFile); - var md5 = CacheFileUtil.GenMd5AndAddCache(file, content); + var md5 = CacheFileUtil.GenStringOrBytesMd5AndAddCache(file, content); ctx.GenDataFilesInOutputDataDir.Add(new FileInfo() { FilePath = file, MD5 = md5 }); })); } @@ -779,32 +779,30 @@ namespace {ctx.TopModule} private async Task GenJsonDataMonolithic(GenContext ctx) { var exportTables = ctx.ExportTables; - List> allJsonTask = new List>(); + var allJsonTask = new List>(); foreach (var c in exportTables) { allJsonTask.Add(Task.Run(() => { - return DataExporterUtil.ToOutputData(c, ctx.Assembly.GetTableExportDataList(c), "data_json"); + return (string)DataExporterUtil.ToOutputData(c, ctx.Assembly.GetTableExportDataList(c), "data_json"); })); } - await Task.WhenAll(allJsonTask); - int estimatedCapacity = allJsonTask.Sum(t => t.Result.Length + 100); - var sb = new MemoryStream(estimatedCapacity); - sb.Write(System.Text.Encoding.UTF8.GetBytes("{\n")); + var lines = new List(); + + lines.Add("{"); for (int i = 0; i < exportTables.Count; i++) { if (i != 0) { - sb.Write(System.Text.Encoding.UTF8.GetBytes((",\n"))); + lines.Add(","); } - sb.Write(System.Text.Encoding.UTF8.GetBytes("\"" + exportTables[i].Name + "\":")); - sb.Write(allJsonTask[i].Result); + lines.Add($"\"{exportTables[i].NeedExport}\":"); + lines.Add(await allJsonTask[i]); } - sb.Write(System.Text.Encoding.UTF8.GetBytes("\n}")); + lines.Add("}"); - var content = sb.ToArray(); - s_logger.Debug("estimated size:{0} actual size:{1}", estimatedCapacity, content.Length); + var content = string.Join('\n', lines); var outputFile = ctx.GenArgs.OutputDataJsonMonolithicFile; var md5 = CacheFileUtil.GenMd5AndAddCache(outputFile, content); ctx.GenScatteredFiles.Add(new FileInfo() { FilePath = outputFile, MD5 = md5 }); diff --git a/src/Luban.Job.Cfg/Source/Utils/DataExporterUtil.cs b/src/Luban.Job.Cfg/Source/Utils/DataExporterUtil.cs index 6d52a61..250fb69 100644 --- a/src/Luban.Job.Cfg/Source/Utils/DataExporterUtil.cs +++ b/src/Luban.Job.Cfg/Source/Utils/DataExporterUtil.cs @@ -17,7 +17,7 @@ namespace Luban.Job.Cfg.Utils { public static class DataExporterUtil { - public static byte[] ToOutputData(DefTable table, List records, string dataType) + public static object ToOutputData(DefTable table, List records, string dataType) { switch (dataType) { @@ -40,7 +40,7 @@ namespace Luban.Job.Cfg.Utils }); JsonExportor.Ins.WriteList(records, table.Assembly, jsonWriter); jsonWriter.Flush(); - return DataUtil.StreamToBytes(ss); + return System.Text.Encoding.UTF8.GetString(DataUtil.StreamToBytes(ss)); } case "data_lua": { @@ -63,7 +63,7 @@ namespace Luban.Job.Cfg.Utils throw new NotSupportedException(); } } - return System.Text.Encoding.UTF8.GetBytes(string.Join('\n', content)); + return string.Join('\n', content); } default: { diff --git a/src/Luban.Job.Cfg/Source/Utils/DataUtil.cs b/src/Luban.Job.Cfg/Source/Utils/DataUtil.cs index a1d8ec4..e4e306a 100644 --- a/src/Luban.Job.Cfg/Source/Utils/DataUtil.cs +++ b/src/Luban.Job.Cfg/Source/Utils/DataUtil.cs @@ -100,6 +100,26 @@ namespace Luban.Job.Cfg.Utils } } + + public static bool IsIgnoreTag(string tagName) + { + return !string.IsNullOrEmpty(tagName) && + ( + tagName.Equals("false", System.StringComparison.OrdinalIgnoreCase) + || tagName.Equals("no", System.StringComparison.OrdinalIgnoreCase) + || tagName.Equals("##", System.StringComparison.Ordinal) + || tagName.Equals("", System.StringComparison.Ordinal) + ); + } + + public static bool IsTestTag(string tagName) + { + return !string.IsNullOrEmpty(tagName) && + (tagName.Equals("test", System.StringComparison.OrdinalIgnoreCase) + || tagName.Equals("", System.StringComparison.Ordinal) + ); + } + //public static string Data2String(DType data) //{ // var s = new StringBuilder(); diff --git a/src/Luban.Job.Common/Source/Utils/CacheFileUtil.cs b/src/Luban.Job.Common/Source/Utils/CacheFileUtil.cs index 1d38741..b602ffc 100644 --- a/src/Luban.Job.Common/Source/Utils/CacheFileUtil.cs +++ b/src/Luban.Job.Common/Source/Utils/CacheFileUtil.cs @@ -25,6 +25,16 @@ namespace Luban.Job.Common.Utils } } + public static string GenStringOrBytesMd5AndAddCache(string fileName, object content) + { + switch (content) + { + case string s: return GenMd5AndAddCache(fileName, s); + case byte[] bs: return GenMd5AndAddCache(fileName, bs); + default: throw new System.NotSupportedException(); + } + } + public static string GenMd5AndAddCache(string fileName, string content) { content = content.Replace("\r\n", "\n");