From 2ca0cebe99cf50c7341e7192ccc1adc48e6fd01c Mon Sep 17 00:00:00 2001 From: walon Date: Fri, 30 Jul 2021 18:22:08 +0800 Subject: [PATCH] =?UTF-8?q?=E3=80=90=E7=89=B9=E6=80=A7=E3=80=91=E6=94=AF?= =?UTF-8?q?=E6=8C=81=E4=BB=8Eexcel=E6=96=87=E4=BB=B6=E7=9A=84=E6=A0=87?= =?UTF-8?q?=E9=A2=98=E5=A4=B4=E9=87=8C=E7=9B=B4=E6=8E=A5=E8=AF=BB=E5=8F=96?= =?UTF-8?q?=E5=AE=9A=E4=B9=89=EF=BC=8C=E5=AE=9A=E4=B9=89=E5=92=8C=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E4=B8=80=E4=BD=93=E4=BA=86=E3=80=82=E7=AE=80=E5=8C=96?= =?UTF-8?q?=E5=86=99=E8=AE=B0=E5=BD=95=E5=AE=9A=E4=B9=89=E7=9A=84=E9=BA=BB?= =?UTF-8?q?=E7=83=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../DataSources/Excel/ExcelDataSource.cs | 38 +++++- .../Source/DataSources/Excel/Sheet.cs | 19 ++- src/Luban.Job.Cfg/Source/Defs/CfgDefLoader.cs | 129 +++++++++++++++++- src/Luban.Job.Cfg/Source/JobController.cs | 1 + src/Luban.Job.Cfg/Source/RawDefs/Table.cs | 2 + .../Source/Utils/TypescriptStringTemplate.cs | 12 +- .../Source/Generate/TypescriptRender.cs | 2 +- src/Luban.Job.Db/Source/JobController.cs | 22 +-- 8 files changed, 199 insertions(+), 26 deletions(-) diff --git a/src/Luban.Job.Cfg/Source/DataSources/Excel/ExcelDataSource.cs b/src/Luban.Job.Cfg/Source/DataSources/Excel/ExcelDataSource.cs index c3a6cb7..5e69ef1 100644 --- a/src/Luban.Job.Cfg/Source/DataSources/Excel/ExcelDataSource.cs +++ b/src/Luban.Job.Cfg/Source/DataSources/Excel/ExcelDataSource.cs @@ -52,10 +52,46 @@ namespace Luban.Job.Cfg.DataSources.Excel } } + public Sheet LoadFirstSheet(string rawUrl, string sheetName, Stream stream) + { + s_logger.Trace("{filename} {sheet}", rawUrl, sheetName); + RawUrl = rawUrl; + string ext = Path.GetExtension(rawUrl); + using (var reader = ext != ".csv" ? ExcelReaderFactory.CreateReader(stream) : ExcelReaderFactory.CreateCsvReader(stream)) + { + do + { + if (sheetName == null || reader.Name == sheetName) + { + try + { + var sheet = ReadSheetHeader(rawUrl, reader); + if (sheet != null) + { + return sheet; + } + } + catch (Exception e) + { + throw new Exception($"excel:{rawUrl} sheet:{reader.Name} 读取失败. ==> {e.Message}", e); + } + + } + } while (reader.NextResult()); + } + throw new Exception($"excel:{rawUrl} 不包含有效的单元薄(有效单元薄的A0单元格必须是##)."); + } + private Sheet ReadSheet(string url, IExcelDataReader reader) { var sheet = new Sheet(url, reader.Name ?? ""); - return sheet.Load(reader) ? sheet : null; + return sheet.Load(reader, false) ? sheet : null; + } + + private Sheet ReadSheetHeader(string url, IExcelDataReader reader) + { + var sheet = new Sheet(url, reader.Name ?? ""); + return sheet.Load(reader, true) ? sheet : null; } public override List ReadMulti(TBean type) diff --git a/src/Luban.Job.Cfg/Source/DataSources/Excel/Sheet.cs b/src/Luban.Job.Cfg/Source/DataSources/Excel/Sheet.cs index 44891f9..7fed36f 100644 --- a/src/Luban.Job.Cfg/Source/DataSources/Excel/Sheet.cs +++ b/src/Luban.Job.Cfg/Source/DataSources/Excel/Sheet.cs @@ -25,6 +25,10 @@ namespace Luban.Job.Cfg.DataSources.Excel private Title _rootTitle; + public List RootFields => _rootTitle.SubTitleList; + + public List<List<Cell>> RowColumns => _rowColumns; + public class Title { public int FromIndex { get; set; } @@ -213,7 +217,7 @@ namespace Luban.Job.Cfg.DataSources.Excel this.Name = name; } - public bool Load(IExcelDataReader reader) + public bool Load(IExcelDataReader reader, bool headerOnly) { //s_logger.Info("read sheet:{sheet}", reader.Name); if (!ParseMeta(reader)) @@ -221,7 +225,7 @@ namespace Luban.Job.Cfg.DataSources.Excel return false; } - LoadRemainRows(reader); + LoadRemainRows(reader, headerOnly); return true; } @@ -368,7 +372,7 @@ namespace Luban.Job.Cfg.DataSources.Excel } - private void LoadRemainRows(IExcelDataReader reader) + private void LoadRemainRows(IExcelDataReader reader, bool headerOnly) { // TODO 优化性能 // 几个思路 @@ -508,7 +512,14 @@ namespace Luban.Job.Cfg.DataSources.Excel //} // 删除标题行 - this._rowColumns.RemoveRange(0, Math.Min(TitleRows + titleRowNum - 1, this._rowColumns.Count)); + if (headerOnly) + { + this._rowColumns.RemoveRange(0, Math.Min(titleRowNum, this._rowColumns.Count)); + } + else + { + this._rowColumns.RemoveRange(0, Math.Min(TitleRows + titleRowNum - 1, this._rowColumns.Count)); + } // 删除忽略的记录行 this._rowColumns.RemoveAll(row => AbstractDataSource.IsIgnoreTag(GetRowTag(row))); diff --git a/src/Luban.Job.Cfg/Source/Defs/CfgDefLoader.cs b/src/Luban.Job.Cfg/Source/Defs/CfgDefLoader.cs index b40d5b8..4462f0f 100644 --- a/src/Luban.Job.Cfg/Source/Defs/CfgDefLoader.cs +++ b/src/Luban.Job.Cfg/Source/Defs/CfgDefLoader.cs @@ -1,11 +1,15 @@ using Luban.Common.Utils; +using Luban.Job.Cfg.DataSources.Excel; using Luban.Job.Cfg.RawDefs; +using Luban.Job.Cfg.Utils; using Luban.Job.Common.Defs; using Luban.Job.Common.RawDefs; using Luban.Server.Common; using System; using System.Collections.Generic; +using System.IO; using System.Linq; +using System.Threading.Tasks; using System.Xml.Linq; namespace Luban.Job.Cfg.Defs @@ -152,9 +156,6 @@ namespace Luban.Job.Cfg.Defs _cfgServices.Add(new Service() { Name = name, Manager = manager, Groups = groups, Refs = refs }); } - private readonly List<string> _tableOptionalAttrs = new List<string> { "index", "mode", "group", "branch_input", "comment" }; - private readonly List<string> _tableRequireAttrs = new List<string> { "name", "value", "input" }; - private readonly Dictionary<string, Table> _name2CfgTable = new Dictionary<string, Table>(); @@ -213,6 +214,9 @@ namespace Luban.Job.Cfg.Defs return mode; } + private readonly List<string> _tableOptionalAttrs = new List<string> { "index", "mode", "group", "branch_input", "comment", "define_from_file" }; + private readonly List<string> _tableRequireAttrs = new List<string> { "name", "value", "input" }; + private void AddTable(XElement e) { ValidAttrKeys(e, _tableOptionalAttrs, _tableRequireAttrs); @@ -222,6 +226,7 @@ namespace Luban.Job.Cfg.Defs Name = XmlUtil.GetRequiredAttribute(e, "name"), Namespace = CurNamespace, ValueType = XmlUtil.GetRequiredAttribute(e, "value"), + LoadDefineFromFile = XmlUtil.GetOptionBoolAttribute(e, "define_from_file"), Index = XmlUtil.GetOptionalAttribute(e, "index"), Groups = CreateGroups(XmlUtil.GetOptionalAttribute(e, "group")), Comment = XmlUtil.GetOptionalAttribute(e, "comment"), @@ -266,6 +271,124 @@ namespace Luban.Job.Cfg.Defs + private async Task<CfgBean> LoadDefineFromFileAsync(Table table, string dataDir) + { + var inputFileInfos = await DataLoaderUtil.CollectInputFilesAsync(this.Agent, table.InputFiles, dataDir); + var file = inputFileInfos[0]; + + var source = new ExcelDataSource(); + var stream = new MemoryStream(await this.Agent.GetFromCacheOrReadAllBytesAsync(file.ActualFile, file.MD5)); + var sheet = source.LoadFirstSheet(file.OriginFile, file.SheetName, stream); + + var cb = new CfgBean() { Namespace = table.Namespace, Name = table.ValueType, }; + + var rc = sheet.RowColumns; + var attrRow = sheet.RowColumns[0]; + var titleRow = sheet.RowColumns[1]; + var descRow = sheet.RowColumns[2]; + foreach (var f in sheet.RootFields) + { + var cf = new CfgField() { Name = f.Name, Id = 0 }; + + + string[] attrs = attrRow[f.FromIndex].Value?.ToString().Split('&'); + + if (attrs.Length == 0 || string.IsNullOrWhiteSpace(attrs[0])) + { + throw new Exception($"table:{table.Name} file:{file.OriginFile} title:{f.Name} type missing!"); + } + + // 优先取desc行，如果为空,则取title行 + + cf.Comment = descRow[f.FromIndex].Value?.ToString(); + if (string.IsNullOrWhiteSpace(cf.Comment)) + { + cf.Comment = titleRow[f.FromIndex].Value?.ToString(); + } + if (string.IsNullOrWhiteSpace(cf.Comment)) + { + cf.Comment = ""; + } + + cf.Type = attrs[0]; + + for (int i = 1; i < attrs.Length; i++) + { + var pair = attrs[i].Split('='); + if (pair.Length != 2) + { + throw new Exception($"table:{table.Name} file:{file.OriginFile} title:{f.Name} attr: '{attrs[i]}' is invalid!"); + } + var attrName = pair[0]; + var attrValue = pair[1]; + switch (attrName) + { + case "index": + { + cf.Index = attrValue; + break; + } + case "sep": + { + cf.Sep = attrValue; + break; + } + case "ref": + case "path": + case "range": + { + var validator = new Validator() { Type = attrName, Rule = attrValue }; + cf.Validators.Add(validator); + cf.ValueValidators.Add(validator); + break; + } + case "multi_lines": + { + cf.IsMultiRow = attrValue == "1" || attrValue.Equals("true", StringComparison.OrdinalIgnoreCase); + break; + } + case "group": + { + cf.Groups = attrValue.Split(',').Select(s => s.Trim()).Where(s => !string.IsNullOrWhiteSpace(s)).ToList(); + break; + } + case "comment": + { + cf.Comment = attrValue; + break; + } + case "convert": + { + cf.Converter = attrValue; + break; + } + default: + { + throw new Exception($"table:{table.Name} file:{file.OriginFile} title:{f.Name} attr: '{attrs[i]}' is invalid!"); + } + } + } + + cb.Fields.Add(cf); + } + return cb; + } + + public async Task LoadDefinesFromFileAsync(string dataDir) + { + var loadTasks = new List<Task<CfgBean>>(); + foreach (var table in this._cfgTables.Where(t => t.LoadDefineFromFile)) + { + loadTasks.Add(Task.Run(async () => await this.LoadDefineFromFileAsync(table, dataDir))); + } + + foreach (var task in loadTasks) + { + this._beans.Add(await task); + } + } + + private static readonly List<string> _fieldOptionalAttrs = new List<string> { "index", "sep", "validator", "key_validator", "value_validator", "ref", "path", "range", "multi_rows", "group", "res", "convert", "comment" }; diff --git a/src/Luban.Job.Cfg/Source/JobController.cs b/src/Luban.Job.Cfg/Source/JobController.cs index fd78588..75a8d5d 100644 --- a/src/Luban.Job.Cfg/Source/JobController.cs +++ b/src/Luban.Job.Cfg/Source/JobController.cs @@ -261,6 +261,7 @@ namespace Luban.Job.Cfg timer.StartPhase("build defines"); var loader = new CfgDefLoader(agent); await loader.LoadAsync(args.DefineFile); + await loader.LoadDefinesFromFileAsync(inputDataDir); timer.EndPhaseAndLog(); var rawDefines = loader.BuildDefines(); diff --git a/src/Luban.Job.Cfg/Source/RawDefs/Table.cs b/src/Luban.Job.Cfg/Source/RawDefs/Table.cs index 20f54ad..31821e8 100644 --- a/src/Luban.Job.Cfg/Source/RawDefs/Table.cs +++ b/src/Luban.Job.Cfg/Source/RawDefs/Table.cs @@ -29,6 +29,8 @@ namespace Luban.Job.Cfg.RawDefs public string ValueType { get; set; } + public bool LoadDefineFromFile { get; set; } + public ETableMode Mode { get; set; } public string Comment { get; set; } diff --git a/src/Luban.Job.Common/Source/Utils/TypescriptStringTemplate.cs b/src/Luban.Job.Common/Source/Utils/TypescriptStringTemplate.cs index 8c79b56..26faf6b 100644 --- a/src/Luban.Job.Common/Source/Utils/TypescriptStringTemplate.cs +++ b/src/Luban.Job.Common/Source/Utils/TypescriptStringTemplate.cs @@ -18,7 +18,7 @@ import ByteBuf = Bright.Serialization.ByteBuf"; } public const string BrightByteBufPathImportsFormat = "import ByteBuf from '{0}/serialization/ByteBuf'"; - public const string BrightByteBufPackageImportsFormat = "import ByteBuf from '{0}'"; + public const string BrightByteBufPackageImportsFormat = "import {{ByteBuf}} from '{0}'"; public static string GetVectorImports(string path, string package) { @@ -31,9 +31,9 @@ import Vector3 from '{0}/math/Vector3' import Vector4 from '{0}/math/Vector4' "; public const string VectorPackageImportsFormat = @" -import Vector2 from '{0}' -import Vector3 from '{0}' -import Vector4 from '{0}' +import {{Vector2}} from '{0}' +import {{Vector3}} from '{0}' +import {{Vector4}} from '{0}' "; public static string GetSerializeImports(string path, string package) { @@ -41,14 +41,14 @@ import Vector4 from '{0}' } public const string SerializePathImportsFormat = @"import BeanBase from '{0}/serialization/BeanBase'"; - public const string SerializePackageImportsFormat = @"import BeanBase from '{0}'"; + public const string SerializePackageImportsFormat = @"import {{BeanBase}} from '{0}'"; public static string GetProtocolImports(string path, string package) { return string.IsNullOrEmpty(package) ? string.Format(ProtocolPathImportsFormat, path) : string.Format(ProtocolPackageImportsFormat, package); } public const string ProtocolPathImportsFormat = "import Protocol from '{0}/net/Protocol'"; - public const string ProtocolPackageImportsFormat = "import Protocol from '{0}'"; + public const string ProtocolPackageImportsFormat = "import {{Protocol}} from '{0}'"; public const string SerializeTypes = @" export interface ISerializable { diff --git a/src/Luban.Job.Db/Source/Generate/TypescriptRender.cs b/src/Luban.Job.Db/Source/Generate/TypescriptRender.cs index fed0c30..9c9c246 100644 --- a/src/Luban.Job.Db/Source/Generate/TypescriptRender.cs +++ b/src/Luban.Job.Db/Source/Generate/TypescriptRender.cs @@ -106,7 +106,7 @@ export {{x.ts_class_modifier}} class {{name}} extends {{if parent_def_type}} {{x if (value == null) throw new Error() {{~end~}} if (this.isManaged) { - let txn = TransactionContext.current + let txn = TransactionContext.current! txn.putFieldLong(this.getObjectId() + {{field.id}}, new {{name}}.{{field.log_type}}(this, value)) {{~if ctype.need_set_children_root~}} value?.initRoot(this.getRoot()) diff --git a/src/Luban.Job.Db/Source/JobController.cs b/src/Luban.Job.Db/Source/JobController.cs index b74f90e..472cee8 100644 --- a/src/Luban.Job.Db/Source/JobController.cs +++ b/src/Luban.Job.Db/Source/JobController.cs @@ -162,19 +162,19 @@ namespace Luban.Job.Db else { fileContent.Add($"import {{FieldLogger, FieldLoggerGeneric1, FieldLoggerGeneric2}} from '{brightPackageName}'"); - fileContent.Add($"import TxnBeanBase from '{brightPackageName}'"); + fileContent.Add($"import {{TxnBeanBase}} from '{brightPackageName}'"); fileContent.Add($"import {{TxnTable, TxnTableGeneric}} from '{brightPackageName}'"); - fileContent.Add($"import TransactionContext from '{brightPackageName}'"); + fileContent.Add($"import {{TransactionContext}} from '{brightPackageName}'"); fileContent.Add($"import {{FieldTag}} from '{brightPackageName}'"); - fileContent.Add($"import TKey from '{brightPackageName}'"); - fileContent.Add($"import PList from '{brightPackageName}'"); - fileContent.Add($"import PList1 from '{brightPackageName}'"); - fileContent.Add($"import PList2 from '{brightPackageName}'"); - fileContent.Add($"import PSet from '{brightPackageName}'"); - fileContent.Add($"import PMap from '{brightPackageName}'"); - fileContent.Add($"import PMap1 from '{brightPackageName}'"); - fileContent.Add($"import PMap2 from '{brightPackageName}'"); - fileContent.Add($"import SerializeFactory from '{brightPackageName}'"); + fileContent.Add($"import {{TKey}} from '{brightPackageName}'"); + fileContent.Add($"import {{PList}} from '{brightPackageName}'"); + fileContent.Add($"import {{PList1}} from '{brightPackageName}'"); + fileContent.Add($"import {{PList2}} from '{brightPackageName}'"); + fileContent.Add($"import {{PSet}} from '{brightPackageName}'"); + fileContent.Add($"import {{PMap}} from '{brightPackageName}'"); + fileContent.Add($"import {{PMap1}} from '{brightPackageName}'"); + fileContent.Add($"import {{PMap2}} from '{brightPackageName}'"); + fileContent.Add($"import {{SerializeFactory}} from '{brightPackageName}'"); } fileContent.Add($"export namespace {ass.TopModule} {{");