[new] 支持纯流式模式的excel解析方式

main
walon 2022-08-10 15:22:51 +08:00
parent 154d55f372
commit c60f91fdd4
13 changed files with 232 additions and 53 deletions

View File

@ -15,7 +15,7 @@ using System.Threading.Tasks;
namespace Luban.Job.Cfg.DataCreators namespace Luban.Job.Cfg.DataCreators
{ {
class SheetDataCreator : ITypeFuncVisitor<Sheet, TitleRow, DType> class SheetDataCreator : ITypeFuncVisitor<RowColumnSheet, TitleRow, DType>
{ {
public static SheetDataCreator Ins { get; } = new(); public static SheetDataCreator Ins { get; } = new();
@ -37,7 +37,7 @@ namespace Luban.Job.Cfg.DataCreators
} }
} }
public DType Accept(TBool type, Sheet sheet, TitleRow row) public DType Accept(TBool type, RowColumnSheet sheet, TitleRow row)
{ {
object x = row.Current; object x = row.Current;
if (CheckNull(type.IsNullable, x)) if (CheckNull(type.IsNullable, x))
@ -56,7 +56,7 @@ namespace Luban.Job.Cfg.DataCreators
return DBool.ValueOf(bool.Parse(x.ToString())); return DBool.ValueOf(bool.Parse(x.ToString()));
} }
public DType Accept(TByte type, Sheet sheet, TitleRow row) public DType Accept(TByte type, RowColumnSheet sheet, TitleRow row)
{ {
object x = row.Current; object x = row.Current;
if (CheckNull(type.IsNullable, x)) if (CheckNull(type.IsNullable, x))
@ -70,7 +70,7 @@ namespace Luban.Job.Cfg.DataCreators
return DByte.ValueOf(byte.Parse(x.ToString())); return DByte.ValueOf(byte.Parse(x.ToString()));
} }
public DType Accept(TShort type, Sheet sheet, TitleRow row) public DType Accept(TShort type, RowColumnSheet sheet, TitleRow row)
{ {
object x = row.Current; object x = row.Current;
if (CheckNull(type.IsNullable, x)) if (CheckNull(type.IsNullable, x))
@ -85,7 +85,7 @@ namespace Luban.Job.Cfg.DataCreators
return DShort.ValueOf(short.Parse(x.ToString())); return DShort.ValueOf(short.Parse(x.ToString()));
} }
public DType Accept(TFshort type, Sheet sheet, TitleRow row) public DType Accept(TFshort type, RowColumnSheet sheet, TitleRow row)
{ {
object x = row.Current; object x = row.Current;
if (CheckNull(type.IsNullable, x)) if (CheckNull(type.IsNullable, x))
@ -99,7 +99,7 @@ namespace Luban.Job.Cfg.DataCreators
return DFshort.ValueOf(short.Parse(x.ToString())); return DFshort.ValueOf(short.Parse(x.ToString()));
} }
public DType Accept(TInt type, Sheet sheet, TitleRow row) public DType Accept(TInt type, RowColumnSheet sheet, TitleRow row)
{ {
object x = row.Current; object x = row.Current;
if (CheckNull(type.IsNullable, x)) if (CheckNull(type.IsNullable, x))
@ -114,7 +114,7 @@ namespace Luban.Job.Cfg.DataCreators
return DInt.ValueOf(int.Parse(x.ToString())); return DInt.ValueOf(int.Parse(x.ToString()));
} }
public DType Accept(TFint type, Sheet sheet, TitleRow row) public DType Accept(TFint type, RowColumnSheet sheet, TitleRow row)
{ {
object x = row.Current; object x = row.Current;
if (CheckNull(type.IsNullable, x)) if (CheckNull(type.IsNullable, x))
@ -129,7 +129,7 @@ namespace Luban.Job.Cfg.DataCreators
return DFint.ValueOf(int.Parse(x.ToString())); return DFint.ValueOf(int.Parse(x.ToString()));
} }
public DType Accept(TLong type, Sheet sheet, TitleRow row) public DType Accept(TLong type, RowColumnSheet sheet, TitleRow row)
{ {
object x = row.Current; object x = row.Current;
if (CheckNull(type.IsNullable, x)) if (CheckNull(type.IsNullable, x))
@ -144,7 +144,7 @@ namespace Luban.Job.Cfg.DataCreators
return DLong.ValueOf(long.Parse(x.ToString())); return DLong.ValueOf(long.Parse(x.ToString()));
} }
public DType Accept(TFlong type, Sheet sheet, TitleRow row) public DType Accept(TFlong type, RowColumnSheet sheet, TitleRow row)
{ {
object x = row.Current; object x = row.Current;
if (CheckNull(type.IsNullable, x)) if (CheckNull(type.IsNullable, x))
@ -159,7 +159,7 @@ namespace Luban.Job.Cfg.DataCreators
return DFlong.ValueOf(long.Parse(x.ToString())); return DFlong.ValueOf(long.Parse(x.ToString()));
} }
public DType Accept(TFloat type, Sheet sheet, TitleRow row) public DType Accept(TFloat type, RowColumnSheet sheet, TitleRow row)
{ {
object x = row.Current; object x = row.Current;
if (CheckNull(type.IsNullable, x)) if (CheckNull(type.IsNullable, x))
@ -174,7 +174,7 @@ namespace Luban.Job.Cfg.DataCreators
return DFloat.ValueOf(float.Parse(x.ToString())); return DFloat.ValueOf(float.Parse(x.ToString()));
} }
public DType Accept(TDouble type, Sheet sheet, TitleRow row) public DType Accept(TDouble type, RowColumnSheet sheet, TitleRow row)
{ {
object x = row.Current; object x = row.Current;
if (CheckNull(type.IsNullable, x)) if (CheckNull(type.IsNullable, x))
@ -189,7 +189,7 @@ namespace Luban.Job.Cfg.DataCreators
return DDouble.ValueOf(double.Parse(x.ToString())); return DDouble.ValueOf(double.Parse(x.ToString()));
} }
public DType Accept(TEnum type, Sheet sheet, TitleRow row) public DType Accept(TEnum type, RowColumnSheet sheet, TitleRow row)
{ {
object x = row.Current; object x = row.Current;
if (CheckNull(type.IsNullable, x)) if (CheckNull(type.IsNullable, x))
@ -227,7 +227,7 @@ namespace Luban.Job.Cfg.DataCreators
} }
} }
public DType Accept(TString type, Sheet sheet, TitleRow row) public DType Accept(TString type, RowColumnSheet sheet, TitleRow row)
{ {
object x = row.Current; object x = row.Current;
if (CheckDefault(x)) if (CheckDefault(x))
@ -249,12 +249,12 @@ namespace Luban.Job.Cfg.DataCreators
return DString.ValueOf(s); return DString.ValueOf(s);
} }
public DType Accept(TBytes type, Sheet sheet, TitleRow row) public DType Accept(TBytes type, RowColumnSheet sheet, TitleRow row)
{ {
throw new NotSupportedException(); throw new NotSupportedException();
} }
public DType Accept(TText type, Sheet sheet, TitleRow row) public DType Accept(TText type, RowColumnSheet sheet, TitleRow row)
{ {
if (string.IsNullOrEmpty(row.SelfTitle.SepOr(type.GetTag("sep")))) if (string.IsNullOrEmpty(row.SelfTitle.SepOr(type.GetTag("sep"))))
{ {
@ -279,7 +279,7 @@ namespace Luban.Job.Cfg.DataCreators
} }
} }
public DType Accept(TDateTime type, Sheet sheet, TitleRow row) public DType Accept(TDateTime type, RowColumnSheet sheet, TitleRow row)
{ {
var d = row.Current; var d = row.Current;
if (CheckNull(type.IsNullable, d)) if (CheckNull(type.IsNullable, d))
@ -293,7 +293,7 @@ namespace Luban.Job.Cfg.DataCreators
return DataUtil.CreateDateTime(d.ToString()); return DataUtil.CreateDateTime(d.ToString());
} }
public DType Accept(TVector2 type, Sheet sheet, TitleRow row) public DType Accept(TVector2 type, RowColumnSheet sheet, TitleRow row)
{ {
var d = row.Current; var d = row.Current;
if (CheckNull(type.IsNullable, d)) if (CheckNull(type.IsNullable, d))
@ -308,7 +308,7 @@ namespace Luban.Job.Cfg.DataCreators
return DataUtil.CreateVector(type, d.ToString()); return DataUtil.CreateVector(type, d.ToString());
} }
public DType Accept(TVector3 type, Sheet sheet, TitleRow row) public DType Accept(TVector3 type, RowColumnSheet sheet, TitleRow row)
{ {
var d = row.Current; var d = row.Current;
if (CheckNull(type.IsNullable, d)) if (CheckNull(type.IsNullable, d))
@ -323,7 +323,7 @@ namespace Luban.Job.Cfg.DataCreators
return DataUtil.CreateVector(type, d.ToString()); return DataUtil.CreateVector(type, d.ToString());
} }
public DType Accept(TVector4 type, Sheet sheet, TitleRow row) public DType Accept(TVector4 type, RowColumnSheet sheet, TitleRow row)
{ {
var d = row.Current; var d = row.Current;
if (CheckNull(type.IsNullable, d)) if (CheckNull(type.IsNullable, d))
@ -338,7 +338,7 @@ namespace Luban.Job.Cfg.DataCreators
return DataUtil.CreateVector(type, d.ToString()); return DataUtil.CreateVector(type, d.ToString());
} }
private List<DType> CreateBeanFields(DefBean bean, Sheet sheet, TitleRow row) private List<DType> CreateBeanFields(DefBean bean, RowColumnSheet sheet, TitleRow row)
{ {
var list = new List<DType>(); var list = new List<DType>();
foreach (DefField f in bean.HierarchyFields) foreach (DefField f in bean.HierarchyFields)
@ -369,7 +369,7 @@ namespace Luban.Job.Cfg.DataCreators
return list; return list;
} }
public DType Accept(TBean type, Sheet sheet, TitleRow row) public DType Accept(TBean type, RowColumnSheet sheet, TitleRow row)
{ {
string sep = row.SelfTitle.Sep;// type.GetBeanAs<DefBean>().Sep; string sep = row.SelfTitle.Sep;// type.GetBeanAs<DefBean>().Sep;
@ -479,7 +479,7 @@ namespace Luban.Job.Cfg.DataCreators
return datas; return datas;
} }
private List<DType> ReadCollectionDatas(TType type, TType elementType, Sheet sheet, TitleRow row) private List<DType> ReadCollectionDatas(TType type, TType elementType, RowColumnSheet sheet, TitleRow row)
{ {
if (row.Row != null) if (row.Row != null)
{ {
@ -518,23 +518,23 @@ namespace Luban.Job.Cfg.DataCreators
} }
} }
public DType Accept(TArray type, Sheet sheet, TitleRow row) public DType Accept(TArray type, RowColumnSheet sheet, TitleRow row)
{ {
//string sep = DataUtil.GetSep(type); //string sep = DataUtil.GetSep(type);
return new DArray(type, ReadCollectionDatas(type, type.ElementType, sheet, row)); return new DArray(type, ReadCollectionDatas(type, type.ElementType, sheet, row));
} }
public DType Accept(TList type, Sheet sheet, TitleRow row) public DType Accept(TList type, RowColumnSheet sheet, TitleRow row)
{ {
return new DList(type, ReadCollectionDatas(type, type.ElementType, sheet, row)); return new DList(type, ReadCollectionDatas(type, type.ElementType, sheet, row));
} }
public DType Accept(TSet type, Sheet sheet, TitleRow row) public DType Accept(TSet type, RowColumnSheet sheet, TitleRow row)
{ {
return new DSet(type, ReadCollectionDatas(type, type.ElementType, sheet, row)); return new DSet(type, ReadCollectionDatas(type, type.ElementType, sheet, row));
} }
public DType Accept(TMap type, Sheet sheet, TitleRow row) public DType Accept(TMap type, RowColumnSheet sheet, TitleRow row)
{ {
string sep = row.SelfTitle.Sep; string sep = row.SelfTitle.Sep;
@ -563,7 +563,7 @@ namespace Luban.Job.Cfg.DataCreators
foreach (var e in row.Fields) foreach (var e in row.Fields)
{ {
var keyData = type.KeyType.Apply(StringDataCreator.Ins, e.Key); var keyData = type.KeyType.Apply(StringDataCreator.Ins, e.Key);
if (Sheet.IsBlankRow(e.Value.Row, e.Value.SelfTitle.FromIndex, e.Value.SelfTitle.ToIndex)) if (RowColumnSheet.IsBlankRow(e.Value.Row, e.Value.SelfTitle.FromIndex, e.Value.SelfTitle.ToIndex))
{ {
continue; continue;
} }

View File

@ -1,5 +1,8 @@
using Luban.Job.Cfg.DataCreators; using Luban.Job.Cfg.DataCreators;
using Luban.Job.Cfg.DataSources.Excel;
using Luban.Job.Cfg.Defs;
using System; using System;
using System.Collections.Generic;
using System.IO; using System.IO;
namespace Luban.Job.Cfg.DataSources namespace Luban.Job.Cfg.DataSources
@ -18,7 +21,35 @@ namespace Luban.Job.Cfg.DataSources
".asset", ".asset",
}; };
public static AbstractDataSource Create(string url, string sheetName, Stream stream) private static string GetSheetParserMode(Dictionary<string, string> options)
{
if (options != null && options.TryGetValue("parser_mode", out var modeStr))
{
return modeStr;
}
//options = DefAssembly.LocalAssebmly.Options;
//if (options != null && options.TryGetValue("sheet.parser_mode", out modeStr))
//{
// return modeStr;
//}
return "";
}
private static bool IsColumnMode(string mode)
{
if (string.IsNullOrEmpty(mode))
{
return true;
}
switch(mode.ToLowerInvariant())
{
case "column": return true;
case "stream": return false;
default: throw new Exception($"unknown parser_mode:{mode}");
}
}
public static AbstractDataSource Create(string url, string sheetName, Dictionary<string, string> options, Stream stream)
{ {
try try
{ {
@ -29,7 +60,11 @@ namespace Luban.Job.Cfg.DataSources
case "csv": case "csv":
case "xls": case "xls":
case "xlsx": case "xlsx":
case "xlsm": source = new Excel.ExcelDataSource(); break; case "xlsm":
{
source = IsColumnMode(GetSheetParserMode(options)) ? new ExcelRowColumnDataSource() : new ExcelStreamDataSource();
break;
}
case "xml": source = new Xml.XmlDataSource(); break; case "xml": source = new Xml.XmlDataSource(); break;
case "lua": source = new Lua.LuaDataSource(); break; case "lua": source = new Lua.LuaDataSource(); break;
case "json": source = new Json.JsonDataSource(); break; case "json": source = new Json.JsonDataSource(); break;

View File

@ -10,11 +10,11 @@ using System.IO;
namespace Luban.Job.Cfg.DataSources.Excel namespace Luban.Job.Cfg.DataSources.Excel
{ {
class ExcelDataSource : AbstractDataSource class ExcelRowColumnDataSource : AbstractDataSource
{ {
private static readonly NLog.Logger s_logger = NLog.LogManager.GetCurrentClassLogger(); private static readonly NLog.Logger s_logger = NLog.LogManager.GetCurrentClassLogger();
private readonly List<Sheet> _sheets = new List<Sheet>(); private readonly List<RowColumnSheet> _sheets = new List<RowColumnSheet>();
public override void Load(string rawUrl, string sheetName, Stream stream) public override void Load(string rawUrl, string sheetName, Stream stream)
@ -25,7 +25,7 @@ namespace Luban.Job.Cfg.DataSources.Excel
foreach (RawSheet rawSheet in SheetLoadUtil.LoadRawSheets(rawUrl, sheetName, stream)) foreach (RawSheet rawSheet in SheetLoadUtil.LoadRawSheets(rawUrl, sheetName, stream))
{ {
var sheet = new Sheet(rawUrl, sheetName); var sheet = new RowColumnSheet(rawUrl, sheetName);
sheet.Load(rawSheet); sheet.Load(rawSheet);
_sheets.Add(sheet); _sheets.Add(sheet);
} }
@ -40,7 +40,7 @@ namespace Luban.Job.Cfg.DataSources.Excel
{ {
foreach (RawSheet rawSheet in rawSheets) foreach (RawSheet rawSheet in rawSheets)
{ {
var sheet = new Sheet("__intern__", rawSheet.TableName); var sheet = new RowColumnSheet("__intern__", rawSheet.TableName);
sheet.Load(rawSheet); sheet.Load(rawSheet);
_sheets.Add(sheet); _sheets.Add(sheet);
} }

View File

@ -0,0 +1,76 @@
using ExcelDataReader;
using Luban.Job.Cfg.DataCreators;
using Luban.Job.Cfg.Datas;
using Luban.Job.Cfg.Utils;
using Luban.Job.Common.Types;
using System;
using System.Collections.Generic;
using System.IO;
namespace Luban.Job.Cfg.DataSources.Excel
{
class ExcelStreamDataSource : AbstractDataSource
{
private static readonly NLog.Logger s_logger = NLog.LogManager.GetCurrentClassLogger();
private readonly List<StreamSheet> _sheets = new List<StreamSheet>();
public override void Load(string rawUrl, string sheetName, Stream stream)
{
s_logger.Trace("{filename} {sheet}", rawUrl, sheetName);
RawUrl = rawUrl;
foreach (RawSheet rawSheet in SheetLoadUtil.LoadRawSheets(rawUrl, sheetName, stream))
{
var sheet = new StreamSheet(rawUrl, sheetName);
sheet.Load(rawSheet);
_sheets.Add(sheet);
}
if (_sheets.Count == 0)
{
throw new Exception($"excel:{rawUrl} 不包含有效的单元薄(有效单元薄的A0单元格必须是##).");
}
}
public RawSheetTableDefInfo LoadTableDefInfo(string rawUrl, string sheetName, Stream stream)
{
return SheetLoadUtil.LoadSheetTableDefInfo(rawUrl, sheetName, stream);
}
public override List<Record> ReadMulti(TBean type)
{
var datas = new List<Record>();
foreach (var sheet in _sheets)
{
try
{
var stream = sheet.Stream;
while(!stream.TryReadEOF())
{
var data = (DBean)type.Apply(ExcelStreamDataCreator.Ins, stream);
datas.Add(new Record(data, sheet.RawUrl, null));
}
}
catch (DataCreateException dce)
{
dce.OriginDataLocation = sheet.RawUrl;
throw;
}
catch (Exception e)
{
throw new Exception($"sheet:{sheet.Name}", e);
}
}
return datas;
}
public override Record ReadOne(TBean type)
{
throw new Exception($"excel不支持单例读取模式");
}
}
}

View File

@ -12,7 +12,7 @@ using System.Linq;
namespace Luban.Job.Cfg.DataSources.Excel namespace Luban.Job.Cfg.DataSources.Excel
{ {
class Sheet class RowColumnSheet
{ {
private static readonly NLog.Logger s_logger = NLog.LogManager.GetCurrentClassLogger(); private static readonly NLog.Logger s_logger = NLog.LogManager.GetCurrentClassLogger();
@ -22,7 +22,7 @@ namespace Luban.Job.Cfg.DataSources.Excel
public List<(string Tag, TitleRow Row)> Rows { get; } = new(); public List<(string Tag, TitleRow Row)> Rows { get; } = new();
public Sheet(string rawUrl, string name) public RowColumnSheet(string rawUrl, string name)
{ {
this.RawUrl = rawUrl; this.RawUrl = rawUrl;
this.Name = name; this.Name = name;

View File

@ -0,0 +1,37 @@
using Bright.Collections;
using ExcelDataReader;
using Luban.Job.Cfg.DataCreators;
using Luban.Job.Cfg.Datas;
using Luban.Job.Cfg.Utils;
using Luban.Job.Common.Types;
using Luban.Job.Common.Utils;
using System;
using System.Collections.Generic;
using System.Linq;
namespace Luban.Job.Cfg.DataSources.Excel
{
class StreamSheet
{
private static readonly NLog.Logger s_logger = NLog.LogManager.GetCurrentClassLogger();
public string Name { get; }
public string RawUrl { get; }
public ExcelStream Stream { get; private set; }
public StreamSheet(string rawUrl, string name)
{
this.RawUrl = rawUrl;
this.Name = name;
}
public void Load(RawSheet rawSheet)
{
Title title = rawSheet.Title;
Stream = new ExcelStream(rawSheet.Cells, 1, title.ToIndex, "", "");
}
}
}

View File

@ -75,11 +75,11 @@ namespace Luban.Job.Cfg.DataSources.Excel
{ {
if (Row != null) if (Row != null)
{ {
return Sheet.IsBlankRow(Row, SelfTitle.FromIndex, SelfTitle.ToIndex); return RowColumnSheet.IsBlankRow(Row, SelfTitle.FromIndex, SelfTitle.ToIndex);
} }
if (Rows != null) if (Rows != null)
{ {
return Sheet.IsBlankRow(Rows[0], SelfTitle.FromIndex, SelfTitle.ToIndex); return RowColumnSheet.IsBlankRow(Rows[0], SelfTitle.FromIndex, SelfTitle.ToIndex);
} }
if (Fields != null) if (Fields != null)
{ {

View File

@ -236,7 +236,7 @@ namespace Luban.Job.Cfg.Defs
return mode; return mode;
} }
private readonly List<string> _tableOptionalAttrs = new List<string> { "index", "mode", "group", "patch_input", "comment", "define_from_file", "output" }; private readonly List<string> _tableOptionalAttrs = new List<string> { "index", "mode", "group", "patch_input", "comment", "define_from_file", "output", "parser_mode" };
private readonly List<string> _tableRequireAttrs = new List<string> { "name", "value", "input" }; private readonly List<string> _tableRequireAttrs = new List<string> { "name", "value", "input" };
private void AddTable(string defineFile, XElement e) private void AddTable(string defineFile, XElement e)
@ -254,11 +254,12 @@ namespace Luban.Job.Cfg.Defs
string mode = XmlUtil.GetOptionalAttribute(e, "mode"); string mode = XmlUtil.GetOptionalAttribute(e, "mode");
string tags = XmlUtil.GetOptionalAttribute(e, "tags"); string tags = XmlUtil.GetOptionalAttribute(e, "tags");
string output = XmlUtil.GetOptionalAttribute(e, "output"); string output = XmlUtil.GetOptionalAttribute(e, "output");
AddTable(defineFile, name, module, valueType, index, mode, group, comment, defineFromFile, input, patchInput, tags, output); string options = XmlUtil.GetOptionalAttribute(e, "options");
AddTable(defineFile, name, module, valueType, index, mode, group, comment, defineFromFile, input, patchInput, tags, output, options);
} }
private void AddTable(string defineFile, string name, string module, string valueType, string index, string mode, string group, private void AddTable(string defineFile, string name, string module, string valueType, string index, string mode, string group,
string comment, bool defineFromExcel, string input, string patchInput, string tags, string outputFileName) string comment, bool defineFromExcel, string input, string patchInput, string tags, string outputFileName, string options)
{ {
var p = new Table() var p = new Table()
{ {
@ -272,6 +273,7 @@ namespace Luban.Job.Cfg.Defs
Mode = ConvertMode(defineFile, name, mode, index), Mode = ConvertMode(defineFile, name, mode, index),
Tags = tags, Tags = tags,
OutputFile = outputFileName, OutputFile = outputFileName,
Options = options,
}; };
if (string.IsNullOrWhiteSpace(name)) if (string.IsNullOrWhiteSpace(name))
{ {
@ -318,7 +320,7 @@ namespace Luban.Job.Cfg.Defs
RawSheetTableDefInfo tableDefInfo; RawSheetTableDefInfo tableDefInfo;
if (!ExcelTableValueTypeDefInfoCacheManager.Instance.TryGetTableDefInfo(file.MD5, file.SheetName, out tableDefInfo)) if (!ExcelTableValueTypeDefInfoCacheManager.Instance.TryGetTableDefInfo(file.MD5, file.SheetName, out tableDefInfo))
{ {
var source = new ExcelDataSource(); var source = new ExcelRowColumnDataSource();
var stream = new MemoryStream(await this.Agent.GetFromCacheOrReadAllBytesAsync(file.ActualFile, file.MD5)); var stream = new MemoryStream(await this.Agent.GetFromCacheOrReadAllBytesAsync(file.ActualFile, file.MD5));
tableDefInfo = source.LoadTableDefInfo(file.OriginFile, file.SheetName, stream); tableDefInfo = source.LoadTableDefInfo(file.OriginFile, file.SheetName, stream);
ExcelTableValueTypeDefInfoCacheManager.Instance.AddTableDefInfoToCache(file.MD5, file.SheetName, tableDefInfo); ExcelTableValueTypeDefInfoCacheManager.Instance.AddTableDefInfoToCache(file.MD5, file.SheetName, tableDefInfo);
@ -436,6 +438,7 @@ namespace Luban.Job.Cfg.Defs
new CfgField() { Name = "output", Type = "string" }, new CfgField() { Name = "output", Type = "string" },
new CfgField() { Name = "patch_input", Type = "string" }, new CfgField() { Name = "patch_input", Type = "string" },
new CfgField() { Name = "tags", Type = "string" }, new CfgField() { Name = "tags", Type = "string" },
new CfgField() { Name = "options", Type = "string" },
} }
}) })
{ {
@ -448,10 +451,10 @@ namespace Luban.Job.Cfg.Defs
foreach (var file in inputFileInfos) foreach (var file in inputFileInfos)
{ {
var source = new ExcelDataSource(); var source = new ExcelRowColumnDataSource();
var bytes = await this.Agent.GetFromCacheOrReadAllBytesAsync(file.ActualFile, file.MD5); var bytes = await this.Agent.GetFromCacheOrReadAllBytesAsync(file.ActualFile, file.MD5);
(var actualFile, var sheetName) = FileUtil.SplitFileAndSheetName(FileUtil.Standardize(file.OriginFile)); (var actualFile, var sheetName) = FileUtil.SplitFileAndSheetName(FileUtil.Standardize(file.OriginFile));
var records = DataLoaderUtil.LoadCfgRecords(tableRecordType, actualFile, sheetName, bytes, true); var records = DataLoaderUtil.LoadCfgRecords(tableRecordType, actualFile, sheetName, bytes, true, null);
foreach (var r in records) foreach (var r in records)
{ {
DBean data = r.Data; DBean data = r.Data;
@ -473,7 +476,8 @@ namespace Luban.Job.Cfg.Defs
string patchInput = (data.GetField("patch_input") as DString).Value.Trim(); string patchInput = (data.GetField("patch_input") as DString).Value.Trim();
string tags = (data.GetField("tags") as DString).Value.Trim(); string tags = (data.GetField("tags") as DString).Value.Trim();
string outputFile = (data.GetField("output") as DString).Value.Trim(); string outputFile = (data.GetField("output") as DString).Value.Trim();
AddTable(file.OriginFile, name, module, valueType, index, mode, group, comment, isDefineFromExcel, inputFile, patchInput, tags, outputFile); string options = (data.GetField("options") as DString).Value.Trim();
AddTable(file.OriginFile, name, module, valueType, index, mode, group, comment, isDefineFromExcel, inputFile, patchInput, tags, outputFile, options);
}; };
} }
} }
@ -547,10 +551,10 @@ namespace Luban.Job.Cfg.Defs
foreach (var file in inputFileInfos) foreach (var file in inputFileInfos)
{ {
var source = new ExcelDataSource(); var source = new ExcelRowColumnDataSource();
var bytes = await this.Agent.GetFromCacheOrReadAllBytesAsync(file.ActualFile, file.MD5); var bytes = await this.Agent.GetFromCacheOrReadAllBytesAsync(file.ActualFile, file.MD5);
(var actualFile, var sheetName) = FileUtil.SplitFileAndSheetName(FileUtil.Standardize(file.OriginFile)); (var actualFile, var sheetName) = FileUtil.SplitFileAndSheetName(FileUtil.Standardize(file.OriginFile));
var records = DataLoaderUtil.LoadCfgRecords(tableRecordType, actualFile, sheetName, bytes, true); var records = DataLoaderUtil.LoadCfgRecords(tableRecordType, actualFile, sheetName, bytes, true, null);
foreach (var r in records) foreach (var r in records)
{ {
@ -659,9 +663,9 @@ namespace Luban.Job.Cfg.Defs
foreach (var file in inputFileInfos) foreach (var file in inputFileInfos)
{ {
var source = new ExcelDataSource(); var source = new ExcelRowColumnDataSource();
var bytes = await this.Agent.GetFromCacheOrReadAllBytesAsync(file.ActualFile, file.MD5); var bytes = await this.Agent.GetFromCacheOrReadAllBytesAsync(file.ActualFile, file.MD5);
var records = DataLoaderUtil.LoadCfgRecords(tableRecordType, file.OriginFile, null, bytes, true); var records = DataLoaderUtil.LoadCfgRecords(tableRecordType, file.OriginFile, null, bytes, true, null);
foreach (var r in records) foreach (var r in records)
{ {

View File

@ -62,7 +62,8 @@ namespace Luban.Job.Cfg.Defs
private readonly ConcurrentDictionary<string, TableDataInfo> _recordsByTables = new(); private readonly ConcurrentDictionary<string, TableDataInfo> _recordsByTables = new();
public Dictionary<string, DefTable> CfgTablesByName = new(); public Dictionary<string, DefTable> CfgTablesByName { get; } = new();
public Dictionary<string, DefTable> CfgTablesByFullName { get; } = new Dictionary<string, DefTable>(); public Dictionary<string, DefTable> CfgTablesByFullName { get; } = new Dictionary<string, DefTable>();
public RawTextTable RawTextTable { get; } = new RawTextTable(); public RawTextTable RawTextTable { get; } = new RawTextTable();

View File

@ -1,4 +1,5 @@
using Bright.Collections; using Bright.Collections;
using Luban.Job.Cfg.DataSources.Excel;
using Luban.Job.Cfg.RawDefs; using Luban.Job.Cfg.RawDefs;
using Luban.Job.Cfg.TypeVisitors; using Luban.Job.Cfg.TypeVisitors;
using Luban.Job.Common.Types; using Luban.Job.Common.Types;
@ -28,8 +29,26 @@ namespace Luban.Job.Cfg.Defs
Comment = b.Comment; Comment = b.Comment;
Tags = DefUtil.ParseAttrs(b.Tags); Tags = DefUtil.ParseAttrs(b.Tags);
_outputFile = b.OutputFile; _outputFile = b.OutputFile;
ParseOptions(b.Options);
} }
private void ParseOptions(string optionsStr)
{
foreach(var kvStr in optionsStr.Split('|', ';', ','))
{
if (string.IsNullOrWhiteSpace(kvStr))
{
continue;
}
string[] entry = kvStr.Split('=');
if (entry.Length != 2)
{
throw new Exception($"table:{FullName} options:'{optionsStr}' invalid");
}
Options[entry[0]] = entry[1];
}
}
public string Index { get; private set; } public string Index { get; private set; }
@ -37,6 +56,8 @@ namespace Luban.Job.Cfg.Defs
public ETableMode Mode { get; } public ETableMode Mode { get; }
public Dictionary<string, string> Options { get; } = new Dictionary<string, string>();
public bool IsMapTable => Mode == ETableMode.MAP; public bool IsMapTable => Mode == ETableMode.MAP;
public bool IsOneValueTable => Mode == ETableMode.ONE; public bool IsOneValueTable => Mode == ETableMode.ONE;

View File

@ -1,3 +1,4 @@
using Luban.Job.Cfg.DataSources.Excel;
using System.Collections.Generic; using System.Collections.Generic;
namespace Luban.Job.Cfg.RawDefs namespace Luban.Job.Cfg.RawDefs
@ -34,6 +35,8 @@ namespace Luban.Job.Cfg.RawDefs
public ETableMode Mode { get; set; } public ETableMode Mode { get; set; }
public string Options { get; set; }
public string Comment { get; set; } public string Comment { get; set; }
public string Tags { get; set; } public string Tags { get; set; }

View File

@ -4,6 +4,7 @@ using Luban.Job.Cfg.Cache;
using Luban.Job.Cfg.DataCreators; using Luban.Job.Cfg.DataCreators;
using Luban.Job.Cfg.Datas; using Luban.Job.Cfg.Datas;
using Luban.Job.Cfg.DataSources; using Luban.Job.Cfg.DataSources;
using Luban.Job.Cfg.DataSources.Excel;
using Luban.Job.Cfg.Defs; using Luban.Job.Cfg.Defs;
using Luban.Job.Common.Types; using Luban.Job.Common.Types;
using Luban.Job.Common.Utils; using Luban.Job.Common.Utils;
@ -95,7 +96,7 @@ namespace Luban.Job.Cfg.Utils
file.OriginFile, file.OriginFile,
file.SheetName, file.SheetName,
await agent.GetFromCacheOrReadAllBytesAsync(file.ActualFile, file.MD5), await agent.GetFromCacheOrReadAllBytesAsync(file.ActualFile, file.MD5),
IsMultiRecordFile(file.ActualFile, file.SheetName)); IsMultiRecordFile(file.ActualFile, file.SheetName), table.Options);
FileRecordCacheManager.Ins.AddCacheLoadedRecords(table, file.MD5, file.SheetName, res); FileRecordCacheManager.Ins.AddCacheLoadedRecords(table, file.MD5, file.SheetName, res);
@ -188,10 +189,10 @@ namespace Luban.Job.Cfg.Utils
await Task.WhenAll(genDataTasks.ToArray()); await Task.WhenAll(genDataTasks.ToArray());
} }
public static List<Record> LoadCfgRecords(TBean recordType, string originFile, string sheetName, byte[] content, bool multiRecord) public static List<Record> LoadCfgRecords(TBean recordType, string originFile, string sheetName, byte[] content, bool multiRecord, Dictionary<string, string> options)
{ {
// (md5,sheet,multiRecord,exportTestData) -> (valuetype, List<(datas)>) // (md5,sheet,multiRecord,exportTestData) -> (valuetype, List<(datas)>)
var dataSource = DataSourceFactory.Create(originFile, sheetName, new MemoryStream(content)); var dataSource = DataSourceFactory.Create(originFile, sheetName, options, new MemoryStream(content));
try try
{ {
if (multiRecord) if (multiRecord)

View File

@ -1,4 +1,5 @@
using Luban.Job.Cfg.Datas; using Luban.Job.Cfg.Datas;
using Luban.Job.Cfg.DataSources.Excel;
using Luban.Job.Cfg.Defs; using Luban.Job.Cfg.Defs;
using Luban.Job.Cfg.RawDefs; using Luban.Job.Cfg.RawDefs;
using Luban.Job.Cfg.Utils; using Luban.Job.Cfg.Utils;
@ -65,7 +66,7 @@ namespace Luban.Job.Cfg.l10n
public void LoadFromFile(string fileName, byte[] bytes) public void LoadFromFile(string fileName, byte[] bytes)
{ {
var records = DataLoaderUtil.LoadCfgRecords(_textRowType, fileName, null, bytes, true); var records = DataLoaderUtil.LoadCfgRecords(_textRowType, fileName, null, bytes, true, null);
foreach (var r in records) foreach (var r in records)
{ {
//s_logger.Info("== read text:{}", r.Data); //s_logger.Info("== read text:{}", r.Data);