[new] 支持纯流式模式的excel解析方式
parent
154d55f372
commit
c60f91fdd4
|
|
@ -15,7 +15,7 @@ using System.Threading.Tasks;
|
||||||
|
|
||||||
namespace Luban.Job.Cfg.DataCreators
|
namespace Luban.Job.Cfg.DataCreators
|
||||||
{
|
{
|
||||||
class SheetDataCreator : ITypeFuncVisitor<Sheet, TitleRow, DType>
|
class SheetDataCreator : ITypeFuncVisitor<RowColumnSheet, TitleRow, DType>
|
||||||
{
|
{
|
||||||
public static SheetDataCreator Ins { get; } = new();
|
public static SheetDataCreator Ins { get; } = new();
|
||||||
|
|
||||||
|
|
@ -37,7 +37,7 @@ namespace Luban.Job.Cfg.DataCreators
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public DType Accept(TBool type, Sheet sheet, TitleRow row)
|
public DType Accept(TBool type, RowColumnSheet sheet, TitleRow row)
|
||||||
{
|
{
|
||||||
object x = row.Current;
|
object x = row.Current;
|
||||||
if (CheckNull(type.IsNullable, x))
|
if (CheckNull(type.IsNullable, x))
|
||||||
|
|
@ -56,7 +56,7 @@ namespace Luban.Job.Cfg.DataCreators
|
||||||
return DBool.ValueOf(bool.Parse(x.ToString()));
|
return DBool.ValueOf(bool.Parse(x.ToString()));
|
||||||
}
|
}
|
||||||
|
|
||||||
public DType Accept(TByte type, Sheet sheet, TitleRow row)
|
public DType Accept(TByte type, RowColumnSheet sheet, TitleRow row)
|
||||||
{
|
{
|
||||||
object x = row.Current;
|
object x = row.Current;
|
||||||
if (CheckNull(type.IsNullable, x))
|
if (CheckNull(type.IsNullable, x))
|
||||||
|
|
@ -70,7 +70,7 @@ namespace Luban.Job.Cfg.DataCreators
|
||||||
return DByte.ValueOf(byte.Parse(x.ToString()));
|
return DByte.ValueOf(byte.Parse(x.ToString()));
|
||||||
}
|
}
|
||||||
|
|
||||||
public DType Accept(TShort type, Sheet sheet, TitleRow row)
|
public DType Accept(TShort type, RowColumnSheet sheet, TitleRow row)
|
||||||
{
|
{
|
||||||
object x = row.Current;
|
object x = row.Current;
|
||||||
if (CheckNull(type.IsNullable, x))
|
if (CheckNull(type.IsNullable, x))
|
||||||
|
|
@ -85,7 +85,7 @@ namespace Luban.Job.Cfg.DataCreators
|
||||||
return DShort.ValueOf(short.Parse(x.ToString()));
|
return DShort.ValueOf(short.Parse(x.ToString()));
|
||||||
}
|
}
|
||||||
|
|
||||||
public DType Accept(TFshort type, Sheet sheet, TitleRow row)
|
public DType Accept(TFshort type, RowColumnSheet sheet, TitleRow row)
|
||||||
{
|
{
|
||||||
object x = row.Current;
|
object x = row.Current;
|
||||||
if (CheckNull(type.IsNullable, x))
|
if (CheckNull(type.IsNullable, x))
|
||||||
|
|
@ -99,7 +99,7 @@ namespace Luban.Job.Cfg.DataCreators
|
||||||
return DFshort.ValueOf(short.Parse(x.ToString()));
|
return DFshort.ValueOf(short.Parse(x.ToString()));
|
||||||
}
|
}
|
||||||
|
|
||||||
public DType Accept(TInt type, Sheet sheet, TitleRow row)
|
public DType Accept(TInt type, RowColumnSheet sheet, TitleRow row)
|
||||||
{
|
{
|
||||||
object x = row.Current;
|
object x = row.Current;
|
||||||
if (CheckNull(type.IsNullable, x))
|
if (CheckNull(type.IsNullable, x))
|
||||||
|
|
@ -114,7 +114,7 @@ namespace Luban.Job.Cfg.DataCreators
|
||||||
return DInt.ValueOf(int.Parse(x.ToString()));
|
return DInt.ValueOf(int.Parse(x.ToString()));
|
||||||
}
|
}
|
||||||
|
|
||||||
public DType Accept(TFint type, Sheet sheet, TitleRow row)
|
public DType Accept(TFint type, RowColumnSheet sheet, TitleRow row)
|
||||||
{
|
{
|
||||||
object x = row.Current;
|
object x = row.Current;
|
||||||
if (CheckNull(type.IsNullable, x))
|
if (CheckNull(type.IsNullable, x))
|
||||||
|
|
@ -129,7 +129,7 @@ namespace Luban.Job.Cfg.DataCreators
|
||||||
return DFint.ValueOf(int.Parse(x.ToString()));
|
return DFint.ValueOf(int.Parse(x.ToString()));
|
||||||
}
|
}
|
||||||
|
|
||||||
public DType Accept(TLong type, Sheet sheet, TitleRow row)
|
public DType Accept(TLong type, RowColumnSheet sheet, TitleRow row)
|
||||||
{
|
{
|
||||||
object x = row.Current;
|
object x = row.Current;
|
||||||
if (CheckNull(type.IsNullable, x))
|
if (CheckNull(type.IsNullable, x))
|
||||||
|
|
@ -144,7 +144,7 @@ namespace Luban.Job.Cfg.DataCreators
|
||||||
return DLong.ValueOf(long.Parse(x.ToString()));
|
return DLong.ValueOf(long.Parse(x.ToString()));
|
||||||
}
|
}
|
||||||
|
|
||||||
public DType Accept(TFlong type, Sheet sheet, TitleRow row)
|
public DType Accept(TFlong type, RowColumnSheet sheet, TitleRow row)
|
||||||
{
|
{
|
||||||
object x = row.Current;
|
object x = row.Current;
|
||||||
if (CheckNull(type.IsNullable, x))
|
if (CheckNull(type.IsNullable, x))
|
||||||
|
|
@ -159,7 +159,7 @@ namespace Luban.Job.Cfg.DataCreators
|
||||||
return DFlong.ValueOf(long.Parse(x.ToString()));
|
return DFlong.ValueOf(long.Parse(x.ToString()));
|
||||||
}
|
}
|
||||||
|
|
||||||
public DType Accept(TFloat type, Sheet sheet, TitleRow row)
|
public DType Accept(TFloat type, RowColumnSheet sheet, TitleRow row)
|
||||||
{
|
{
|
||||||
object x = row.Current;
|
object x = row.Current;
|
||||||
if (CheckNull(type.IsNullable, x))
|
if (CheckNull(type.IsNullable, x))
|
||||||
|
|
@ -174,7 +174,7 @@ namespace Luban.Job.Cfg.DataCreators
|
||||||
return DFloat.ValueOf(float.Parse(x.ToString()));
|
return DFloat.ValueOf(float.Parse(x.ToString()));
|
||||||
}
|
}
|
||||||
|
|
||||||
public DType Accept(TDouble type, Sheet sheet, TitleRow row)
|
public DType Accept(TDouble type, RowColumnSheet sheet, TitleRow row)
|
||||||
{
|
{
|
||||||
object x = row.Current;
|
object x = row.Current;
|
||||||
if (CheckNull(type.IsNullable, x))
|
if (CheckNull(type.IsNullable, x))
|
||||||
|
|
@ -189,7 +189,7 @@ namespace Luban.Job.Cfg.DataCreators
|
||||||
return DDouble.ValueOf(double.Parse(x.ToString()));
|
return DDouble.ValueOf(double.Parse(x.ToString()));
|
||||||
}
|
}
|
||||||
|
|
||||||
public DType Accept(TEnum type, Sheet sheet, TitleRow row)
|
public DType Accept(TEnum type, RowColumnSheet sheet, TitleRow row)
|
||||||
{
|
{
|
||||||
object x = row.Current;
|
object x = row.Current;
|
||||||
if (CheckNull(type.IsNullable, x))
|
if (CheckNull(type.IsNullable, x))
|
||||||
|
|
@ -227,7 +227,7 @@ namespace Luban.Job.Cfg.DataCreators
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public DType Accept(TString type, Sheet sheet, TitleRow row)
|
public DType Accept(TString type, RowColumnSheet sheet, TitleRow row)
|
||||||
{
|
{
|
||||||
object x = row.Current;
|
object x = row.Current;
|
||||||
if (CheckDefault(x))
|
if (CheckDefault(x))
|
||||||
|
|
@ -249,12 +249,12 @@ namespace Luban.Job.Cfg.DataCreators
|
||||||
return DString.ValueOf(s);
|
return DString.ValueOf(s);
|
||||||
}
|
}
|
||||||
|
|
||||||
public DType Accept(TBytes type, Sheet sheet, TitleRow row)
|
public DType Accept(TBytes type, RowColumnSheet sheet, TitleRow row)
|
||||||
{
|
{
|
||||||
throw new NotSupportedException();
|
throw new NotSupportedException();
|
||||||
}
|
}
|
||||||
|
|
||||||
public DType Accept(TText type, Sheet sheet, TitleRow row)
|
public DType Accept(TText type, RowColumnSheet sheet, TitleRow row)
|
||||||
{
|
{
|
||||||
if (string.IsNullOrEmpty(row.SelfTitle.SepOr(type.GetTag("sep"))))
|
if (string.IsNullOrEmpty(row.SelfTitle.SepOr(type.GetTag("sep"))))
|
||||||
{
|
{
|
||||||
|
|
@ -279,7 +279,7 @@ namespace Luban.Job.Cfg.DataCreators
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public DType Accept(TDateTime type, Sheet sheet, TitleRow row)
|
public DType Accept(TDateTime type, RowColumnSheet sheet, TitleRow row)
|
||||||
{
|
{
|
||||||
var d = row.Current;
|
var d = row.Current;
|
||||||
if (CheckNull(type.IsNullable, d))
|
if (CheckNull(type.IsNullable, d))
|
||||||
|
|
@ -293,7 +293,7 @@ namespace Luban.Job.Cfg.DataCreators
|
||||||
return DataUtil.CreateDateTime(d.ToString());
|
return DataUtil.CreateDateTime(d.ToString());
|
||||||
}
|
}
|
||||||
|
|
||||||
public DType Accept(TVector2 type, Sheet sheet, TitleRow row)
|
public DType Accept(TVector2 type, RowColumnSheet sheet, TitleRow row)
|
||||||
{
|
{
|
||||||
var d = row.Current;
|
var d = row.Current;
|
||||||
if (CheckNull(type.IsNullable, d))
|
if (CheckNull(type.IsNullable, d))
|
||||||
|
|
@ -308,7 +308,7 @@ namespace Luban.Job.Cfg.DataCreators
|
||||||
return DataUtil.CreateVector(type, d.ToString());
|
return DataUtil.CreateVector(type, d.ToString());
|
||||||
}
|
}
|
||||||
|
|
||||||
public DType Accept(TVector3 type, Sheet sheet, TitleRow row)
|
public DType Accept(TVector3 type, RowColumnSheet sheet, TitleRow row)
|
||||||
{
|
{
|
||||||
var d = row.Current;
|
var d = row.Current;
|
||||||
if (CheckNull(type.IsNullable, d))
|
if (CheckNull(type.IsNullable, d))
|
||||||
|
|
@ -323,7 +323,7 @@ namespace Luban.Job.Cfg.DataCreators
|
||||||
return DataUtil.CreateVector(type, d.ToString());
|
return DataUtil.CreateVector(type, d.ToString());
|
||||||
}
|
}
|
||||||
|
|
||||||
public DType Accept(TVector4 type, Sheet sheet, TitleRow row)
|
public DType Accept(TVector4 type, RowColumnSheet sheet, TitleRow row)
|
||||||
{
|
{
|
||||||
var d = row.Current;
|
var d = row.Current;
|
||||||
if (CheckNull(type.IsNullable, d))
|
if (CheckNull(type.IsNullable, d))
|
||||||
|
|
@ -338,7 +338,7 @@ namespace Luban.Job.Cfg.DataCreators
|
||||||
return DataUtil.CreateVector(type, d.ToString());
|
return DataUtil.CreateVector(type, d.ToString());
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<DType> CreateBeanFields(DefBean bean, Sheet sheet, TitleRow row)
|
private List<DType> CreateBeanFields(DefBean bean, RowColumnSheet sheet, TitleRow row)
|
||||||
{
|
{
|
||||||
var list = new List<DType>();
|
var list = new List<DType>();
|
||||||
foreach (DefField f in bean.HierarchyFields)
|
foreach (DefField f in bean.HierarchyFields)
|
||||||
|
|
@ -369,7 +369,7 @@ namespace Luban.Job.Cfg.DataCreators
|
||||||
return list;
|
return list;
|
||||||
}
|
}
|
||||||
|
|
||||||
public DType Accept(TBean type, Sheet sheet, TitleRow row)
|
public DType Accept(TBean type, RowColumnSheet sheet, TitleRow row)
|
||||||
{
|
{
|
||||||
string sep = row.SelfTitle.Sep;// type.GetBeanAs<DefBean>().Sep;
|
string sep = row.SelfTitle.Sep;// type.GetBeanAs<DefBean>().Sep;
|
||||||
|
|
||||||
|
|
@ -479,7 +479,7 @@ namespace Luban.Job.Cfg.DataCreators
|
||||||
return datas;
|
return datas;
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<DType> ReadCollectionDatas(TType type, TType elementType, Sheet sheet, TitleRow row)
|
private List<DType> ReadCollectionDatas(TType type, TType elementType, RowColumnSheet sheet, TitleRow row)
|
||||||
{
|
{
|
||||||
if (row.Row != null)
|
if (row.Row != null)
|
||||||
{
|
{
|
||||||
|
|
@ -518,23 +518,23 @@ namespace Luban.Job.Cfg.DataCreators
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public DType Accept(TArray type, Sheet sheet, TitleRow row)
|
public DType Accept(TArray type, RowColumnSheet sheet, TitleRow row)
|
||||||
{
|
{
|
||||||
//string sep = DataUtil.GetSep(type);
|
//string sep = DataUtil.GetSep(type);
|
||||||
return new DArray(type, ReadCollectionDatas(type, type.ElementType, sheet, row));
|
return new DArray(type, ReadCollectionDatas(type, type.ElementType, sheet, row));
|
||||||
}
|
}
|
||||||
|
|
||||||
public DType Accept(TList type, Sheet sheet, TitleRow row)
|
public DType Accept(TList type, RowColumnSheet sheet, TitleRow row)
|
||||||
{
|
{
|
||||||
return new DList(type, ReadCollectionDatas(type, type.ElementType, sheet, row));
|
return new DList(type, ReadCollectionDatas(type, type.ElementType, sheet, row));
|
||||||
}
|
}
|
||||||
|
|
||||||
public DType Accept(TSet type, Sheet sheet, TitleRow row)
|
public DType Accept(TSet type, RowColumnSheet sheet, TitleRow row)
|
||||||
{
|
{
|
||||||
return new DSet(type, ReadCollectionDatas(type, type.ElementType, sheet, row));
|
return new DSet(type, ReadCollectionDatas(type, type.ElementType, sheet, row));
|
||||||
}
|
}
|
||||||
|
|
||||||
public DType Accept(TMap type, Sheet sheet, TitleRow row)
|
public DType Accept(TMap type, RowColumnSheet sheet, TitleRow row)
|
||||||
{
|
{
|
||||||
string sep = row.SelfTitle.Sep;
|
string sep = row.SelfTitle.Sep;
|
||||||
|
|
||||||
|
|
@ -563,7 +563,7 @@ namespace Luban.Job.Cfg.DataCreators
|
||||||
foreach (var e in row.Fields)
|
foreach (var e in row.Fields)
|
||||||
{
|
{
|
||||||
var keyData = type.KeyType.Apply(StringDataCreator.Ins, e.Key);
|
var keyData = type.KeyType.Apply(StringDataCreator.Ins, e.Key);
|
||||||
if (Sheet.IsBlankRow(e.Value.Row, e.Value.SelfTitle.FromIndex, e.Value.SelfTitle.ToIndex))
|
if (RowColumnSheet.IsBlankRow(e.Value.Row, e.Value.SelfTitle.FromIndex, e.Value.SelfTitle.ToIndex))
|
||||||
{
|
{
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,8 @@
|
||||||
using Luban.Job.Cfg.DataCreators;
|
using Luban.Job.Cfg.DataCreators;
|
||||||
|
using Luban.Job.Cfg.DataSources.Excel;
|
||||||
|
using Luban.Job.Cfg.Defs;
|
||||||
using System;
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
using System.IO;
|
using System.IO;
|
||||||
|
|
||||||
namespace Luban.Job.Cfg.DataSources
|
namespace Luban.Job.Cfg.DataSources
|
||||||
|
|
@ -18,7 +21,35 @@ namespace Luban.Job.Cfg.DataSources
|
||||||
".asset",
|
".asset",
|
||||||
};
|
};
|
||||||
|
|
||||||
public static AbstractDataSource Create(string url, string sheetName, Stream stream)
|
private static string GetSheetParserMode(Dictionary<string, string> options)
|
||||||
|
{
|
||||||
|
if (options != null && options.TryGetValue("parser_mode", out var modeStr))
|
||||||
|
{
|
||||||
|
return modeStr;
|
||||||
|
}
|
||||||
|
//options = DefAssembly.LocalAssebmly.Options;
|
||||||
|
//if (options != null && options.TryGetValue("sheet.parser_mode", out modeStr))
|
||||||
|
//{
|
||||||
|
// return modeStr;
|
||||||
|
//}
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool IsColumnMode(string mode)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrEmpty(mode))
|
||||||
|
{
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
switch(mode.ToLowerInvariant())
|
||||||
|
{
|
||||||
|
case "column": return true;
|
||||||
|
case "stream": return false;
|
||||||
|
default: throw new Exception($"unknown parser_mode:{mode}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static AbstractDataSource Create(string url, string sheetName, Dictionary<string, string> options, Stream stream)
|
||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
|
|
@ -29,7 +60,11 @@ namespace Luban.Job.Cfg.DataSources
|
||||||
case "csv":
|
case "csv":
|
||||||
case "xls":
|
case "xls":
|
||||||
case "xlsx":
|
case "xlsx":
|
||||||
case "xlsm": source = new Excel.ExcelDataSource(); break;
|
case "xlsm":
|
||||||
|
{
|
||||||
|
source = IsColumnMode(GetSheetParserMode(options)) ? new ExcelRowColumnDataSource() : new ExcelStreamDataSource();
|
||||||
|
break;
|
||||||
|
}
|
||||||
case "xml": source = new Xml.XmlDataSource(); break;
|
case "xml": source = new Xml.XmlDataSource(); break;
|
||||||
case "lua": source = new Lua.LuaDataSource(); break;
|
case "lua": source = new Lua.LuaDataSource(); break;
|
||||||
case "json": source = new Json.JsonDataSource(); break;
|
case "json": source = new Json.JsonDataSource(); break;
|
||||||
|
|
|
||||||
|
|
@ -10,11 +10,11 @@ using System.IO;
|
||||||
namespace Luban.Job.Cfg.DataSources.Excel
|
namespace Luban.Job.Cfg.DataSources.Excel
|
||||||
{
|
{
|
||||||
|
|
||||||
class ExcelDataSource : AbstractDataSource
|
class ExcelRowColumnDataSource : AbstractDataSource
|
||||||
{
|
{
|
||||||
private static readonly NLog.Logger s_logger = NLog.LogManager.GetCurrentClassLogger();
|
private static readonly NLog.Logger s_logger = NLog.LogManager.GetCurrentClassLogger();
|
||||||
|
|
||||||
private readonly List<Sheet> _sheets = new List<Sheet>();
|
private readonly List<RowColumnSheet> _sheets = new List<RowColumnSheet>();
|
||||||
|
|
||||||
|
|
||||||
public override void Load(string rawUrl, string sheetName, Stream stream)
|
public override void Load(string rawUrl, string sheetName, Stream stream)
|
||||||
|
|
@ -25,7 +25,7 @@ namespace Luban.Job.Cfg.DataSources.Excel
|
||||||
|
|
||||||
foreach (RawSheet rawSheet in SheetLoadUtil.LoadRawSheets(rawUrl, sheetName, stream))
|
foreach (RawSheet rawSheet in SheetLoadUtil.LoadRawSheets(rawUrl, sheetName, stream))
|
||||||
{
|
{
|
||||||
var sheet = new Sheet(rawUrl, sheetName);
|
var sheet = new RowColumnSheet(rawUrl, sheetName);
|
||||||
sheet.Load(rawSheet);
|
sheet.Load(rawSheet);
|
||||||
_sheets.Add(sheet);
|
_sheets.Add(sheet);
|
||||||
}
|
}
|
||||||
|
|
@ -40,7 +40,7 @@ namespace Luban.Job.Cfg.DataSources.Excel
|
||||||
{
|
{
|
||||||
foreach (RawSheet rawSheet in rawSheets)
|
foreach (RawSheet rawSheet in rawSheets)
|
||||||
{
|
{
|
||||||
var sheet = new Sheet("__intern__", rawSheet.TableName);
|
var sheet = new RowColumnSheet("__intern__", rawSheet.TableName);
|
||||||
sheet.Load(rawSheet);
|
sheet.Load(rawSheet);
|
||||||
_sheets.Add(sheet);
|
_sheets.Add(sheet);
|
||||||
}
|
}
|
||||||
|
|
@ -0,0 +1,76 @@
|
||||||
|
using ExcelDataReader;
|
||||||
|
using Luban.Job.Cfg.DataCreators;
|
||||||
|
using Luban.Job.Cfg.Datas;
|
||||||
|
using Luban.Job.Cfg.Utils;
|
||||||
|
using Luban.Job.Common.Types;
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.IO;
|
||||||
|
|
||||||
|
namespace Luban.Job.Cfg.DataSources.Excel
|
||||||
|
{
|
||||||
|
|
||||||
|
class ExcelStreamDataSource : AbstractDataSource
|
||||||
|
{
|
||||||
|
private static readonly NLog.Logger s_logger = NLog.LogManager.GetCurrentClassLogger();
|
||||||
|
|
||||||
|
private readonly List<StreamSheet> _sheets = new List<StreamSheet>();
|
||||||
|
|
||||||
|
|
||||||
|
public override void Load(string rawUrl, string sheetName, Stream stream)
|
||||||
|
{
|
||||||
|
s_logger.Trace("{filename} {sheet}", rawUrl, sheetName);
|
||||||
|
RawUrl = rawUrl;
|
||||||
|
|
||||||
|
|
||||||
|
foreach (RawSheet rawSheet in SheetLoadUtil.LoadRawSheets(rawUrl, sheetName, stream))
|
||||||
|
{
|
||||||
|
var sheet = new StreamSheet(rawUrl, sheetName);
|
||||||
|
sheet.Load(rawSheet);
|
||||||
|
_sheets.Add(sheet);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (_sheets.Count == 0)
|
||||||
|
{
|
||||||
|
throw new Exception($"excel:{rawUrl} 不包含有效的单元薄(有效单元薄的A0单元格必须是##).");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public RawSheetTableDefInfo LoadTableDefInfo(string rawUrl, string sheetName, Stream stream)
|
||||||
|
{
|
||||||
|
return SheetLoadUtil.LoadSheetTableDefInfo(rawUrl, sheetName, stream);
|
||||||
|
}
|
||||||
|
|
||||||
|
public override List<Record> ReadMulti(TBean type)
|
||||||
|
{
|
||||||
|
var datas = new List<Record>();
|
||||||
|
foreach (var sheet in _sheets)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var stream = sheet.Stream;
|
||||||
|
while(!stream.TryReadEOF())
|
||||||
|
{
|
||||||
|
var data = (DBean)type.Apply(ExcelStreamDataCreator.Ins, stream);
|
||||||
|
datas.Add(new Record(data, sheet.RawUrl, null));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch (DataCreateException dce)
|
||||||
|
{
|
||||||
|
dce.OriginDataLocation = sheet.RawUrl;
|
||||||
|
throw;
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
throw new Exception($"sheet:{sheet.Name}", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return datas;
|
||||||
|
}
|
||||||
|
|
||||||
|
public override Record ReadOne(TBean type)
|
||||||
|
{
|
||||||
|
throw new Exception($"excel不支持单例读取模式");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -12,7 +12,7 @@ using System.Linq;
|
||||||
namespace Luban.Job.Cfg.DataSources.Excel
|
namespace Luban.Job.Cfg.DataSources.Excel
|
||||||
{
|
{
|
||||||
|
|
||||||
class Sheet
|
class RowColumnSheet
|
||||||
{
|
{
|
||||||
private static readonly NLog.Logger s_logger = NLog.LogManager.GetCurrentClassLogger();
|
private static readonly NLog.Logger s_logger = NLog.LogManager.GetCurrentClassLogger();
|
||||||
|
|
||||||
|
|
@ -22,7 +22,7 @@ namespace Luban.Job.Cfg.DataSources.Excel
|
||||||
|
|
||||||
public List<(string Tag, TitleRow Row)> Rows { get; } = new();
|
public List<(string Tag, TitleRow Row)> Rows { get; } = new();
|
||||||
|
|
||||||
public Sheet(string rawUrl, string name)
|
public RowColumnSheet(string rawUrl, string name)
|
||||||
{
|
{
|
||||||
this.RawUrl = rawUrl;
|
this.RawUrl = rawUrl;
|
||||||
this.Name = name;
|
this.Name = name;
|
||||||
|
|
@ -0,0 +1,37 @@
|
||||||
|
using Bright.Collections;
|
||||||
|
using ExcelDataReader;
|
||||||
|
using Luban.Job.Cfg.DataCreators;
|
||||||
|
using Luban.Job.Cfg.Datas;
|
||||||
|
using Luban.Job.Cfg.Utils;
|
||||||
|
using Luban.Job.Common.Types;
|
||||||
|
using Luban.Job.Common.Utils;
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
|
||||||
|
namespace Luban.Job.Cfg.DataSources.Excel
|
||||||
|
{
|
||||||
|
|
||||||
|
class StreamSheet
|
||||||
|
{
|
||||||
|
private static readonly NLog.Logger s_logger = NLog.LogManager.GetCurrentClassLogger();
|
||||||
|
|
||||||
|
public string Name { get; }
|
||||||
|
|
||||||
|
public string RawUrl { get; }
|
||||||
|
|
||||||
|
public ExcelStream Stream { get; private set; }
|
||||||
|
|
||||||
|
public StreamSheet(string rawUrl, string name)
|
||||||
|
{
|
||||||
|
this.RawUrl = rawUrl;
|
||||||
|
this.Name = name;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Load(RawSheet rawSheet)
|
||||||
|
{
|
||||||
|
Title title = rawSheet.Title;
|
||||||
|
Stream = new ExcelStream(rawSheet.Cells, 1, title.ToIndex, "", "");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -75,11 +75,11 @@ namespace Luban.Job.Cfg.DataSources.Excel
|
||||||
{
|
{
|
||||||
if (Row != null)
|
if (Row != null)
|
||||||
{
|
{
|
||||||
return Sheet.IsBlankRow(Row, SelfTitle.FromIndex, SelfTitle.ToIndex);
|
return RowColumnSheet.IsBlankRow(Row, SelfTitle.FromIndex, SelfTitle.ToIndex);
|
||||||
}
|
}
|
||||||
if (Rows != null)
|
if (Rows != null)
|
||||||
{
|
{
|
||||||
return Sheet.IsBlankRow(Rows[0], SelfTitle.FromIndex, SelfTitle.ToIndex);
|
return RowColumnSheet.IsBlankRow(Rows[0], SelfTitle.FromIndex, SelfTitle.ToIndex);
|
||||||
}
|
}
|
||||||
if (Fields != null)
|
if (Fields != null)
|
||||||
{
|
{
|
||||||
|
|
|
||||||
|
|
@ -236,7 +236,7 @@ namespace Luban.Job.Cfg.Defs
|
||||||
return mode;
|
return mode;
|
||||||
}
|
}
|
||||||
|
|
||||||
private readonly List<string> _tableOptionalAttrs = new List<string> { "index", "mode", "group", "patch_input", "comment", "define_from_file", "output" };
|
private readonly List<string> _tableOptionalAttrs = new List<string> { "index", "mode", "group", "patch_input", "comment", "define_from_file", "output", "parser_mode" };
|
||||||
private readonly List<string> _tableRequireAttrs = new List<string> { "name", "value", "input" };
|
private readonly List<string> _tableRequireAttrs = new List<string> { "name", "value", "input" };
|
||||||
|
|
||||||
private void AddTable(string defineFile, XElement e)
|
private void AddTable(string defineFile, XElement e)
|
||||||
|
|
@ -254,11 +254,12 @@ namespace Luban.Job.Cfg.Defs
|
||||||
string mode = XmlUtil.GetOptionalAttribute(e, "mode");
|
string mode = XmlUtil.GetOptionalAttribute(e, "mode");
|
||||||
string tags = XmlUtil.GetOptionalAttribute(e, "tags");
|
string tags = XmlUtil.GetOptionalAttribute(e, "tags");
|
||||||
string output = XmlUtil.GetOptionalAttribute(e, "output");
|
string output = XmlUtil.GetOptionalAttribute(e, "output");
|
||||||
AddTable(defineFile, name, module, valueType, index, mode, group, comment, defineFromFile, input, patchInput, tags, output);
|
string options = XmlUtil.GetOptionalAttribute(e, "options");
|
||||||
|
AddTable(defineFile, name, module, valueType, index, mode, group, comment, defineFromFile, input, patchInput, tags, output, options);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void AddTable(string defineFile, string name, string module, string valueType, string index, string mode, string group,
|
private void AddTable(string defineFile, string name, string module, string valueType, string index, string mode, string group,
|
||||||
string comment, bool defineFromExcel, string input, string patchInput, string tags, string outputFileName)
|
string comment, bool defineFromExcel, string input, string patchInput, string tags, string outputFileName, string options)
|
||||||
{
|
{
|
||||||
var p = new Table()
|
var p = new Table()
|
||||||
{
|
{
|
||||||
|
|
@ -272,6 +273,7 @@ namespace Luban.Job.Cfg.Defs
|
||||||
Mode = ConvertMode(defineFile, name, mode, index),
|
Mode = ConvertMode(defineFile, name, mode, index),
|
||||||
Tags = tags,
|
Tags = tags,
|
||||||
OutputFile = outputFileName,
|
OutputFile = outputFileName,
|
||||||
|
Options = options,
|
||||||
};
|
};
|
||||||
if (string.IsNullOrWhiteSpace(name))
|
if (string.IsNullOrWhiteSpace(name))
|
||||||
{
|
{
|
||||||
|
|
@ -318,7 +320,7 @@ namespace Luban.Job.Cfg.Defs
|
||||||
RawSheetTableDefInfo tableDefInfo;
|
RawSheetTableDefInfo tableDefInfo;
|
||||||
if (!ExcelTableValueTypeDefInfoCacheManager.Instance.TryGetTableDefInfo(file.MD5, file.SheetName, out tableDefInfo))
|
if (!ExcelTableValueTypeDefInfoCacheManager.Instance.TryGetTableDefInfo(file.MD5, file.SheetName, out tableDefInfo))
|
||||||
{
|
{
|
||||||
var source = new ExcelDataSource();
|
var source = new ExcelRowColumnDataSource();
|
||||||
var stream = new MemoryStream(await this.Agent.GetFromCacheOrReadAllBytesAsync(file.ActualFile, file.MD5));
|
var stream = new MemoryStream(await this.Agent.GetFromCacheOrReadAllBytesAsync(file.ActualFile, file.MD5));
|
||||||
tableDefInfo = source.LoadTableDefInfo(file.OriginFile, file.SheetName, stream);
|
tableDefInfo = source.LoadTableDefInfo(file.OriginFile, file.SheetName, stream);
|
||||||
ExcelTableValueTypeDefInfoCacheManager.Instance.AddTableDefInfoToCache(file.MD5, file.SheetName, tableDefInfo);
|
ExcelTableValueTypeDefInfoCacheManager.Instance.AddTableDefInfoToCache(file.MD5, file.SheetName, tableDefInfo);
|
||||||
|
|
@ -436,6 +438,7 @@ namespace Luban.Job.Cfg.Defs
|
||||||
new CfgField() { Name = "output", Type = "string" },
|
new CfgField() { Name = "output", Type = "string" },
|
||||||
new CfgField() { Name = "patch_input", Type = "string" },
|
new CfgField() { Name = "patch_input", Type = "string" },
|
||||||
new CfgField() { Name = "tags", Type = "string" },
|
new CfgField() { Name = "tags", Type = "string" },
|
||||||
|
new CfgField() { Name = "options", Type = "string" },
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
{
|
{
|
||||||
|
|
@ -448,10 +451,10 @@ namespace Luban.Job.Cfg.Defs
|
||||||
|
|
||||||
foreach (var file in inputFileInfos)
|
foreach (var file in inputFileInfos)
|
||||||
{
|
{
|
||||||
var source = new ExcelDataSource();
|
var source = new ExcelRowColumnDataSource();
|
||||||
var bytes = await this.Agent.GetFromCacheOrReadAllBytesAsync(file.ActualFile, file.MD5);
|
var bytes = await this.Agent.GetFromCacheOrReadAllBytesAsync(file.ActualFile, file.MD5);
|
||||||
(var actualFile, var sheetName) = FileUtil.SplitFileAndSheetName(FileUtil.Standardize(file.OriginFile));
|
(var actualFile, var sheetName) = FileUtil.SplitFileAndSheetName(FileUtil.Standardize(file.OriginFile));
|
||||||
var records = DataLoaderUtil.LoadCfgRecords(tableRecordType, actualFile, sheetName, bytes, true);
|
var records = DataLoaderUtil.LoadCfgRecords(tableRecordType, actualFile, sheetName, bytes, true, null);
|
||||||
foreach (var r in records)
|
foreach (var r in records)
|
||||||
{
|
{
|
||||||
DBean data = r.Data;
|
DBean data = r.Data;
|
||||||
|
|
@ -473,7 +476,8 @@ namespace Luban.Job.Cfg.Defs
|
||||||
string patchInput = (data.GetField("patch_input") as DString).Value.Trim();
|
string patchInput = (data.GetField("patch_input") as DString).Value.Trim();
|
||||||
string tags = (data.GetField("tags") as DString).Value.Trim();
|
string tags = (data.GetField("tags") as DString).Value.Trim();
|
||||||
string outputFile = (data.GetField("output") as DString).Value.Trim();
|
string outputFile = (data.GetField("output") as DString).Value.Trim();
|
||||||
AddTable(file.OriginFile, name, module, valueType, index, mode, group, comment, isDefineFromExcel, inputFile, patchInput, tags, outputFile);
|
string options = (data.GetField("options") as DString).Value.Trim();
|
||||||
|
AddTable(file.OriginFile, name, module, valueType, index, mode, group, comment, isDefineFromExcel, inputFile, patchInput, tags, outputFile, options);
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -547,10 +551,10 @@ namespace Luban.Job.Cfg.Defs
|
||||||
|
|
||||||
foreach (var file in inputFileInfos)
|
foreach (var file in inputFileInfos)
|
||||||
{
|
{
|
||||||
var source = new ExcelDataSource();
|
var source = new ExcelRowColumnDataSource();
|
||||||
var bytes = await this.Agent.GetFromCacheOrReadAllBytesAsync(file.ActualFile, file.MD5);
|
var bytes = await this.Agent.GetFromCacheOrReadAllBytesAsync(file.ActualFile, file.MD5);
|
||||||
(var actualFile, var sheetName) = FileUtil.SplitFileAndSheetName(FileUtil.Standardize(file.OriginFile));
|
(var actualFile, var sheetName) = FileUtil.SplitFileAndSheetName(FileUtil.Standardize(file.OriginFile));
|
||||||
var records = DataLoaderUtil.LoadCfgRecords(tableRecordType, actualFile, sheetName, bytes, true);
|
var records = DataLoaderUtil.LoadCfgRecords(tableRecordType, actualFile, sheetName, bytes, true, null);
|
||||||
|
|
||||||
foreach (var r in records)
|
foreach (var r in records)
|
||||||
{
|
{
|
||||||
|
|
@ -659,9 +663,9 @@ namespace Luban.Job.Cfg.Defs
|
||||||
|
|
||||||
foreach (var file in inputFileInfos)
|
foreach (var file in inputFileInfos)
|
||||||
{
|
{
|
||||||
var source = new ExcelDataSource();
|
var source = new ExcelRowColumnDataSource();
|
||||||
var bytes = await this.Agent.GetFromCacheOrReadAllBytesAsync(file.ActualFile, file.MD5);
|
var bytes = await this.Agent.GetFromCacheOrReadAllBytesAsync(file.ActualFile, file.MD5);
|
||||||
var records = DataLoaderUtil.LoadCfgRecords(tableRecordType, file.OriginFile, null, bytes, true);
|
var records = DataLoaderUtil.LoadCfgRecords(tableRecordType, file.OriginFile, null, bytes, true, null);
|
||||||
|
|
||||||
foreach (var r in records)
|
foreach (var r in records)
|
||||||
{
|
{
|
||||||
|
|
|
||||||
|
|
@ -62,7 +62,8 @@ namespace Luban.Job.Cfg.Defs
|
||||||
|
|
||||||
private readonly ConcurrentDictionary<string, TableDataInfo> _recordsByTables = new();
|
private readonly ConcurrentDictionary<string, TableDataInfo> _recordsByTables = new();
|
||||||
|
|
||||||
public Dictionary<string, DefTable> CfgTablesByName = new();
|
public Dictionary<string, DefTable> CfgTablesByName { get; } = new();
|
||||||
|
|
||||||
public Dictionary<string, DefTable> CfgTablesByFullName { get; } = new Dictionary<string, DefTable>();
|
public Dictionary<string, DefTable> CfgTablesByFullName { get; } = new Dictionary<string, DefTable>();
|
||||||
|
|
||||||
public RawTextTable RawTextTable { get; } = new RawTextTable();
|
public RawTextTable RawTextTable { get; } = new RawTextTable();
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
using Bright.Collections;
|
using Bright.Collections;
|
||||||
|
using Luban.Job.Cfg.DataSources.Excel;
|
||||||
using Luban.Job.Cfg.RawDefs;
|
using Luban.Job.Cfg.RawDefs;
|
||||||
using Luban.Job.Cfg.TypeVisitors;
|
using Luban.Job.Cfg.TypeVisitors;
|
||||||
using Luban.Job.Common.Types;
|
using Luban.Job.Common.Types;
|
||||||
|
|
@ -28,8 +29,26 @@ namespace Luban.Job.Cfg.Defs
|
||||||
Comment = b.Comment;
|
Comment = b.Comment;
|
||||||
Tags = DefUtil.ParseAttrs(b.Tags);
|
Tags = DefUtil.ParseAttrs(b.Tags);
|
||||||
_outputFile = b.OutputFile;
|
_outputFile = b.OutputFile;
|
||||||
|
|
||||||
|
ParseOptions(b.Options);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void ParseOptions(string optionsStr)
|
||||||
|
{
|
||||||
|
foreach(var kvStr in optionsStr.Split('|', ';', ','))
|
||||||
|
{
|
||||||
|
if (string.IsNullOrWhiteSpace(kvStr))
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
string[] entry = kvStr.Split('=');
|
||||||
|
if (entry.Length != 2)
|
||||||
|
{
|
||||||
|
throw new Exception($"table:{FullName} options:'{optionsStr}' invalid");
|
||||||
|
}
|
||||||
|
Options[entry[0]] = entry[1];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public string Index { get; private set; }
|
public string Index { get; private set; }
|
||||||
|
|
||||||
|
|
@ -37,6 +56,8 @@ namespace Luban.Job.Cfg.Defs
|
||||||
|
|
||||||
public ETableMode Mode { get; }
|
public ETableMode Mode { get; }
|
||||||
|
|
||||||
|
public Dictionary<string, string> Options { get; } = new Dictionary<string, string>();
|
||||||
|
|
||||||
public bool IsMapTable => Mode == ETableMode.MAP;
|
public bool IsMapTable => Mode == ETableMode.MAP;
|
||||||
|
|
||||||
public bool IsOneValueTable => Mode == ETableMode.ONE;
|
public bool IsOneValueTable => Mode == ETableMode.ONE;
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
using Luban.Job.Cfg.DataSources.Excel;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
|
|
||||||
namespace Luban.Job.Cfg.RawDefs
|
namespace Luban.Job.Cfg.RawDefs
|
||||||
|
|
@ -34,6 +35,8 @@ namespace Luban.Job.Cfg.RawDefs
|
||||||
|
|
||||||
public ETableMode Mode { get; set; }
|
public ETableMode Mode { get; set; }
|
||||||
|
|
||||||
|
public string Options { get; set; }
|
||||||
|
|
||||||
public string Comment { get; set; }
|
public string Comment { get; set; }
|
||||||
|
|
||||||
public string Tags { get; set; }
|
public string Tags { get; set; }
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,7 @@ using Luban.Job.Cfg.Cache;
|
||||||
using Luban.Job.Cfg.DataCreators;
|
using Luban.Job.Cfg.DataCreators;
|
||||||
using Luban.Job.Cfg.Datas;
|
using Luban.Job.Cfg.Datas;
|
||||||
using Luban.Job.Cfg.DataSources;
|
using Luban.Job.Cfg.DataSources;
|
||||||
|
using Luban.Job.Cfg.DataSources.Excel;
|
||||||
using Luban.Job.Cfg.Defs;
|
using Luban.Job.Cfg.Defs;
|
||||||
using Luban.Job.Common.Types;
|
using Luban.Job.Common.Types;
|
||||||
using Luban.Job.Common.Utils;
|
using Luban.Job.Common.Utils;
|
||||||
|
|
@ -95,7 +96,7 @@ namespace Luban.Job.Cfg.Utils
|
||||||
file.OriginFile,
|
file.OriginFile,
|
||||||
file.SheetName,
|
file.SheetName,
|
||||||
await agent.GetFromCacheOrReadAllBytesAsync(file.ActualFile, file.MD5),
|
await agent.GetFromCacheOrReadAllBytesAsync(file.ActualFile, file.MD5),
|
||||||
IsMultiRecordFile(file.ActualFile, file.SheetName));
|
IsMultiRecordFile(file.ActualFile, file.SheetName), table.Options);
|
||||||
|
|
||||||
FileRecordCacheManager.Ins.AddCacheLoadedRecords(table, file.MD5, file.SheetName, res);
|
FileRecordCacheManager.Ins.AddCacheLoadedRecords(table, file.MD5, file.SheetName, res);
|
||||||
|
|
||||||
|
|
@ -188,10 +189,10 @@ namespace Luban.Job.Cfg.Utils
|
||||||
await Task.WhenAll(genDataTasks.ToArray());
|
await Task.WhenAll(genDataTasks.ToArray());
|
||||||
}
|
}
|
||||||
|
|
||||||
public static List<Record> LoadCfgRecords(TBean recordType, string originFile, string sheetName, byte[] content, bool multiRecord)
|
public static List<Record> LoadCfgRecords(TBean recordType, string originFile, string sheetName, byte[] content, bool multiRecord, Dictionary<string, string> options)
|
||||||
{
|
{
|
||||||
// (md5,sheet,multiRecord,exportTestData) -> (valuetype, List<(datas)>)
|
// (md5,sheet,multiRecord,exportTestData) -> (valuetype, List<(datas)>)
|
||||||
var dataSource = DataSourceFactory.Create(originFile, sheetName, new MemoryStream(content));
|
var dataSource = DataSourceFactory.Create(originFile, sheetName, options, new MemoryStream(content));
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
if (multiRecord)
|
if (multiRecord)
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
using Luban.Job.Cfg.Datas;
|
using Luban.Job.Cfg.Datas;
|
||||||
|
using Luban.Job.Cfg.DataSources.Excel;
|
||||||
using Luban.Job.Cfg.Defs;
|
using Luban.Job.Cfg.Defs;
|
||||||
using Luban.Job.Cfg.RawDefs;
|
using Luban.Job.Cfg.RawDefs;
|
||||||
using Luban.Job.Cfg.Utils;
|
using Luban.Job.Cfg.Utils;
|
||||||
|
|
@ -65,7 +66,7 @@ namespace Luban.Job.Cfg.l10n
|
||||||
|
|
||||||
public void LoadFromFile(string fileName, byte[] bytes)
|
public void LoadFromFile(string fileName, byte[] bytes)
|
||||||
{
|
{
|
||||||
var records = DataLoaderUtil.LoadCfgRecords(_textRowType, fileName, null, bytes, true);
|
var records = DataLoaderUtil.LoadCfgRecords(_textRowType, fileName, null, bytes, true, null);
|
||||||
foreach (var r in records)
|
foreach (var r in records)
|
||||||
{
|
{
|
||||||
//s_logger.Info("== read text:{}", r.Data);
|
//s_logger.Info("== read text:{}", r.Data);
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue