xiang.li
/
luban
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

【特性】新增list表，支持多key联合索引与多key独立索引

main
walon 2021-12-01 12:33:31 +08:00
parent 967c45dda1
commit ba4bb014a9
9 changed files with 386 additions and 194 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/Luban.Job.Cfg/Source/DataSources/Record.cs
View File

@ -11,8 +11,6 @@ namespace Luban.Job.Cfg.DataSources
public List<string> Tags { get; }
public int Index { get; set; }
public bool IsNotFiltered(List<string> excludeTags)
{
if (Tags == null)

28
src/Luban.Job.Cfg/Source/Defs/CfgDefLoader.cs
View File

@ -183,29 +183,45 @@ namespace Luban.Job.Cfg.Defs
private ETableMode ConvertMode(string defineFile, string tableName, string modeStr, string indexStr)
{
ETableMode mode;
string[] indexs = indexStr.Split(',');
switch (modeStr)
{
case "1":
case "one":
case "single":
case "singleton":
{
if (!string.IsNullOrWhiteSpace(indexStr))
{
throw new Exception($"定义文件:{defineFile} table:'{tableName}' mode=one 是单例表，不支持定义index属性");
throw new Exception($"定义文件:{defineFile} table:'{tableName}' mode={modeStr} 是单例表，不支持定义index属性");
}
mode = ETableMode.ONE;
break;
}
case "map":
{
//if ((string.IsNullOrWhiteSpace(indexStr) || indexStr.Split(',').Length != 1))
//{
// throw new Exception($"定义文件:{CurImportFile} table:{tableName} 是单键表，必须在index属性里指定1个key");
//}
if ((string.IsNullOrWhiteSpace(indexStr) || indexs.Length != 1))
{
throw new Exception($"定义文件:'{defineFile}' table:'{tableName}' 是单键表，index:'{indexStr}'不能包含多个key");
}
mode = ETableMode.MAP;
break;
}
case "list":
{
mode = ETableMode.LIST;
break;
}
case "":
{
mode = ETableMode.MAP;
if (string.IsNullOrWhiteSpace(indexStr) || indexs.Length == 1)
{
mode = ETableMode.MAP;
}
else
{
mode = ETableMode.LIST;
}
break;
}
default:

19
src/Luban.Job.Cfg/Source/Defs/DefAssembly.cs
View File

@ -16,25 +16,6 @@ using System.Linq;
namespace Luban.Job.Cfg.Defs
{
public class TableDataInfo
{
public DefTable Table { get; }
public List<Record> MainRecords { get; }
public List<Record> PatchRecords { get; }
public List<Record> FinalRecords { get; set; }
public Dictionary<DType, Record> FinalRecordMap { get; set; }
public TableDataInfo(DefTable table, List<Record> mainRecords, List<Record> patchRecords)
{
Table = table;
MainRecords = mainRecords;
PatchRecords = patchRecords;
}
}
public class DefAssembly : DefAssemblyBase
{

35
src/Luban.Job.Cfg/Source/Defs/DefTable.cs
View File

@ -4,6 +4,7 @@ using Luban.Job.Common.Types;
using Luban.Job.Common.Utils;
using System;
using System.Collections.Generic;
using System.Linq;
namespace Luban.Job.Cfg.Defs
{
@ -37,6 +38,10 @@ namespace Luban.Job.Cfg.Defs
public bool IsOneValueTable => Mode == ETableMode.ONE;
public bool IsSingletonTable => Mode == ETableMode.ONE;
public bool IsListTable => Mode == ETableMode.LIST;
public List<string> InputFiles { get; }
private readonly Dictionary<string, List<string>> _patchInputFiles;
@ -47,13 +52,17 @@ namespace Luban.Job.Cfg.Defs
public TType KeyTType { get; private set; }
public DefField IndexField { get; private set; }
public int IndexFieldIdIndex { get; private set; }
public TBean ValueTType { get; private set; }
public TType Type { get; private set; }
public DefField IndexField { get; private set; }
public bool IsUnionIndex { get; private set; }
public int IndexFieldIdIndex { get; private set; }
public List<(TType Type, DefField IndexField, int IndexFieldIdIndex)> IndexList { get; } = new();
public bool NeedExport => Assembly.NeedExport(this.Groups);
@ -87,12 +96,14 @@ namespace Luban.Job.Cfg.Defs
{
case ETableMode.ONE:
{
IsUnionIndex = false;
KeyTType = null;
Type = ValueTType;
break;
}
case ETableMode.MAP:
{
IsUnionIndex = true;
if (!string.IsNullOrWhiteSpace(Index))
{
if (ValueTType.GetBeanAs<DefBean>().TryGetField(Index, out var f, out var i))
@ -119,6 +130,26 @@ namespace Luban.Job.Cfg.Defs
Type = TMap.Create(false, null, KeyTType, ValueTType, false);
break;
}
case ETableMode.LIST:
{
var indexs = Index.Split(',', '|', '+', '&').Where(s => !string.IsNullOrWhiteSpace(s)).Select(s => s.Trim()).ToList();
foreach (var idx in indexs)
{
if (ValueTType.GetBeanAs<DefBean>().TryGetField(idx, out var f, out var i))
{
IndexField = f;
IndexFieldIdIndex = i;
this.IndexList.Add((f.CType, f, i));
}
else
{
throw new Exception($"table:'{FullName}' index:'{idx}' 字段不存在");
}
}
// 如果不是 union index, 每个key必须唯一，否则 (key1,..,key n)唯一
IsUnionIndex = IndexList.Count > 1 && !Index.Contains('|');
break;
}
default: throw new Exception($"unknown mode:'{Mode}'");
}
}

177
src/Luban.Job.Cfg/Source/Defs/TableDataInfo.cs 100644
View File

@ -0,0 +1,177 @@
using Luban.Job.Cfg.Datas;
using Luban.Job.Cfg.DataSources;
#if !LUBAN_LITE
#endif
using Luban.Job.Cfg.RawDefs;
using Luban.Job.Cfg.Utils;
using System;
using System.Collections.Generic;
using System.Linq;
namespace Luban.Job.Cfg.Defs
{
public class TableDataInfo
{
private static readonly NLog.Logger s_logger = NLog.LogManager.GetCurrentClassLogger();
public DefTable Table { get; }
public List<Record> MainRecords { get; }
public List<Record> PatchRecords { get; }
public List<Record> FinalRecords { get; private set; }
public Dictionary<DType, Record> FinalRecordMap { get; private set; }
public Dictionary<string, Dictionary<DType, Record>> FinalRecordMapByIndexs { get; private set; }
public TableDataInfo(DefTable table, List<Record> mainRecords, List<Record> patchRecords)
{
Table = table;
MainRecords = mainRecords;
PatchRecords = patchRecords;
BuildIndexs();
}
private void BuildIndexs()
{
#if !LUBAN_LITE
List<Record> mainRecords = MainRecords;
List<Record> patchRecords = PatchRecords;
// 这么大费周张是为了保证被覆盖的id仍然保持原来的顺序，而不是出现在最后
int index = 0;
var recordIndex = new Dictionary<Record, int>();
var overrideRecords = new HashSet<Record>();
foreach (var r in mainRecords)
{
recordIndex.Add(r, index++);
}
if (patchRecords != null)
{
foreach (var r in patchRecords)
{
recordIndex.Add(r, index++);
}
}
var table = Table;
// TODO 有一个微妙的问题，ref检查虽然通过，但ref的记录有可能未导出
switch (Table.Mode)
{
case ETableMode.ONE:
{
// TODO 如果此单例表使用tag,有多个记录，则patchRecords会覆盖全部。
// 好像也挺有道理的，毕竟没有key，无法区分覆盖哪个
if (patchRecords != null && patchRecords.Count > 0)
{
mainRecords = patchRecords;
}
FinalRecords = mainRecords;
break;
}
case ETableMode.MAP:
{
var recordMap = new Dictionary<DType, Record>();
foreach (Record r in mainRecords)
{
DType key = r.Data.Fields[table.IndexFieldIdIndex];
if (!recordMap.TryAdd(key, r))
{
throw new Exception($@"配置表 '{table.FullName}' 主文件 主键字段:'{table.Index}' 主键值:'{key}' 重复.
记录1 来自文件:{r.Source}
记录2 来自文件:{recordMap[key].Source}
");
}
}
if (patchRecords != null && patchRecords.Count > 0)
{
foreach (Record r in patchRecords)
{
DType key = r.Data.Fields[table.IndexFieldIdIndex];
if (recordMap.TryGetValue(key, out var old))
{
if (overrideRecords.Contains(old))
{
throw new Exception($"配置表 '{table.FullName}' 主文件 主键字段:'{table.Index}' 主键值:'{key}' 被patch多次覆盖，请检查patch是否有重复记录");
}
s_logger.Debug("配置表 {} 分支文件 主键:{} 覆盖 主文件记录", table.FullName, key);
mainRecords[recordIndex[old]] = r;
}
else
{
mainRecords.Add(r);
}
overrideRecords.Add(r);
recordMap[key] = r;
}
}
FinalRecords = mainRecords;
FinalRecordMap = recordMap;
break;
}
case ETableMode.LIST:
{
if (patchRecords != null && patchRecords.Count > 0)
{
throw new Exception($"配置表 '{table.FullName}' 是list表.不支持patch");
}
var recordMapByIndexs = new Dictionary<string, Dictionary<DType, Record>>();
if (table.IsUnionIndex)
{
var unionRecordMap = new Dictionary<List<DType>, Record>(ListEqualityComparer<DType>.Default); // comparetor
foreach (Record r in mainRecords)
{
var unionKeys = table.IndexList.Select(idx => r.Data.Fields[idx.IndexFieldIdIndex]).ToList();
if (!unionRecordMap.TryAdd(unionKeys, r))
{
throw new Exception($@"配置表 '{table.FullName}' 主文件 主键字段:'{table.Index}' 主键值:'{Bright.Common.StringUtil.CollectionToString(unionKeys)}' 重复.
记录1 来自文件:{r.Source}
记录2 来自文件:{unionRecordMap[unionKeys].Source}
");
}
}
// 联合索引的 独立子索引允许有重复key
foreach (var indexInfo in table.IndexList)
{
var recordMap = new Dictionary<DType, Record>();
foreach (Record r in mainRecords)
{
DType key = r.Data.Fields[indexInfo.IndexFieldIdIndex];
recordMap[key] = r;
}
recordMapByIndexs.Add(indexInfo.IndexField.Name, recordMap);
}
}
else
{
foreach (var indexInfo in table.IndexList)
{
var recordMap = new Dictionary<DType, Record>();
foreach (Record r in mainRecords)
{
DType key = r.Data.Fields[indexInfo.IndexFieldIdIndex];
if (!recordMap.TryAdd(key, r))
{
throw new Exception($@"配置表 '{table.FullName}' 主文件 主键字段:'{indexInfo.IndexField.Name}' 主键值:'{key}' 重复.
记录1 来自文件:{r.Source}
记录2 来自文件:{recordMap[key].Source}
");
}
}
recordMapByIndexs.Add(indexInfo.IndexField.Name, recordMap);
}
}
this.FinalRecordMapByIndexs = recordMapByIndexs;
FinalRecords = mainRecords;
break;
}
default: throw new Exception($"unknown mode:{Table.Mode}");
}
#endif
}
}
}

1
src/Luban.Job.Cfg/Source/RawDefs/Table.cs
View File

@ -6,6 +6,7 @@ namespace Luban.Job.Cfg.RawDefs
{
ONE,
MAP,
LIST,
}
public class CfgInputFile

30
src/Luban.Job.Cfg/Source/Utils/ListEqualityComparer.cs 100644
View File

@ -0,0 +1,30 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace Luban.Job.Cfg.Utils
{
public class ListEqualityComparer<T> : IEqualityComparer<List<T>>
{
public static ListEqualityComparer<T> Default { get; } = new ListEqualityComparer<T>();
public bool Equals(List<T> x, List<T> y)
{
return x.Count == y.Count && System.Linq.Enumerable.SequenceEqual(x, y);
}
public int GetHashCode([DisallowNull] List<T> obj)
{
int hash = 17;
foreach (T x in obj)
{
hash = hash * 23 + x.GetHashCode();
}
return hash;
}
}
}

133
src/Luban.Job.Cfg/Source/ValidatorContext.cs
View File

@ -61,45 +61,31 @@ namespace Luban.Job.Cfg
public async Task ValidateTables(IEnumerable<DefTable> tables)
{
var tasks = new List<Task>();
foreach (var t in tables)
{
var tasks = new List<Task>();
foreach (var t in tables)
tasks.Add(Task.Run(() =>
{
tasks.Add(Task.Run(() =>
var records = t.Assembly.GetTableAllDataList(t);
var visitor = new ValidatorVisitor(this);
try
{
ValidateTableModeIndex(t);
}));
}
await Task.WhenAll(tasks);
}
{
var tasks = new List<Task>();
foreach (var t in tables)
{
tasks.Add(Task.Run(() =>
{
var records = t.Assembly.GetTableAllDataList(t);
var visitor = new ValidatorVisitor(this);
try
{
CurrentVisitor = visitor;
visitor.ValidateTable(t, records);
CurrentVisitor = visitor;
visitor.ValidateTable(t, records);
#if !LUBAN_LITE
if (this.Assembly.NeedL10nTextTranslate)
{
ValidateText(t, records);
}
#endif
}
finally
if (this.Assembly.NeedL10nTextTranslate)
{
CurrentVisitor = null;
ValidateText(t, records);
}
}));
}
await Task.WhenAll(tasks);
#endif
}
finally
{
CurrentVisitor = null;
}
}));
}
await Task.WhenAll(tasks);
if (!string.IsNullOrWhiteSpace(RootDir))
{
@ -181,88 +167,5 @@ namespace Luban.Job.Cfg
}
}
private void ValidateTableModeIndex(DefTable table)
{
var tableDataInfo = Assembly.GetTableDataInfo(table);
List<Record> mainRecords = tableDataInfo.MainRecords;
List<Record> patchRecords = tableDataInfo.PatchRecords;
// 这么大费周张是为了保证被覆盖的id仍然保持原来的顺序，而不是出现在最后
int index = 0;
foreach (var r in mainRecords)
{
r.Index = index++;
}
if (patchRecords != null)
{
foreach (var r in patchRecords)
{
r.Index = index++;
}
}
var mainRecordMap = new Dictionary<DType, Record>();
switch (table.Mode)
{
case ETableMode.ONE:
{
//if (mainRecords.Count != 1)
//{
// throw new Exception($"配置表 {table.FullName} 是单值表 mode=one,但主文件数据个数:{mainRecords.Count} != 1");
//}
//if (patchRecords != null && patchRecords.Count != 1)
//{
// throw new Exception($"配置表 {table.FullName} 是单值表 mode=one,但分支文件数据个数:{patchRecords.Count} != 1");
//}
if (patchRecords != null)
{
mainRecords = patchRecords;
}
break;
}
case ETableMode.MAP:
{
foreach (Record r in mainRecords)
{
DType key = r.Data.Fields[table.IndexFieldIdIndex];
if (!mainRecordMap.TryAdd(key, r))
{
throw new Exception($@"配置表 '{table.FullName}' 主文件 主键字段:'{table.Index}' 主键值:'{key}' 重复.
记录1 来自文件:{r.Source}
记录2 来自文件:{mainRecordMap[key].Source}
");
}
}
if (patchRecords != null)
{
var patchRecordMap = new Dictionary<DType, Record>();
foreach (Record r in patchRecords)
{
DType key = r.Data.Fields[table.IndexFieldIdIndex];
if (!patchRecordMap.TryAdd(key, r))
{
throw new Exception($@"配置表 '{table.FullName}' 分支文件 主键字段:'{table.Index}' 主键值:'{key}' 重复.
记录1 来自文件:{r.Source}
记录2 来自文件:{patchRecordMap[key].Source}
");
}
if (mainRecordMap.TryGetValue(key, out var old))
{
s_logger.Debug("配置表 {} 分支文件 主键:{} 覆盖 主文件记录", table.FullName, key);
mainRecords[old.Index] = r;
}
mainRecordMap[key] = r;
}
}
break;
}
}
#if !LUBAN_LITE
tableDataInfo.FinalRecords = mainRecords;
tableDataInfo.FinalRecordMap = mainRecordMap;
#endif
}
}
}

155
src/Luban.Job.Cfg/Source/Validators/RefValidator.cs
View File

@ -1,6 +1,7 @@
using Luban.Job.Cfg.Datas;
using Luban.Job.Cfg.DataVisitors;
using Luban.Job.Cfg.Defs;
using Luban.Job.Cfg.RawDefs;
using Luban.Job.Cfg.Utils;
using Luban.Job.Common.Defs;
using Luban.Job.Common.Types;
@ -17,11 +18,7 @@ namespace Luban.Job.Cfg.Validators
public static string GetActualTableName(string table)
{
#if !LUBAN_LITE
return table.EndsWith("?") ? table[0..^1] : table;
#else
return table.EndsWith("?") ? table.Substring(0, table.Length - 1) : table;
#endif
}
public List<string> Tables { get; }
@ -45,57 +42,78 @@ namespace Luban.Job.Cfg.Validators
}
var assembly = ctx.Assembly;
#if !LUBAN_LITE
foreach (var table in Tables)
{
bool zeroAble;
string actualTable;
if (table.EndsWith("?"))
{
zeroAble = true;
#if !LUBAN_LITE
actualTable = table[0..^1];
#else
actualTable = table.Substring(0, table.Length - 1);
#endif
}
else
{
zeroAble = false;
actualTable = table;
}
var (actualTable, field, zeroAble) = ParseRefString(table);
if (zeroAble && key.Apply(IsDefaultValue.Ins))
{
return;
}
DefTable ct = assembly.GetCfgTable(actualTable);
#if !LUBAN_LITE
var recordMap = assembly.GetTableDataInfo(ct).FinalRecordMap;
if (/*recordMap != null &&*/ recordMap.ContainsKey(key))
switch (ct.Mode)
{
return;
case ETableMode.ONE:
{
throw new NotSupportedException($"{actualTable} 是singleton表，不支持ref");
}
case ETableMode.MAP:
{
var recordMap = assembly.GetTableDataInfo(ct).FinalRecordMap;
if (/*recordMap != null &&*/ recordMap.ContainsKey(key))
{
return;
}
break;
}
case ETableMode.LIST:
{
var recordMap = assembly.GetTableDataInfo(ct).FinalRecordMapByIndexs[field];
if (recordMap.ContainsKey(key))
{
return;
}
break;
}
default: throw new NotSupportedException();
}
#endif
}
string source = ValidatorContext.CurrentVisitor.CurrentValidateRecord.Source;
foreach (var table in Tables)
{
string actualTable;
if (table.EndsWith("?"))
{
#if !LUBAN_LITE
actualTable = table[0..^1];
#else
actualTable = table.Substring(0, table.Length - 1);
#endif
}
else
{
actualTable = table;
}
var (actualTable, field, zeroAble) = ParseRefString(table);
DefTable ct = assembly.GetCfgTable(actualTable);
string source = ValidatorContext.CurrentVisitor.CurrentValidateRecord.Source;
assembly.Agent.Error("记录 {0} = {1} (来自文件:{2}) 在引用表:{3} 中不存在", ValidatorContext.CurrentRecordPath, key, source, table);
}
#endif
}
private (string TableName, string FieldName, bool IgnoreDefault) ParseRefString(string refStr)
{
bool ignoreDefault = false;
if (refStr.EndsWith("?"))
{
refStr = refStr.Substring(0, refStr.Length - 1);
ignoreDefault = true;
}
string tableName;
string fieldName;
int sepIndex = refStr.IndexOf('@');
if (sepIndex >= 0)
{
tableName = refStr.Substring(sepIndex + 1);
fieldName = refStr.Substring(0, sepIndex);
}
else
{
tableName = refStr;
fieldName = "";
}
return (tableName, fieldName, ignoreDefault);
}
public void Compile(DefFieldBase def)
@ -110,28 +128,65 @@ namespace Luban.Job.Cfg.Validators
var assembly = ((DefField)def).Assembly;
foreach (var table in Tables)
{
#if !LUBAN_LITE
string actualTable = table.EndsWith("?") ? table[0..^1] : table;
#else
string actualTable = table.EndsWith("?") ? table.Substring(0, table.Length - 1) : table;
#endif
var (actualTable, indexName, ignoreDefault) = ParseRefString(table);
var ct = assembly.GetCfgTable(actualTable);
if (ct == null)
{
throw new Exception($"结构:{hostTypeName} 字段:{fieldName} ref:{table} 不存在");
throw new Exception($"结构:{hostTypeName} 字段:{fieldName} ref:{actualTable} 不存在");
}
if (!ct.NeedExport)
{
throw new Exception($"type:'{hostTypeName}' field:'{fieldName}' ref 引用的表:'{table}' 没有导出");
throw new Exception($"type:'{hostTypeName}' field:'{fieldName}' ref 引用的表:'{actualTable}' 没有导出");
}
if (ct.IsOneValueTable)
{
throw new Exception($"结构:{hostTypeName} 字段:{fieldName} ref:{table} 是单值表，不能执行引用检查");
if (string.IsNullOrEmpty(fieldName))
{
throw new Exception($"结构:{hostTypeName} 字段:{fieldName} ref:{actualTable} 是singleton表，索引字段不能为空");
}
else
{
if (!ct.ValueTType.Bean.TryGetField(fieldName, out var indexField, out _))
{
throw new Exception($"结构:{hostTypeName} 字段:{fieldName} ref:{actualTable} value_type:{ct.ValueTType.Bean.FullName} 未包含索引字段:{fieldName}");
}
if (!(indexField.CType is TMap tmap))
{
throw new Exception($"结构:{hostTypeName} 字段:{fieldName} ref:{actualTable} value_type:{ct.ValueTType.Bean.FullName} 索引字段:{fieldName} type:{indexField.CType.TypeName} 不是map类型");
}
if (tmap.KeyType.TypeName != Type.TypeName)
{
throw new Exception($"结构:{hostTypeName} 字段:{fieldName} 类型:'{Type.TypeName}' 与被引用的表:{actualTable} value_type:{ct.ValueTType.Bean.FullName} 索引字段:{fieldName} key_type:{tmap.KeyType.TypeName} 不一致");
}
}
}
var keyType = ct.KeyTType;
if (keyType.GetType() != Type.GetType())
else if (ct.IsMapTable)
{
throw new Exception($"type:'{hostTypeName}' field:'{fieldName}' 类型:'{Type.GetType()}' 与 被引用的表:'{ct.FullName}' key类型:'{keyType.GetType()}' 不一致");
if (!string.IsNullOrEmpty(indexName))
{
throw new Exception($"结构:{hostTypeName} 字段:{fieldName} ref:{actualTable} 是map表，不能索引子字段");
}
var keyType = ct.KeyTType;
if (keyType.TypeName != Type.TypeName)
{
throw new Exception($"type:'{hostTypeName}' field:'{fieldName}' 类型:'{Type.TypeName}' 与 被引用的map表:'{actualTable}' key类型:'{keyType.TypeName}' 不一致");
}
}
else
{
if (string.IsNullOrEmpty(indexName))
{
throw new Exception($"结构:{hostTypeName} 字段:{fieldName} ref:{actualTable} 是list表，必须显式指定索引字段");
}
var indexField = ct.IndexList.Find(k => k.IndexField.Name == indexName);
if (indexField.Type == null)
{
throw new Exception($"结构:{hostTypeName} 字段:{fieldName} 索引字段:{indexName} 不是被引用的list表:{actualTable} 的索引字段，合法值为'{ct.Index}'之一");
}
if (indexField.Type.TypeName != Type.TypeName)
{
throw new Exception($"type:'{hostTypeName}' field:'{fieldName}' 类型:'{Type.TypeName}' 与 被引用的list表:'{actualTable}' key:{indexName} 类型:'{indexField.Type.TypeName}' 不一致");
}
}
}
}