xiang.li
/
luban
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

【修复】修复lambda函数中捕捉了foreach循环变量的问题(其实好像也没啥问题，只要不修改它？) 

【优化】新增cfg data 文件的cache.只要文件记录相同，则直接返回生成好的内容(之前虽然缓存了加载记录，但仍然要Export一遍)
main
walon 2021-08-11 19:34:22 +08:00
parent 9673664147
commit 73e98662ab
13 changed files with 82 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
src/Luban.Job.Cfg/Source/Cache/FileRecordCacheManager.cs
View File

@ -35,7 +35,7 @@ namespace Luban.Job.Cfg.Cache
}
}
private readonly ConcurrentDictionary<(string MD5, string SheetName), FileRecordCache> _caches = new();
private readonly ConcurrentDictionary<(string TableName, string MD5, string SheetName), FileRecordCache> _caches = new();
private readonly object _shrinkLocker = new object();
@ -43,7 +43,7 @@ namespace Luban.Job.Cfg.Cache
{
// TODO text localization check
cacheRecords = null;
if (!_caches.TryGetValue((md5, sheetName), out var r))
if (!_caches.TryGetValue((table.FullName, md5, sheetName), out var r))
{
return false;
}
@ -64,7 +64,7 @@ namespace Luban.Job.Cfg.Cache
{
lock (_shrinkLocker)
{
_caches[(md5, sheetName)] = new FileRecordCache(table, cacheRecords);
_caches[(table.FullName, md5, sheetName)] = new FileRecordCache(table, cacheRecords);
if (_caches.Count > CACHE_FILE_HIGH_WATER_MARK)
{
s_logger.Info("ShrinkCaches. cache count > high CACHE_FILE_HIGH_WATER_MARK:{}", CACHE_FILE_HIGH_WATER_MARK);
@ -73,6 +73,37 @@ namespace Luban.Job.Cfg.Cache
}
}
private readonly ConcurrentDictionary<(string TableFullName, string DataType), (DefTable Table, List<Record> Records, string Md5)> _tableCaches = new();
public bool TryGetRecordOutputData(DefTable table, List<Record> records, string dataType, out string md5)
{
if (_tableCaches.TryGetValue((table.FullName, dataType), out var cacheInfo))
{
var cacheAss = cacheInfo.Table.Assembly;
var curAss = table.Assembly;
if (cacheAss.TimeZone == curAss.TimeZone
&& cacheAss.TargetBranch == null && curAss.TargetBranch == null
&& !cacheAss.NeedL10nTextTranslate && !curAss.NeedL10nTextTranslate
&& records.Count == cacheInfo.Records.Count && records.SequenceEqual(cacheInfo.Records))
{
md5 = cacheInfo.Md5;
s_logger.Debug("find output data cache. table:{} dataType:{} md5:{}", table.FullName, dataType, md5);
return true;
}
}
md5 = null;
return false;
}
public void AddCachedRecordOutputData(DefTable table, List<Record> records, string dataType, string md5)
{
var curAss = table.Assembly;
if (curAss.TargetBranch == null && !curAss.NeedL10nTextTranslate)
{
_tableCaches[(table.FullName, dataType)] = (table, records, md5);
s_logger.Debug("add output data cache. table:{} dataType:{} md5:{}", table.FullName, dataType, md5);
}
}
private void ShrinkCaches()
{
if (_caches.Count < CACHE_FILE_HIGH_WATER_MARK)

2
src/Luban.Job.Cfg/Source/DataCreators/ExcelDataCreator.cs
View File

@ -364,7 +364,7 @@ namespace Luban.Job.Cfg.DataCreators
throw new InvalidExcelDataException("字段不是nullable类型，不能为null");
}
}
return new DString(s);
return DString.ValueOf(s);
}
public DType Accept(TBytes type, object converter, ExcelStream x, DefAssembly ass)

2
src/Luban.Job.Cfg/Source/DataCreators/JsonDataCreator.cs
View File

@ -71,7 +71,7 @@ namespace Luban.Job.Cfg.DataCreators
public DType Accept(TString type, JsonElement x, DefAssembly ass)
{
return new DString(x.GetString());
return DString.ValueOf(x.GetString());
}
public DType Accept(TBytes type, JsonElement x, DefAssembly ass)

2
src/Luban.Job.Cfg/Source/DataCreators/LuaDataCreator.cs
View File

@ -110,7 +110,7 @@ namespace Luban.Job.Cfg.DataCreators
{
if (x is string s)
{
return new DString(s);
return DString.ValueOf(s);
}
else
{

2
src/Luban.Job.Cfg/Source/DataCreators/StringDataCreator.cs
View File

@ -137,7 +137,7 @@ namespace Luban.Job.Cfg.DataCreators
public DType Accept(TString type, string x)
{
return new DString(x);
return DString.ValueOf(x);
}
public DType Accept(TBytes type, string x)

2
src/Luban.Job.Cfg/Source/DataCreators/XmlDataCreator.cs
View File

@ -72,7 +72,7 @@ namespace Luban.Job.Cfg.DataCreators
public DType Accept(TString type, XElement x, DefAssembly ass)
{
return new DString(x.Value);
return DString.ValueOf(x.Value);
}
public DType Accept(TBytes type, XElement x, DefAssembly ass)

2
src/Luban.Job.Cfg/Source/Datas/DString.cs
View File

@ -15,7 +15,7 @@ namespace Luban.Job.Cfg.Datas
return new DString(s);
}
public DString(string x) : base(x)
private DString(string x) : base(x)
{
}

2
src/Luban.Job.Cfg/Source/Defs/DefAssembly.cs
View File

@ -85,6 +85,8 @@ namespace Luban.Job.Cfg.Defs
public NotConvertTextSet NotConvertTextSet { get; private set; }
public bool NeedL10nTextTranslate => ExportTextTable != null;
public void InitL10n(string textValueFieldName)
{
ExportTextTable = new TextTable(this, textValueFieldName);

41
src/Luban.Job.Cfg/Source/JobController.cs
View File

@ -2,6 +2,7 @@ using Bright.Time;
using CommandLine;
using Luban.Common.Protos;
using Luban.Common.Utils;
using Luban.Job.Cfg.Cache;
using Luban.Job.Cfg.DataCreators;
using Luban.Job.Cfg.Defs;
using Luban.Job.Cfg.Generate;
@ -476,10 +477,11 @@ namespace Luban.Job.Cfg
ctx.Lan = GetLanguage(genType);
foreach (var c in ctx.ExportTypes)
{
var t = c;
ctx.Tasks.Add(Task.Run(() =>
{
var content = FileHeaderUtil.ConcatAutoGenerationHeader(ctx.Render.RenderAny(c), ctx.Lan);
var file = RenderFileUtil.GetDefTypePath(c.FullName, ctx.Lan);
var content = FileHeaderUtil.ConcatAutoGenerationHeader(ctx.Render.RenderAny(t), ctx.Lan);
var file = RenderFileUtil.GetDefTypePath(t.FullName, ctx.Lan);
var md5 = CacheFileUtil.GenMd5AndAddCache(file, content);
ctx.GenCodeFilesInOutputCodeDir.Add(new FileInfo() { FilePath = file, MD5 = md5 });
}));
@ -713,10 +715,11 @@ namespace {ctx.TopModule}
foreach (var c in renderTypes)
{
var table = c;
ctx.Tasks.Add(Task.Run(() =>
{
var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(c), ELanguage.CPP);
var file = "editor_" + RenderFileUtil.GetUeCppDefTypeHeaderFilePath(c.FullName);
var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(table), ELanguage.CPP);
var file = "editor_" + RenderFileUtil.GetUeCppDefTypeHeaderFilePath(table.FullName);
var md5 = CacheFileUtil.GenMd5AndAddCache(file, content);
ctx.GenCodeFilesInOutputCodeDir.Add(new FileInfo() { FilePath = file, MD5 = md5 });
}));
@ -745,10 +748,11 @@ namespace {ctx.TopModule}
var render = new EditorCsRender();
foreach (var c in ctx.Assembly.Types.Values)
{
var type = c;
ctx.Tasks.Add(Task.Run(() =>
{
var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(c), ELanguage.CS);
var file = RenderFileUtil.GetDefTypePath(c.FullName, ELanguage.CS);
var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(type), ELanguage.CS);
var file = RenderFileUtil.GetDefTypePath(type.FullName, ELanguage.CS);
var md5 = CacheFileUtil.GenMd5AndAddCache(file, content);
ctx.GenCodeFilesInOutputCodeDir.Add(new FileInfo() { FilePath = file, MD5 = md5 });
}));
@ -760,15 +764,16 @@ namespace {ctx.TopModule}
var render = new UE4BpCppRender();
foreach (var c in ctx.ExportTypes)
{
if (!(c is DefEnum || c is DefBean))
var type = c;
if (!(type is DefEnum || type is DefBean))
{
continue;
}
ctx.Tasks.Add(Task.Run(() =>
{
var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(c), ELanguage.CPP);
var file = "bp_" + RenderFileUtil.GetUeCppDefTypeHeaderFilePath(c.FullName);
var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(type), ELanguage.CPP);
var file = "bp_" + RenderFileUtil.GetUeCppDefTypeHeaderFilePath(type.FullName);
var md5 = CacheFileUtil.GenMd5AndAddCache(file, content);
ctx.GenCodeFilesInOutputCodeDir.Add(new FileInfo() { FilePath = file, MD5 = md5 });
}));
@ -780,11 +785,17 @@ namespace {ctx.TopModule}
string genType = ctx.GenType;
foreach (var c in ctx.ExportTables)
{
var table = c;
ctx.Tasks.Add(Task.Run(() =>
{
var content = DataExporterUtil.ToOutputData(c, ctx.Assembly.GetTableExportDataList(c), genType);
var file = GetOutputFileName(genType, c.OutputDataFile);
var md5 = CacheFileUtil.GenStringOrBytesMd5AndAddCache(file, content);
var file = GetOutputFileName(genType, table.OutputDataFile);
var records = ctx.Assembly.GetTableExportDataList(table);
if (!FileRecordCacheManager.Ins.TryGetRecordOutputData(table, records, genType, out string md5))
{
var content = DataExporterUtil.ToOutputData(table, records, genType);
md5 = CacheFileUtil.GenStringOrBytesMd5AndAddCache(file, content);
FileRecordCacheManager.Ins.AddCachedRecordOutputData(table, records, genType, md5);
}
ctx.GenDataFilesInOutputDataDir.Add(new FileInfo() { FilePath = file, MD5 = md5 });
}));
}
@ -796,9 +807,10 @@ namespace {ctx.TopModule}
var allJsonTask = new List<Task<string>>();
foreach (var c in exportTables)
{
var table = c;
allJsonTask.Add(Task.Run(() =>
{
return (string)DataExporterUtil.ToOutputData(c, ctx.Assembly.GetTableExportDataList(c), "data_json");
return (string)DataExporterUtil.ToOutputData(table, ctx.Assembly.GetTableExportDataList(table), "data_json");
}));
}
@ -827,9 +839,10 @@ namespace {ctx.TopModule}
var genDataTasks = new List<Task<List<ResourceInfo>>>();
foreach (var c in ctx.ExportTables)
{
var table = c;
genDataTasks.Add(Task.Run(() =>
{
return DataExporterUtil.ExportResourceList(ctx.Assembly.GetTableExportDataList(c));
return DataExporterUtil.ExportResourceList(ctx.Assembly.GetTableExportDataList(table));
}));
}

5
src/Luban.Job.Cfg/Source/Utils/DataLoaderUtil.cs
View File

@ -157,14 +157,15 @@ namespace Luban.Job.Cfg.Utils
foreach (DefTable c in exportTables)
{
var table = c;
genDataTasks.Add(Task.Run(async () =>
{
long beginTime = TimeUtil.NowMillis;
await LoadTableAsync(agent, c, dataDir, branchName, branchDataDir, exportTestData);
await LoadTableAsync(agent, table, dataDir, branchName, branchDataDir, exportTestData);
long endTime = TimeUtil.NowMillis;
if (endTime - beginTime > 100)
{
ctx.Info("====== load {0} cost {1} ms ======", c.FullName, (endTime - beginTime));
ctx.Info("====== load {0} cost {1} ms ======", table.FullName, (endTime - beginTime));
}
}));
}

5
src/Luban.Job.Cfg/Source/ValidatorContext.cs
View File

@ -87,7 +87,10 @@ namespace Luban.Job.Cfg
{
CurrentVisitor = visitor;
visitor.ValidateTable(t, records);
ValidateText(t, records);
if (this.Assembly.NeedL10nTextTranslate)
{
ValidateText(t, records);
}
}
finally
{

5
src/Luban.Job.Db/Source/JobController.cs
View File

@ -105,10 +105,11 @@ namespace Luban.Job.Db
var render = new AsyncCsRender();
foreach (var c in ass.Types.Values)
{
var type = c;
tasks.Add(Task.Run(() =>
{
var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(c), Common.ELanguage.CS);
var file = RenderFileUtil.GetDefTypePath(c.FullName, Common.ELanguage.CS);
var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(type), Common.ELanguage.CS);
var file = RenderFileUtil.GetDefTypePath(type.FullName, Common.ELanguage.CS);
var md5 = CacheFileUtil.GenMd5AndAddCache(file, content);
genCodeFiles.Add(new FileInfo() { FilePath = file, MD5 = md5 });
}));

5
src/Luban.Job.Proto/Source/JobController.cs
View File

@ -116,10 +116,11 @@ namespace Luban.Job.Proto
var render = new CsRender();
foreach (var c in ass.Types.Values)
{
var type = c;
tasks.Add(Task.Run(() =>
{
var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(c), Common.ELanguage.CS);
var file = RenderFileUtil.GetDefTypePath(c.FullName, Common.ELanguage.CS);
var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(type), Common.ELanguage.CS);
var file = RenderFileUtil.GetDefTypePath(type.FullName, Common.ELanguage.CS);
var md5 = CacheFileUtil.GenMd5AndAddCache(file, content);
genCodeFiles.Add(new FileInfo() { FilePath = file, MD5 = md5 });
}));