xiang.li
/
luban
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

【修复】修复lambda函数中捕捉了foreach循环变量的问题(其实好像也没啥问题，只要不修改它？) 

【优化】新增cfg data 文件的cache.只要文件记录相同，则直接返回生成好的内容(之前虽然缓存了加载记录，但仍然要Export一遍)
main
walon 2021-08-11 19:34:22 +08:00
parent 9673664147
commit 73e98662ab
13 changed files with 82 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
src/Luban.Job.Cfg/Source/Cache/FileRecordCacheManager.cs
View File

@ -35,7 +35,7 @@ namespace Luban.Job.Cfg.Cache
} }
} }
private readonly ConcurrentDictionary<(string MD5, string SheetName), FileRecordCache> _caches = new(); private readonly ConcurrentDictionary<(string TableName, string MD5, string SheetName), FileRecordCache> _caches = new();
private readonly object _shrinkLocker = new object(); private readonly object _shrinkLocker = new object();
@ -43,7 +43,7 @@ namespace Luban.Job.Cfg.Cache
{ {
// TODO text localization check // TODO text localization check
cacheRecords = null; cacheRecords = null;
if (!_caches.TryGetValue((md5, sheetName), out var r)) if (!_caches.TryGetValue((table.FullName, md5, sheetName), out var r))
{ {
return false; return false;
} }
@ -64,7 +64,7 @@ namespace Luban.Job.Cfg.Cache
{ {
lock (_shrinkLocker) lock (_shrinkLocker)
{ {
_caches[(md5, sheetName)] = new FileRecordCache(table, cacheRecords); _caches[(table.FullName, md5, sheetName)] = new FileRecordCache(table, cacheRecords);
if (_caches.Count > CACHE_FILE_HIGH_WATER_MARK) if (_caches.Count > CACHE_FILE_HIGH_WATER_MARK)
{ {
s_logger.Info("ShrinkCaches. cache count > high CACHE_FILE_HIGH_WATER_MARK:{}", CACHE_FILE_HIGH_WATER_MARK); s_logger.Info("ShrinkCaches. cache count > high CACHE_FILE_HIGH_WATER_MARK:{}", CACHE_FILE_HIGH_WATER_MARK);
@ -73,6 +73,37 @@ namespace Luban.Job.Cfg.Cache
} }
} }
private readonly ConcurrentDictionary<(string TableFullName, string DataType), (DefTable Table, List<Record> Records, string Md5)> _tableCaches = new();
public bool TryGetRecordOutputData(DefTable table, List<Record> records, string dataType, out string md5)
{
if (_tableCaches.TryGetValue((table.FullName, dataType), out var cacheInfo))
{
var cacheAss = cacheInfo.Table.Assembly;
var curAss = table.Assembly;
if (cacheAss.TimeZone == curAss.TimeZone
&& cacheAss.TargetBranch == null && curAss.TargetBranch == null
&& !cacheAss.NeedL10nTextTranslate && !curAss.NeedL10nTextTranslate
&& records.Count == cacheInfo.Records.Count && records.SequenceEqual(cacheInfo.Records))
{
md5 = cacheInfo.Md5;
s_logger.Debug("find output data cache. table:{} dataType:{} md5:{}", table.FullName, dataType, md5);
return true;
}
}
md5 = null;
return false;
}
public void AddCachedRecordOutputData(DefTable table, List<Record> records, string dataType, string md5)
{
var curAss = table.Assembly;
if (curAss.TargetBranch == null && !curAss.NeedL10nTextTranslate)
{
_tableCaches[(table.FullName, dataType)] = (table, records, md5);
s_logger.Debug("add output data cache. table:{} dataType:{} md5:{}", table.FullName, dataType, md5);
}
}
private void ShrinkCaches() private void ShrinkCaches()
{ {
if (_caches.Count < CACHE_FILE_HIGH_WATER_MARK) if (_caches.Count < CACHE_FILE_HIGH_WATER_MARK)

2
src/Luban.Job.Cfg/Source/DataCreators/ExcelDataCreator.cs
View File

@ -364,7 +364,7 @@ namespace Luban.Job.Cfg.DataCreators
throw new InvalidExcelDataException("字段不是nullable类型，不能为null"); throw new InvalidExcelDataException("字段不是nullable类型，不能为null");
} }
} }
return new DString(s); return DString.ValueOf(s);
} }
public DType Accept(TBytes type, object converter, ExcelStream x, DefAssembly ass) public DType Accept(TBytes type, object converter, ExcelStream x, DefAssembly ass)

2
src/Luban.Job.Cfg/Source/DataCreators/JsonDataCreator.cs
View File

@ -71,7 +71,7 @@ namespace Luban.Job.Cfg.DataCreators
public DType Accept(TString type, JsonElement x, DefAssembly ass) public DType Accept(TString type, JsonElement x, DefAssembly ass)
{ {
return new DString(x.GetString()); return DString.ValueOf(x.GetString());
} }
public DType Accept(TBytes type, JsonElement x, DefAssembly ass) public DType Accept(TBytes type, JsonElement x, DefAssembly ass)

2
src/Luban.Job.Cfg/Source/DataCreators/LuaDataCreator.cs
View File

@ -110,7 +110,7 @@ namespace Luban.Job.Cfg.DataCreators
{ {
if (x is string s) if (x is string s)
{ {
return new DString(s); return DString.ValueOf(s);
} }
else else
{ {

2
src/Luban.Job.Cfg/Source/DataCreators/StringDataCreator.cs
View File

@ -137,7 +137,7 @@ namespace Luban.Job.Cfg.DataCreators
public DType Accept(TString type, string x) public DType Accept(TString type, string x)
{ {
return new DString(x); return DString.ValueOf(x);
} }
public DType Accept(TBytes type, string x) public DType Accept(TBytes type, string x)

2
src/Luban.Job.Cfg/Source/DataCreators/XmlDataCreator.cs
View File

@ -72,7 +72,7 @@ namespace Luban.Job.Cfg.DataCreators
public DType Accept(TString type, XElement x, DefAssembly ass) public DType Accept(TString type, XElement x, DefAssembly ass)
{ {
return new DString(x.Value); return DString.ValueOf(x.Value);
} }
public DType Accept(TBytes type, XElement x, DefAssembly ass) public DType Accept(TBytes type, XElement x, DefAssembly ass)

2
src/Luban.Job.Cfg/Source/Datas/DString.cs
View File

@ -15,7 +15,7 @@ namespace Luban.Job.Cfg.Datas
return new DString(s); return new DString(s);
} }
public DString(string x) : base(x) private DString(string x) : base(x)
{ {
} }

2
src/Luban.Job.Cfg/Source/Defs/DefAssembly.cs
View File

@ -85,6 +85,8 @@ namespace Luban.Job.Cfg.Defs
public NotConvertTextSet NotConvertTextSet { get; private set; } public NotConvertTextSet NotConvertTextSet { get; private set; }
public bool NeedL10nTextTranslate => ExportTextTable != null;
public void InitL10n(string textValueFieldName) public void InitL10n(string textValueFieldName)
{ {
ExportTextTable = new TextTable(this, textValueFieldName); ExportTextTable = new TextTable(this, textValueFieldName);

41
src/Luban.Job.Cfg/Source/JobController.cs
View File

@ -2,6 +2,7 @@ using Bright.Time;
using CommandLine; using CommandLine;
using Luban.Common.Protos; using Luban.Common.Protos;
using Luban.Common.Utils; using Luban.Common.Utils;
using Luban.Job.Cfg.Cache;
using Luban.Job.Cfg.DataCreators; using Luban.Job.Cfg.DataCreators;
using Luban.Job.Cfg.Defs; using Luban.Job.Cfg.Defs;
using Luban.Job.Cfg.Generate; using Luban.Job.Cfg.Generate;
@ -476,10 +477,11 @@ namespace Luban.Job.Cfg
ctx.Lan = GetLanguage(genType); ctx.Lan = GetLanguage(genType);
foreach (var c in ctx.ExportTypes) foreach (var c in ctx.ExportTypes)
{ {
var t = c;
ctx.Tasks.Add(Task.Run(() => ctx.Tasks.Add(Task.Run(() =>
{ {
var content = FileHeaderUtil.ConcatAutoGenerationHeader(ctx.Render.RenderAny(c), ctx.Lan); var content = FileHeaderUtil.ConcatAutoGenerationHeader(ctx.Render.RenderAny(t), ctx.Lan);
var file = RenderFileUtil.GetDefTypePath(c.FullName, ctx.Lan); var file = RenderFileUtil.GetDefTypePath(t.FullName, ctx.Lan);
var md5 = CacheFileUtil.GenMd5AndAddCache(file, content); var md5 = CacheFileUtil.GenMd5AndAddCache(file, content);
ctx.GenCodeFilesInOutputCodeDir.Add(new FileInfo() { FilePath = file, MD5 = md5 }); ctx.GenCodeFilesInOutputCodeDir.Add(new FileInfo() { FilePath = file, MD5 = md5 });
})); }));
@ -713,10 +715,11 @@ namespace {ctx.TopModule}
foreach (var c in renderTypes) foreach (var c in renderTypes)
{ {
var table = c;
ctx.Tasks.Add(Task.Run(() => ctx.Tasks.Add(Task.Run(() =>
{ {
var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(c), ELanguage.CPP); var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(table), ELanguage.CPP);
var file = "editor_" + RenderFileUtil.GetUeCppDefTypeHeaderFilePath(c.FullName); var file = "editor_" + RenderFileUtil.GetUeCppDefTypeHeaderFilePath(table.FullName);
var md5 = CacheFileUtil.GenMd5AndAddCache(file, content); var md5 = CacheFileUtil.GenMd5AndAddCache(file, content);
ctx.GenCodeFilesInOutputCodeDir.Add(new FileInfo() { FilePath = file, MD5 = md5 }); ctx.GenCodeFilesInOutputCodeDir.Add(new FileInfo() { FilePath = file, MD5 = md5 });
})); }));
@ -745,10 +748,11 @@ namespace {ctx.TopModule}
var render = new EditorCsRender(); var render = new EditorCsRender();
foreach (var c in ctx.Assembly.Types.Values) foreach (var c in ctx.Assembly.Types.Values)
{ {
var type = c;
ctx.Tasks.Add(Task.Run(() => ctx.Tasks.Add(Task.Run(() =>
{ {
var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(c), ELanguage.CS); var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(type), ELanguage.CS);
var file = RenderFileUtil.GetDefTypePath(c.FullName, ELanguage.CS); var file = RenderFileUtil.GetDefTypePath(type.FullName, ELanguage.CS);
var md5 = CacheFileUtil.GenMd5AndAddCache(file, content); var md5 = CacheFileUtil.GenMd5AndAddCache(file, content);
ctx.GenCodeFilesInOutputCodeDir.Add(new FileInfo() { FilePath = file, MD5 = md5 }); ctx.GenCodeFilesInOutputCodeDir.Add(new FileInfo() { FilePath = file, MD5 = md5 });
})); }));
@ -760,15 +764,16 @@ namespace {ctx.TopModule}
var render = new UE4BpCppRender(); var render = new UE4BpCppRender();
foreach (var c in ctx.ExportTypes) foreach (var c in ctx.ExportTypes)
{ {
if (!(c is DefEnum || c is DefBean)) var type = c;
if (!(type is DefEnum || type is DefBean))
{ {
continue; continue;
} }
ctx.Tasks.Add(Task.Run(() => ctx.Tasks.Add(Task.Run(() =>
{ {
var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(c), ELanguage.CPP); var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(type), ELanguage.CPP);
var file = "bp_" + RenderFileUtil.GetUeCppDefTypeHeaderFilePath(c.FullName); var file = "bp_" + RenderFileUtil.GetUeCppDefTypeHeaderFilePath(type.FullName);
var md5 = CacheFileUtil.GenMd5AndAddCache(file, content); var md5 = CacheFileUtil.GenMd5AndAddCache(file, content);
ctx.GenCodeFilesInOutputCodeDir.Add(new FileInfo() { FilePath = file, MD5 = md5 }); ctx.GenCodeFilesInOutputCodeDir.Add(new FileInfo() { FilePath = file, MD5 = md5 });
})); }));
@ -780,11 +785,17 @@ namespace {ctx.TopModule}
string genType = ctx.GenType; string genType = ctx.GenType;
foreach (var c in ctx.ExportTables) foreach (var c in ctx.ExportTables)
{ {
var table = c;
ctx.Tasks.Add(Task.Run(() => ctx.Tasks.Add(Task.Run(() =>
{ {
var content = DataExporterUtil.ToOutputData(c, ctx.Assembly.GetTableExportDataList(c), genType); var file = GetOutputFileName(genType, table.OutputDataFile);
var file = GetOutputFileName(genType, c.OutputDataFile); var records = ctx.Assembly.GetTableExportDataList(table);
var md5 = CacheFileUtil.GenStringOrBytesMd5AndAddCache(file, content); if (!FileRecordCacheManager.Ins.TryGetRecordOutputData(table, records, genType, out string md5))
{
var content = DataExporterUtil.ToOutputData(table, records, genType);
md5 = CacheFileUtil.GenStringOrBytesMd5AndAddCache(file, content);
FileRecordCacheManager.Ins.AddCachedRecordOutputData(table, records, genType, md5);
}
ctx.GenDataFilesInOutputDataDir.Add(new FileInfo() { FilePath = file, MD5 = md5 }); ctx.GenDataFilesInOutputDataDir.Add(new FileInfo() { FilePath = file, MD5 = md5 });
})); }));
} }
@ -796,9 +807,10 @@ namespace {ctx.TopModule}
var allJsonTask = new List<Task<string>>(); var allJsonTask = new List<Task<string>>();
foreach (var c in exportTables) foreach (var c in exportTables)
{ {
var table = c;
allJsonTask.Add(Task.Run(() => allJsonTask.Add(Task.Run(() =>
{ {
return (string)DataExporterUtil.ToOutputData(c, ctx.Assembly.GetTableExportDataList(c), "data_json"); return (string)DataExporterUtil.ToOutputData(table, ctx.Assembly.GetTableExportDataList(table), "data_json");
})); }));
} }
@ -827,9 +839,10 @@ namespace {ctx.TopModule}
var genDataTasks = new List<Task<List<ResourceInfo>>>(); var genDataTasks = new List<Task<List<ResourceInfo>>>();
foreach (var c in ctx.ExportTables) foreach (var c in ctx.ExportTables)
{ {
var table = c;
genDataTasks.Add(Task.Run(() => genDataTasks.Add(Task.Run(() =>
{ {
return DataExporterUtil.ExportResourceList(ctx.Assembly.GetTableExportDataList(c)); return DataExporterUtil.ExportResourceList(ctx.Assembly.GetTableExportDataList(table));
})); }));
} }

5
src/Luban.Job.Cfg/Source/Utils/DataLoaderUtil.cs
View File

@ -157,14 +157,15 @@ namespace Luban.Job.Cfg.Utils
foreach (DefTable c in exportTables) foreach (DefTable c in exportTables)
{ {
var table = c;
genDataTasks.Add(Task.Run(async () => genDataTasks.Add(Task.Run(async () =>
{ {
long beginTime = TimeUtil.NowMillis; long beginTime = TimeUtil.NowMillis;
await LoadTableAsync(agent, c, dataDir, branchName, branchDataDir, exportTestData); await LoadTableAsync(agent, table, dataDir, branchName, branchDataDir, exportTestData);
long endTime = TimeUtil.NowMillis; long endTime = TimeUtil.NowMillis;
if (endTime - beginTime > 100) if (endTime - beginTime > 100)
{ {
ctx.Info("====== load {0} cost {1} ms ======", c.FullName, (endTime - beginTime)); ctx.Info("====== load {0} cost {1} ms ======", table.FullName, (endTime - beginTime));
} }
})); }));
} }

5
src/Luban.Job.Cfg/Source/ValidatorContext.cs
View File

@ -87,7 +87,10 @@ namespace Luban.Job.Cfg
{ {
CurrentVisitor = visitor; CurrentVisitor = visitor;
visitor.ValidateTable(t, records); visitor.ValidateTable(t, records);
ValidateText(t, records); if (this.Assembly.NeedL10nTextTranslate)
{
ValidateText(t, records);
}
} }
finally finally
{ {

5
src/Luban.Job.Db/Source/JobController.cs
View File

@ -105,10 +105,11 @@ namespace Luban.Job.Db
var render = new AsyncCsRender(); var render = new AsyncCsRender();
foreach (var c in ass.Types.Values) foreach (var c in ass.Types.Values)
{ {
var type = c;
tasks.Add(Task.Run(() => tasks.Add(Task.Run(() =>
{ {
var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(c), Common.ELanguage.CS); var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(type), Common.ELanguage.CS);
var file = RenderFileUtil.GetDefTypePath(c.FullName, Common.ELanguage.CS); var file = RenderFileUtil.GetDefTypePath(type.FullName, Common.ELanguage.CS);
var md5 = CacheFileUtil.GenMd5AndAddCache(file, content); var md5 = CacheFileUtil.GenMd5AndAddCache(file, content);
genCodeFiles.Add(new FileInfo() { FilePath = file, MD5 = md5 }); genCodeFiles.Add(new FileInfo() { FilePath = file, MD5 = md5 });
})); }));

5
src/Luban.Job.Proto/Source/JobController.cs
View File

@ -116,10 +116,11 @@ namespace Luban.Job.Proto
var render = new CsRender(); var render = new CsRender();
foreach (var c in ass.Types.Values) foreach (var c in ass.Types.Values)
{ {
var type = c;
tasks.Add(Task.Run(() => tasks.Add(Task.Run(() =>
{ {
var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(c), Common.ELanguage.CS); var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(type), Common.ELanguage.CS);
var file = RenderFileUtil.GetDefTypePath(c.FullName, Common.ELanguage.CS); var file = RenderFileUtil.GetDefTypePath(type.FullName, Common.ELanguage.CS);
var md5 = CacheFileUtil.GenMd5AndAddCache(file, content); var md5 = CacheFileUtil.GenMd5AndAddCache(file, content);
genCodeFiles.Add(new FileInfo() { FilePath = file, MD5 = md5 }); genCodeFiles.Add(new FileInfo() { FilePath = file, MD5 = md5 });
})); }));