diff --git a/src/Luban.Job.Cfg/Source/Cache/FileRecordCacheManager.cs b/src/Luban.Job.Cfg/Source/Cache/FileRecordCacheManager.cs index bfe2314..de428b6 100644 --- a/src/Luban.Job.Cfg/Source/Cache/FileRecordCacheManager.cs +++ b/src/Luban.Job.Cfg/Source/Cache/FileRecordCacheManager.cs @@ -35,7 +35,7 @@ namespace Luban.Job.Cfg.Cache } } - private readonly ConcurrentDictionary<(string MD5, string SheetName), FileRecordCache> _caches = new(); + private readonly ConcurrentDictionary<(string TableName, string MD5, string SheetName), FileRecordCache> _caches = new(); private readonly object _shrinkLocker = new object(); @@ -43,7 +43,7 @@ namespace Luban.Job.Cfg.Cache { // TODO text localization check cacheRecords = null; - if (!_caches.TryGetValue((md5, sheetName), out var r)) + if (!_caches.TryGetValue((table.FullName, md5, sheetName), out var r)) { return false; } @@ -64,7 +64,7 @@ namespace Luban.Job.Cfg.Cache { lock (_shrinkLocker) { - _caches[(md5, sheetName)] = new FileRecordCache(table, cacheRecords); + _caches[(table.FullName, md5, sheetName)] = new FileRecordCache(table, cacheRecords); if (_caches.Count > CACHE_FILE_HIGH_WATER_MARK) { s_logger.Info("ShrinkCaches. cache count > high CACHE_FILE_HIGH_WATER_MARK:{}", CACHE_FILE_HIGH_WATER_MARK); @@ -73,6 +73,37 @@ namespace Luban.Job.Cfg.Cache } } + private readonly ConcurrentDictionary<(string TableFullName, string DataType), (DefTable Table, List Records, string Md5)> _tableCaches = new(); + public bool TryGetRecordOutputData(DefTable table, List records, string dataType, out string md5) + { + if (_tableCaches.TryGetValue((table.FullName, dataType), out var cacheInfo)) + { + var cacheAss = cacheInfo.Table.Assembly; + var curAss = table.Assembly; + if (cacheAss.TimeZone == curAss.TimeZone + && cacheAss.TargetBranch == null && curAss.TargetBranch == null + && !cacheAss.NeedL10nTextTranslate && !curAss.NeedL10nTextTranslate + && records.Count == cacheInfo.Records.Count && records.SequenceEqual(cacheInfo.Records)) + { + md5 = cacheInfo.Md5; + s_logger.Debug("find output data cache. table:{} dataType:{} md5:{}", table.FullName, dataType, md5); + return true; + } + } + md5 = null; + return false; + } + + public void AddCachedRecordOutputData(DefTable table, List records, string dataType, string md5) + { + var curAss = table.Assembly; + if (curAss.TargetBranch == null && !curAss.NeedL10nTextTranslate) + { + _tableCaches[(table.FullName, dataType)] = (table, records, md5); + s_logger.Debug("add output data cache. table:{} dataType:{} md5:{}", table.FullName, dataType, md5); + } + } + private void ShrinkCaches() { if (_caches.Count < CACHE_FILE_HIGH_WATER_MARK) diff --git a/src/Luban.Job.Cfg/Source/DataCreators/ExcelDataCreator.cs b/src/Luban.Job.Cfg/Source/DataCreators/ExcelDataCreator.cs index 3d80727..9d45984 100644 --- a/src/Luban.Job.Cfg/Source/DataCreators/ExcelDataCreator.cs +++ b/src/Luban.Job.Cfg/Source/DataCreators/ExcelDataCreator.cs @@ -364,7 +364,7 @@ namespace Luban.Job.Cfg.DataCreators throw new InvalidExcelDataException("字段不是nullable类型,不能为null"); } } - return new DString(s); + return DString.ValueOf(s); } public DType Accept(TBytes type, object converter, ExcelStream x, DefAssembly ass) diff --git a/src/Luban.Job.Cfg/Source/DataCreators/JsonDataCreator.cs b/src/Luban.Job.Cfg/Source/DataCreators/JsonDataCreator.cs index 19d3718..a65293a 100644 --- a/src/Luban.Job.Cfg/Source/DataCreators/JsonDataCreator.cs +++ b/src/Luban.Job.Cfg/Source/DataCreators/JsonDataCreator.cs @@ -71,7 +71,7 @@ namespace Luban.Job.Cfg.DataCreators public DType Accept(TString type, JsonElement x, DefAssembly ass) { - return new DString(x.GetString()); + return DString.ValueOf(x.GetString()); } public DType Accept(TBytes type, JsonElement x, DefAssembly ass) diff --git a/src/Luban.Job.Cfg/Source/DataCreators/LuaDataCreator.cs b/src/Luban.Job.Cfg/Source/DataCreators/LuaDataCreator.cs index 4715d45..4edfeda 100644 --- a/src/Luban.Job.Cfg/Source/DataCreators/LuaDataCreator.cs +++ b/src/Luban.Job.Cfg/Source/DataCreators/LuaDataCreator.cs @@ -110,7 +110,7 @@ namespace Luban.Job.Cfg.DataCreators { if (x is string s) { - return new DString(s); + return DString.ValueOf(s); } else { diff --git a/src/Luban.Job.Cfg/Source/DataCreators/StringDataCreator.cs b/src/Luban.Job.Cfg/Source/DataCreators/StringDataCreator.cs index fd469cd..2458e1e 100644 --- a/src/Luban.Job.Cfg/Source/DataCreators/StringDataCreator.cs +++ b/src/Luban.Job.Cfg/Source/DataCreators/StringDataCreator.cs @@ -137,7 +137,7 @@ namespace Luban.Job.Cfg.DataCreators public DType Accept(TString type, string x) { - return new DString(x); + return DString.ValueOf(x); } public DType Accept(TBytes type, string x) diff --git a/src/Luban.Job.Cfg/Source/DataCreators/XmlDataCreator.cs b/src/Luban.Job.Cfg/Source/DataCreators/XmlDataCreator.cs index 2cdf27e..734dbbf 100644 --- a/src/Luban.Job.Cfg/Source/DataCreators/XmlDataCreator.cs +++ b/src/Luban.Job.Cfg/Source/DataCreators/XmlDataCreator.cs @@ -72,7 +72,7 @@ namespace Luban.Job.Cfg.DataCreators public DType Accept(TString type, XElement x, DefAssembly ass) { - return new DString(x.Value); + return DString.ValueOf(x.Value); } public DType Accept(TBytes type, XElement x, DefAssembly ass) diff --git a/src/Luban.Job.Cfg/Source/Datas/DString.cs b/src/Luban.Job.Cfg/Source/Datas/DString.cs index ed57e41..8140154 100644 --- a/src/Luban.Job.Cfg/Source/Datas/DString.cs +++ b/src/Luban.Job.Cfg/Source/Datas/DString.cs @@ -15,7 +15,7 @@ namespace Luban.Job.Cfg.Datas return new DString(s); } - public DString(string x) : base(x) + private DString(string x) : base(x) { } diff --git a/src/Luban.Job.Cfg/Source/Defs/DefAssembly.cs b/src/Luban.Job.Cfg/Source/Defs/DefAssembly.cs index 576f864..a9cf7a9 100644 --- a/src/Luban.Job.Cfg/Source/Defs/DefAssembly.cs +++ b/src/Luban.Job.Cfg/Source/Defs/DefAssembly.cs @@ -85,6 +85,8 @@ namespace Luban.Job.Cfg.Defs public NotConvertTextSet NotConvertTextSet { get; private set; } + public bool NeedL10nTextTranslate => ExportTextTable != null; + public void InitL10n(string textValueFieldName) { ExportTextTable = new TextTable(this, textValueFieldName); diff --git a/src/Luban.Job.Cfg/Source/JobController.cs b/src/Luban.Job.Cfg/Source/JobController.cs index 3c84160..3b3538f 100644 --- a/src/Luban.Job.Cfg/Source/JobController.cs +++ b/src/Luban.Job.Cfg/Source/JobController.cs @@ -2,6 +2,7 @@ using Bright.Time; using CommandLine; using Luban.Common.Protos; using Luban.Common.Utils; +using Luban.Job.Cfg.Cache; using Luban.Job.Cfg.DataCreators; using Luban.Job.Cfg.Defs; using Luban.Job.Cfg.Generate; @@ -476,10 +477,11 @@ namespace Luban.Job.Cfg ctx.Lan = GetLanguage(genType); foreach (var c in ctx.ExportTypes) { + var t = c; ctx.Tasks.Add(Task.Run(() => { - var content = FileHeaderUtil.ConcatAutoGenerationHeader(ctx.Render.RenderAny(c), ctx.Lan); - var file = RenderFileUtil.GetDefTypePath(c.FullName, ctx.Lan); + var content = FileHeaderUtil.ConcatAutoGenerationHeader(ctx.Render.RenderAny(t), ctx.Lan); + var file = RenderFileUtil.GetDefTypePath(t.FullName, ctx.Lan); var md5 = CacheFileUtil.GenMd5AndAddCache(file, content); ctx.GenCodeFilesInOutputCodeDir.Add(new FileInfo() { FilePath = file, MD5 = md5 }); })); @@ -713,10 +715,11 @@ namespace {ctx.TopModule} foreach (var c in renderTypes) { + var table = c; ctx.Tasks.Add(Task.Run(() => { - var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(c), ELanguage.CPP); - var file = "editor_" + RenderFileUtil.GetUeCppDefTypeHeaderFilePath(c.FullName); + var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(table), ELanguage.CPP); + var file = "editor_" + RenderFileUtil.GetUeCppDefTypeHeaderFilePath(table.FullName); var md5 = CacheFileUtil.GenMd5AndAddCache(file, content); ctx.GenCodeFilesInOutputCodeDir.Add(new FileInfo() { FilePath = file, MD5 = md5 }); })); @@ -745,10 +748,11 @@ namespace {ctx.TopModule} var render = new EditorCsRender(); foreach (var c in ctx.Assembly.Types.Values) { + var type = c; ctx.Tasks.Add(Task.Run(() => { - var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(c), ELanguage.CS); - var file = RenderFileUtil.GetDefTypePath(c.FullName, ELanguage.CS); + var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(type), ELanguage.CS); + var file = RenderFileUtil.GetDefTypePath(type.FullName, ELanguage.CS); var md5 = CacheFileUtil.GenMd5AndAddCache(file, content); ctx.GenCodeFilesInOutputCodeDir.Add(new FileInfo() { FilePath = file, MD5 = md5 }); })); @@ -760,15 +764,16 @@ namespace {ctx.TopModule} var render = new UE4BpCppRender(); foreach (var c in ctx.ExportTypes) { - if (!(c is DefEnum || c is DefBean)) + var type = c; + if (!(type is DefEnum || type is DefBean)) { continue; } ctx.Tasks.Add(Task.Run(() => { - var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(c), ELanguage.CPP); - var file = "bp_" + RenderFileUtil.GetUeCppDefTypeHeaderFilePath(c.FullName); + var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(type), ELanguage.CPP); + var file = "bp_" + RenderFileUtil.GetUeCppDefTypeHeaderFilePath(type.FullName); var md5 = CacheFileUtil.GenMd5AndAddCache(file, content); ctx.GenCodeFilesInOutputCodeDir.Add(new FileInfo() { FilePath = file, MD5 = md5 }); })); @@ -780,11 +785,17 @@ namespace {ctx.TopModule} string genType = ctx.GenType; foreach (var c in ctx.ExportTables) { + var table = c; ctx.Tasks.Add(Task.Run(() => { - var content = DataExporterUtil.ToOutputData(c, ctx.Assembly.GetTableExportDataList(c), genType); - var file = GetOutputFileName(genType, c.OutputDataFile); - var md5 = CacheFileUtil.GenStringOrBytesMd5AndAddCache(file, content); + var file = GetOutputFileName(genType, table.OutputDataFile); + var records = ctx.Assembly.GetTableExportDataList(table); + if (!FileRecordCacheManager.Ins.TryGetRecordOutputData(table, records, genType, out string md5)) + { + var content = DataExporterUtil.ToOutputData(table, records, genType); + md5 = CacheFileUtil.GenStringOrBytesMd5AndAddCache(file, content); + FileRecordCacheManager.Ins.AddCachedRecordOutputData(table, records, genType, md5); + } ctx.GenDataFilesInOutputDataDir.Add(new FileInfo() { FilePath = file, MD5 = md5 }); })); } @@ -796,9 +807,10 @@ namespace {ctx.TopModule} var allJsonTask = new List>(); foreach (var c in exportTables) { + var table = c; allJsonTask.Add(Task.Run(() => { - return (string)DataExporterUtil.ToOutputData(c, ctx.Assembly.GetTableExportDataList(c), "data_json"); + return (string)DataExporterUtil.ToOutputData(table, ctx.Assembly.GetTableExportDataList(table), "data_json"); })); } @@ -827,9 +839,10 @@ namespace {ctx.TopModule} var genDataTasks = new List>>(); foreach (var c in ctx.ExportTables) { + var table = c; genDataTasks.Add(Task.Run(() => { - return DataExporterUtil.ExportResourceList(ctx.Assembly.GetTableExportDataList(c)); + return DataExporterUtil.ExportResourceList(ctx.Assembly.GetTableExportDataList(table)); })); } diff --git a/src/Luban.Job.Cfg/Source/Utils/DataLoaderUtil.cs b/src/Luban.Job.Cfg/Source/Utils/DataLoaderUtil.cs index a39f034..0bc9e17 100644 --- a/src/Luban.Job.Cfg/Source/Utils/DataLoaderUtil.cs +++ b/src/Luban.Job.Cfg/Source/Utils/DataLoaderUtil.cs @@ -157,14 +157,15 @@ namespace Luban.Job.Cfg.Utils foreach (DefTable c in exportTables) { + var table = c; genDataTasks.Add(Task.Run(async () => { long beginTime = TimeUtil.NowMillis; - await LoadTableAsync(agent, c, dataDir, branchName, branchDataDir, exportTestData); + await LoadTableAsync(agent, table, dataDir, branchName, branchDataDir, exportTestData); long endTime = TimeUtil.NowMillis; if (endTime - beginTime > 100) { - ctx.Info("====== load {0} cost {1} ms ======", c.FullName, (endTime - beginTime)); + ctx.Info("====== load {0} cost {1} ms ======", table.FullName, (endTime - beginTime)); } })); } diff --git a/src/Luban.Job.Cfg/Source/ValidatorContext.cs b/src/Luban.Job.Cfg/Source/ValidatorContext.cs index 93d0cd3..b4fa8ba 100644 --- a/src/Luban.Job.Cfg/Source/ValidatorContext.cs +++ b/src/Luban.Job.Cfg/Source/ValidatorContext.cs @@ -87,7 +87,10 @@ namespace Luban.Job.Cfg { CurrentVisitor = visitor; visitor.ValidateTable(t, records); - ValidateText(t, records); + if (this.Assembly.NeedL10nTextTranslate) + { + ValidateText(t, records); + } } finally { diff --git a/src/Luban.Job.Db/Source/JobController.cs b/src/Luban.Job.Db/Source/JobController.cs index 8e36803..8aabc6d 100644 --- a/src/Luban.Job.Db/Source/JobController.cs +++ b/src/Luban.Job.Db/Source/JobController.cs @@ -105,10 +105,11 @@ namespace Luban.Job.Db var render = new AsyncCsRender(); foreach (var c in ass.Types.Values) { + var type = c; tasks.Add(Task.Run(() => { - var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(c), Common.ELanguage.CS); - var file = RenderFileUtil.GetDefTypePath(c.FullName, Common.ELanguage.CS); + var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(type), Common.ELanguage.CS); + var file = RenderFileUtil.GetDefTypePath(type.FullName, Common.ELanguage.CS); var md5 = CacheFileUtil.GenMd5AndAddCache(file, content); genCodeFiles.Add(new FileInfo() { FilePath = file, MD5 = md5 }); })); diff --git a/src/Luban.Job.Proto/Source/JobController.cs b/src/Luban.Job.Proto/Source/JobController.cs index 88b65c3..5a6fe8c 100644 --- a/src/Luban.Job.Proto/Source/JobController.cs +++ b/src/Luban.Job.Proto/Source/JobController.cs @@ -116,10 +116,11 @@ namespace Luban.Job.Proto var render = new CsRender(); foreach (var c in ass.Types.Values) { + var type = c; tasks.Add(Task.Run(() => { - var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(c), Common.ELanguage.CS); - var file = RenderFileUtil.GetDefTypePath(c.FullName, Common.ELanguage.CS); + var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(type), Common.ELanguage.CS); + var file = RenderFileUtil.GetDefTypePath(type.FullName, Common.ELanguage.CS); var md5 = CacheFileUtil.GenMd5AndAddCache(file, content); genCodeFiles.Add(new FileInfo() { FilePath = file, MD5 = md5 }); }));