xiang.li
/
luban
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

- 支持零散文件形式的输出文件 

- gen_types中  output_data_json_monolithic_file 和 output_data_resource_list_file 改成零散文件的输出形式，不必再指定输出目录。
main
walon 2021-04-08 11:50:54 +08:00
parent ad965cf8e4
commit 0cbf47e23c
4 changed files with 61 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/Luban.Client/Source/Program.cs
View File

@ -210,6 +210,11 @@ Options:
tasks.Add(DownloadFileUtil.DownloadGeneratedFiles(fg.Dir, fg.Files)); tasks.Add(DownloadFileUtil.DownloadGeneratedFiles(fg.Dir, fg.Files));
} }
foreach (var f in res.ScatteredFiles)
{
tasks.Add(DownloadFileUtil.DownloadGeneratedFile(f));
}
Task.WaitAll(tasks.ToArray()); Task.WaitAll(tasks.ToArray());
return 0; return 0;
} }

2
src/Luban.Client/Source/Utils/DownloadFileUtil.cs
View File

@ -35,9 +35,7 @@ namespace Luban.Client.Common.Utils
} }
await Task.WhenAll(tasks); await Task.WhenAll(tasks);
// todo 感觉有点问题哈，不是每个生成目录都需要clean up 的
FileCleaner.Clean(outputDir, newFiles); FileCleaner.Clean(outputDir, newFiles);
} }
public static async Task DownloadGeneratedFile(FileInfo file) public static async Task DownloadGeneratedFile(FileInfo file)

8
src/Luban.Common/Source/Protos/GenJob.cs
View File

@ -66,7 +66,9 @@ namespace Luban.Common.Protos
public string ErrMsg { get; set; } public string ErrMsg { get; set; }
public List<FileGroup> FileGroups { get; set; } public List<FileGroup> FileGroups { get; set; } = new List<FileGroup>();
public List<FileInfo> ScatteredFiles { get; set; } = new List<FileInfo>();
public override int GetTypeId() public override int GetTypeId()
{ {
@ -78,12 +80,14 @@ namespace Luban.Common.Protos
os.WriteInt((int)ErrCode); os.WriteInt((int)ErrCode);
os.WriteString(ErrMsg); os.WriteString(ErrMsg);
Bright.Common.SerializationUtil.Serialize(os, FileGroups); Bright.Common.SerializationUtil.Serialize(os, FileGroups);
Bright.Common.SerializationUtil.Serialize(os, ScatteredFiles);
} }
public override void Deserialize(ByteBuf os) public override void Deserialize(ByteBuf os)
{ {
ErrCode = (EErrorCode)os.ReadInt(); ErrCode = (EErrorCode)os.ReadInt();
ErrMsg = os.ReadString(); ErrMsg = os.ReadString();
Bright.Common.SerializationUtil.Deserialize(os, FileGroups = new List<FileGroup>()); Bright.Common.SerializationUtil.Deserialize(os, FileGroups);
Bright.Common.SerializationUtil.Deserialize(os, ScatteredFiles);
} }
} }

81
src/Luban.Job.Cfg/Source/JobController.cs
View File

@ -49,6 +49,12 @@ namespace Luban.Job.Cfg
[Option("output_data_dir", Required = true, HelpText = "output data directory")] [Option("output_data_dir", Required = true, HelpText = "output data directory")]
public string OutputDataDir { get; set; } public string OutputDataDir { get; set; }
[Option("output_data_resource_list_file", Required = false, HelpText = "output resource list file")]
public string OutputDataResourceListFile { get; set; }
[Option("output_data_json_monolithic_file", Required = false, HelpText = "output monolithic json file")]
public string OutputDataJsonMonolithicFile { get; set; }
[Option("gen_types", Required = true, HelpText = "code_cs_bin,code_cs_json,code_lua_bin,data_bin,data_lua,data_json,data_json_monolithic . can be multi")] [Option("gen_types", Required = true, HelpText = "code_cs_bin,code_cs_json,code_lua_bin,data_bin,data_lua,data_json,data_json_monolithic . can be multi")]
public string GenType { get; set; } public string GenType { get; set; }
@ -143,7 +149,7 @@ namespace Luban.Job.Cfg
errMsg = "--outputcodedir missing"; errMsg = "--outputcodedir missing";
return false; return false;
} }
else if (genTypes.Any(t => t.StartsWith("data_", StringComparison.Ordinal))) if (genTypes.Any(t => t.StartsWith("data_", StringComparison.Ordinal)))
{ {
if (string.IsNullOrWhiteSpace(inputDataDir)) if (string.IsNullOrWhiteSpace(inputDataDir))
{ {
@ -155,6 +161,16 @@ namespace Luban.Job.Cfg
errMsg = "--outputdatadir missing"; errMsg = "--outputdatadir missing";
return false; return false;
} }
if (genTypes.Contains("data_resources") && string.IsNullOrWhiteSpace(result.OutputDataResourceListFile))
{
errMsg = "--output_data_resource_list_file missing";
return false;
}
if (genTypes.Contains("data_json_monolithic") && string.IsNullOrWhiteSpace(result.OutputDataJsonMonolithicFile))
{
errMsg = "--output_data_json_monolithic_file missing";
return false;
}
} }
return true; return true;
@ -229,8 +245,9 @@ namespace Luban.Job.Cfg
var tasks = new List<Task>(); var tasks = new List<Task>();
var genCodeFiles = new ConcurrentBag<FileInfo>(); var genCodeFilesInOutputCodeDir = new ConcurrentBag<FileInfo>();
var genDataFiles = new ConcurrentBag<FileInfo>(); var genDataFilesInOutputDataDir = new ConcurrentBag<FileInfo>();
var genScatteredFiles = new ConcurrentBag<FileInfo>();
foreach (var genType in genTypes) foreach (var genType in genTypes)
{ {
@ -238,7 +255,6 @@ namespace Luban.Job.Cfg
{ {
case "code_cs_bin": case "code_cs_bin":
case "code_cs_json": case "code_cs_json":
case "code_java_bin": case "code_java_bin":
{ {
ICodeRender render = CreateCodeRender(genType); ICodeRender render = CreateCodeRender(genType);
@ -251,7 +267,7 @@ namespace Luban.Job.Cfg
var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(c), lan); var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(c), lan);
var file = RenderFileUtil.GetDefTypePath(c.FullName, lan); var file = RenderFileUtil.GetDefTypePath(c.FullName, lan);
var md5 = CacheFileUtil.GenMd5AndAddCache(file, content); var md5 = CacheFileUtil.GenMd5AndAddCache(file, content);
genCodeFiles.Add(new FileInfo() { FilePath = file, MD5 = md5 }); genCodeFilesInOutputCodeDir.Add(new FileInfo() { FilePath = file, MD5 = md5 });
})); }));
} }
@ -262,7 +278,7 @@ namespace Luban.Job.Cfg
var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderService(name, module, exportTables), lan); var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderService(name, module, exportTables), lan);
var file = RenderFileUtil.GetDefTypePath(name, lan); var file = RenderFileUtil.GetDefTypePath(name, lan);
var md5 = CacheFileUtil.GenMd5AndAddCache(file, content); var md5 = CacheFileUtil.GenMd5AndAddCache(file, content);
genCodeFiles.Add(new FileInfo() { FilePath = file, MD5 = md5 }); genCodeFilesInOutputCodeDir.Add(new FileInfo() { FilePath = file, MD5 = md5 });
})); }));
break; break;
@ -275,7 +291,7 @@ namespace Luban.Job.Cfg
var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAll(ass.Types.Values.ToList()), ELanguage.LUA); var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAll(ass.Types.Values.ToList()), ELanguage.LUA);
var file = "Types.lua"; var file = "Types.lua";
var md5 = CacheFileUtil.GenMd5AndAddCache(file, content); var md5 = CacheFileUtil.GenMd5AndAddCache(file, content);
genCodeFiles.Add(new FileInfo() { FilePath = file, MD5 = md5 }); genCodeFilesInOutputCodeDir.Add(new FileInfo() { FilePath = file, MD5 = md5 });
})); }));
break; break;
} }
@ -289,7 +305,7 @@ namespace Luban.Job.Cfg
var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(c), ELanguage.GO); var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(c), ELanguage.GO);
var file = RenderFileUtil.GetDefTypePath(c.FullName, ELanguage.GO); var file = RenderFileUtil.GetDefTypePath(c.FullName, ELanguage.GO);
var md5 = CacheFileUtil.GenMd5AndAddCache(file, content); var md5 = CacheFileUtil.GenMd5AndAddCache(file, content);
genCodeFiles.Add(new FileInfo() { FilePath = file, MD5 = md5 }); genCodeFilesInOutputCodeDir.Add(new FileInfo() { FilePath = file, MD5 = md5 });
})); }));
} }
@ -300,7 +316,7 @@ namespace Luban.Job.Cfg
var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderService(name, module, exportTables), ELanguage.GO); var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderService(name, module, exportTables), ELanguage.GO);
var file = RenderFileUtil.GetDefTypePath(name, ELanguage.GO); var file = RenderFileUtil.GetDefTypePath(name, ELanguage.GO);
var md5 = CacheFileUtil.GenMd5AndAddCache(file, content); var md5 = CacheFileUtil.GenMd5AndAddCache(file, content);
genCodeFiles.Add(new FileInfo() { FilePath = file, MD5 = md5 }); genCodeFilesInOutputCodeDir.Add(new FileInfo() { FilePath = file, MD5 = md5 });
})); }));
break; break;
} }
@ -375,7 +391,7 @@ namespace {ass.TopModule}
var content = FileHeaderUtil.ConcatAutoGenerationHeader(string.Join('\n', headerFileContent), ELanguage.CPP); var content = FileHeaderUtil.ConcatAutoGenerationHeader(string.Join('\n', headerFileContent), ELanguage.CPP);
var file = "gen_types.h"; var file = "gen_types.h";
var md5 = CacheFileUtil.GenMd5AndAddCache(file, content); var md5 = CacheFileUtil.GenMd5AndAddCache(file, content);
genCodeFiles.Add(new FileInfo() { FilePath = file, MD5 = md5 }); genCodeFilesInOutputCodeDir.Add(new FileInfo() { FilePath = file, MD5 = md5 });
})); }));
var beanTypes = exportTypes.Where(c => c is DefBean).ToList(); var beanTypes = exportTypes.Where(c => c is DefBean).ToList();
@ -393,7 +409,7 @@ namespace {ass.TopModule}
ELanguage.CPP); ELanguage.CPP);
var file = $"gen_stub_{index}.cpp"; var file = $"gen_stub_{index}.cpp";
var md5 = CacheFileUtil.GenMd5AndAddCache(file, content); var md5 = CacheFileUtil.GenMd5AndAddCache(file, content);
genCodeFiles.Add(new FileInfo() { FilePath = file, MD5 = md5 }); genCodeFilesInOutputCodeDir.Add(new FileInfo() { FilePath = file, MD5 = md5 });
})); }));
} }
break; break;
@ -514,7 +530,7 @@ export class Vector2 {
var content = FileHeaderUtil.ConcatAutoGenerationHeader(string.Join('\n', fileContent), ELanguage.TYPESCRIPT); var content = FileHeaderUtil.ConcatAutoGenerationHeader(string.Join('\n', fileContent), ELanguage.TYPESCRIPT);
var file = "Types.ts"; var file = "Types.ts";
var md5 = CacheFileUtil.GenMd5AndAddCache(file, content); var md5 = CacheFileUtil.GenMd5AndAddCache(file, content);
genCodeFiles.Add(new FileInfo() { FilePath = file, MD5 = md5 }); genCodeFilesInOutputCodeDir.Add(new FileInfo() { FilePath = file, MD5 = md5 });
})); }));
break; break;
} }
@ -617,7 +633,7 @@ class Vector4:
var content = FileHeaderUtil.ConcatAutoGenerationHeader(string.Join('\n', fileContent), ELanguage.PYTHON); var content = FileHeaderUtil.ConcatAutoGenerationHeader(string.Join('\n', fileContent), ELanguage.PYTHON);
var file = "Types.py"; var file = "Types.py";
var md5 = CacheFileUtil.GenMd5AndAddCache(file, content); var md5 = CacheFileUtil.GenMd5AndAddCache(file, content);
genCodeFiles.Add(new FileInfo() { FilePath = file, MD5 = md5 }); genCodeFilesInOutputCodeDir.Add(new FileInfo() { FilePath = file, MD5 = md5 });
{ {
@ -625,7 +641,7 @@ class Vector4:
var initFile = "__init__.py"; var initFile = "__init__.py";
var initMd5 = CacheFileUtil.GenMd5AndAddCache(initFile, moduleInitContent); var initMd5 = CacheFileUtil.GenMd5AndAddCache(initFile, moduleInitContent);
genCodeFiles.Add(new FileInfo() { FilePath = initFile, MD5 = initMd5 }); genCodeFilesInOutputCodeDir.Add(new FileInfo() { FilePath = initFile, MD5 = initMd5 });
} }
})); }));
break; break;
@ -732,7 +748,7 @@ class Vector4:
var content = FileHeaderUtil.ConcatAutoGenerationHeader(string.Join('\n', fileContent), ELanguage.PYTHON); var content = FileHeaderUtil.ConcatAutoGenerationHeader(string.Join('\n', fileContent), ELanguage.PYTHON);
var file = "Types.py"; var file = "Types.py";
var md5 = CacheFileUtil.GenMd5AndAddCache(file, content); var md5 = CacheFileUtil.GenMd5AndAddCache(file, content);
genCodeFiles.Add(new FileInfo() { FilePath = file, MD5 = md5 }); genCodeFilesInOutputCodeDir.Add(new FileInfo() { FilePath = file, MD5 = md5 });
})); }));
break; break;
} }
@ -750,7 +766,7 @@ class Vector4:
var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(c), ELanguage.CPP); var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(c), ELanguage.CPP);
var file = "editor_" + RenderFileUtil.GetUeCppDefTypeHeaderFilePath(c.FullName); var file = "editor_" + RenderFileUtil.GetUeCppDefTypeHeaderFilePath(c.FullName);
var md5 = CacheFileUtil.GenMd5AndAddCache(file, content); var md5 = CacheFileUtil.GenMd5AndAddCache(file, content);
genCodeFiles.Add(new FileInfo() { FilePath = file, MD5 = md5 }); genCodeFilesInOutputCodeDir.Add(new FileInfo() { FilePath = file, MD5 = md5 });
})); }));
} }
@ -767,7 +783,7 @@ class Vector4:
ELanguage.CPP); ELanguage.CPP);
var file = $"stub_{index}.cpp"; var file = $"stub_{index}.cpp";
var md5 = CacheFileUtil.GenMd5AndAddCache(file, content); var md5 = CacheFileUtil.GenMd5AndAddCache(file, content);
genCodeFiles.Add(new FileInfo() { FilePath = file, MD5 = md5 }); genCodeFilesInOutputCodeDir.Add(new FileInfo() { FilePath = file, MD5 = md5 });
})); }));
} }
break; break;
@ -782,7 +798,7 @@ class Vector4:
var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(c), ELanguage.CS); var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(c), ELanguage.CS);
var file = RenderFileUtil.GetDefTypePath(c.FullName, ELanguage.GO); var file = RenderFileUtil.GetDefTypePath(c.FullName, ELanguage.GO);
var md5 = CacheFileUtil.GenMd5AndAddCache(file, content); var md5 = CacheFileUtil.GenMd5AndAddCache(file, content);
genCodeFiles.Add(new FileInfo() { FilePath = file, MD5 = md5 }); genCodeFilesInOutputCodeDir.Add(new FileInfo() { FilePath = file, MD5 = md5 });
})); }));
} }
break; break;
@ -802,7 +818,7 @@ class Vector4:
var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(c), ELanguage.CPP); var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderAny(c), ELanguage.CPP);
var file = "bp_" + RenderFileUtil.GetUeCppDefTypeHeaderFilePath(c.FullName); var file = "bp_" + RenderFileUtil.GetUeCppDefTypeHeaderFilePath(c.FullName);
var md5 = CacheFileUtil.GenMd5AndAddCache(file, content); var md5 = CacheFileUtil.GenMd5AndAddCache(file, content);
genCodeFiles.Add(new FileInfo() { FilePath = file, MD5 = md5 }); genCodeFilesInOutputCodeDir.Add(new FileInfo() { FilePath = file, MD5 = md5 });
})); }));
} }
break; break;
@ -819,7 +835,7 @@ class Vector4:
var file = genType.EndsWith("json") ? c.JsonOutputDataFile : c.OutputDataFile; var file = genType.EndsWith("json") ? c.JsonOutputDataFile : c.OutputDataFile;
var md5 = FileUtil.CalcMD5(content); var md5 = FileUtil.CalcMD5(content);
CacheManager.Ins.AddCache(file, md5, content); CacheManager.Ins.AddCache(file, md5, content);
genDataFiles.Add(new FileInfo() { FilePath = file, MD5 = md5 }); genDataFilesInOutputDataDir.Add(new FileInfo() { FilePath = file, MD5 = md5 });
})); }));
} }
break; break;
@ -854,9 +870,9 @@ class Vector4:
var content = sb.ToArray(); var content = sb.ToArray();
s_logger.Debug("estimated size:{0} actual size:{1}", estimatedCapacity, content.Length); s_logger.Debug("estimated size:{0} actual size:{1}", estimatedCapacity, content.Length);
var md5 = FileUtil.CalcMD5(content); var md5 = FileUtil.CalcMD5(content);
var outputFile = "tables.json"; var outputFile = args.OutputDataJsonMonolithicFile;
CacheManager.Ins.AddCache(outputFile, md5, content); CacheManager.Ins.AddCache(outputFile, md5, content);
genDataFiles.Add(new FileInfo() { FilePath = outputFile, MD5 = md5 }); genScatteredFiles.Add(new FileInfo() { FilePath = outputFile, MD5 = md5 });
break; break;
} }
case "data_lua": case "data_lua":
@ -869,7 +885,7 @@ class Vector4:
var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderDefines(ass.Types.Values.ToList()), ELanguage.LUA); var content = FileHeaderUtil.ConcatAutoGenerationHeader(render.RenderDefines(ass.Types.Values.ToList()), ELanguage.LUA);
var file = "Types.lua"; var file = "Types.lua";
var md5 = CacheFileUtil.GenMd5AndAddCache(file, content); var md5 = CacheFileUtil.GenMd5AndAddCache(file, content);
genDataFiles.Add(new FileInfo() { FilePath = file, MD5 = md5 }); genDataFilesInOutputDataDir.Add(new FileInfo() { FilePath = file, MD5 = md5 });
})); }));
@ -881,7 +897,7 @@ class Vector4:
var file = $"{c.Name}.lua"; var file = $"{c.Name}.lua";
var md5 = FileUtil.CalcMD5(content); var md5 = FileUtil.CalcMD5(content);
CacheManager.Ins.AddCache(file, md5, content); CacheManager.Ins.AddCache(file, md5, content);
genDataFiles.Add(new FileInfo() { FilePath = file, MD5 = md5 }); genDataFilesInOutputDataDir.Add(new FileInfo() { FilePath = file, MD5 = md5 });
})); }));
} }
break; break;
@ -912,13 +928,12 @@ class Vector4:
} }
} }
} }
var file = "resources.txt"; var file = args.OutputDataResourceListFile;
var contents = System.Text.Encoding.UTF8.GetBytes(string.Join("\n", resourceLines)); var contents = System.Text.Encoding.UTF8.GetBytes(string.Join("\n", resourceLines));
var md5 = FileUtil.CalcMD5(contents); var md5 = FileUtil.CalcMD5(contents);
CacheManager.Ins.AddCache(file, md5, contents); CacheManager.Ins.AddCache(file, md5, contents);
// 不这么处理??? genScatteredFiles.Add(new FileInfo() { FilePath = file, MD5 = md5 });
genDataFiles.Add(new FileInfo() { FilePath = file, MD5 = md5 });
})); }));
break; break;
} }
@ -932,13 +947,17 @@ class Vector4:
} }
await Task.WhenAll(tasks.ToArray()); await Task.WhenAll(tasks.ToArray());
if (genCodeFiles.Count > 0) if (!genCodeFilesInOutputCodeDir.IsEmpty)
{ {
res.FileGroups.Add(new FileGroup() { Dir = outputCodeDir, Files = genCodeFiles.ToList() }); res.FileGroups.Add(new FileGroup() { Dir = outputCodeDir, Files = genCodeFilesInOutputCodeDir.ToList() });
} }
if (genDataFiles.Count > 0) if (!genDataFilesInOutputDataDir.IsEmpty)
{ {
res.FileGroups.Add(new FileGroup() { Dir = outputDataDir, Files = genDataFiles.ToList() }); res.FileGroups.Add(new FileGroup() { Dir = outputDataDir, Files = genDataFilesInOutputDataDir.ToList() });
}
if (!genScatteredFiles.IsEmpty)
{
res.ScatteredFiles.AddRange(genScatteredFiles);
} }
} }
catch (Exception e) catch (Exception e)