From d873aa3a6c9fd3508f1aaabb81e5945d09a8714e Mon Sep 17 00:00:00 2001 From: walon Date: Thu, 17 Jun 2021 14:57:39 +0800 Subject: [PATCH] =?UTF-8?q?=E3=80=90=E7=89=B9=E6=80=A7=E3=80=91=20cfg=20?= =?UTF-8?q?=E5=AE=9E=E7=8E=B0=E9=9D=99=E6=80=81=E6=9C=AC=E5=9C=B0=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/Datas/l10n/TextTable_CN.xlsx | Bin 0 -> 9689 bytes config/L10N_生成.bat | 14 + config/l10n/NotLocalized_CN.txt | 9 + .../DataVisitors/TextValidatorVisitor.cs | 2 +- src/Luban.Job.Cfg/Source/Datas/DText.cs | 2 +- src/Luban.Job.Cfg/Source/Defs/DefAssembly.cs | 12 +- src/Luban.Job.Cfg/Source/JobController.cs | 262 +++--------------- .../Source/Utils/DataExporterUtil.cs | 100 +++++++ .../Source/Utils/DataLoaderUtil.cs | 197 +++++++++++++ src/Luban.Job.Cfg/Source/ValidatorContext.cs | 1 - .../Source/i10n/NotConvertTextSet.cs | 18 -- src/Luban.Job.Cfg/Source/i10n/TextTable.cs | 33 --- .../Source/l10n/NotConvertTextSet.cs | 31 +++ .../Source/{i10n => l10n}/RawTextTable.cs | 2 +- src/Luban.Job.Cfg/Source/l10n/TextTable.cs | 83 ++++++ src/Luban.Server.Common/Source/RemoteAgent.cs | 2 +- 16 files changed, 480 insertions(+), 288 deletions(-) create mode 100644 config/Datas/l10n/TextTable_CN.xlsx create mode 100644 config/L10N_生成.bat create mode 100644 config/l10n/NotLocalized_CN.txt create mode 100644 src/Luban.Job.Cfg/Source/Utils/DataExporterUtil.cs create mode 100644 src/Luban.Job.Cfg/Source/Utils/DataLoaderUtil.cs delete mode 100644 src/Luban.Job.Cfg/Source/i10n/NotConvertTextSet.cs delete mode 100644 src/Luban.Job.Cfg/Source/i10n/TextTable.cs create mode 100644 src/Luban.Job.Cfg/Source/l10n/NotConvertTextSet.cs rename src/Luban.Job.Cfg/Source/{i10n => l10n}/RawTextTable.cs (97%) create mode 100644 src/Luban.Job.Cfg/Source/l10n/TextTable.cs diff --git a/config/Datas/l10n/TextTable_CN.xlsx b/config/Datas/l10n/TextTable_CN.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..d9c009acc4ab39b991603773ffd590100aeaa146 GIT binary patch literal 9689 zcmeHt^;=YH_x?~rBi$uK!_X~_fPjP`-AJdDfaHuwht$v|-JQ~i3?T^82uO*9Al>pA zJ?H&;4(ELTg7@7&?0wC(XWiG{&$I5e?zPs|P(eW@1^@vV007`FKt>ok>Wl;cprQc) zL;wtALpc{`4=ZO6)5ktft=x^dyqz5BU!x+k<^Yfp|Nr0dU%Ue4j|ZH4c<|eGx1{cO z=!EBnmX=~rw^AE2enn65j@M&sT&G~Y@@KJrq+Ns2&Gg7|)R6g5oV;R=SzlIq)sgE$ zU!BnmyC}NCZC~R0LAE?Jzg#D}?p35<4ms}L9`IuW`orcq3od(x8wMs)!FnwIjqU@r zP1&+ADLJNg_AjN#=QG<913mi4Dr}zQ79!p+G{Kb0i@I%XkNEg?7^~9VX|%Z#Povq0 zYcyKwFX)uE2FHnWjB9Ku#N_Ct?aGmgo%rZ*_(L%9Yj}wd=xu_4+@OWl!Dc?J-Z^hS zvu#E4*38w)Mo-hLd(g30QQA4`leJzg_G_`&hpFy|I|A^6ig56tfz^)MDdrDvax1@M z7tZ`cGRWvx`Yhg8`#$Z#815nXb_;;T)^7wy)b16?UorQObcD&ilU*j_#Y~8Z#AeI^ zlD*8y;wr_CaZn*RJ92KeGbd5h=VkMF3z@HF=sv^t@u+?vcy@Dx0?_yyO&hd%Kt~AQ zdx*G;ji9Nio0X$GH`mYe|LFN&%*nsJdU?F6dM^)7$limiQ26QO!Yh1P6)%|wZ|*)0 z2vYiVry)9zk!qong&O}co&lNEvTNJ8R!phmBXkmL(jPjq%> zH<*HJ`SM$WXOqX1C#j0cKCI8$W7*4^i*r>5mOu(0_wU!>k8MvTHRU%EwYJIB+scsU^*-98q51fV-ZsJOXi9XyJPbGo7TiDE2`oSIV9-mWcYuSq0 zRa<1ai81(?S~_+e-cRo&y7mLBWxzEV1PIUgMiu*+a*hKH>IL?PGJW0(WA>K8zYjno z>53N-8S9@U$pV3NfG7aK5)c5mgIF1FM{X|{HwOzB7l)sDtX$jJC7p-ZKfC@W=t`{y z^Hq>Ci!5QqxRADCQ~Uii>Q&S-GyZvjlG5u_h_F;D?>4a!yUq{g)R0Lj@90 z+eHoRBUP{%|LcYwrl;UZVgC*O$!CBXg6L{56xIU+nDK7@xW~t!d2~Wwble+MT4*k2 ze`c3Qtu{xBj)K!S3Tbr3ah4xM)c6OSsG%Rp+i=*h1UgB$LSxH44wB2`70cH!qdo^J zfaoh^I4Dr}xW`r$o$t1aEJcI_iYia%8J|KLxwLQ(DD8J@31PaTX+0X>&g(E2ybetS z%y2=}%&g7ciHAui>>UOEPpV|#Ao0fv@%iREI#8a7cW&U4^@4a(kF(Fyo%YUMTn=bh zr;-PI?H{PIu#eC>WL5MPgug`0&24)ooII7@B^@+MJhMgO$VxthF(#XLjt9SOJLDxlw0EN7Gy3Jo zOfGr_W{x6*(L-0tb8pe`Yg<^z56-mU019xAmAEk%>~Lni-xxT1BSMHiV6zHd)th=e z@1Q!1`$?7V{Fxdq#@1K-?`RS+M_$Or`l8MQOh&!!_yg~`UKtfRFkJR#4 z;`}GgAR$yBf_VSkN4ch|%3FjA!u}D;W^_FgsMg7IYD;wNnAD zZ)&1XW3yhrX_1>h0cJye=*A8yFqOn{eE?%$C+>2bb<3Y%9JF0otNM&Mbmv{*tW%e% z(3T2H_=R}Thr$)<0@>o-*I%c;(cWA3ki8sa>zqH}96H*wUK{A2Tte*hKV2p`jWq}Z z5oHw+1q3S8A2`^bF-Q(v$whk3D^T zgkt2%RV}84nslyDV~8*yXyL|~a4YK(>QU<0RPmj`o;i&Y@+yn9S*}`JdGEne?Wh31 zn+dABDt1N%&7Vo4$FkUuc&^XPYu69t*Awlr@!3i8j2s9=V;DriHMK1{Bzk;YD&@ped{ z=pUKs&X<4Q{^N~VTL)+}Bo&yIb9MUY00@Q!B4aONk~W$Z8u1y_2`B`N0er36nZ7Ek z>-=EWF=97wINyPh+Ybxs?9WNqVWrP_!cuG-b1}gY~;eO9nZ7tg(bMJNn$n zIc0v3V%JOvce%98v5=P6ienBVYDdsS4@7hKr7!9d>4?Mf)?=mVFWK{7Y+RQ;X*E8) z@d?^fr8PEu+MVaHZCZpx(w+`HTg)G8h~~D<=v%PG#@KYG{XBm_Zd703a8J9#dA7}g z%7k0-3`qHTew^I+!`Hbs?9JiPD3)&-10H8UhR?oRLob2x{Hsc+s`4XIMVdkn(+0+O zp@%zf@6jmp5CSB%Q`@y=Qa5p9S&qJhCViaayBLS`e4`W@ z{@@tSmXLi{zrkgAL3M&LReic^W$YdFPd25~J%oB1=fgjw-N-|Y_{fcpQz=2+1+Wx5 zI9VzZSN8aGX3=j_L7Rd9%AVKNH+M|X0f6`PKkMyZ?D4R*a;B?6V@^c3Z=zpOh~+Bl5FlPSR1jj>4kG7}yW762G%)C7J7|Bf0*0Yh!l;?^m*gf$u(+-UTiHxY3 z6vmIRKz9*V0YCKziBSYrn9k?eoL63rO99&qbMXW*HRuQ7GqPZ*c(+Dv&Ike=kSpZ< zQ{*L8^4PdwMM-dss2H%F7p$XPV8E-uxZjT2mvb^_DXTDZc|FD*=y&|R>5gQGRG_b@ zVs_8{aAJG2`{x~LNQO|#dyx%@h!E}XGb*pzfK>o1|cb6k!rsGlpu9p@B zPFAvyonHyA0WVk*hEU&yl1Q**)0Q*iLV1cCbT0&h20e{B7psmPIQ%Jt6hL}*_K_8J zCYwNtDL?@oO{?#*NrU4u?)9W#`h4z1!*|NWtvgfSU&xTP*)9vR_Jul}Re#($a=fNY zTybzXdtS1Dy#))vjbK&6oBmKxt@Djb_YL$R`GRXtx;am=F5kK+R;_uGv3&%l%37gO z;*CN!9V`)EleSBOL z7`P8P-MC}=xJodak<>{n>#xYyEq4QdeuSF0>-`O`>8v z%W{NImW})R7ja2@k!em=QGQqu_F@pIAj{{>=eBZ#u)RGY#n{u`jo2=(W)x)K$FCZ# zwgcX7#EV#W5cZKijpdy)|FjvzVb9gu+v~&UsXns7ybF|7tZaG1T7do;)-T*XSb z2bMa`rp%bOf~unK-;c-ptbEsBma}s=j=%=0ntv*}FBT?Z%*TkX!>}%Uk1?c~(+DVA z3|`t(v0KNC^Ix&y0TEo2U_g%O3Lh|s|*Xq8@U6hfaTs1b9i`k}}gB9%ODz=^mpiyu;6}D@q zy}P}mIogw1h3(w8=W{~7Gc?cL$}I!ztj58=bicF?D;AW=;*mo|3yE}=d5aau^?F1b zEaWw)x}%yNii8C!TMSn^7+%?UJWKTBo>lCdAki1l{)#hgifSZKa}B_E$Il+I4o^+0%R*4Z}X*)#sPb!3_^hI;5=6xhhOr9u-+$dpcX5QXZvm z%2iMjcOuEnA6ipA@+5y`iYH5)WgToXo2obFp*N{9<-Yij5OpL&Z$3zZxHR`fK1gFHh~GlZF0>%-f7&gTvG7|9$U0f%#< zujIY`-oO^ngdy2$t!94;r7Y*OWOEn=S#9Y3L@}cTwLu_nT#9evTd*lOFbP*+jQX_` zhi(x5qTXtnG>1!?E564)xHn%n$S!1o{85$~9b{Rn@@YU>pDArQ<1BSgJ6&{aJBrwb zFn+$C9ybSiOn6Y_3%9NJypZ&2lDPGAuPw7d+v&E5anZyWkXWR~v)3t>Sv`W`mBVA| z9CYOBvuWMA+g#!)r6?y!>I*h>lY{O+yDv6{ee7Ek9toS<_ttRB4&)|k*$bSn)xd6* z4qPpGgp)qX;;R$>F(}ORsA-#-iW%oNAvakoESLj6GCTBo(duo`?V}m@XQed zPif9=<;=5-FQ3{bts5Prqx)J56?s-q;3^l?`+__3fl zgV&l=ZOAIv6L7Z}SRe~>)W!bBQx3Z{Z<*MByxZN)kk?FI*mQ!@+>;ZzvuhGtwXoDb zWSE8xA|*@h$mX{1J-+sDS!wO0D^2%V0!fQ`IV8!{&FxHJy|?$(z<8dikULx9iPdh< zv`BT-YMv+Byy2@zNnKyIkTc~%Y`^Z@gg>D?c%@LMafdzWnTCt-h0LXi&bzKbtqZsx z((dFlCqzr_U&L&)?}6wBqV^#`q-5Mb(y_aTkE50QPX$}AJ>Wc#kosg_rSRK03>;e; z@3IxIVqm;NMvF&oA1AaTTDEvq&(ZX?A~`@wy&b*%$RXm0Uo6qmP$q~A^LY*Y?nb*X zcK?7KSY%GqDJ7l_PaQd`nDg<#8egAFrMpX1T+Mx0Fo%*>TOKgmdNiBq-L6Qx!6SPD z=?4ibNe$a2d+8SPXuK@;s!SP=m9Z_%%aS|v9h8+E$LpN|Hb_GSw#G z_$t>z6u6SG5{3$omBV@>6}H=FvaQH~>#y20oOH)VsUTmlScA$=iyESl4Bd==yl1%^ zlhdsk=^a$M#MWh5iCQFyl_pyTpnVWBk4%x~Ht@12u;J%rRY z4k%~iK$+uCf-|@r)sZlTC9bLdX&`Xf?kmoB-Vt9+J6wI_%qJx<(Tvy5{_TcxYlLZS zN!W^kZS515WD2gM-7Xk)mbo67vG%4|$;VTPOc^?)JyVV?Zl+~Z-`{9V$*x){4Zt;( zdW;88Xcv%s*jP7ocbD5McFm_r;;G`WeIFH)XH~151YkluhDfUxEHRfI+0c?S#7n^( zM%={J1y1c-Sj0!?1}7{;d94?^XkD||*CwCQA@GJr5rLl;e_S14Qg(u5B5_w?1F&(X zKUkk6NJCu{Lw#c;+G zQ)wGIhPEdk9R1|nBb1H{f4h3P^D(V}MMPh$4VC2FU%E7c~97w?>N>YB~ z$cMJeDCEOF20mo2)33C&Br4!>sxfG29n>s=#DAzqGG4${p38}yE9ITn%Yzz%MIw}6 z@)L`z82L{utEpL_G}E*YGtCTS5T}Gd^obHNM>^t{HsvPw!_$p~1$Xk;KcH)!><)%T z9$2oSP~6ELTvnN-@ph9g9PF2yV}&)aEsy2|e!ECsKa5sKdQ6&nlzqB(;vp{LWkHv> zdpSRQRw~sJm}w+6vO##!PSL=O+K{3*&0Id9*Dsh2o%TEYy? z+8Zbq7jwY*YsR$c4sPe<2<(l5nJsI2kgI44?>U#>kNv#tK_I|uHRsS(-w zM2ra*e`WrWt0N6XO-(tx|CJsIcFoI2jMwD70dHq(VlYcyNq(7C3isg|IFF?IP+n^9 zbYO~J&#l(5$GPUO`$9}YQ~$?Kj6a>Yb6)4jgM-mvCDB)AlCK|#!sWjm1X;*zd!1xC z_~W4#It>pUo?!ml3YeGzfqsZL|Bea(u>7$CPZ8edVdbV}<>B$u4Y_aq=H;@mbH*}* zn?10EKVdb8+rX_f%rMMYE#Rn!BwZF*IF&e7I3?0!;yDq7^pTKmv~nQhU{@Es2ksUs z6U_7@5OfcO!qUs>OcX;5Q~*8;!C+>l$7P@(r>1xq#Ml6U-^T}xMLi&2@Cy^?e=xvG zjtd#XC&xuXw3`3z1=uogkfjjYr~NaxAo^;SE>ASvT%NjfKXGxh`g5rIzjhj8>7wFI zReE`d`z{ZAH=-m-h~n>P_+bw$=s1qQy8p)C_oGx<()s7r_Kh{K(RN0| zY)cwHNR4)$8?hoPT8=k}-4i}m`6-x#Sg$|P%rlQ((Wctg_|u}Lk=7!)w@@^2mAx9m z!lu$PGBzfv%G-E_5kCvRmq>OR9XEVFPF-z%^@0Y#5}dCGeF7B?xo1|)!ax}ZRex}8 zw?#`Y5<jg-NQY)%wWXp1`}zIyAbs1OBy>mX7m6_GJDz*;4OZ z=8KX8CaX0UDnxP^h%($zh6M{*y(qGh7B=mz7p6>oOd-uur!m`=Hb-&j_+ee4IdW@} z2d&=ToWjKq9uS5y;;z1l=iGFYPgrZGy4dz7v$uu29!Z6>8RsXyFRo z8>e|b!-Xm38X&wC@!^HE&r4$k;U8v4(!*D+!F-1@1BjnwMuXqLn0|O+kuJ>;?T)LI$?EOWY zY5Kq9Wt`v=Qwu!r$ITzLD>zeD@S_j?LIuM{*b%@f#3E{R{AxwD?!4aU1mS3I8`F0H8|)0Q@8U-!}hyJqd%C}v{~5 exportTables = ass.Types.Values.Where(t => t is DefTable ct && ct.NeedExport).Select(t => (DefTable)t).ToList(); - var genDataTasks = new List(); - var outputDataFiles = new ConcurrentBag(); - long genDataStartTime = TimeUtil.NowMillis; + [Option("input_l10n_text_files", Required = false, HelpText = "input l10n text table files. can be multi, sep by ','")] + public string InputTextTableFiles { get; set; } - foreach (DefTable c in exportTables) - { - genDataTasks.Add(Task.Run(async () => - { - long beginTime = TimeUtil.NowMillis; - await LoadTableAsync(agent, c, dataDir, exportTestData); - long endTime = TimeUtil.NowMillis; - if (endTime - beginTime > 100) - { - ctx.Info("====== load {0} cost {1} ms ======", c.FullName, (endTime - beginTime)); - } - })); - } - await Task.WhenAll(genDataTasks.ToArray()); + [Option("output_l10n_not_converted_text_file", Required = false, HelpText = "the file save not converted l10n texts.")] + public string OutputNotConvertTextFile { get; set; } } private ICodeRender CreateCodeRender(string genType) @@ -171,6 +147,12 @@ namespace Luban.Job.Cfg errMsg = "--output_data_json_monolithic_file missing"; return false; } + + if (string.IsNullOrWhiteSpace(result.InputTextTableFiles) ^ string.IsNullOrWhiteSpace(result.OutputNotConvertTextFile)) + { + errMsg = "--input_l10n_text_files must be provided with --output_l10n_not_converted_text_file"; + return false; + } } return true; @@ -225,6 +207,7 @@ namespace Luban.Job.Cfg bool hasLoadCfgData = false; + bool needL10NTextConvert = !string.IsNullOrWhiteSpace(args.InputTextTableFiles); async Task CheckLoadCfgDataAsync() { @@ -233,9 +216,15 @@ namespace Luban.Job.Cfg hasLoadCfgData = true; var timer = new ProfileTimer(); timer.StartPhase("load config data"); - await LoadCfgDataAsync(agent, ass, args.InputDataDir, args.ExportTestData); + await DataLoaderUtil.LoadCfgDataAsync(agent, ass, args.InputDataDir, args.ExportTestData); timer.EndPhaseAndLog(); + if (needL10NTextConvert) + { + ass.InitL10n(); + await DataLoaderUtil.LoadTextTablesAsync(agent, ass, args.InputDataDir, args.InputTextTableFiles); + } + timer.StartPhase("validate"); var validateCtx = new ValidatorContext(ass, args.ValidateRootDir); await validateCtx.ValidateTables(exportTables); @@ -944,7 +933,7 @@ class Vector4: { tasks.Add(Task.Run(() => { - var content = ToOutputData(c, ass.GetTableDataList(c), genType); + var content = DataExporterUtil.ToOutputData(c, ass.GetTableDataList(c), genType); var file = genType.EndsWith("json") ? c.JsonOutputDataFile : c.OutputDataFile; var md5 = FileUtil.CalcMD5(content); CacheManager.Ins.AddCache(file, md5, content); @@ -961,7 +950,7 @@ class Vector4: { allJsonTask.Add(Task.Run(() => { - return ToOutputData(c, ass.GetTableDataList(c), "data_json"); + return DataExporterUtil.ToOutputData(c, ass.GetTableDataList(c), "data_json"); })); } await Task.WhenAll(allJsonTask); @@ -1006,7 +995,7 @@ class Vector4: { tasks.Add(Task.Run(() => { - var content = ToOutputData(c, ass.GetTableDataList(c), genType); + var content = DataExporterUtil.ToOutputData(c, ass.GetTableDataList(c), genType); var file = $"{c.Name}.lua"; var md5 = FileUtil.CalcMD5(content); CacheManager.Ins.AddCache(file, md5, content); @@ -1023,7 +1012,7 @@ class Vector4: { genDataTasks.Add(Task.Run(() => { - return ExportResourceList(ass.GetTableDataList(c)); + return DataExporterUtil.ExportResourceList(ass.GetTableDataList(c)); })); } @@ -1060,6 +1049,16 @@ class Vector4: } await Task.WhenAll(tasks.ToArray()); + if (needL10NTextConvert) + { + var notConvertTextList = DataExporterUtil.GenNotConvertTextList(ass.NotConvertTextSet); + var md5 = FileUtil.CalcMD5(notConvertTextList); + string outputNotConvertTextFile = args.OutputNotConvertTextFile; + CacheManager.Ins.AddCache(outputNotConvertTextFile, md5, notConvertTextList); + + genScatteredFiles.Add(new FileInfo() { FilePath = outputNotConvertTextFile, MD5 = md5 }); + } + if (!genCodeFilesInOutputCodeDir.IsEmpty) { res.FileGroups.Add(new FileGroup() { Dir = outputCodeDir, Files = genCodeFilesInOutputCodeDir.ToList() }); @@ -1082,200 +1081,5 @@ class Vector4: agent.Session.ReplyRpc(rpc, res); } - - public string GetActualFileName(string file) - { - int index = file.IndexOf('@'); - return index >= 0 ? file[(index + 1)..] : file; - } - - private List LoadCfgRecords(DefTable table, string originFile, string sheetName, byte[] content, bool multiRecord, bool exportTestData) - { - // (md5,sheet,multiRecord,exportTestData) -> (valuetype, List<(datas)>) - var dataSource = DataSourceFactory.Create(originFile, sheetName, new MemoryStream(content), exportTestData); - try - { - List datas; - if (multiRecord) - { - datas = dataSource.ReadMulti(table.ValueTType); - } - else - { - datas = new List { dataSource.ReadOne(table.ValueTType) }; - } - var records = new List(datas.Count); - foreach (var data in datas) - { - records.Add(new Record((DBean)data, originFile)); - } - return records; - } - catch (Exception e) - { - throw new Exception($"配置文件:{originFile} 生成失败. ==> {e.Message}", e); - } - } - - class InputFileInfo - { - public string MD5 { get; set; } - - public string OriginFile { get; set; } - - public string ActualFile { get; set; } - - public string SheetName { get; set; } - } - - - - private async Task> CollectInputFilesAsync(RemoteAgent agent, DefTable table, string dataDir) - { - var collectTasks = new List>>(); - foreach (var file in table.InputFiles) - { - (var actualFile, var sheetName) = RenderFileUtil.SplitFileAndSheetName(FileUtil.Standardize(file)); - var actualFullPath = FileUtil.Combine(dataDir, actualFile); - var originFullPath = FileUtil.Combine(dataDir, file); - //s_logger.Info("== get input file:{file} actualFile:{actual}", file, actualFile); - - collectTasks.Add(Task.Run(async () => - { - var fileOrDirContent = await agent.GetFileOrDirectoryAsync(actualFullPath); - if (fileOrDirContent.IsFile) - { - return new List { new InputFileInfo() { OriginFile = file, ActualFile = actualFullPath, SheetName = sheetName, MD5 = fileOrDirContent.Md5 } }; - } - else - { - return fileOrDirContent.SubFiles.Select(f => new InputFileInfo() { OriginFile = f.FilePath, ActualFile = f.FilePath, MD5 = f.MD5 }).ToList(); - } - })); - } - - var allFiles = new List(); - foreach (var t in collectTasks) - { - allFiles.AddRange(await t); - } - return allFiles; - } - - public async Task LoadTableAsync(RemoteAgent agent, DefTable table, string dataDir, bool exportTestData) - { - var tasks = new List>>(); - - var inputFiles = await CollectInputFilesAsync(agent, table, dataDir); - - // check cache (table, exporttestdata) -> (list, List) - // (md5, sheetName,exportTestData) -> (value_type, List) - - foreach (var file in inputFiles) - { - var actualFile = file.ActualFile; - //s_logger.Info("== get input file:{file} actualFile:{actual}", file, actualFile); - - tasks.Add(Task.Run(async () => - { - if (FileRecordCacheManager.Ins.TryGetCacheLoadedRecords(table, file.MD5, actualFile, file.SheetName, exportTestData, out var cacheRecords)) - { - return cacheRecords; - } - var res = LoadCfgRecords(table, - file.OriginFile, - file.SheetName, - await agent.GetFromCacheOrReadAllBytesAsync(file.ActualFile, file.MD5), - RenderFileUtil.IsExcelFile(file.ActualFile), - exportTestData); - - FileRecordCacheManager.Ins.AddCacheLoadedRecords(table, file.MD5, file.SheetName, exportTestData, res); - - return res; - })); - } - - var records = new List(tasks.Count); - foreach (var task in tasks) - { - records.AddRange(await task); - } - - s_logger.Trace("== load recors. count:{count}", records.Count); - - table.Assembly.AddDataTable(table, records); - - s_logger.Trace("table:{name} record num:{num}", table.FullName, records.Count); - } - - private byte[] ToOutputData(DefTable table, List records, string dataType) - { - switch (dataType) - { - case "data_bin": - { - var buf = ThreadLocalTemporalByteBufPool.Alloc(1024 * 1024); - BinaryExportor.Ins.WriteList(records, table.Assembly, buf); - var bytes = buf.CopyData(); - ThreadLocalTemporalByteBufPool.Free(buf); - return bytes; - } - case "data_json": - { - var ss = new MemoryStream(); - var jsonWriter = new Utf8JsonWriter(ss, new JsonWriterOptions() - { - Indented = true, - SkipValidation = false, - Encoder = System.Text.Encodings.Web.JavaScriptEncoder.Create(System.Text.Unicode.UnicodeRanges.All), - }); - JsonExportor.Ins.WriteList(records, table.Assembly, jsonWriter); - jsonWriter.Flush(); - return DataUtil.StreamToBytes(ss); - } - case "data_lua": - { - var content = new List(); - - switch (table.Mode) - { - case ETableMode.ONE: - { - LuaExportor.Ins.ExportTableOne(table, records, content); - break; - } - case ETableMode.MAP: - { - LuaExportor.Ins.ExportTableOneKeyMap(table, records, content); - break; - } - case ETableMode.BMAP: - { - LuaExportor.Ins.ExportTableTwoKeyMap(table, records, content); - break; - } - default: - { - throw new NotSupportedException(); - } - } - return System.Text.Encoding.UTF8.GetBytes(string.Join('\n', content)); - } - default: - { - throw new ArgumentException($"not support datatype:{dataType}"); - } - } - } - - private List ExportResourceList(List records) - { - var resList = new List(); - foreach (Record res in records) - { - ResourceExportor.Ins.Accept(res.Data, null, resList); - } - return resList; - } } } diff --git a/src/Luban.Job.Cfg/Source/Utils/DataExporterUtil.cs b/src/Luban.Job.Cfg/Source/Utils/DataExporterUtil.cs new file mode 100644 index 0000000..a14efbf --- /dev/null +++ b/src/Luban.Job.Cfg/Source/Utils/DataExporterUtil.cs @@ -0,0 +1,100 @@ +using Bright.Serialization; +using Luban.Config.Common.RawDefs; +using Luban.Job.Cfg.Datas; +using Luban.Job.Cfg.DataVisitors; +using Luban.Job.Cfg.Defs; +using Luban.Job.Cfg.l10n; +using Luban.Job.Cfg.RawDefs; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Threading.Tasks; + +namespace Luban.Job.Cfg.Utils +{ + public static class DataExporterUtil + { + public static byte[] ToOutputData(DefTable table, List records, string dataType) + { + switch (dataType) + { + case "data_bin": + { + var buf = ThreadLocalTemporalByteBufPool.Alloc(1024 * 1024); + BinaryExportor.Ins.WriteList(records, table.Assembly, buf); + var bytes = buf.CopyData(); + ThreadLocalTemporalByteBufPool.Free(buf); + return bytes; + } + case "data_json": + { + var ss = new MemoryStream(); + var jsonWriter = new Utf8JsonWriter(ss, new JsonWriterOptions() + { + Indented = true, + SkipValidation = false, + Encoder = System.Text.Encodings.Web.JavaScriptEncoder.Create(System.Text.Unicode.UnicodeRanges.All), + }); + JsonExportor.Ins.WriteList(records, table.Assembly, jsonWriter); + jsonWriter.Flush(); + return DataUtil.StreamToBytes(ss); + } + case "data_lua": + { + var content = new List(); + + switch (table.Mode) + { + case ETableMode.ONE: + { + LuaExportor.Ins.ExportTableOne(table, records, content); + break; + } + case ETableMode.MAP: + { + LuaExportor.Ins.ExportTableOneKeyMap(table, records, content); + break; + } + case ETableMode.BMAP: + { + LuaExportor.Ins.ExportTableTwoKeyMap(table, records, content); + break; + } + default: + { + throw new NotSupportedException(); + } + } + return System.Text.Encoding.UTF8.GetBytes(string.Join('\n', content)); + } + default: + { + throw new ArgumentException($"not support datatype:{dataType}"); + } + } + } + + public static List ExportResourceList(List records) + { + var resList = new List(); + foreach (Record res in records) + { + ResourceExportor.Ins.Accept(res.Data, null, resList); + } + return resList; + } + + public static byte[] GenNotConvertTextList(NotConvertTextSet notConvertSet) + { + StringBuilder sb = new StringBuilder(); + foreach (var e in notConvertSet.SortedEntry) + { + sb.Append(e.Key).Append('|').Append(e.Value).Append('\n'); + } + return System.Text.Encoding.UTF8.GetBytes(sb.ToString()); + } + } +} diff --git a/src/Luban.Job.Cfg/Source/Utils/DataLoaderUtil.cs b/src/Luban.Job.Cfg/Source/Utils/DataLoaderUtil.cs new file mode 100644 index 0000000..ea72aa6 --- /dev/null +++ b/src/Luban.Job.Cfg/Source/Utils/DataLoaderUtil.cs @@ -0,0 +1,197 @@ +using Bright.Time; +using Luban.Common.Utils; +using Luban.Job.Cfg.Cache; +using Luban.Job.Cfg.Datas; +using Luban.Job.Cfg.DataSources; +using Luban.Job.Cfg.Defs; +using Luban.Job.Common.Types; +using Luban.Job.Common.Utils; +using Luban.Server.Common; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Luban.Job.Cfg.Utils +{ + public static class DataLoaderUtil + { + private static readonly NLog.Logger s_logger = NLog.LogManager.GetCurrentClassLogger(); + + public class InputFileInfo + { + public string MD5 { get; set; } + + public string OriginFile { get; set; } + + public string ActualFile { get; set; } + + public string SheetName { get; set; } + } + + public static async Task> CollectInputFilesAsync(RemoteAgent agent, IEnumerable files, string dataDir) + { + var collectTasks = new List>>(); + foreach (var file in files) + { + (var actualFile, var sheetName) = RenderFileUtil.SplitFileAndSheetName(FileUtil.Standardize(file)); + var actualFullPath = FileUtil.Combine(dataDir, actualFile); + var originFullPath = FileUtil.Combine(dataDir, file); + //s_logger.Info("== get input file:{file} actualFile:{actual}", file, actualFile); + + collectTasks.Add(Task.Run(async () => + { + var fileOrDirContent = await agent.GetFileOrDirectoryAsync(actualFullPath); + if (fileOrDirContent.IsFile) + { + return new List { new InputFileInfo() { OriginFile = file, ActualFile = actualFullPath, SheetName = sheetName, MD5 = fileOrDirContent.Md5 } }; + } + else + { + return fileOrDirContent.SubFiles.Select(f => new InputFileInfo() { OriginFile = f.FilePath, ActualFile = f.FilePath, MD5 = f.MD5 }).ToList(); + } + })); + } + + var allFiles = new List(); + foreach (var t in collectTasks) + { + allFiles.AddRange(await t); + } + return allFiles; + } + + //private async Task> CollectInputFilesAsync(RemoteAgent agent, DefTable table, string dataDir) + //{ + // var collectTasks = new List>>(); + // foreach (var file in table.InputFiles) + // return CollectInputFilesAsync(agent, table.InputFiles, dataDir) + //} + + public static async Task LoadTableAsync(RemoteAgent agent, DefTable table, string dataDir, bool exportTestData) + { + var tasks = new List>>(); + + var inputFiles = await CollectInputFilesAsync(agent, table.InputFiles, dataDir); + + // check cache (table, exporttestdata) -> (list, List) + // (md5, sheetName,exportTestData) -> (value_type, List) + + foreach (var file in inputFiles) + { + var actualFile = file.ActualFile; + //s_logger.Info("== get input file:{file} actualFile:{actual}", file, actualFile); + + tasks.Add(Task.Run(async () => + { + if (FileRecordCacheManager.Ins.TryGetCacheLoadedRecords(table, file.MD5, actualFile, file.SheetName, exportTestData, out var cacheRecords)) + { + return cacheRecords; + } + var res = LoadCfgRecords(table.ValueTType, + file.OriginFile, + file.SheetName, + await agent.GetFromCacheOrReadAllBytesAsync(file.ActualFile, file.MD5), + RenderFileUtil.IsExcelFile(file.ActualFile), + exportTestData); + + FileRecordCacheManager.Ins.AddCacheLoadedRecords(table, file.MD5, file.SheetName, exportTestData, res); + + return res; + })); + } + + var records = new List(tasks.Count); + foreach (var task in tasks) + { + records.AddRange(await task); + } + + s_logger.Trace("== load recors. count:{count}", records.Count); + + table.Assembly.AddDataTable(table, records); + + s_logger.Trace("table:{name} record num:{num}", table.FullName, records.Count); + } + + public static async Task LoadCfgDataAsync(RemoteAgent agent, DefAssembly ass, string dataDir, bool exportTestData) + { + var ctx = agent; + List exportTables = ass.Types.Values.Where(t => t is DefTable ct && ct.NeedExport).Select(t => (DefTable)t).ToList(); + var genDataTasks = new List(); + var outputDataFiles = new ConcurrentBag(); + long genDataStartTime = TimeUtil.NowMillis; + + foreach (DefTable c in exportTables) + { + genDataTasks.Add(Task.Run(async () => + { + long beginTime = TimeUtil.NowMillis; + await LoadTableAsync(agent, c, dataDir, exportTestData); + long endTime = TimeUtil.NowMillis; + if (endTime - beginTime > 100) + { + ctx.Info("====== load {0} cost {1} ms ======", c.FullName, (endTime - beginTime)); + } + })); + } + await Task.WhenAll(genDataTasks.ToArray()); + } + + public static List LoadCfgRecords(TBean recordType, string originFile, string sheetName, byte[] content, bool multiRecord, bool exportTestData) + { + // (md5,sheet,multiRecord,exportTestData) -> (valuetype, List<(datas)>) + var dataSource = DataSourceFactory.Create(originFile, sheetName, new MemoryStream(content), exportTestData); + try + { + List datas; + if (multiRecord) + { + datas = dataSource.ReadMulti(recordType); + } + else + { + datas = new List { dataSource.ReadOne(recordType) }; + } + var records = new List(datas.Count); + foreach (var data in datas) + { + records.Add(new Record((DBean)data, originFile)); + } + return records; + } + catch (Exception e) + { + throw new Exception($"配置文件:{originFile} 生成失败. ==> {e.Message}", e); + } + } + + public static async Task LoadTextTablesAsync(RemoteAgent agent, DefAssembly ass, string baseDir, string textTableFiles) + { + var tasks = new List>(); + var files = textTableFiles.Split(','); + foreach (var file in await CollectInputFilesAsync(agent, files, baseDir)) + { + tasks.Add(agent.GetFromCacheOrReadAllBytesAsync(file.ActualFile, file.MD5)); + } + + var textTable = ass.ExportTextTable; + for (int i = 0; i < tasks.Count; i++) + { + var bytes = await tasks[i]; + try + { + textTable.LoadFromFile(files[i], bytes); + } + catch (Exception e) + { + throw new Exception($"load text table file:{files[i]} fail. ==> {e.Message} "); + } + } + } + + } +} diff --git a/src/Luban.Job.Cfg/Source/ValidatorContext.cs b/src/Luban.Job.Cfg/Source/ValidatorContext.cs index c5de99f..ba6152a 100644 --- a/src/Luban.Job.Cfg/Source/ValidatorContext.cs +++ b/src/Luban.Job.Cfg/Source/ValidatorContext.cs @@ -69,7 +69,6 @@ namespace Luban.Job.Cfg { var records = t.Assembly.GetTableDataList(t); ValidateTableModeIndex(t, records); - })); } await Task.WhenAll(tasks); diff --git a/src/Luban.Job.Cfg/Source/i10n/NotConvertTextSet.cs b/src/Luban.Job.Cfg/Source/i10n/NotConvertTextSet.cs deleted file mode 100644 index 53b438e..0000000 --- a/src/Luban.Job.Cfg/Source/i10n/NotConvertTextSet.cs +++ /dev/null @@ -1,18 +0,0 @@ -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Luban.Job.Cfg.i10n -{ - public class NotConvertTextSet - { - private readonly ConcurrentDictionary _notConvertTexts = new(); - public void Add(string key, string text) - { - _notConvertTexts.TryAdd(key, text); - } - } -} diff --git a/src/Luban.Job.Cfg/Source/i10n/TextTable.cs b/src/Luban.Job.Cfg/Source/i10n/TextTable.cs deleted file mode 100644 index 759c21b..0000000 --- a/src/Luban.Job.Cfg/Source/i10n/TextTable.cs +++ /dev/null @@ -1,33 +0,0 @@ -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Luban.Job.Cfg.i10n -{ - public class TextTable - { - private readonly Dictionary _key2Texts = new(); - - public TextTable() - { - _key2Texts.Add("test/a", "这是本地化数据 test/a"); - _key2Texts.Add("name", "这是本地化数据 name"); - } - - public void AddText(string key, string text) - { - if (!_key2Texts.TryAdd(key, text)) - { - throw new Exception($"text key:{key} 重复"); - } - } - - public bool TryGetText(string key, out string text) - { - return _key2Texts.TryGetValue(key, out text); - } - } -} diff --git a/src/Luban.Job.Cfg/Source/l10n/NotConvertTextSet.cs b/src/Luban.Job.Cfg/Source/l10n/NotConvertTextSet.cs new file mode 100644 index 0000000..568df59 --- /dev/null +++ b/src/Luban.Job.Cfg/Source/l10n/NotConvertTextSet.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Luban.Job.Cfg.l10n +{ + public class NotConvertTextSet + { + private readonly ConcurrentDictionary _notConvertTexts = new(); + public void Add(string key, string text) + { + if (key != "") + { + _notConvertTexts.TryAdd(key, text); + } + } + + public List> SortedEntry + { + get + { + var list = _notConvertTexts.ToList(); + list.Sort((a, b) => a.Key.CompareTo(b.Key)); + return list; + } + } + } +} diff --git a/src/Luban.Job.Cfg/Source/i10n/RawTextTable.cs b/src/Luban.Job.Cfg/Source/l10n/RawTextTable.cs similarity index 97% rename from src/Luban.Job.Cfg/Source/i10n/RawTextTable.cs rename to src/Luban.Job.Cfg/Source/l10n/RawTextTable.cs index bc56f6f..d675fd7 100644 --- a/src/Luban.Job.Cfg/Source/i10n/RawTextTable.cs +++ b/src/Luban.Job.Cfg/Source/l10n/RawTextTable.cs @@ -6,7 +6,7 @@ using System.Linq; using System.Text; using System.Threading.Tasks; -namespace Luban.Job.Cfg.i10n +namespace Luban.Job.Cfg.l10n { public class RawTextTable { diff --git a/src/Luban.Job.Cfg/Source/l10n/TextTable.cs b/src/Luban.Job.Cfg/Source/l10n/TextTable.cs new file mode 100644 index 0000000..65ee6b9 --- /dev/null +++ b/src/Luban.Job.Cfg/Source/l10n/TextTable.cs @@ -0,0 +1,83 @@ +using Luban.Config.Common.RawDefs; +using Luban.Job.Cfg.Datas; +using Luban.Job.Cfg.DataVisitors; +using Luban.Job.Cfg.Defs; +using Luban.Job.Cfg.Utils; +using Luban.Job.Common.Types; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Luban.Job.Cfg.l10n +{ + public class TextTable + { + private static readonly NLog.Logger s_logger = NLog.LogManager.GetCurrentClassLogger(); + + public DefAssembly Assembly { get; } + + private readonly Dictionary _key2Texts = new(); + + private readonly TBean _textRowType; + + public TextTable(DefAssembly ass) + { + this.Assembly = ass; + var defTextRowType = new DefBean(new CfgBean() + { + Namespace = "__intern__", + Name = "__TextInfo__", + Parent = "", + Alias = "", + IsValueType = false, + Sep = "", + TypeId = 0, + IsSerializeCompatible = false, + Fields = new List + { + new CfgField() { Name = "key", Type = "string" }, + //new Common.RawDefs.Field() { Id = 1, Name = "origin_text", Type = "string" }, + new CfgField() { Name = "text", Type = "string" }, + } + }) + { + AssemblyBase = ass, + }; + defTextRowType.PreCompile(); + defTextRowType.Compile(); + defTextRowType.PostCompile(); + _textRowType = new TBean(defTextRowType, false); + } + + public void AddText(string key, string text) + { + if (!_key2Texts.TryAdd(key, text)) + { + throw new Exception($"text key:{key} 重复"); + } + } + + public bool TryGetText(string key, out string text) + { + return _key2Texts.TryGetValue(key, out text); + } + + public void LoadFromFile(string fileName, byte[] bytes) + { + var records = DataLoaderUtil.LoadCfgRecords(_textRowType, fileName, null, bytes, true, false); + foreach (var r in records) + { + //s_logger.Info("== read text:{}", r.Data); + string key = (r.Data.Fields[0] as DString).Value; + string text = (r.Data.Fields[1] as DString).Value; + if (!_key2Texts.TryAdd(key, text)) + { + throw new Exception($"TextTableFile:{fileName} key:{key} text:{text} 重复"); + } + } + } + } +} diff --git a/src/Luban.Server.Common/Source/RemoteAgent.cs b/src/Luban.Server.Common/Source/RemoteAgent.cs index 9a24d3b..b778342 100644 --- a/src/Luban.Server.Common/Source/RemoteAgent.cs +++ b/src/Luban.Server.Common/Source/RemoteAgent.cs @@ -100,7 +100,7 @@ namespace Luban.Server.Common GET_INPUT_FILE_TIMEOUT); if (res.Err != Luban.Common.EErrorCode.OK) { - throw new ReadRemoteFailException($"{res.Err}"); + throw new ReadRemoteFailException($"ReadFile:{file} fail. {res.Err}"); } s_logger.Trace("read GetFileOrDirectoryAsync end. file:{file} cost:{time}", file, TimeUtil.NowMillis - t1); return res;