xiang.li
/
luban
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

【修复】解决不同平台下生成json和lua数据文件（换行符）不同的问题

main
walon 2021-08-02 13:46:59 +08:00
parent 39d976436c
commit cd6f0f1282
9 changed files with 57 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
src/Luban.Job.Cfg/Source/DataSources/AbstractDataSource.cs
View File

@ -9,25 +9,6 @@ namespace Luban.Job.Cfg.DataSources
{
public const string TAG_KEY = "__tag__";
public static bool IsIgnoreTag(string tagName)
{
return !string.IsNullOrWhiteSpace(tagName) &&
(
tagName.Equals("false", System.StringComparison.OrdinalIgnoreCase)
|| tagName.Equals("no", System.StringComparison.OrdinalIgnoreCase)
|| tagName.Equals("##", System.StringComparison.Ordinal)
|| tagName.Equals("·ñ", System.StringComparison.Ordinal)
);
}
public static bool IsTestTag(string tagName)
{
return !string.IsNullOrWhiteSpace(tagName) &&
(tagName.Equals("test", System.StringComparison.OrdinalIgnoreCase)
|| tagName.Equals("²âÊÔ", System.StringComparison.Ordinal)
);
}
public string RawUrl { get; protected set; }
public abstract Record ReadOne(TBean type);

7
src/Luban.Job.Cfg/Source/DataSources/Excel/Sheet.cs
View File

@ -2,6 +2,7 @@ using ExcelDataReader;
using Luban.Job.Cfg.DataCreators;
using Luban.Job.Cfg.Datas;
using Luban.Job.Cfg.TypeVisitors;
using Luban.Job.Cfg.Utils;
using Luban.Job.Common.Types;
using System;
using System.Collections.Generic;
@ -529,7 +530,7 @@ namespace Luban.Job.Cfg.DataSources.Excel
// 删除所有标题行，包含字段名行、属性行、标题、描述等等非有效数据行
this._rowColumns.RemoveRange(0, Math.Min(TitleRows + titleRowNum - 1, this._rowColumns.Count));
// 删除忽略的记录行
this._rowColumns.RemoveAll(row => AbstractDataSource.IsIgnoreTag(GetRowTag(row)));
this._rowColumns.RemoveAll(row => DataUtil.IsIgnoreTag(GetRowTag(row)));
}
}
@ -629,7 +630,7 @@ namespace Luban.Job.Cfg.DataSources.Excel
{
return null;
}
bool isTest = AbstractDataSource.IsTestTag(GetRowTag(row));
bool isTest = DataUtil.IsTestTag(GetRowTag(row));
var data = (DBean)ExcelNamedRowDataCreator.Ins.ReadExcel(new NamedRow(_rootTitle, row), type);
return new Record(data, RawUrl, isTest);
}
@ -640,7 +641,7 @@ namespace Luban.Job.Cfg.DataSources.Excel
{
return null;
}
bool isTest = AbstractDataSource.IsTestTag(GetRowTag(rows[0]));
bool isTest = DataUtil.IsTestTag(GetRowTag(rows[0]));
var data = (DBean)ExcelNamedRowDataCreator.Ins.ReadExcel(new NamedRow(_rootTitle, rows), type);
return new Record(data, RawUrl, isTest);
}

5
src/Luban.Job.Cfg/Source/DataSources/Json/JsonDataSource.cs
View File

@ -2,6 +2,7 @@ using Luban.Job.Cfg.DataCreators;
using Luban.Job.Cfg.Datas;
using Luban.Job.Cfg.Defs;
using Luban.Job.Cfg.TypeVisitors;
using Luban.Job.Cfg.Utils;
using Luban.Job.Common.Types;
using System;
using System.Collections.Generic;
@ -31,11 +32,11 @@ namespace Luban.Job.Cfg.DataSources.Json
if (_data.TryGetProperty(TAG_KEY, out var tagEle))
{
var tagName = tagEle.GetString();
if (IsIgnoreTag(tagName))
if (DataUtil.IsIgnoreTag(tagName))
{
return null;
}
isTest = IsTestTag(tagName);
isTest = DataUtil.IsTestTag(tagName);
}
var data = (DBean)type.Apply(JsonDataCreator.Ins, _data, (DefAssembly)type.Bean.AssemblyBase);

5
src/Luban.Job.Cfg/Source/DataSources/Lua/LuaDataSource.cs
View File

@ -2,6 +2,7 @@ using Luban.Job.Cfg.DataCreators;
using Luban.Job.Cfg.Datas;
using Luban.Job.Cfg.Defs;
using Luban.Job.Cfg.TypeVisitors;
using Luban.Job.Cfg.Utils;
using Luban.Job.Common.Types;
using Neo.IronLua;
using System.Collections.Generic;
@ -48,12 +49,12 @@ namespace Luban.Job.Cfg.DataSources.Lua
protected Record ReadRecord(LuaTable table, TBean type)
{
string tagName = table.GetValue(TAG_KEY)?.ToString();
if (IsIgnoreTag(tagName))
if (DataUtil.IsIgnoreTag(tagName))
{
return null;
}
var data = (DBean)type.Apply(LuaDataCreator.Ins, table, (DefAssembly)type.Bean.AssemblyBase);
var isTest = IsTestTag(tagName);
var isTest = DataUtil.IsTestTag(tagName);
return new Record(data, RawUrl, isTest);
}
}

5
src/Luban.Job.Cfg/Source/DataSources/Xml/XmlDataSource.cs
View File

@ -2,6 +2,7 @@ using Luban.Job.Cfg.DataCreators;
using Luban.Job.Cfg.Datas;
using Luban.Job.Cfg.Defs;
using Luban.Job.Cfg.TypeVisitors;
using Luban.Job.Cfg.Utils;
using Luban.Job.Common.Types;
using System;
using System.Collections.Generic;
@ -28,12 +29,12 @@ namespace Luban.Job.Cfg.DataSources.Xml
public override Record ReadOne(TBean type)
{
string tagName = _doc.Element(TAG_KEY)?.Value;
if (IsIgnoreTag(tagName))
if (DataUtil.IsIgnoreTag(tagName))
{
return null;
}
var data = (DBean)type.Apply(XmlDataCreator.Ins, _doc, (DefAssembly)type.Bean.AssemblyBase);
bool isTest = IsTestTag(tagName);
bool isTest = DataUtil.IsTestTag(tagName);
return new Record(data, RawUrl, isTest);
}
}

24
src/Luban.Job.Cfg/Source/JobController.cs
View File

@ -770,7 +770,7 @@ namespace {ctx.TopModule}
{
var content = DataExporterUtil.ToOutputData(c, ctx.Assembly.GetTableExportDataList(c), genType);
var file = GetOutputFileName(genType, c.OutputDataFile);
var md5 = CacheFileUtil.GenMd5AndAddCache(file, content);
var md5 = CacheFileUtil.GenStringOrBytesMd5AndAddCache(file, content);
ctx.GenDataFilesInOutputDataDir.Add(new FileInfo() { FilePath = file, MD5 = md5 });
}));
}
@ -779,32 +779,30 @@ namespace {ctx.TopModule}
private async Task GenJsonDataMonolithic(GenContext ctx)
{
var exportTables = ctx.ExportTables;
List<Task<byte[]>> allJsonTask = new List<Task<byte[]>>();
var allJsonTask = new List<Task<string>>();
foreach (var c in exportTables)
{
allJsonTask.Add(Task.Run(() =>
{
return DataExporterUtil.ToOutputData(c, ctx.Assembly.GetTableExportDataList(c), "data_json");
return (string)DataExporterUtil.ToOutputData(c, ctx.Assembly.GetTableExportDataList(c), "data_json");
}));
}
await Task.WhenAll(allJsonTask);
int estimatedCapacity = allJsonTask.Sum(t => t.Result.Length + 100);
var sb = new MemoryStream(estimatedCapacity);
sb.Write(System.Text.Encoding.UTF8.GetBytes("{\n"));
var lines = new List<string>();
lines.Add("{");
for (int i = 0; i < exportTables.Count; i++)
{
if (i != 0)
{
sb.Write(System.Text.Encoding.UTF8.GetBytes((",\n")));
lines.Add(",");
}
sb.Write(System.Text.Encoding.UTF8.GetBytes("\"" + exportTables[i].Name + "\":"));
sb.Write(allJsonTask[i].Result);
lines.Add($"\"{exportTables[i].NeedExport}\":");
lines.Add(await allJsonTask[i]);
}
sb.Write(System.Text.Encoding.UTF8.GetBytes("\n}"));
lines.Add("}");
var content = sb.ToArray();
s_logger.Debug("estimated size:{0} actual size:{1}", estimatedCapacity, content.Length);
var content = string.Join('\n', lines);
var outputFile = ctx.GenArgs.OutputDataJsonMonolithicFile;
var md5 = CacheFileUtil.GenMd5AndAddCache(outputFile, content);
ctx.GenScatteredFiles.Add(new FileInfo() { FilePath = outputFile, MD5 = md5 });

6
src/Luban.Job.Cfg/Source/Utils/DataExporterUtil.cs
View File

@ -17,7 +17,7 @@ namespace Luban.Job.Cfg.Utils
{
public static class DataExporterUtil
{
public static byte[] ToOutputData(DefTable table, List<Record> records, string dataType)
public static object ToOutputData(DefTable table, List<Record> records, string dataType)
{
switch (dataType)
{
@ -40,7 +40,7 @@ namespace Luban.Job.Cfg.Utils
});
JsonExportor.Ins.WriteList(records, table.Assembly, jsonWriter);
jsonWriter.Flush();
return DataUtil.StreamToBytes(ss);
return System.Text.Encoding.UTF8.GetString(DataUtil.StreamToBytes(ss));
}
case "data_lua":
{
@ -63,7 +63,7 @@ namespace Luban.Job.Cfg.Utils
throw new NotSupportedException();
}
}
return System.Text.Encoding.UTF8.GetBytes(string.Join('\n', content));
return string.Join('\n', content);
}
default:
{

20
src/Luban.Job.Cfg/Source/Utils/DataUtil.cs
View File

@ -100,6 +100,26 @@ namespace Luban.Job.Cfg.Utils
}
}
public static bool IsIgnoreTag(string tagName)
{
return !string.IsNullOrEmpty(tagName) &&
(
tagName.Equals("false", System.StringComparison.OrdinalIgnoreCase)
|| tagName.Equals("no", System.StringComparison.OrdinalIgnoreCase)
|| tagName.Equals("##", System.StringComparison.Ordinal)
|| tagName.Equals("·ñ", System.StringComparison.Ordinal)
);
}
public static bool IsTestTag(string tagName)
{
return !string.IsNullOrEmpty(tagName) &&
(tagName.Equals("test", System.StringComparison.OrdinalIgnoreCase)
|| tagName.Equals("²âÊÔ", System.StringComparison.Ordinal)
);
}
//public static string Data2String(DType data)
//{
// var s = new StringBuilder();

10
src/Luban.Job.Common/Source/Utils/CacheFileUtil.cs
View File

@ -25,6 +25,16 @@ namespace Luban.Job.Common.Utils
}
}
public static string GenStringOrBytesMd5AndAddCache(string fileName, object content)
{
switch (content)
{
case string s: return GenMd5AndAddCache(fileName, s);
case byte[] bs: return GenMd5AndAddCache(fileName, bs);
default: throw new System.NotSupportedException();
}
}
public static string GenMd5AndAddCache(string fileName, string content)
{
content = content.Replace("\r\n", "\n");