xiang.li
/
luban
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

【优化】GetImportFileOrDirectory 新增文件后缀过滤，只获取支持的文件 

【优化】CommonDefLoader import目录时，只获取.xml后缀文件
main
walon 2021-08-23 10:48:08 +08:00
parent af4038e810
commit 8530840ddf
6 changed files with 46 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/Luban.Client/Source/Net/GenClient.cs
View File

@ -10,6 +10,7 @@ using Luban.Common.Utils;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
@ -100,6 +101,7 @@ namespace Luban.Client.Common.Net
{
long t1 = TimeUtil.NowMillis;
var file = rpc.Arg.FileOrDirName;
var suffixes = rpc.Arg.InclusiveSuffixs;
var re = new GetImportFileOrDirectoryRes()
{
SubFiles = new List<Luban.Common.Protos.FileInfo>(),
@ -113,7 +115,7 @@ namespace Luban.Client.Common.Net
re.IsFile = false;
foreach (var subFile in Directory.GetFiles(file, "*", SearchOption.AllDirectories))
{
if (FileUtil.IsValidInputFile(subFile))
if (FileUtil.IsValidInputFile(subFile) && (suffixes.Count == 0 || suffixes.Any(s => subFile.EndsWith(s))))
{
var md5 = await CacheMetaManager.Ins.GetOrUpdateFileMd5Async(subFile);
re.SubFiles.Add(new Luban.Common.Protos.FileInfo() { FilePath = FileUtil.Standardize(subFile), MD5 = md5 });

4
src/Luban.Common/Source/Protos/GetImportFileOrDirectory.cs
View File

@ -9,6 +9,8 @@ namespace Luban.Common.Protos
{
public string FileOrDirName { get; set; }
public List<string> InclusiveSuffixs { get; set; } = new List<string>();
public override int GetTypeId()
{
return 0;
@ -17,10 +19,12 @@ namespace Luban.Common.Protos
public override void Serialize(ByteBuf os)
{
os.WriteString(FileOrDirName);
Bright.Common.SerializationUtil.Serialize(os, InclusiveSuffixs);
}
public override void Deserialize(ByteBuf os)
{
FileOrDirName = os.ReadString();
Bright.Common.SerializationUtil.Deserialize(os, InclusiveSuffixs);
}
}

14
src/Luban.Job.Cfg/Source/DataSources/DataSourceFactory.cs
View File

@ -6,6 +6,18 @@ namespace Luban.Job.Cfg.DataSources
{
static class DataSourceFactory
{
public static readonly string[] validDataSourceSuffixes = new string[]
{
".xlsx",
".xls",
".csv",
".xml",
".lua",
".json",
".yml",
".bin",
};
public static AbstractDataSource Create(string url, string sheetName, Stream stream, bool exportTestData)
{
try
@ -20,7 +32,7 @@ namespace Luban.Job.Cfg.DataSources
case "xml": source = new Xml.XmlDataSource(); break;
case "lua": source = new Lua.LuaDataSource(); break;
case "json": source = new Json.JsonDataSource(); break;
case "b": source = new Binary.BinaryDataSource(); break;
case "bin": source = new Binary.BinaryDataSource(); break;
case "yml": source = new Yaml.YamlDataSource(); break;
default: throw new Exception($"不支持的文件类型:{url}");
}

2
src/Luban.Job.Cfg/Source/Utils/DataLoaderUtil.cs
View File

@ -45,7 +45,7 @@ namespace Luban.Job.Cfg.Utils
collectTasks.Add(Task.Run(async () =>
{
var fileOrDirContent = await agent.GetFileOrDirectoryAsync(actualFullPath);
var fileOrDirContent = await agent.GetFileOrDirectoryAsync(actualFullPath, DataSourceFactory.validDataSourceSuffixes);
if (fileOrDirContent.IsFile)
{
return new List<InputFileInfo> { new InputFileInfo() { OriginFile = file, ActualFile = actualFullPath, SheetName = sheetName, MD5 = fileOrDirContent.Md5 } };

2
src/Luban.Job.Common/Source/Defs/CommonDefLoader.cs
View File

@ -124,7 +124,7 @@ namespace Luban.Job.Common.Defs
var xmlFullPath = FileUtil.Combine(RootDir, xmlFile);
s_logger.Trace("import {file} {full_path}", xmlFile, xmlFullPath);
var fileOrDirContent = await Agent.GetFileOrDirectoryAsync(xmlFullPath);
var fileOrDirContent = await Agent.GetFileOrDirectoryAsync(xmlFullPath, ".xml");
if (fileOrDirContent.IsFile)
{

55
src/Luban.Server.Common/Source/RemoteAgent.cs
View File

@ -2,8 +2,10 @@ using Bright.Net.ServiceModes.Managers;
using Bright.Time;
using Luban.Common.Protos;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.Serialization;
using System.Threading.Tasks;
using System.Xml.Linq;
@ -37,9 +39,9 @@ namespace Luban.Server.Common
private readonly bool _trace;
private readonly Dictionary<string, Task<byte[]>> _remoteReadAllBytesTasks = new Dictionary<string, Task<byte[]>>();
private readonly ConcurrentDictionary<string, Task<byte[]>> _remoteReadAllBytesTasks = new();
private readonly Dictionary<string, Task<GetImportFileOrDirectoryRes>> _getImportFileOrDirTasks = new Dictionary<string, Task<GetImportFileOrDirectoryRes>>();
private readonly ConcurrentDictionary<string, Task<GetImportFileOrDirectoryRes>> _getImportFileOrDirTasks = new();
public RemoteAgent(SessionBase session, bool trace)
{
@ -63,40 +65,35 @@ namespace Luban.Server.Common
public Task<byte[]> ReadAllBytesAsync(string file)
{
lock (_remoteReadAllBytesTasks)
return _remoteReadAllBytesTasks.GetOrAdd(file, f =>
{
if (!_remoteReadAllBytesTasks.TryGetValue(file, out var task))
return Task.Run(async () =>
{
task = Task.Run(async () =>
long t1 = TimeUtil.NowMillis;
var res = await Session.CallRpcAsync<GetInputFile, GetInputFileArg, GetInputFileRes>(new GetInputFileArg() { File = f }, GET_INPUT_FILE_TIMEOUT);
if (res.Err != Luban.Common.EErrorCode.OK)
{
long t1 = TimeUtil.NowMillis;
var res = await Session.CallRpcAsync<GetInputFile, GetInputFileArg, GetInputFileRes>(new GetInputFileArg() { File = file }, GET_INPUT_FILE_TIMEOUT);
if (res.Err != Luban.Common.EErrorCode.OK)
{
throw new ReadRemoteFailException($"{res.Err}");
}
s_logger.Info("read remote file:{file} cost:{time}", file, TimeUtil.NowMillis - t1);
return res.Content;
});
task.ConfigureAwait(false);
_remoteReadAllBytesTasks.Add(file, task);
}
return task;
}
throw new ReadRemoteFailException($"{res.Err}");
}
s_logger.Info("read remote file:{file} cost:{time}", f, TimeUtil.NowMillis - t1);
return res.Content;
});
});
}
public Task<GetImportFileOrDirectoryRes> GetFileOrDirectoryAsync(string file)
public Task<GetImportFileOrDirectoryRes> GetFileOrDirectoryAsync(string file, params string[] searchPatterns)
{
lock (_getImportFileOrDirTasks)
{
if (!_getImportFileOrDirTasks.TryGetValue(file, out var task))
return _getImportFileOrDirTasks.GetOrAdd(file, f =>
{
task = Task.Run(async () =>
return Task.Run(async () =>
{
long t1 = TimeUtil.NowMillis;
var res = await Session.CallRpcAsync<GetImportFileOrDirectory, GetImportFileOrDirectoryArg, GetImportFileOrDirectoryRes>(
new GetImportFileOrDirectoryArg() { FileOrDirName = file },
new GetImportFileOrDirectoryArg()
{
FileOrDirName = file,
InclusiveSuffixs = new List<string>(searchPatterns.Select(s => s.Trim()).Where(s => !string.IsNullOrWhiteSpace(s))),
},
GET_INPUT_FILE_TIMEOUT);
if (res.Err != Luban.Common.EErrorCode.OK)
{
@ -105,11 +102,7 @@ namespace Luban.Server.Common
s_logger.Trace("read GetFileOrDirectoryAsync end. file:{file} cost:{time}", file, TimeUtil.NowMillis - t1);
return res;
});
_getImportFileOrDirTasks.Add(file, task);
}
return task;
}
});
}
const int QUERY_FILE_EXISTS_TIMEOUT = 10;