diff --git a/src/Luban.Client/Source/Net/GenClient.cs b/src/Luban.Client/Source/Net/GenClient.cs index 61f73e7..612a790 100644 --- a/src/Luban.Client/Source/Net/GenClient.cs +++ b/src/Luban.Client/Source/Net/GenClient.cs @@ -10,6 +10,7 @@ using Luban.Common.Utils; using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Net; using System.Threading.Tasks; @@ -100,6 +101,7 @@ namespace Luban.Client.Common.Net { long t1 = TimeUtil.NowMillis; var file = rpc.Arg.FileOrDirName; + var suffixes = rpc.Arg.InclusiveSuffixs; var re = new GetImportFileOrDirectoryRes() { SubFiles = new List(), @@ -113,7 +115,7 @@ namespace Luban.Client.Common.Net re.IsFile = false; foreach (var subFile in Directory.GetFiles(file, "*", SearchOption.AllDirectories)) { - if (FileUtil.IsValidInputFile(subFile)) + if (FileUtil.IsValidInputFile(subFile) && (suffixes.Count == 0 || suffixes.Any(s => subFile.EndsWith(s)))) { var md5 = await CacheMetaManager.Ins.GetOrUpdateFileMd5Async(subFile); re.SubFiles.Add(new Luban.Common.Protos.FileInfo() { FilePath = FileUtil.Standardize(subFile), MD5 = md5 }); diff --git a/src/Luban.Common/Source/Protos/GetImportFileOrDirectory.cs b/src/Luban.Common/Source/Protos/GetImportFileOrDirectory.cs index f54966b..4c16972 100644 --- a/src/Luban.Common/Source/Protos/GetImportFileOrDirectory.cs +++ b/src/Luban.Common/Source/Protos/GetImportFileOrDirectory.cs @@ -9,6 +9,8 @@ namespace Luban.Common.Protos { public string FileOrDirName { get; set; } + public List InclusiveSuffixs { get; set; } = new List(); + public override int GetTypeId() { return 0; @@ -17,10 +19,12 @@ namespace Luban.Common.Protos public override void Serialize(ByteBuf os) { os.WriteString(FileOrDirName); + Bright.Common.SerializationUtil.Serialize(os, InclusiveSuffixs); } public override void Deserialize(ByteBuf os) { FileOrDirName = os.ReadString(); + Bright.Common.SerializationUtil.Deserialize(os, InclusiveSuffixs); } } diff --git a/src/Luban.Job.Cfg/Source/DataSources/DataSourceFactory.cs b/src/Luban.Job.Cfg/Source/DataSources/DataSourceFactory.cs index ad4ec5c..e9f6c4a 100644 --- a/src/Luban.Job.Cfg/Source/DataSources/DataSourceFactory.cs +++ b/src/Luban.Job.Cfg/Source/DataSources/DataSourceFactory.cs @@ -6,6 +6,18 @@ namespace Luban.Job.Cfg.DataSources { static class DataSourceFactory { + public static readonly string[] validDataSourceSuffixes = new string[] + { + ".xlsx", + ".xls", + ".csv", + ".xml", + ".lua", + ".json", + ".yml", + ".bin", + }; + public static AbstractDataSource Create(string url, string sheetName, Stream stream, bool exportTestData) { try @@ -20,7 +32,7 @@ namespace Luban.Job.Cfg.DataSources case "xml": source = new Xml.XmlDataSource(); break; case "lua": source = new Lua.LuaDataSource(); break; case "json": source = new Json.JsonDataSource(); break; - case "b": source = new Binary.BinaryDataSource(); break; + case "bin": source = new Binary.BinaryDataSource(); break; case "yml": source = new Yaml.YamlDataSource(); break; default: throw new Exception($"不支持的文件类型:{url}"); } diff --git a/src/Luban.Job.Cfg/Source/Utils/DataLoaderUtil.cs b/src/Luban.Job.Cfg/Source/Utils/DataLoaderUtil.cs index 0bc9e17..657043a 100644 --- a/src/Luban.Job.Cfg/Source/Utils/DataLoaderUtil.cs +++ b/src/Luban.Job.Cfg/Source/Utils/DataLoaderUtil.cs @@ -45,7 +45,7 @@ namespace Luban.Job.Cfg.Utils collectTasks.Add(Task.Run(async () => { - var fileOrDirContent = await agent.GetFileOrDirectoryAsync(actualFullPath); + var fileOrDirContent = await agent.GetFileOrDirectoryAsync(actualFullPath, DataSourceFactory.validDataSourceSuffixes); if (fileOrDirContent.IsFile) { return new List { new InputFileInfo() { OriginFile = file, ActualFile = actualFullPath, SheetName = sheetName, MD5 = fileOrDirContent.Md5 } }; diff --git a/src/Luban.Job.Common/Source/Defs/CommonDefLoader.cs b/src/Luban.Job.Common/Source/Defs/CommonDefLoader.cs index 1ad6a50..85bd9df 100644 --- a/src/Luban.Job.Common/Source/Defs/CommonDefLoader.cs +++ b/src/Luban.Job.Common/Source/Defs/CommonDefLoader.cs @@ -124,7 +124,7 @@ namespace Luban.Job.Common.Defs var xmlFullPath = FileUtil.Combine(RootDir, xmlFile); s_logger.Trace("import {file} {full_path}", xmlFile, xmlFullPath); - var fileOrDirContent = await Agent.GetFileOrDirectoryAsync(xmlFullPath); + var fileOrDirContent = await Agent.GetFileOrDirectoryAsync(xmlFullPath, ".xml"); if (fileOrDirContent.IsFile) { diff --git a/src/Luban.Server.Common/Source/RemoteAgent.cs b/src/Luban.Server.Common/Source/RemoteAgent.cs index b778342..8694245 100644 --- a/src/Luban.Server.Common/Source/RemoteAgent.cs +++ b/src/Luban.Server.Common/Source/RemoteAgent.cs @@ -2,8 +2,10 @@ using Bright.Net.ServiceModes.Managers; using Bright.Time; using Luban.Common.Protos; using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Runtime.Serialization; using System.Threading.Tasks; using System.Xml.Linq; @@ -37,9 +39,9 @@ namespace Luban.Server.Common private readonly bool _trace; - private readonly Dictionary> _remoteReadAllBytesTasks = new Dictionary>(); + private readonly ConcurrentDictionary> _remoteReadAllBytesTasks = new(); - private readonly Dictionary> _getImportFileOrDirTasks = new Dictionary>(); + private readonly ConcurrentDictionary> _getImportFileOrDirTasks = new(); public RemoteAgent(SessionBase session, bool trace) { @@ -63,40 +65,35 @@ namespace Luban.Server.Common public Task ReadAllBytesAsync(string file) { - lock (_remoteReadAllBytesTasks) + return _remoteReadAllBytesTasks.GetOrAdd(file, f => { - if (!_remoteReadAllBytesTasks.TryGetValue(file, out var task)) + return Task.Run(async () => { - task = Task.Run(async () => + long t1 = TimeUtil.NowMillis; + var res = await Session.CallRpcAsync(new GetInputFileArg() { File = f }, GET_INPUT_FILE_TIMEOUT); + if (res.Err != Luban.Common.EErrorCode.OK) { - long t1 = TimeUtil.NowMillis; - var res = await Session.CallRpcAsync(new GetInputFileArg() { File = file }, GET_INPUT_FILE_TIMEOUT); - if (res.Err != Luban.Common.EErrorCode.OK) - { - throw new ReadRemoteFailException($"{res.Err}"); - } - s_logger.Info("read remote file:{file} cost:{time}", file, TimeUtil.NowMillis - t1); - return res.Content; - }); - task.ConfigureAwait(false); - _remoteReadAllBytesTasks.Add(file, task); - } - return task; - } - + throw new ReadRemoteFailException($"{res.Err}"); + } + s_logger.Info("read remote file:{file} cost:{time}", f, TimeUtil.NowMillis - t1); + return res.Content; + }); + }); } - public Task GetFileOrDirectoryAsync(string file) + public Task GetFileOrDirectoryAsync(string file, params string[] searchPatterns) { - lock (_getImportFileOrDirTasks) - { - if (!_getImportFileOrDirTasks.TryGetValue(file, out var task)) + return _getImportFileOrDirTasks.GetOrAdd(file, f => { - task = Task.Run(async () => + return Task.Run(async () => { long t1 = TimeUtil.NowMillis; var res = await Session.CallRpcAsync( - new GetImportFileOrDirectoryArg() { FileOrDirName = file }, + new GetImportFileOrDirectoryArg() + { + FileOrDirName = file, + InclusiveSuffixs = new List(searchPatterns.Select(s => s.Trim()).Where(s => !string.IsNullOrWhiteSpace(s))), + }, GET_INPUT_FILE_TIMEOUT); if (res.Err != Luban.Common.EErrorCode.OK) { @@ -105,11 +102,7 @@ namespace Luban.Server.Common s_logger.Trace("read GetFileOrDirectoryAsync end. file:{file} cost:{time}", file, TimeUtil.NowMillis - t1); return res; }); - _getImportFileOrDirTasks.Add(file, task); - } - return task; - } - + }); } const int QUERY_FILE_EXISTS_TIMEOUT = 10;