【优化】GetImportFileOrDirectory 新增文件后缀过滤，只获取支持的文件

【优化】CommonDefLoader import目录时，只获取.xml后缀文件

src/Luban.Client/Source/Net/GenClient.cs

@@ -10,6 +10,7 @@ using Luban.Common.Utils;
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Net;
 using System.Threading.Tasks;
 
@@ -100,6 +101,7 @@ namespace Luban.Client.Common.Net
         {
             long t1 = TimeUtil.NowMillis;
             var file = rpc.Arg.FileOrDirName;
+            var suffixes = rpc.Arg.InclusiveSuffixs;
             var re = new GetImportFileOrDirectoryRes()
             {
                 SubFiles = new List<Luban.Common.Protos.FileInfo>(),
@@ -113,7 +115,7 @@ namespace Luban.Client.Common.Net
                     re.IsFile = false;
                     foreach (var subFile in Directory.GetFiles(file, "*", SearchOption.AllDirectories))
                     {
-                        if (FileUtil.IsValidInputFile(subFile))
+                        if (FileUtil.IsValidInputFile(subFile) && (suffixes.Count == 0 || suffixes.Any(s => subFile.EndsWith(s))))
                         {
                             var md5 = await CacheMetaManager.Ins.GetOrUpdateFileMd5Async(subFile);
                             re.SubFiles.Add(new Luban.Common.Protos.FileInfo() { FilePath = FileUtil.Standardize(subFile), MD5 = md5 });

src/Luban.Common/Source/Protos/GetImportFileOrDirectory.cs

@@ -9,6 +9,8 @@ namespace Luban.Common.Protos
     {
         public string FileOrDirName { get; set; }
 
+        public List<string> InclusiveSuffixs { get; set; } = new List<string>();
+
         public override int GetTypeId()
         {
             return 0;
@@ -17,10 +19,12 @@ namespace Luban.Common.Protos
         public override void Serialize(ByteBuf os)
         {
             os.WriteString(FileOrDirName);
+            Bright.Common.SerializationUtil.Serialize(os, InclusiveSuffixs);
         }
         public override void Deserialize(ByteBuf os)
         {
             FileOrDirName = os.ReadString();
+            Bright.Common.SerializationUtil.Deserialize(os, InclusiveSuffixs);
         }
     }
 

src/Luban.Job.Cfg/Source/DataSources/DataSourceFactory.cs

@@ -6,6 +6,18 @@ namespace Luban.Job.Cfg.DataSources
 {
     static class DataSourceFactory
     {
+        public static readonly string[] validDataSourceSuffixes = new string[]
+        {
+            ".xlsx",
+            ".xls",
+            ".csv",
+            ".xml",
+            ".lua",
+            ".json",
+            ".yml",
+            ".bin",
+        };
+
         public static AbstractDataSource Create(string url, string sheetName, Stream stream, bool exportTestData)
         {
             try
@@ -20,7 +32,7 @@ namespace Luban.Job.Cfg.DataSources
                     case "xml": source = new Xml.XmlDataSource(); break;
                     case "lua": source = new Lua.LuaDataSource(); break;
                     case "json": source = new Json.JsonDataSource(); break;
-                    case "b": source = new Binary.BinaryDataSource(); break;
+                    case "bin": source = new Binary.BinaryDataSource(); break;
                     case "yml": source = new Yaml.YamlDataSource(); break;
                     default: throw new Exception($"不支持的文件类型:{url}");
                 }

src/Luban.Job.Cfg/Source/Utils/DataLoaderUtil.cs

@@ -45,7 +45,7 @@ namespace Luban.Job.Cfg.Utils
 
                 collectTasks.Add(Task.Run(async () =>
                 {
-                    var fileOrDirContent = await agent.GetFileOrDirectoryAsync(actualFullPath);
+                    var fileOrDirContent = await agent.GetFileOrDirectoryAsync(actualFullPath, DataSourceFactory.validDataSourceSuffixes);
                     if (fileOrDirContent.IsFile)
                     {
                         return new List<InputFileInfo> { new InputFileInfo() { OriginFile = file, ActualFile = actualFullPath, SheetName = sheetName, MD5 = fileOrDirContent.Md5 } };

src/Luban.Job.Common/Source/Defs/CommonDefLoader.cs

@@ -124,7 +124,7 @@ namespace Luban.Job.Common.Defs
             var xmlFullPath = FileUtil.Combine(RootDir, xmlFile);
             s_logger.Trace("import {file} {full_path}", xmlFile, xmlFullPath);
 
-            var fileOrDirContent = await Agent.GetFileOrDirectoryAsync(xmlFullPath);
+            var fileOrDirContent = await Agent.GetFileOrDirectoryAsync(xmlFullPath, ".xml");
 
             if (fileOrDirContent.IsFile)
             {

src/Luban.Server.Common/Source/RemoteAgent.cs

@@ -2,8 +2,10 @@ using Bright.Net.ServiceModes.Managers;
 using Bright.Time;
 using Luban.Common.Protos;
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Runtime.Serialization;
 using System.Threading.Tasks;
 using System.Xml.Linq;
@@ -37,9 +39,9 @@ namespace Luban.Server.Common
 
         private readonly bool _trace;
 
-        private readonly Dictionary<string, Task<byte[]>> _remoteReadAllBytesTasks = new Dictionary<string, Task<byte[]>>();
+        private readonly ConcurrentDictionary<string, Task<byte[]>> _remoteReadAllBytesTasks = new();
 
-        private readonly Dictionary<string, Task<GetImportFileOrDirectoryRes>> _getImportFileOrDirTasks = new Dictionary<string, Task<GetImportFileOrDirectoryRes>>();
+        private readonly ConcurrentDictionary<string, Task<GetImportFileOrDirectoryRes>> _getImportFileOrDirTasks = new();
 
         public RemoteAgent(SessionBase session, bool trace)
         {
@@ -63,40 +65,35 @@ namespace Luban.Server.Common
 
         public Task<byte[]> ReadAllBytesAsync(string file)
         {
-            lock (_remoteReadAllBytesTasks)
+            return _remoteReadAllBytesTasks.GetOrAdd(file, f =>
             {
-                if (!_remoteReadAllBytesTasks.TryGetValue(file, out var task))
+                return Task.Run(async () =>
                 {
-                    task = Task.Run(async () =>
+                    long t1 = TimeUtil.NowMillis;
+                    var res = await Session.CallRpcAsync<GetInputFile, GetInputFileArg, GetInputFileRes>(new GetInputFileArg() { File = f }, GET_INPUT_FILE_TIMEOUT);
+                    if (res.Err != Luban.Common.EErrorCode.OK)
                     {
-                        long t1 = TimeUtil.NowMillis;
+                        throw new ReadRemoteFailException($"{res.Err}");
-                        var res = await Session.CallRpcAsync<GetInputFile, GetInputFileArg, GetInputFileRes>(new GetInputFileArg() { File = file }, GET_INPUT_FILE_TIMEOUT);
+                    }
-                        if (res.Err != Luban.Common.EErrorCode.OK)
+                    s_logger.Info("read remote file:{file} cost:{time}", f, TimeUtil.NowMillis - t1);
-                        {
+                    return res.Content;
-                            throw new ReadRemoteFailException($"{res.Err}");
+                });
-                        }
+            });
-                        s_logger.Info("read remote file:{file} cost:{time}", file, TimeUtil.NowMillis - t1);
-                        return res.Content;
-                    });
-                    task.ConfigureAwait(false);
-                    _remoteReadAllBytesTasks.Add(file, task);
-                }
-                return task;
-            }
-
         }
 
-        public Task<GetImportFileOrDirectoryRes> GetFileOrDirectoryAsync(string file)
+        public Task<GetImportFileOrDirectoryRes> GetFileOrDirectoryAsync(string file, params string[] searchPatterns)
         {
-            lock (_getImportFileOrDirTasks)
+            return _getImportFileOrDirTasks.GetOrAdd(file, f =>
-            {
-                if (!_getImportFileOrDirTasks.TryGetValue(file, out var task))
                 {
-                    task = Task.Run(async () =>
+                    return Task.Run(async () =>
                     {
                         long t1 = TimeUtil.NowMillis;
                         var res = await Session.CallRpcAsync<GetImportFileOrDirectory, GetImportFileOrDirectoryArg, GetImportFileOrDirectoryRes>(
-                            new GetImportFileOrDirectoryArg() { FileOrDirName = file },
+                            new GetImportFileOrDirectoryArg()
+                            {
+                                FileOrDirName = file,
+                                InclusiveSuffixs = new List<string>(searchPatterns.Select(s => s.Trim()).Where(s => !string.IsNullOrWhiteSpace(s))),
+                            },
                             GET_INPUT_FILE_TIMEOUT);
                         if (res.Err != Luban.Common.EErrorCode.OK)
                         {
@@ -105,11 +102,7 @@ namespace Luban.Server.Common
                         s_logger.Trace("read GetFileOrDirectoryAsync end. file:{file} cost:{time}", file, TimeUtil.NowMillis - t1);
                         return res;
                     });
-                    _getImportFileOrDirTasks.Add(file, task);
+                });
-                }
-                return task;
-            }
-
         }
 
         const int QUERY_FILE_EXISTS_TIMEOUT = 10;