From b9b421c63ce5a8ae6ab4c7393bf7fd84b8901e6f Mon Sep 17 00:00:00 2001 From: lilin Date: Tue, 1 Apr 2025 20:37:12 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8F=90=E4=BA=A4=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controller/TsFilesController.java | 17 ++ .../service/ITsFilesService.java | 8 + .../service/impl/TsFilesServiceImpl.java | 162 ++++++++++++++++-- 3 files changed, 173 insertions(+), 14 deletions(-) diff --git a/java/src/main/java/com/yfd/platform/modules/experimentalData/controller/TsFilesController.java b/java/src/main/java/com/yfd/platform/modules/experimentalData/controller/TsFilesController.java index be3e05c..f1d1035 100644 --- a/java/src/main/java/com/yfd/platform/modules/experimentalData/controller/TsFilesController.java +++ b/java/src/main/java/com/yfd/platform/modules/experimentalData/controller/TsFilesController.java @@ -10,6 +10,7 @@ import com.yfd.platform.modules.experimentalData.domain.*; import com.yfd.platform.modules.experimentalData.service.ITsFilesService; import com.yfd.platform.modules.specialDocument.domain.Files; import com.yfd.platform.modules.specialDocument.service.IFilesService; +import com.yfd.platform.modules.storage.model.result.FileItemResult; import io.swagger.annotations.ApiOperation; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; @@ -540,4 +541,20 @@ public class TsFilesController { + /********************************** + * 用途说明: 获取文件url + * 参数说明 id 文件夹ID + * 返回值说明: com.yfd.platform.config.ResponseResult 返回文件信息 + ***********************************/ + @Log(module = "实验数据管理", value = "获取文件url！") + @PostMapping("/obtainUrl") + @ApiOperation("获取文件url") + public ResponseResult obtainUrl(String id,String type) { + if (StrUtil.isBlank(id) && StrUtil.isBlank(type)) { + return ResponseResult.error("参数为空"); + } + //查询本地树和minio树 + FileItemResult fileItemResult = tsFilesService.obtainUrl(id,type); + return ResponseResult.successData(fileItemResult); + } } diff --git a/java/src/main/java/com/yfd/platform/modules/experimentalData/service/ITsFilesService.java b/java/src/main/java/com/yfd/platform/modules/experimentalData/service/ITsFilesService.java index 9d826ad..4ef6aa9 100644 --- a/java/src/main/java/com/yfd/platform/modules/experimentalData/service/ITsFilesService.java +++ b/java/src/main/java/com/yfd/platform/modules/experimentalData/service/ITsFilesService.java @@ -4,6 +4,7 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.yfd.platform.config.ResponseResult; import com.yfd.platform.modules.experimentalData.domain.*; import com.baomidou.mybatisplus.extension.service.IService; +import com.yfd.platform.modules.storage.model.result.FileItemResult; import org.springframework.web.bind.annotation.RequestParam; import java.io.FileNotFoundException; @@ -215,4 +216,11 @@ public interface ITsFilesService extends IService { * 返回值说明: com.yfd.platform.config.ResponseResult 返回双树数据 ***********************************/ DualTreeResponse listLocalAndBackup(String taskId, String nodeId); + + /********************************** + * 用途说明: 获取文件url + * 参数说明 id 文件夹ID + * 返回值说明: com.yfd.platform.config.ResponseResult 返回文件信息 + ***********************************/ + FileItemResult obtainUrl(String id, String type); } diff --git a/java/src/main/java/com/yfd/platform/modules/experimentalData/service/impl/TsFilesServiceImpl.java b/java/src/main/java/com/yfd/platform/modules/experimentalData/service/impl/TsFilesServiceImpl.java index 31067e7..a401442 100644 --- a/java/src/main/java/com/yfd/platform/modules/experimentalData/service/impl/TsFilesServiceImpl.java +++ b/java/src/main/java/com/yfd/platform/modules/experimentalData/service/impl/TsFilesServiceImpl.java @@ -3,7 +3,6 @@ package com.yfd.platform.modules.experimentalData.service.impl; import cn.hutool.core.collection.CollUtil; - import java.nio.charset.StandardCharsets; import java.nio.file.*; import java.nio.file.attribute.BasicFileAttributes; @@ -2518,16 +2517,8 @@ public class TsFilesServiceImpl extends ServiceImpl impl TsFiles tsFiles = new TsFiles(); try { // 获取 MinIO 和本地文件列表（并行处理） -// List fileItemListMinio = Collections.synchronizedList(new ArrayList<>()); -// List fileItemListLocal = Collections.synchronizedList(new ArrayList<>()); CopyOnWriteArrayList fileItemListMinio = new CopyOnWriteArrayList<>(); CopyOnWriteArrayList fileItemListLocal = new CopyOnWriteArrayList<>(); - - - // 优化点3: 使用 CompletableFuture 进行并行任务管理 -// final List minioFiles = Collections.synchronizedList(new ArrayList<>()); -// final List localFiles = Collections.synchronizedList(new ArrayList<>()); - if (StringUtils.isNoneEmpty(nodeId, taskId)) { // 并行查询数据库记录 tsFilesMapper.selectList(buildTaskQuery(nodeId, taskId)).parallelStream() @@ -2557,6 +2548,7 @@ public class TsFilesServiceImpl extends ServiceImpl impl deduplicateFileList(fileItemListMinio); deduplicateFileList(fileItemListLocal); + // 对比文件（并行计算 MD5） tsFiles = compareFiles(fileItemListMinio, fileItemListLocal, filePathConfig.getValue(), bucketConfig.getValue(), nodeId, taskId); @@ -2583,7 +2575,6 @@ public class TsFilesServiceImpl extends ServiceImpl impl .eq("task_id", taskId) .eq("parent_id", "00") .eq("node_id", nodeId); - //.and(wrapper -> wrapper.eq("work_path", "/").or().eq("backup_path", "/")); } // 辅助方法：处理文件列表（并行安全） @@ -2593,9 +2584,6 @@ public class TsFilesServiceImpl extends ServiceImpl impl } // 辅助方法：获取文件列表并添加到集合 - // 辅助方法：获取文件列表并添加到集合（优化版） - - // 优化点7：提取文件处理核心逻辑 private void processFileList(String path, String fileName, String storageKey, List targetList, String isFile, String nodeId, String taskId) throws Exception { if (StringUtils.isNotEmpty(path)) { AbstractBaseFileService service = storageSourceContext.getByStorageKey(storageKey); @@ -2773,6 +2761,12 @@ public class TsFilesServiceImpl extends ServiceImpl impl String bucketName, String nodeId, String taskId) { TsFiles result = new TsFiles(); + + Map backupPathMap = preloadTsFiles(nodeId, taskId, "backup_path"); + Map workPathMap = preloadTsFiles(nodeId, taskId, "work_path"); + + + // 使用并行流构建 Map（路径 + 文件名 -> 文件） ConcurrentMap minioMap = minioFiles.parallelStream() .collect(Collectors.toConcurrentMap( @@ -4356,6 +4350,24 @@ public class TsFilesServiceImpl extends ServiceImpl impl return new DualTreeResponse(localTrees, minioTrees); } + /********************************** + * 用途说明: 获取文件url + * 参数说明 id 文件夹ID + * 返回值说明: com.yfd.platform.config.ResponseResult 返回文件信息 + ***********************************/ + @Override + public FileItemResult obtainUrl(String id,String type) { + FileItemResult fileItemResult = new FileItemResult(); + TsFiles tsFiles = tsFilesMapper.selectById(id); + String fileNameData = tsFiles.getFileName(); + String workPath = tsFiles.getWorkPath(); + String path = workPath + fileNameData; + //准备获取文件的信息 + AbstractBaseFileService fileService = storageSourceContext.getByStorageKey(type); + fileItemResult = fileService.getFileItem(path); + return fileItemResult; + } + /** * 递归构建树形结构（内存操作，不再查询数据库） * @@ -4366,7 +4378,8 @@ public class TsFilesServiceImpl extends ServiceImpl impl */ private TreeDTO buildTree(TsFiles current, Map> parentChildrenMap, boolean isLocal) { // 1. 转换为DTO并检查路径有效性 - TreeDTO dto = convertToDTO(current, isLocal); + TreeDTO dto = convertToDTOs(current, isLocal); + dto.setType(current.getIsFile()); if (dto.getPath() == null || dto.getPath().trim().isEmpty()) { LOGGER.warn("由于路径为空，跳过节点｛｝", current.getId()); return null; @@ -4405,6 +4418,127 @@ public class TsFilesServiceImpl extends ServiceImpl impl return dto; } + /** + * 将 FileNode 转换为 TreeDTO + * + * @param node 当前节点 + * @param isLocal 是否是本地树（true: workPath, false: backupPath） + * @return 转换后的 DTO + */ + private TreeDTO convertToDTOs(TsFiles node, boolean isLocal) { +// //查询字典表获取压缩文件后缀 +// QueryWrapper queryWrapperSysDictionary = new QueryWrapper<>(); +// queryWrapperSysDictionary.eq("parentcode", "compressType"); +// queryWrapperSysDictionary.orderByAsc("orderno"); +// List sysDictionaryItems = sysDictionaryItemsMapper.selectList(queryWrapperSysDictionary); +// int count = 0; +// +// if ("FOLDER".equals(node.getIsFile())) { +// QueryWrapper queryWrapper = new QueryWrapper<>(); +// queryWrapper.eq("parent_id", node.getId()); +// if (isLocal) { +// queryWrapper.isNotNull("work_path").ne("work_path", ""); +// } else { +// queryWrapper.isNotNull("backup_path").ne("backup_path", ""); +// } +// count = tsFilesMapper.selectCount(queryWrapper); +// } + + TreeDTO dto = new TreeDTO(); + // 复制公共字段 + dto.setId(node.getId()); +// dto.setNodeId(node.getNodeId()); +// dto.setTaskId(node.getTaskId()); + dto.setIsFile(node.getIsFile()); + dto.setParentId(node.getParentId()); + dto.setFileName(node.getFileName()); + dto.setFileSize(node.getFileSize()); +// dto.setKeywords(node.getKeywords()); +// dto.setDescription(node.getDescription()); + dto.setUploadTime(node.getUploadTime()); +// dto.setUploader(node.getUploader()); + dto.setUpdateTime(node.getUpdateTime()); +// dto.setNumber(count); + //查询本地 + if (isLocal) { + String workPath = node.getWorkPath(); + if (StringUtils.isEmpty(workPath)) { + dto.setPath(null); + return dto; + } + String fileNameData = node.getFileName(); + //主要是用于文件路径加名称 + String path = workPath + fileNameData; + //准备获取文件的信息 + if ("FILE".equals(node.getIsFile())) { + dto.setUrl(null); + dto.setType(null); +// +// LOGGER.info("查询本地树的时候" + path); +// AbstractBaseFileService fileService = storageSourceContext.getByStorageKey("local"); +// FileItemResult fileItemResult = fileService.getFileItem(path); +// if (fileItemResult != null || fileItemResult.getName() != null) { +// dto.setUrl(fileItemResult.getUrl()); +// dto.setType(fileItemResult.getType().getValue()); +//// //如果是压缩文件 类型就给zip +//// boolean isValid = hasValidExtension(fileItemResult.getName(), sysDictionaryItems); +//// if (isValid) { +//// dto.setType("ZIP"); +//// } else { +//// dto.setType(fileItemResult.getType().getValue()); +//// } +// } else { +// dto.setUrl(null); +// dto.setType(null); +// } + + } + } else { + String backupPath = node.getBackupPath(); + if (backupPath == null || StringUtils.isEmpty(backupPath)) { + dto.setPath(null); + return dto; + } + String fileNameData = node.getFileName(); + //主要是用于文件路径加名称 + String path = backupPath + fileNameData; + //准备获取文件的信息 + if ("FILE".equals(node.getIsFile())) { + dto.setUrl(null); + dto.setType(null); +// LOGGER.info("查询minio树的时候" + path); +// AbstractBaseFileService fileService = storageSourceContext.getByStorageKey("minio"); +// FileItemResult fileItemResult = fileService.getFileItem(path); +// if (fileItemResult == null) { +// dto.setUrl(null); +// dto.setType(null); +// } else { +// +// dto.setUrl(fileItemResult.getUrl()); +// dto.setType(fileItemResult.getType().getValue()); +//// //如果是压缩文件 类型就给zip +//// boolean isValid = hasValidExtension(fileItemResult.getName(), sysDictionaryItems); +//// if (isValid) { +//// dto.setType("ZIP"); +//// } else { +//// dto.setType(fileItemResult.getType().getValue()); +//// } +// } + } + } + // 设置路径字段 + if (isLocal) { + String ProcessingPath = processingPath(node.getWorkPath(), node.getNodeId()); + dto.setPath(ProcessingPath); + } else { + String ProcessingPath = processingPath(node.getBackupPath(), node.getNodeId()); + dto.setPath(ProcessingPath); + } + + return dto; + } + + /** * 将 FileNode 转换为 TreeDTO *