提交代码

This commit is contained in:
lilin 2025-04-01 20:37:12 +08:00
parent d81dc8683f
commit b9b421c63c
3 changed files with 173 additions and 14 deletions

View File

@ -10,6 +10,7 @@ import com.yfd.platform.modules.experimentalData.domain.*;
import com.yfd.platform.modules.experimentalData.service.ITsFilesService; import com.yfd.platform.modules.experimentalData.service.ITsFilesService;
import com.yfd.platform.modules.specialDocument.domain.Files; import com.yfd.platform.modules.specialDocument.domain.Files;
import com.yfd.platform.modules.specialDocument.service.IFilesService; import com.yfd.platform.modules.specialDocument.service.IFilesService;
import com.yfd.platform.modules.storage.model.result.FileItemResult;
import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiOperation;
import org.springframework.http.HttpStatus; import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity; import org.springframework.http.ResponseEntity;
@ -540,4 +541,20 @@ public class TsFilesController {
/**********************************
* 用途说明: 获取文件url
* 参数说明 id 文件夹ID
* 返回值说明: com.yfd.platform.config.ResponseResult 返回文件信息
***********************************/
@Log(module = "实验数据管理", value = "获取文件url")
@PostMapping("/obtainUrl")
@ApiOperation("获取文件url")
public ResponseResult obtainUrl(String id,String type) {
if (StrUtil.isBlank(id) && StrUtil.isBlank(type)) {
return ResponseResult.error("参数为空");
}
//查询本地树和minio树
FileItemResult fileItemResult = tsFilesService.obtainUrl(id,type);
return ResponseResult.successData(fileItemResult);
}
} }

View File

@ -4,6 +4,7 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.yfd.platform.config.ResponseResult; import com.yfd.platform.config.ResponseResult;
import com.yfd.platform.modules.experimentalData.domain.*; import com.yfd.platform.modules.experimentalData.domain.*;
import com.baomidou.mybatisplus.extension.service.IService; import com.baomidou.mybatisplus.extension.service.IService;
import com.yfd.platform.modules.storage.model.result.FileItemResult;
import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RequestParam;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
@ -215,4 +216,11 @@ public interface ITsFilesService extends IService<TsFiles> {
* 返回值说明: com.yfd.platform.config.ResponseResult 返回双树数据 * 返回值说明: com.yfd.platform.config.ResponseResult 返回双树数据
***********************************/ ***********************************/
DualTreeResponse listLocalAndBackup(String taskId, String nodeId); DualTreeResponse listLocalAndBackup(String taskId, String nodeId);
/**********************************
* 用途说明: 获取文件url
* 参数说明 id 文件夹ID
* 返回值说明: com.yfd.platform.config.ResponseResult 返回文件信息
***********************************/
FileItemResult obtainUrl(String id, String type);
} }

View File

@ -3,7 +3,6 @@ package com.yfd.platform.modules.experimentalData.service.impl;
import cn.hutool.core.collection.CollUtil; import cn.hutool.core.collection.CollUtil;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.nio.file.*; import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.BasicFileAttributes;
@ -2518,16 +2517,8 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
TsFiles tsFiles = new TsFiles(); TsFiles tsFiles = new TsFiles();
try { try {
// 获取 MinIO 和本地文件列表并行处理 // 获取 MinIO 和本地文件列表并行处理
// List<FileItemResult> fileItemListMinio = Collections.synchronizedList(new ArrayList<>());
// List<FileItemResult> fileItemListLocal = Collections.synchronizedList(new ArrayList<>());
CopyOnWriteArrayList<FileItemResult> fileItemListMinio = new CopyOnWriteArrayList<>(); CopyOnWriteArrayList<FileItemResult> fileItemListMinio = new CopyOnWriteArrayList<>();
CopyOnWriteArrayList<FileItemResult> fileItemListLocal = new CopyOnWriteArrayList<>(); CopyOnWriteArrayList<FileItemResult> fileItemListLocal = new CopyOnWriteArrayList<>();
// 优化点3: 使用 CompletableFuture 进行并行任务管理
// final List<FileItemResult> minioFiles = Collections.synchronizedList(new ArrayList<>());
// final List<FileItemResult> localFiles = Collections.synchronizedList(new ArrayList<>());
if (StringUtils.isNoneEmpty(nodeId, taskId)) { if (StringUtils.isNoneEmpty(nodeId, taskId)) {
// 并行查询数据库记录 // 并行查询数据库记录
tsFilesMapper.selectList(buildTaskQuery(nodeId, taskId)).parallelStream() tsFilesMapper.selectList(buildTaskQuery(nodeId, taskId)).parallelStream()
@ -2557,6 +2548,7 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
deduplicateFileList(fileItemListMinio); deduplicateFileList(fileItemListMinio);
deduplicateFileList(fileItemListLocal); deduplicateFileList(fileItemListLocal);
// 对比文件并行计算 MD5 // 对比文件并行计算 MD5
tsFiles = compareFiles(fileItemListMinio, fileItemListLocal, tsFiles = compareFiles(fileItemListMinio, fileItemListLocal,
filePathConfig.getValue(), bucketConfig.getValue(), nodeId, taskId); filePathConfig.getValue(), bucketConfig.getValue(), nodeId, taskId);
@ -2583,7 +2575,6 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
.eq("task_id", taskId) .eq("task_id", taskId)
.eq("parent_id", "00") .eq("parent_id", "00")
.eq("node_id", nodeId); .eq("node_id", nodeId);
//.and(wrapper -> wrapper.eq("work_path", "/").or().eq("backup_path", "/"));
} }
// 辅助方法处理文件列表并行安全 // 辅助方法处理文件列表并行安全
@ -2593,9 +2584,6 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
} }
// 辅助方法获取文件列表并添加到集合 // 辅助方法获取文件列表并添加到集合
// 辅助方法获取文件列表并添加到集合优化版
// 优化点7提取文件处理核心逻辑
private void processFileList(String path, String fileName, String storageKey, List<FileItemResult> targetList, String isFile, String nodeId, String taskId) throws Exception { private void processFileList(String path, String fileName, String storageKey, List<FileItemResult> targetList, String isFile, String nodeId, String taskId) throws Exception {
if (StringUtils.isNotEmpty(path)) { if (StringUtils.isNotEmpty(path)) {
AbstractBaseFileService<?> service = storageSourceContext.getByStorageKey(storageKey); AbstractBaseFileService<?> service = storageSourceContext.getByStorageKey(storageKey);
@ -2773,6 +2761,12 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
String bucketName, String nodeId, String taskId) { String bucketName, String nodeId, String taskId) {
TsFiles result = new TsFiles(); TsFiles result = new TsFiles();
Map<String, TsFiles> backupPathMap = preloadTsFiles(nodeId, taskId, "backup_path");
Map<String, TsFiles> workPathMap = preloadTsFiles(nodeId, taskId, "work_path");
// 使用并行流构建 Map路径 + 文件名 -> 文件 // 使用并行流构建 Map路径 + 文件名 -> 文件
ConcurrentMap<String, FileItemResult> minioMap = minioFiles.parallelStream() ConcurrentMap<String, FileItemResult> minioMap = minioFiles.parallelStream()
.collect(Collectors.toConcurrentMap( .collect(Collectors.toConcurrentMap(
@ -4356,6 +4350,24 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
return new DualTreeResponse(localTrees, minioTrees); return new DualTreeResponse(localTrees, minioTrees);
} }
/**********************************
* 用途说明: 获取文件url
* 参数说明 id 文件夹ID
* 返回值说明: com.yfd.platform.config.ResponseResult 返回文件信息
***********************************/
@Override
public FileItemResult obtainUrl(String id,String type) {
FileItemResult fileItemResult = new FileItemResult();
TsFiles tsFiles = tsFilesMapper.selectById(id);
String fileNameData = tsFiles.getFileName();
String workPath = tsFiles.getWorkPath();
String path = workPath + fileNameData;
//准备获取文件的信息
AbstractBaseFileService<?> fileService = storageSourceContext.getByStorageKey(type);
fileItemResult = fileService.getFileItem(path);
return fileItemResult;
}
/** /**
* 递归构建树形结构内存操作不再查询数据库 * 递归构建树形结构内存操作不再查询数据库
* *
@ -4366,7 +4378,8 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
*/ */
private TreeDTO buildTree(TsFiles current, Map<String, List<TsFiles>> parentChildrenMap, boolean isLocal) { private TreeDTO buildTree(TsFiles current, Map<String, List<TsFiles>> parentChildrenMap, boolean isLocal) {
// 1. 转换为DTO并检查路径有效性 // 1. 转换为DTO并检查路径有效性
TreeDTO dto = convertToDTO(current, isLocal); TreeDTO dto = convertToDTOs(current, isLocal);
dto.setType(current.getIsFile());
if (dto.getPath() == null || dto.getPath().trim().isEmpty()) { if (dto.getPath() == null || dto.getPath().trim().isEmpty()) {
LOGGER.warn("由于路径为空,跳过节点{}", current.getId()); LOGGER.warn("由于路径为空,跳过节点{}", current.getId());
return null; return null;
@ -4405,6 +4418,127 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
return dto; return dto;
} }
/**
* FileNode 转换为 TreeDTO
*
* @param node 当前节点
* @param isLocal 是否是本地树true: workPath, false: backupPath
* @return 转换后的 DTO
*/
private TreeDTO convertToDTOs(TsFiles node, boolean isLocal) {
// //查询字典表获取压缩文件后缀
// QueryWrapper<SysDictionaryItems> queryWrapperSysDictionary = new QueryWrapper<>();
// queryWrapperSysDictionary.eq("parentcode", "compressType");
// queryWrapperSysDictionary.orderByAsc("orderno");
// List<SysDictionaryItems> sysDictionaryItems = sysDictionaryItemsMapper.selectList(queryWrapperSysDictionary);
// int count = 0;
//
// if ("FOLDER".equals(node.getIsFile())) {
// QueryWrapper<TsFiles> queryWrapper = new QueryWrapper<>();
// queryWrapper.eq("parent_id", node.getId());
// if (isLocal) {
// queryWrapper.isNotNull("work_path").ne("work_path", "");
// } else {
// queryWrapper.isNotNull("backup_path").ne("backup_path", "");
// }
// count = tsFilesMapper.selectCount(queryWrapper);
// }
TreeDTO dto = new TreeDTO();
// 复制公共字段
dto.setId(node.getId());
// dto.setNodeId(node.getNodeId());
// dto.setTaskId(node.getTaskId());
dto.setIsFile(node.getIsFile());
dto.setParentId(node.getParentId());
dto.setFileName(node.getFileName());
dto.setFileSize(node.getFileSize());
// dto.setKeywords(node.getKeywords());
// dto.setDescription(node.getDescription());
dto.setUploadTime(node.getUploadTime());
// dto.setUploader(node.getUploader());
dto.setUpdateTime(node.getUpdateTime());
// dto.setNumber(count);
//查询本地
if (isLocal) {
String workPath = node.getWorkPath();
if (StringUtils.isEmpty(workPath)) {
dto.setPath(null);
return dto;
}
String fileNameData = node.getFileName();
//主要是用于文件路径加名称
String path = workPath + fileNameData;
//准备获取文件的信息
if ("FILE".equals(node.getIsFile())) {
dto.setUrl(null);
dto.setType(null);
//
// LOGGER.info("查询本地树的时候" + path);
// AbstractBaseFileService<?> fileService = storageSourceContext.getByStorageKey("local");
// FileItemResult fileItemResult = fileService.getFileItem(path);
// if (fileItemResult != null || fileItemResult.getName() != null) {
// dto.setUrl(fileItemResult.getUrl());
// dto.setType(fileItemResult.getType().getValue());
//// //如果是压缩文件 类型就给zip
//// boolean isValid = hasValidExtension(fileItemResult.getName(), sysDictionaryItems);
//// if (isValid) {
//// dto.setType("ZIP");
//// } else {
//// dto.setType(fileItemResult.getType().getValue());
//// }
// } else {
// dto.setUrl(null);
// dto.setType(null);
// }
}
} else {
String backupPath = node.getBackupPath();
if (backupPath == null || StringUtils.isEmpty(backupPath)) {
dto.setPath(null);
return dto;
}
String fileNameData = node.getFileName();
//主要是用于文件路径加名称
String path = backupPath + fileNameData;
//准备获取文件的信息
if ("FILE".equals(node.getIsFile())) {
dto.setUrl(null);
dto.setType(null);
// LOGGER.info("查询minio树的时候" + path);
// AbstractBaseFileService<?> fileService = storageSourceContext.getByStorageKey("minio");
// FileItemResult fileItemResult = fileService.getFileItem(path);
// if (fileItemResult == null) {
// dto.setUrl(null);
// dto.setType(null);
// } else {
//
// dto.setUrl(fileItemResult.getUrl());
// dto.setType(fileItemResult.getType().getValue());
//// //如果是压缩文件 类型就给zip
//// boolean isValid = hasValidExtension(fileItemResult.getName(), sysDictionaryItems);
//// if (isValid) {
//// dto.setType("ZIP");
//// } else {
//// dto.setType(fileItemResult.getType().getValue());
//// }
// }
}
}
// 设置路径字段
if (isLocal) {
String ProcessingPath = processingPath(node.getWorkPath(), node.getNodeId());
dto.setPath(ProcessingPath);
} else {
String ProcessingPath = processingPath(node.getBackupPath(), node.getNodeId());
dto.setPath(ProcessingPath);
}
return dto;
}
/** /**
* FileNode 转换为 TreeDTO * FileNode 转换为 TreeDTO
* *