提交代码
This commit is contained in:
lilin
parent
d81dc8683f
commit
b9b421c63c
@ -10,6 +10,7 @@ import com.yfd.platform.modules.experimentalData.domain.*;
|
||||
import com.yfd.platform.modules.experimentalData.service.ITsFilesService;
|
||||
import com.yfd.platform.modules.specialDocument.domain.Files;
|
||||
import com.yfd.platform.modules.specialDocument.service.IFilesService;
|
||||
import com.yfd.platform.modules.storage.model.result.FileItemResult;
|
||||
import io.swagger.annotations.ApiOperation;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
@ -540,4 +541,20 @@ public class TsFilesController {
|
||||
|
||||
|
||||
|
||||
/**********************************
|
||||
* 用途说明: 获取文件url
|
||||
* 参数说明 id 文件夹ID
|
||||
* 返回值说明: com.yfd.platform.config.ResponseResult 返回文件信息
|
||||
***********************************/
|
||||
@Log(module = "实验数据管理", value = "获取文件url！")
|
||||
@PostMapping("/obtainUrl")
|
||||
@ApiOperation("获取文件url")
|
||||
public ResponseResult obtainUrl(String id,String type) {
|
||||
if (StrUtil.isBlank(id) && StrUtil.isBlank(type)) {
|
||||
return ResponseResult.error("参数为空");
|
||||
}
|
||||
//查询本地树和minio树
|
||||
FileItemResult fileItemResult = tsFilesService.obtainUrl(id,type);
|
||||
return ResponseResult.successData(fileItemResult);
|
||||
}
|
||||
}
|
||||
|
||||
@ -4,6 +4,7 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
|
||||
import com.yfd.platform.config.ResponseResult;
|
||||
import com.yfd.platform.modules.experimentalData.domain.*;
|
||||
import com.baomidou.mybatisplus.extension.service.IService;
|
||||
import com.yfd.platform.modules.storage.model.result.FileItemResult;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
|
||||
import java.io.FileNotFoundException;
|
||||
@ -215,4 +216,11 @@ public interface ITsFilesService extends IService<TsFiles> {
|
||||
* 返回值说明: com.yfd.platform.config.ResponseResult 返回双树数据
|
||||
***********************************/
|
||||
DualTreeResponse listLocalAndBackup(String taskId, String nodeId);
|
||||
|
||||
/**********************************
|
||||
* 用途说明: 获取文件url
|
||||
* 参数说明 id 文件夹ID
|
||||
* 返回值说明: com.yfd.platform.config.ResponseResult 返回文件信息
|
||||
***********************************/
|
||||
FileItemResult obtainUrl(String id, String type);
|
||||
}
|
||||
|
||||
@ -3,7 +3,6 @@ package com.yfd.platform.modules.experimentalData.service.impl;
|
||||
import cn.hutool.core.collection.CollUtil;
|
||||
|
||||
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.*;
|
||||
import java.nio.file.attribute.BasicFileAttributes;
|
||||
@ -2518,16 +2517,8 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
|
||||
TsFiles tsFiles = new TsFiles();
|
||||
try {
|
||||
// 获取 MinIO 和本地文件列表（并行处理）
|
||||
// List<FileItemResult> fileItemListMinio = Collections.synchronizedList(new ArrayList<>());
|
||||
// List<FileItemResult> fileItemListLocal = Collections.synchronizedList(new ArrayList<>());
|
||||
CopyOnWriteArrayList<FileItemResult> fileItemListMinio = new CopyOnWriteArrayList<>();
|
||||
CopyOnWriteArrayList<FileItemResult> fileItemListLocal = new CopyOnWriteArrayList<>();
|
||||
|
||||
|
||||
// 优化点3: 使用 CompletableFuture 进行并行任务管理
|
||||
// final List<FileItemResult> minioFiles = Collections.synchronizedList(new ArrayList<>());
|
||||
// final List<FileItemResult> localFiles = Collections.synchronizedList(new ArrayList<>());
|
||||
|
||||
if (StringUtils.isNoneEmpty(nodeId, taskId)) {
|
||||
// 并行查询数据库记录
|
||||
tsFilesMapper.selectList(buildTaskQuery(nodeId, taskId)).parallelStream()
|
||||
@ -2557,6 +2548,7 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
|
||||
deduplicateFileList(fileItemListMinio);
|
||||
deduplicateFileList(fileItemListLocal);
|
||||
|
||||
|
||||
// 对比文件（并行计算 MD5）
|
||||
tsFiles = compareFiles(fileItemListMinio, fileItemListLocal,
|
||||
filePathConfig.getValue(), bucketConfig.getValue(), nodeId, taskId);
|
||||
@ -2583,7 +2575,6 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
|
||||
.eq("task_id", taskId)
|
||||
.eq("parent_id", "00")
|
||||
.eq("node_id", nodeId);
|
||||
//.and(wrapper -> wrapper.eq("work_path", "/").or().eq("backup_path", "/"));
|
||||
}
|
||||
|
||||
// 辅助方法：处理文件列表（并行安全）
|
||||
@ -2593,9 +2584,6 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
|
||||
}
|
||||
|
||||
// 辅助方法：获取文件列表并添加到集合
|
||||
// 辅助方法：获取文件列表并添加到集合（优化版）
|
||||
|
||||
// 优化点7：提取文件处理核心逻辑
|
||||
private void processFileList(String path, String fileName, String storageKey, List<FileItemResult> targetList, String isFile, String nodeId, String taskId) throws Exception {
|
||||
if (StringUtils.isNotEmpty(path)) {
|
||||
AbstractBaseFileService<?> service = storageSourceContext.getByStorageKey(storageKey);
|
||||
@ -2773,6 +2761,12 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
|
||||
String bucketName, String nodeId, String taskId) {
|
||||
TsFiles result = new TsFiles();
|
||||
|
||||
|
||||
Map<String, TsFiles> backupPathMap = preloadTsFiles(nodeId, taskId, "backup_path");
|
||||
Map<String, TsFiles> workPathMap = preloadTsFiles(nodeId, taskId, "work_path");
|
||||
|
||||
|
||||
|
||||
// 使用并行流构建 Map（路径 + 文件名 -> 文件）
|
||||
ConcurrentMap<String, FileItemResult> minioMap = minioFiles.parallelStream()
|
||||
.collect(Collectors.toConcurrentMap(
|
||||
@ -4356,6 +4350,24 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
|
||||
return new DualTreeResponse(localTrees, minioTrees);
|
||||
}
|
||||
|
||||
/**********************************
|
||||
* 用途说明: 获取文件url
|
||||
* 参数说明 id 文件夹ID
|
||||
* 返回值说明: com.yfd.platform.config.ResponseResult 返回文件信息
|
||||
***********************************/
|
||||
@Override
|
||||
public FileItemResult obtainUrl(String id,String type) {
|
||||
FileItemResult fileItemResult = new FileItemResult();
|
||||
TsFiles tsFiles = tsFilesMapper.selectById(id);
|
||||
String fileNameData = tsFiles.getFileName();
|
||||
String workPath = tsFiles.getWorkPath();
|
||||
String path = workPath + fileNameData;
|
||||
//准备获取文件的信息
|
||||
AbstractBaseFileService<?> fileService = storageSourceContext.getByStorageKey(type);
|
||||
fileItemResult = fileService.getFileItem(path);
|
||||
return fileItemResult;
|
||||
}
|
||||
|
||||
/**
|
||||
* 递归构建树形结构（内存操作，不再查询数据库）
|
||||
*
|
||||
@ -4366,7 +4378,8 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
|
||||
*/
|
||||
private TreeDTO buildTree(TsFiles current, Map<String, List<TsFiles>> parentChildrenMap, boolean isLocal) {
|
||||
// 1. 转换为DTO并检查路径有效性
|
||||
TreeDTO dto = convertToDTO(current, isLocal);
|
||||
TreeDTO dto = convertToDTOs(current, isLocal);
|
||||
dto.setType(current.getIsFile());
|
||||
if (dto.getPath() == null || dto.getPath().trim().isEmpty()) {
|
||||
LOGGER.warn("由于路径为空，跳过节点｛｝", current.getId());
|
||||
return null;
|
||||
@ -4405,6 +4418,127 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
|
||||
return dto;
|
||||
}
|
||||
|
||||
/**
|
||||
* 将 FileNode 转换为 TreeDTO
|
||||
*
|
||||
* @param node 当前节点
|
||||
* @param isLocal 是否是本地树（true: workPath, false: backupPath）
|
||||
* @return 转换后的 DTO
|
||||
*/
|
||||
private TreeDTO convertToDTOs(TsFiles node, boolean isLocal) {
|
||||
// //查询字典表获取压缩文件后缀
|
||||
// QueryWrapper<SysDictionaryItems> queryWrapperSysDictionary = new QueryWrapper<>();
|
||||
// queryWrapperSysDictionary.eq("parentcode", "compressType");
|
||||
// queryWrapperSysDictionary.orderByAsc("orderno");
|
||||
// List<SysDictionaryItems> sysDictionaryItems = sysDictionaryItemsMapper.selectList(queryWrapperSysDictionary);
|
||||
// int count = 0;
|
||||
//
|
||||
// if ("FOLDER".equals(node.getIsFile())) {
|
||||
// QueryWrapper<TsFiles> queryWrapper = new QueryWrapper<>();
|
||||
// queryWrapper.eq("parent_id", node.getId());
|
||||
// if (isLocal) {
|
||||
// queryWrapper.isNotNull("work_path").ne("work_path", "");
|
||||
// } else {
|
||||
// queryWrapper.isNotNull("backup_path").ne("backup_path", "");
|
||||
// }
|
||||
// count = tsFilesMapper.selectCount(queryWrapper);
|
||||
// }
|
||||
|
||||
TreeDTO dto = new TreeDTO();
|
||||
// 复制公共字段
|
||||
dto.setId(node.getId());
|
||||
// dto.setNodeId(node.getNodeId());
|
||||
// dto.setTaskId(node.getTaskId());
|
||||
dto.setIsFile(node.getIsFile());
|
||||
dto.setParentId(node.getParentId());
|
||||
dto.setFileName(node.getFileName());
|
||||
dto.setFileSize(node.getFileSize());
|
||||
// dto.setKeywords(node.getKeywords());
|
||||
// dto.setDescription(node.getDescription());
|
||||
dto.setUploadTime(node.getUploadTime());
|
||||
// dto.setUploader(node.getUploader());
|
||||
dto.setUpdateTime(node.getUpdateTime());
|
||||
// dto.setNumber(count);
|
||||
//查询本地
|
||||
if (isLocal) {
|
||||
String workPath = node.getWorkPath();
|
||||
if (StringUtils.isEmpty(workPath)) {
|
||||
dto.setPath(null);
|
||||
return dto;
|
||||
}
|
||||
String fileNameData = node.getFileName();
|
||||
//主要是用于文件路径加名称
|
||||
String path = workPath + fileNameData;
|
||||
//准备获取文件的信息
|
||||
if ("FILE".equals(node.getIsFile())) {
|
||||
dto.setUrl(null);
|
||||
dto.setType(null);
|
||||
//
|
||||
// LOGGER.info("查询本地树的时候" + path);
|
||||
// AbstractBaseFileService<?> fileService = storageSourceContext.getByStorageKey("local");
|
||||
// FileItemResult fileItemResult = fileService.getFileItem(path);
|
||||
// if (fileItemResult != null || fileItemResult.getName() != null) {
|
||||
// dto.setUrl(fileItemResult.getUrl());
|
||||
// dto.setType(fileItemResult.getType().getValue());
|
||||
//// //如果是压缩文件 类型就给zip
|
||||
//// boolean isValid = hasValidExtension(fileItemResult.getName(), sysDictionaryItems);
|
||||
//// if (isValid) {
|
||||
//// dto.setType("ZIP");
|
||||
//// } else {
|
||||
//// dto.setType(fileItemResult.getType().getValue());
|
||||
//// }
|
||||
// } else {
|
||||
// dto.setUrl(null);
|
||||
// dto.setType(null);
|
||||
// }
|
||||
|
||||
}
|
||||
} else {
|
||||
String backupPath = node.getBackupPath();
|
||||
if (backupPath == null || StringUtils.isEmpty(backupPath)) {
|
||||
dto.setPath(null);
|
||||
return dto;
|
||||
}
|
||||
String fileNameData = node.getFileName();
|
||||
//主要是用于文件路径加名称
|
||||
String path = backupPath + fileNameData;
|
||||
//准备获取文件的信息
|
||||
if ("FILE".equals(node.getIsFile())) {
|
||||
dto.setUrl(null);
|
||||
dto.setType(null);
|
||||
// LOGGER.info("查询minio树的时候" + path);
|
||||
// AbstractBaseFileService<?> fileService = storageSourceContext.getByStorageKey("minio");
|
||||
// FileItemResult fileItemResult = fileService.getFileItem(path);
|
||||
// if (fileItemResult == null) {
|
||||
// dto.setUrl(null);
|
||||
// dto.setType(null);
|
||||
// } else {
|
||||
//
|
||||
// dto.setUrl(fileItemResult.getUrl());
|
||||
// dto.setType(fileItemResult.getType().getValue());
|
||||
//// //如果是压缩文件 类型就给zip
|
||||
//// boolean isValid = hasValidExtension(fileItemResult.getName(), sysDictionaryItems);
|
||||
//// if (isValid) {
|
||||
//// dto.setType("ZIP");
|
||||
//// } else {
|
||||
//// dto.setType(fileItemResult.getType().getValue());
|
||||
//// }
|
||||
// }
|
||||
}
|
||||
}
|
||||
// 设置路径字段
|
||||
if (isLocal) {
|
||||
String ProcessingPath = processingPath(node.getWorkPath(), node.getNodeId());
|
||||
dto.setPath(ProcessingPath);
|
||||
} else {
|
||||
String ProcessingPath = processingPath(node.getBackupPath(), node.getNodeId());
|
||||
dto.setPath(ProcessingPath);
|
||||
}
|
||||
|
||||
return dto;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 将 FileNode 转换为 TreeDTO
|
||||
*
|
||||
|
||||
Loading…
Reference in New Issue
Block a user