提交代码
This commit is contained in:
parent
d81dc8683f
commit
b9b421c63c
@ -10,6 +10,7 @@ import com.yfd.platform.modules.experimentalData.domain.*;
|
|||||||
import com.yfd.platform.modules.experimentalData.service.ITsFilesService;
|
import com.yfd.platform.modules.experimentalData.service.ITsFilesService;
|
||||||
import com.yfd.platform.modules.specialDocument.domain.Files;
|
import com.yfd.platform.modules.specialDocument.domain.Files;
|
||||||
import com.yfd.platform.modules.specialDocument.service.IFilesService;
|
import com.yfd.platform.modules.specialDocument.service.IFilesService;
|
||||||
|
import com.yfd.platform.modules.storage.model.result.FileItemResult;
|
||||||
import io.swagger.annotations.ApiOperation;
|
import io.swagger.annotations.ApiOperation;
|
||||||
import org.springframework.http.HttpStatus;
|
import org.springframework.http.HttpStatus;
|
||||||
import org.springframework.http.ResponseEntity;
|
import org.springframework.http.ResponseEntity;
|
||||||
@ -540,4 +541,20 @@ public class TsFilesController {
|
|||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/**********************************
|
||||||
|
* 用途说明: 获取文件url
|
||||||
|
* 参数说明 id 文件夹ID
|
||||||
|
* 返回值说明: com.yfd.platform.config.ResponseResult 返回文件信息
|
||||||
|
***********************************/
|
||||||
|
@Log(module = "实验数据管理", value = "获取文件url!")
|
||||||
|
@PostMapping("/obtainUrl")
|
||||||
|
@ApiOperation("获取文件url")
|
||||||
|
public ResponseResult obtainUrl(String id,String type) {
|
||||||
|
if (StrUtil.isBlank(id) && StrUtil.isBlank(type)) {
|
||||||
|
return ResponseResult.error("参数为空");
|
||||||
|
}
|
||||||
|
//查询本地树和minio树
|
||||||
|
FileItemResult fileItemResult = tsFilesService.obtainUrl(id,type);
|
||||||
|
return ResponseResult.successData(fileItemResult);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -4,6 +4,7 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
|
|||||||
import com.yfd.platform.config.ResponseResult;
|
import com.yfd.platform.config.ResponseResult;
|
||||||
import com.yfd.platform.modules.experimentalData.domain.*;
|
import com.yfd.platform.modules.experimentalData.domain.*;
|
||||||
import com.baomidou.mybatisplus.extension.service.IService;
|
import com.baomidou.mybatisplus.extension.service.IService;
|
||||||
|
import com.yfd.platform.modules.storage.model.result.FileItemResult;
|
||||||
import org.springframework.web.bind.annotation.RequestParam;
|
import org.springframework.web.bind.annotation.RequestParam;
|
||||||
|
|
||||||
import java.io.FileNotFoundException;
|
import java.io.FileNotFoundException;
|
||||||
@ -215,4 +216,11 @@ public interface ITsFilesService extends IService<TsFiles> {
|
|||||||
* 返回值说明: com.yfd.platform.config.ResponseResult 返回双树数据
|
* 返回值说明: com.yfd.platform.config.ResponseResult 返回双树数据
|
||||||
***********************************/
|
***********************************/
|
||||||
DualTreeResponse listLocalAndBackup(String taskId, String nodeId);
|
DualTreeResponse listLocalAndBackup(String taskId, String nodeId);
|
||||||
|
|
||||||
|
/**********************************
|
||||||
|
* 用途说明: 获取文件url
|
||||||
|
* 参数说明 id 文件夹ID
|
||||||
|
* 返回值说明: com.yfd.platform.config.ResponseResult 返回文件信息
|
||||||
|
***********************************/
|
||||||
|
FileItemResult obtainUrl(String id, String type);
|
||||||
}
|
}
|
||||||
|
@ -3,7 +3,6 @@ package com.yfd.platform.modules.experimentalData.service.impl;
|
|||||||
import cn.hutool.core.collection.CollUtil;
|
import cn.hutool.core.collection.CollUtil;
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
import java.nio.file.*;
|
import java.nio.file.*;
|
||||||
import java.nio.file.attribute.BasicFileAttributes;
|
import java.nio.file.attribute.BasicFileAttributes;
|
||||||
@ -2518,16 +2517,8 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
|
|||||||
TsFiles tsFiles = new TsFiles();
|
TsFiles tsFiles = new TsFiles();
|
||||||
try {
|
try {
|
||||||
// 获取 MinIO 和本地文件列表(并行处理)
|
// 获取 MinIO 和本地文件列表(并行处理)
|
||||||
// List<FileItemResult> fileItemListMinio = Collections.synchronizedList(new ArrayList<>());
|
|
||||||
// List<FileItemResult> fileItemListLocal = Collections.synchronizedList(new ArrayList<>());
|
|
||||||
CopyOnWriteArrayList<FileItemResult> fileItemListMinio = new CopyOnWriteArrayList<>();
|
CopyOnWriteArrayList<FileItemResult> fileItemListMinio = new CopyOnWriteArrayList<>();
|
||||||
CopyOnWriteArrayList<FileItemResult> fileItemListLocal = new CopyOnWriteArrayList<>();
|
CopyOnWriteArrayList<FileItemResult> fileItemListLocal = new CopyOnWriteArrayList<>();
|
||||||
|
|
||||||
|
|
||||||
// 优化点3: 使用 CompletableFuture 进行并行任务管理
|
|
||||||
// final List<FileItemResult> minioFiles = Collections.synchronizedList(new ArrayList<>());
|
|
||||||
// final List<FileItemResult> localFiles = Collections.synchronizedList(new ArrayList<>());
|
|
||||||
|
|
||||||
if (StringUtils.isNoneEmpty(nodeId, taskId)) {
|
if (StringUtils.isNoneEmpty(nodeId, taskId)) {
|
||||||
// 并行查询数据库记录
|
// 并行查询数据库记录
|
||||||
tsFilesMapper.selectList(buildTaskQuery(nodeId, taskId)).parallelStream()
|
tsFilesMapper.selectList(buildTaskQuery(nodeId, taskId)).parallelStream()
|
||||||
@ -2557,6 +2548,7 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
|
|||||||
deduplicateFileList(fileItemListMinio);
|
deduplicateFileList(fileItemListMinio);
|
||||||
deduplicateFileList(fileItemListLocal);
|
deduplicateFileList(fileItemListLocal);
|
||||||
|
|
||||||
|
|
||||||
// 对比文件(并行计算 MD5)
|
// 对比文件(并行计算 MD5)
|
||||||
tsFiles = compareFiles(fileItemListMinio, fileItemListLocal,
|
tsFiles = compareFiles(fileItemListMinio, fileItemListLocal,
|
||||||
filePathConfig.getValue(), bucketConfig.getValue(), nodeId, taskId);
|
filePathConfig.getValue(), bucketConfig.getValue(), nodeId, taskId);
|
||||||
@ -2583,7 +2575,6 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
|
|||||||
.eq("task_id", taskId)
|
.eq("task_id", taskId)
|
||||||
.eq("parent_id", "00")
|
.eq("parent_id", "00")
|
||||||
.eq("node_id", nodeId);
|
.eq("node_id", nodeId);
|
||||||
//.and(wrapper -> wrapper.eq("work_path", "/").or().eq("backup_path", "/"));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// 辅助方法:处理文件列表(并行安全)
|
// 辅助方法:处理文件列表(并行安全)
|
||||||
@ -2593,9 +2584,6 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 辅助方法:获取文件列表并添加到集合
|
// 辅助方法:获取文件列表并添加到集合
|
||||||
// 辅助方法:获取文件列表并添加到集合(优化版)
|
|
||||||
|
|
||||||
// 优化点7:提取文件处理核心逻辑
|
|
||||||
private void processFileList(String path, String fileName, String storageKey, List<FileItemResult> targetList, String isFile, String nodeId, String taskId) throws Exception {
|
private void processFileList(String path, String fileName, String storageKey, List<FileItemResult> targetList, String isFile, String nodeId, String taskId) throws Exception {
|
||||||
if (StringUtils.isNotEmpty(path)) {
|
if (StringUtils.isNotEmpty(path)) {
|
||||||
AbstractBaseFileService<?> service = storageSourceContext.getByStorageKey(storageKey);
|
AbstractBaseFileService<?> service = storageSourceContext.getByStorageKey(storageKey);
|
||||||
@ -2773,6 +2761,12 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
|
|||||||
String bucketName, String nodeId, String taskId) {
|
String bucketName, String nodeId, String taskId) {
|
||||||
TsFiles result = new TsFiles();
|
TsFiles result = new TsFiles();
|
||||||
|
|
||||||
|
|
||||||
|
Map<String, TsFiles> backupPathMap = preloadTsFiles(nodeId, taskId, "backup_path");
|
||||||
|
Map<String, TsFiles> workPathMap = preloadTsFiles(nodeId, taskId, "work_path");
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
// 使用并行流构建 Map(路径 + 文件名 -> 文件)
|
// 使用并行流构建 Map(路径 + 文件名 -> 文件)
|
||||||
ConcurrentMap<String, FileItemResult> minioMap = minioFiles.parallelStream()
|
ConcurrentMap<String, FileItemResult> minioMap = minioFiles.parallelStream()
|
||||||
.collect(Collectors.toConcurrentMap(
|
.collect(Collectors.toConcurrentMap(
|
||||||
@ -4356,6 +4350,24 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
|
|||||||
return new DualTreeResponse(localTrees, minioTrees);
|
return new DualTreeResponse(localTrees, minioTrees);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**********************************
|
||||||
|
* 用途说明: 获取文件url
|
||||||
|
* 参数说明 id 文件夹ID
|
||||||
|
* 返回值说明: com.yfd.platform.config.ResponseResult 返回文件信息
|
||||||
|
***********************************/
|
||||||
|
@Override
|
||||||
|
public FileItemResult obtainUrl(String id,String type) {
|
||||||
|
FileItemResult fileItemResult = new FileItemResult();
|
||||||
|
TsFiles tsFiles = tsFilesMapper.selectById(id);
|
||||||
|
String fileNameData = tsFiles.getFileName();
|
||||||
|
String workPath = tsFiles.getWorkPath();
|
||||||
|
String path = workPath + fileNameData;
|
||||||
|
//准备获取文件的信息
|
||||||
|
AbstractBaseFileService<?> fileService = storageSourceContext.getByStorageKey(type);
|
||||||
|
fileItemResult = fileService.getFileItem(path);
|
||||||
|
return fileItemResult;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 递归构建树形结构(内存操作,不再查询数据库)
|
* 递归构建树形结构(内存操作,不再查询数据库)
|
||||||
*
|
*
|
||||||
@ -4366,7 +4378,8 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
|
|||||||
*/
|
*/
|
||||||
private TreeDTO buildTree(TsFiles current, Map<String, List<TsFiles>> parentChildrenMap, boolean isLocal) {
|
private TreeDTO buildTree(TsFiles current, Map<String, List<TsFiles>> parentChildrenMap, boolean isLocal) {
|
||||||
// 1. 转换为DTO并检查路径有效性
|
// 1. 转换为DTO并检查路径有效性
|
||||||
TreeDTO dto = convertToDTO(current, isLocal);
|
TreeDTO dto = convertToDTOs(current, isLocal);
|
||||||
|
dto.setType(current.getIsFile());
|
||||||
if (dto.getPath() == null || dto.getPath().trim().isEmpty()) {
|
if (dto.getPath() == null || dto.getPath().trim().isEmpty()) {
|
||||||
LOGGER.warn("由于路径为空,跳过节点{}", current.getId());
|
LOGGER.warn("由于路径为空,跳过节点{}", current.getId());
|
||||||
return null;
|
return null;
|
||||||
@ -4405,6 +4418,127 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
|
|||||||
return dto;
|
return dto;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 将 FileNode 转换为 TreeDTO
|
||||||
|
*
|
||||||
|
* @param node 当前节点
|
||||||
|
* @param isLocal 是否是本地树(true: workPath, false: backupPath)
|
||||||
|
* @return 转换后的 DTO
|
||||||
|
*/
|
||||||
|
private TreeDTO convertToDTOs(TsFiles node, boolean isLocal) {
|
||||||
|
// //查询字典表获取压缩文件后缀
|
||||||
|
// QueryWrapper<SysDictionaryItems> queryWrapperSysDictionary = new QueryWrapper<>();
|
||||||
|
// queryWrapperSysDictionary.eq("parentcode", "compressType");
|
||||||
|
// queryWrapperSysDictionary.orderByAsc("orderno");
|
||||||
|
// List<SysDictionaryItems> sysDictionaryItems = sysDictionaryItemsMapper.selectList(queryWrapperSysDictionary);
|
||||||
|
// int count = 0;
|
||||||
|
//
|
||||||
|
// if ("FOLDER".equals(node.getIsFile())) {
|
||||||
|
// QueryWrapper<TsFiles> queryWrapper = new QueryWrapper<>();
|
||||||
|
// queryWrapper.eq("parent_id", node.getId());
|
||||||
|
// if (isLocal) {
|
||||||
|
// queryWrapper.isNotNull("work_path").ne("work_path", "");
|
||||||
|
// } else {
|
||||||
|
// queryWrapper.isNotNull("backup_path").ne("backup_path", "");
|
||||||
|
// }
|
||||||
|
// count = tsFilesMapper.selectCount(queryWrapper);
|
||||||
|
// }
|
||||||
|
|
||||||
|
TreeDTO dto = new TreeDTO();
|
||||||
|
// 复制公共字段
|
||||||
|
dto.setId(node.getId());
|
||||||
|
// dto.setNodeId(node.getNodeId());
|
||||||
|
// dto.setTaskId(node.getTaskId());
|
||||||
|
dto.setIsFile(node.getIsFile());
|
||||||
|
dto.setParentId(node.getParentId());
|
||||||
|
dto.setFileName(node.getFileName());
|
||||||
|
dto.setFileSize(node.getFileSize());
|
||||||
|
// dto.setKeywords(node.getKeywords());
|
||||||
|
// dto.setDescription(node.getDescription());
|
||||||
|
dto.setUploadTime(node.getUploadTime());
|
||||||
|
// dto.setUploader(node.getUploader());
|
||||||
|
dto.setUpdateTime(node.getUpdateTime());
|
||||||
|
// dto.setNumber(count);
|
||||||
|
//查询本地
|
||||||
|
if (isLocal) {
|
||||||
|
String workPath = node.getWorkPath();
|
||||||
|
if (StringUtils.isEmpty(workPath)) {
|
||||||
|
dto.setPath(null);
|
||||||
|
return dto;
|
||||||
|
}
|
||||||
|
String fileNameData = node.getFileName();
|
||||||
|
//主要是用于文件路径加名称
|
||||||
|
String path = workPath + fileNameData;
|
||||||
|
//准备获取文件的信息
|
||||||
|
if ("FILE".equals(node.getIsFile())) {
|
||||||
|
dto.setUrl(null);
|
||||||
|
dto.setType(null);
|
||||||
|
//
|
||||||
|
// LOGGER.info("查询本地树的时候" + path);
|
||||||
|
// AbstractBaseFileService<?> fileService = storageSourceContext.getByStorageKey("local");
|
||||||
|
// FileItemResult fileItemResult = fileService.getFileItem(path);
|
||||||
|
// if (fileItemResult != null || fileItemResult.getName() != null) {
|
||||||
|
// dto.setUrl(fileItemResult.getUrl());
|
||||||
|
// dto.setType(fileItemResult.getType().getValue());
|
||||||
|
//// //如果是压缩文件 类型就给zip
|
||||||
|
//// boolean isValid = hasValidExtension(fileItemResult.getName(), sysDictionaryItems);
|
||||||
|
//// if (isValid) {
|
||||||
|
//// dto.setType("ZIP");
|
||||||
|
//// } else {
|
||||||
|
//// dto.setType(fileItemResult.getType().getValue());
|
||||||
|
//// }
|
||||||
|
// } else {
|
||||||
|
// dto.setUrl(null);
|
||||||
|
// dto.setType(null);
|
||||||
|
// }
|
||||||
|
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
String backupPath = node.getBackupPath();
|
||||||
|
if (backupPath == null || StringUtils.isEmpty(backupPath)) {
|
||||||
|
dto.setPath(null);
|
||||||
|
return dto;
|
||||||
|
}
|
||||||
|
String fileNameData = node.getFileName();
|
||||||
|
//主要是用于文件路径加名称
|
||||||
|
String path = backupPath + fileNameData;
|
||||||
|
//准备获取文件的信息
|
||||||
|
if ("FILE".equals(node.getIsFile())) {
|
||||||
|
dto.setUrl(null);
|
||||||
|
dto.setType(null);
|
||||||
|
// LOGGER.info("查询minio树的时候" + path);
|
||||||
|
// AbstractBaseFileService<?> fileService = storageSourceContext.getByStorageKey("minio");
|
||||||
|
// FileItemResult fileItemResult = fileService.getFileItem(path);
|
||||||
|
// if (fileItemResult == null) {
|
||||||
|
// dto.setUrl(null);
|
||||||
|
// dto.setType(null);
|
||||||
|
// } else {
|
||||||
|
//
|
||||||
|
// dto.setUrl(fileItemResult.getUrl());
|
||||||
|
// dto.setType(fileItemResult.getType().getValue());
|
||||||
|
//// //如果是压缩文件 类型就给zip
|
||||||
|
//// boolean isValid = hasValidExtension(fileItemResult.getName(), sysDictionaryItems);
|
||||||
|
//// if (isValid) {
|
||||||
|
//// dto.setType("ZIP");
|
||||||
|
//// } else {
|
||||||
|
//// dto.setType(fileItemResult.getType().getValue());
|
||||||
|
//// }
|
||||||
|
// }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// 设置路径字段
|
||||||
|
if (isLocal) {
|
||||||
|
String ProcessingPath = processingPath(node.getWorkPath(), node.getNodeId());
|
||||||
|
dto.setPath(ProcessingPath);
|
||||||
|
} else {
|
||||||
|
String ProcessingPath = processingPath(node.getBackupPath(), node.getNodeId());
|
||||||
|
dto.setPath(ProcessingPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
return dto;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 将 FileNode 转换为 TreeDTO
|
* 将 FileNode 转换为 TreeDTO
|
||||||
*
|
*
|
||||||
|
Loading…
Reference in New Issue
Block a user