From e0cc6d6e60c2901916620196d5534d7e122a9009 Mon Sep 17 00:00:00 2001 From: lilin Date: Thu, 3 Jul 2025 11:56:40 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BB=A3=E7=A0=81=E6=8F=90?= =?UTF-8?q?=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controller/TsNodesController.java | 51 ++--- .../service/impl/TsFilesServiceImpl.java | 215 +++++++++++++----- .../service/impl/TsNodesServiceImpl.java | 104 --------- .../controller/NodesController.java | 23 +- 4 files changed, 186 insertions(+), 207 deletions(-) diff --git a/java/src/main/java/com/yfd/platform/modules/experimentalData/controller/TsNodesController.java b/java/src/main/java/com/yfd/platform/modules/experimentalData/controller/TsNodesController.java index eeff79e..ee8ba31 100644 --- a/java/src/main/java/com/yfd/platform/modules/experimentalData/controller/TsNodesController.java +++ b/java/src/main/java/com/yfd/platform/modules/experimentalData/controller/TsNodesController.java @@ -247,34 +247,33 @@ public class TsNodesController { @PostMapping("/selectTsNodesById") @ApiOperation("查询可不可以修改项目") public ResponseResult selectTsNodesById(String taskId) { - try { - if (StrUtil.isBlank(taskId)) { - return ResponseResult.error("参数为空"); - } - TsTask tsTask = tsTaskService.getById(taskId); - TableNameContextHolder.setTaskCode(tsTask.getTaskCode()); - List tsNodesList = tsNodesService.list(new QueryWrapper().eq("task_id", taskId)); - // 如果节点不为空 就不能初始化了 - if (tsNodesList.size() > 0) { - return ResponseResult.successData(false); - } else { - // 如果节点为空 就判断文件表 - List filesList = tsFilesService.list(new QueryWrapper().eq("task_id", taskId)); - if (filesList.size() > 0) { - return ResponseResult.successData(false); - } else { - return ResponseResult.successData(true); - } - } - } catch (Exception e) { - // 捕获所有异常并记录日志 - // log.error("查询可不可以修改项目时发生异常: {}", e.getMessage(), e); - return ResponseResult.error("系统异常，请稍后再试"); - }finally { - TableNameContextHolder.clear(); - + if (StrUtil.isBlank(taskId)) { + return ResponseResult.error("任务ID不能为空"); } + try { + TsTask tsTask = tsTaskService.getById(taskId); + if (tsTask == null) { + return ResponseResult.error("任务不存在"); + } + + TableNameContextHolder.setTaskCode(tsTask.getTaskCode()); + + // 1. 优先检查节点是否存在 + if (tsNodesService.count(new QueryWrapper().eq("task_id", taskId)) > 0) { + return ResponseResult.successData(false); + } + + // 2. 再检查文件是否存在 + return ResponseResult.successData( + tsFilesService.count(new QueryWrapper().eq("task_id", taskId)) == 0 + ); + + } catch (Exception e) { + return ResponseResult.error("系统异常，请稍后再试"); + } finally { + TableNameContextHolder.clear(); + } } } diff --git a/java/src/main/java/com/yfd/platform/modules/experimentalData/service/impl/TsFilesServiceImpl.java b/java/src/main/java/com/yfd/platform/modules/experimentalData/service/impl/TsFilesServiceImpl.java index caeadf7..20fcccf 100644 --- a/java/src/main/java/com/yfd/platform/modules/experimentalData/service/impl/TsFilesServiceImpl.java +++ b/java/src/main/java/com/yfd/platform/modules/experimentalData/service/impl/TsFilesServiceImpl.java @@ -12,6 +12,7 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.*; import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -85,7 +86,6 @@ import java.sql.Timestamp; import java.time.LocalDateTime; import java.util.zip.*; -import com.fasterxml.jackson.core.type.TypeReference; import org.apache.commons.codec.binary.Hex; @@ -127,9 +127,6 @@ public class TsFilesServiceImpl extends ServiceImpl impl @Resource private TsTaskMapper tsTaskMapper; - @Resource - private TsNodesMapper tsNodesMapper; - // 从数据库获取的压缩类型列表 private List compressSuffixes; private final Set addedEntries = new HashSet<>(); @@ -3711,82 +3708,174 @@ public class TsFilesServiceImpl extends ServiceImpl impl ***********************************/ @Override public String automaticFileBackup(String taskId, String nodeId) { + // 1. 获取任务信息 TsTask tsTask = tsTaskMapper.selectById(taskId); - TableNameContextHolder.setTaskCode(tsTask.getTaskCode()); + final String taskCode = tsTask.getTaskCode(); + // 原子计数器用于多线程统计 + AtomicInteger fileCount = new AtomicInteger(0); + AtomicInteger folderCount = new AtomicInteger(0); + AtomicInteger backupsFileCount = new AtomicInteger(0); + AtomicInteger backupsFolderCount = new AtomicInteger(0); + try { - - //首先查询节点下面所有 备份路径为空的数据 - QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.select("id", "node_id", "task_id", "is_file", "parent_id", "file_name", - "file_size", "work_path", "backup_path", "upload_time"); - queryWrapper.eq("node_id", nodeId); - queryWrapper.eq("task_id", taskId); - queryWrapper.and(wrapper -> wrapper.isNull("backup_path").or().eq("backup_path", "")); - queryWrapper.isNotNull("work_path").ne("work_path", ""); - List tsFilelist = tsFilesMapper.selectList(queryWrapper); - int FileCount = 0, FolderCount = 0; - int BackupsFileCount = 0, BackupsFolderCount = 0; - - if (tsFilelist.isEmpty()) { - return "本地有新增文件 " + FileCount + " 个, 文件夹 " + FolderCount + " 个, 已将 " + BackupsFileCount + " 个文件, " + BackupsFolderCount + " 个文件夹备份成功"; - } - - // 在循环前记录开始时间 + // 1. 使用游标分页查询避免内存溢出 + int batchSize = 1000; // 每批处理量 + long totalProcessed = 0; long startTime = System.currentTimeMillis(); - for (TsFiles tsFiles : tsFilelist) { - ParameterList parameterList = new ParameterList(); - Parameter fileParameter = new Parameter(); - Parameter FolderParameter = new Parameter(); - List fileParameterlist = new ArrayList<>(); - List FolderParameterlist = new ArrayList<>(); - if ("FILE".equals(tsFiles.getIsFile())) { - parameterList.setName(tsFiles.getFileName()); - parameterList.setPath(tsFiles.getWorkPath()); - parameterList.setSize(tsFiles.getFileSize()); - parameterList.setType(tsFiles.getIsFile()); - fileParameterlist.add(parameterList); - fileParameter.setParameterLists(fileParameterlist); - fileParameter.setTaskId(taskId); - Boolean value = uploadToBackupData(fileParameter); - if (value) { - BackupsFileCount++; + // 2. 分批处理循环 + while (true) { + // 5. 在查询前设置上下文 + TableNameContextHolder.setTaskCode(taskCode); + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.select("id", "node_id", "task_id", "is_file", "parent_id", "file_name", + "file_size", "work_path", "backup_path", "upload_time") + .eq("node_id", nodeId) + .eq("task_id", taskId) + .and(wrapper -> wrapper.isNull("backup_path").or().eq("backup_path", "")) + .isNotNull("work_path") + .ne("work_path", "") + .last("LIMIT " + batchSize + " OFFSET " + totalProcessed); + + List batchList = tsFilesMapper.selectList(queryWrapper); + if (batchList.isEmpty()) break; + + // 3. 使用并行流处理批次 + batchList.parallelStream().forEach(tsFile -> { + try { + Parameter parameter = new Parameter(); + parameter.setTaskId(taskId); + ParameterList paramItem = new ParameterList(); + paramItem.setName(tsFile.getFileName()); + paramItem.setPath(tsFile.getWorkPath()); + paramItem.setSize(tsFile.getFileSize()); + paramItem.setType(tsFile.getIsFile()); + + List paramList = new ArrayList<>(1); + paramList.add(paramItem); + parameter.setParameterLists(paramList); + + if ("FILE".equals(tsFile.getIsFile())) { + fileCount.incrementAndGet(); + if (uploadToBackupData(parameter)) { + backupsFileCount.incrementAndGet(); + } + } else { + folderCount.incrementAndGet(); + if (uploadToBackupData(parameter)) { + backupsFolderCount.incrementAndGet(); + } + } + } catch (Exception e) { + LOGGER.error("文件处理失败: ID={}, 名称={}", tsFile.getId(), tsFile.getFileName(), e); } - FileCount++; - } else { - parameterList.setName(tsFiles.getFileName()); - parameterList.setPath(tsFiles.getWorkPath()); - parameterList.setSize(tsFiles.getFileSize()); - parameterList.setType(tsFiles.getIsFile()); - FolderParameterlist.add(parameterList); - FolderParameter.setParameterLists(FolderParameterlist); - FolderParameter.setTaskId(taskId); - Boolean value = uploadToBackupData(FolderParameter); - if (value) { - BackupsFolderCount++; - } - FolderCount++; - } + }); + + totalProcessed += batchList.size(); + LOGGER.debug("已处理: {}/{}", totalProcessed, batchList.size()); } - // 计算总耗时 - long endTime = System.currentTimeMillis(); - long totalTime = endTime - startTime; - LOGGER.info("循环执行耗时: {} 毫秒 (约 {} 秒)", totalTime, totalTime / 1000.0); - //批量修改表结构在上下文中执行更新 + // 4. 批量更新路径 + long updateStart = System.currentTimeMillis(); updateBackupPaths(taskId, nodeId); + LOGGER.info("批量更新耗时: {}ms", System.currentTimeMillis() - updateStart); + + // 5. 性能监控 + long totalTime = System.currentTimeMillis() - startTime; + LOGGER.info("总处理: {}文件/{}文件夹 | 成功: {}文件/{}文件夹 | 总耗时: {}秒", + fileCount.get(), folderCount.get(), + backupsFileCount.get(), backupsFolderCount.get(), + totalTime/1000.0); + + return String.format("本地有新增文件 %d 个, 文件夹 %d 个, 备份成功 %d 文件/%d 文件夹", + fileCount.get(), folderCount.get(), + backupsFileCount.get(), backupsFolderCount.get()); - return "本地有新增文件 " + FileCount + " 个, 新增文件夹 " + FolderCount + " 个, 已将 " + BackupsFileCount + " 个文件, " + BackupsFolderCount + " 个文件夹备份成功"; } catch (Exception e) { - // 日志记录异常信息 LOGGER.error("自动备份失败：任务ID={}, 节点ID={}", taskId, nodeId, e); - // 返回错误信息 return "自动备份失败：" + e.getMessage(); } finally { TableNameContextHolder.clear(); } + + +// TsTask tsTask = tsTaskMapper.selectById(taskId); +// TableNameContextHolder.setTaskCode(tsTask.getTaskCode()); +// try { +// +// //首先查询节点下面所有 备份路径为空的数据 +// QueryWrapper queryWrapper = new QueryWrapper<>(); +// queryWrapper.select("id", "node_id", "task_id", "is_file", "parent_id", "file_name", +// "file_size", "work_path", "backup_path", "upload_time"); +// queryWrapper.eq("node_id", nodeId); +// queryWrapper.eq("task_id", taskId); +// queryWrapper.and(wrapper -> wrapper.isNull("backup_path").or().eq("backup_path", "")); +// queryWrapper.isNotNull("work_path").ne("work_path", ""); +// List tsFilelist = tsFilesMapper.selectList(queryWrapper); +// int FileCount = 0, FolderCount = 0; +// int BackupsFileCount = 0, BackupsFolderCount = 0; +// +// if (tsFilelist.isEmpty()) { +// return "本地有新增文件 " + FileCount + " 个, 文件夹 " + FolderCount + " 个, 已将 " + BackupsFileCount + " 个文件, " + BackupsFolderCount + " 个文件夹备份成功"; +// } +// +// // 在循环前记录开始时间 +// long startTime = System.currentTimeMillis(); +// for (TsFiles tsFiles : tsFilelist) { +// ParameterList parameterList = new ParameterList(); +// Parameter fileParameter = new Parameter(); +// Parameter FolderParameter = new Parameter(); +// List fileParameterlist = new ArrayList<>(); +// List FolderParameterlist = new ArrayList<>(); +// +// if ("FILE".equals(tsFiles.getIsFile())) { +// parameterList.setName(tsFiles.getFileName()); +// parameterList.setPath(tsFiles.getWorkPath()); +// parameterList.setSize(tsFiles.getFileSize()); +// parameterList.setType(tsFiles.getIsFile()); +// fileParameterlist.add(parameterList); +// fileParameter.setParameterLists(fileParameterlist); +// fileParameter.setTaskId(taskId); +// Boolean value = uploadToBackupData(fileParameter); +// if (value) { +// BackupsFileCount++; +// } +// FileCount++; +// } else { +// parameterList.setName(tsFiles.getFileName()); +// parameterList.setPath(tsFiles.getWorkPath()); +// parameterList.setSize(tsFiles.getFileSize()); +// parameterList.setType(tsFiles.getIsFile()); +// FolderParameterlist.add(parameterList); +// FolderParameter.setParameterLists(FolderParameterlist); +// FolderParameter.setTaskId(taskId); +// Boolean value = uploadToBackupData(FolderParameter); +// if (value) { +// BackupsFolderCount++; +// } +// FolderCount++; +// } +// } +// // 计算总耗时 +// long endTime = System.currentTimeMillis(); +// long totalTime = endTime - startTime; +// LOGGER.info("循环执行耗时: {} 毫秒 (约 {} 秒)", totalTime, totalTime / 1000.0); +// +// //批量修改表结构在上下文中执行更新 +// updateBackupPaths(taskId, nodeId); +// +// return "本地有新增文件 " + FileCount + " 个, 新增文件夹 " + FolderCount + " 个, 已将 " + BackupsFileCount + " 个文件, " + BackupsFolderCount + " 个文件夹备份成功"; +// } catch (Exception e) { +// // 日志记录异常信息 +// LOGGER.error("自动备份失败：任务ID={}, 节点ID={}", taskId, nodeId, e); +// // 返回错误信息 +// return "自动备份失败：" + e.getMessage(); +// } finally { +// TableNameContextHolder.clear(); +// } } + // 辅助方法：更新备份路径 private void updateBackupPaths(String taskId, String nodeId) { try { diff --git a/java/src/main/java/com/yfd/platform/modules/experimentalData/service/impl/TsNodesServiceImpl.java b/java/src/main/java/com/yfd/platform/modules/experimentalData/service/impl/TsNodesServiceImpl.java index c849ba4..87a7fa4 100644 --- a/java/src/main/java/com/yfd/platform/modules/experimentalData/service/impl/TsNodesServiceImpl.java +++ b/java/src/main/java/com/yfd/platform/modules/experimentalData/service/impl/TsNodesServiceImpl.java @@ -629,40 +629,6 @@ public class TsNodesServiceImpl extends ServiceImpl impl return nodesList; } - /** - * 递归删除子节点 - * - * @param parentId 父节点ID - */ - private void deleteChildren(String parentId, String taskId) { - // 使用 QueryWrapper 查询当前节点的所有子节点 - QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.eq("parent_id", parentId); // parent_id = #{parentId} - queryWrapper.eq("task_id", taskId); // parent_id = #{parentId} - List children = tsNodesMapper.selectList(queryWrapper); - - // 递归删除每个子节点 - for (TsNodes child : children) { - deleteChildren(child.getNodeId(), child.getTaskId()); // 递归删除子节点的子节点 - tsNodesMapper.deleteById(child.getNodeId()); // 删除当前子节点 - //批量文件的数据 - QueryWrapper queryWrapper1 = new QueryWrapper<>(); - queryWrapper1.eq("node_id", child.getNodeId()); - queryWrapper1.eq("task_id", taskId); - List tsFiles = tsFilesMapper.selectList(queryWrapper1); - - List dataset = new ArrayList<>(); - if (tsFiles != null && !tsFiles.isEmpty()) { - dataset = tsFiles.stream() - .map(TsFiles::getId) // 假设 TsFiles 类中有 getId() 方法 - .collect(Collectors.toList()); - } - //批量删除TsFiles表数据 - if (dataset.size() > 0) { - tsFilesService.deleteTsFilesByIds(dataset, "local", taskId); - } - } - } /********************************** @@ -895,76 +861,6 @@ public class TsNodesServiceImpl extends ServiceImpl impl } -// //查询节点表中 有没有根节点的节点 如果有就直接使用 如果没有就新增 -// LambdaQueryWrapper queryWrapper = new LambdaQueryWrapper<>(); -// queryWrapper.eq(TsNodes::getTaskId, taskId); -// queryWrapper.eq(TsNodes::getParentId, TOP_LEVEL_PARENT_NODE); -// queryWrapper.eq(TsNodes::getNodeName, "根节点"); -// TsNodes nodeData = tsNodesMapper.selectOne(queryWrapper); -// if (nodeData == null) { -// //新增一个根节点 -// TsNodes node = savetsNodes(taskId, TOP_LEVEL_PARENT_NODE, "根节点"); -// -// String finalPath = item.getPath().replace(item.getName(), ""); -// //保存文件信息 -// TsFiles tsFiles1 = new TsFiles(); -// //任务 -// tsFiles1.setTaskId(taskId); -// //节点 -// tsFiles1.setNodeId(node.getNodeId()); -// //文件 文件夹 区分 -// tsFiles1.setIsFile("FILE"); -// //上级ID -// tsFiles1.setParentId("00"); -// //文件名称 -// tsFiles1.setFileName(item.getName()); -// //工作空间路径 -// tsFiles1.setWorkPath(ensurePathFormat(finalPath)); -// tsFiles1.setUploadTime(currentTime); -// tsFiles1.setUploader(loginuser.getUsername()); -// long bytes = item.getSize(); -// // 转换为 KB 并保留两位小数 -// double fileSizeInKB = bytes / (1024.0); -// String fileSizeFormatted = String.format("%.2f", fileSizeInKB); // 保留两位小数 -// tsFiles1.setFileSize(fileSizeFormatted); -// tsFilesMapper.insert(tsFiles1); -// } else { -// //先通过名称 + 节点 + 任务 + 路径 查询 如果有就跳过 没有就继续新增 -// LambdaQueryWrapper queryWrapperFiles = new LambdaQueryWrapper<>(); -// queryWrapperFiles.eq(TsFiles::getNodeId, nodeData.getNodeId()); -// queryWrapperFiles.eq(TsFiles::getTaskId, taskId); -// queryWrapperFiles.eq(TsFiles::getWorkPath, item.getPath()); -// queryWrapperFiles.eq(TsFiles::getFileName, item.getName()); -// TsFiles tsFiles = tsFilesMapper.selectOne(queryWrapperFiles); -// if (tsFiles == null) { -// String finalPath = item.getPath().replace(item.getName(), ""); -// //保存文件信息 -// TsFiles tsFiles1 = new TsFiles(); -// //任务 -// tsFiles1.setTaskId(taskId); -// //节点 -// tsFiles1.setNodeId(nodeData.getNodeId()); -// //文件 文件夹 区分 -// tsFiles1.setIsFile("FILE"); -// //上级ID -// tsFiles1.setParentId("00"); -// //文件名称 -// tsFiles1.setFileName(item.getName()); -// //工作空间路径 -// tsFiles1.setWorkPath(ensurePathFormat(finalPath)); -// tsFiles1.setUploadTime(currentTime); -// tsFiles1.setUploader(loginuser.getUsername()); -// long bytes = item.getSize(); -// // 转换为 KB 并保留两位小数 -// double fileSizeInKB = bytes / (1024.0); -// String fileSizeFormatted = String.format("%.2f", fileSizeInKB); // 保留两位小数 -// tsFiles1.setFileSize(fileSizeFormatted); -// tsFilesMapper.insert(tsFiles1); -// } else { -// continue; -// } -// -// } } } } diff --git a/java/src/main/java/com/yfd/platform/modules/specialDocument/controller/NodesController.java b/java/src/main/java/com/yfd/platform/modules/specialDocument/controller/NodesController.java index 6ca89b9..4383c26 100644 --- a/java/src/main/java/com/yfd/platform/modules/specialDocument/controller/NodesController.java +++ b/java/src/main/java/com/yfd/platform/modules/specialDocument/controller/NodesController.java @@ -295,23 +295,18 @@ public class NodesController { @ApiOperation("查询可不可以修改项目") public ResponseResult selectNodesById(String projectId) { if (StrUtil.isBlank(projectId)) { - return ResponseResult.error("参数为空"); + return ResponseResult.error("项目ID不能为空"); } - List nodesList = nodesService.list(new QueryWrapper().eq("project_id", projectId)); - //如果节点不为空 就不能初始化了 如果大于0是false 代表不能修改 - if (nodesList.size() > 0) { + + // 优先检查节点是否存在 - 使用count避免加载全量数据 + if (nodesService.count(new QueryWrapper().eq("project_id", projectId)) > 0) { return ResponseResult.successData(false); - } else { - //如果节点为空 就判断文件表ew Qu - List filesList = filesService.list(new QueryWrapper().eq("project_id", projectId)); - if (filesList.size() > 0) { - return ResponseResult.successData(false); - - } else { - return ResponseResult.successData(true); - - } } + + // 再检查文件是否存在 + return ResponseResult.successData( + filesService.count(new QueryWrapper().eq("project_id", projectId)) == 0 + ); }