From fc27c0f57199ad184afec62e5cb8f0e3f13416b1 Mon Sep 17 00:00:00 2001 From: lilin Date: Mon, 7 Jul 2025 11:59:28 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BB=A3=E7=A0=81=E6=8F=90?= =?UTF-8?q?=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/impl/TsFilesServiceImpl.java | 107 +++++++++++++----- 1 file changed, 81 insertions(+), 26 deletions(-) diff --git a/java/src/main/java/com/yfd/platform/modules/experimentalData/service/impl/TsFilesServiceImpl.java b/java/src/main/java/com/yfd/platform/modules/experimentalData/service/impl/TsFilesServiceImpl.java index 99788b8..8f6a532 100644 --- a/java/src/main/java/com/yfd/platform/modules/experimentalData/service/impl/TsFilesServiceImpl.java +++ b/java/src/main/java/com/yfd/platform/modules/experimentalData/service/impl/TsFilesServiceImpl.java @@ -3173,37 +3173,83 @@ public class TsFilesServiceImpl extends ServiceImpl impl } // ================ 2. 过滤并处理符合条件的记录 ================ - List filteredRecords = records.stream() + List filteredRecords = records.parallelStream() // 使用并行流提高处理效率（根据场景选择） .filter(tsFile -> { - try { - if("FOLDER".equals(tsFile.getIsFile())){ - return false; - } - // 计算本地文件MD5 - File localFile = new File(filePathConfig.getValue() + tsFile.getWorkPath(), tsFile.getFileName()); - String localMD5 = calculateMD5Data(new FileInputStream(localFile)); - - // 计算MinIO文件MD5 - - String minioMD5 = getMinioMD5Data(bucketConfig.getValue(), tsFile.getBackupPath(), tsFile.getFileName(),storageSource); - - // 路径处理 - // tsFile.setWorkPath(processingPath(tsFile.getWorkPath(), tsFile.getNodeId())); - // tsFile.setBackupPath(processingPath(tsFile.getBackupPath(), tsFile.getNodeId())); - - // 设置MD5字段（即使不满足条件也保留字段） - tsFile.setLocatMd5(localMD5); - tsFile.setMinioMd5(minioMD5); - - // 返回是否满足过滤条件 - return StringUtils.isNoneEmpty(localMD5, minioMD5) && !localMD5.equals(minioMD5); - } catch (Exception e) { - LOGGER.error("MD5计算失败: {}", tsFile.getFileName(), e); - return false; + if ("FOLDER".equals(tsFile.getIsFile())) { + return false; // 跳过文件夹 } + String localMD5 = null; + String minioMD5 = null; + try (InputStream localStream = new FileInputStream( // 自动关闭资源 + new File(filePathConfig.getValue() + tsFile.getWorkPath(), tsFile.getFileName()) + )) { + // 计算本地文件MD5 + localMD5 = calculateMD5Data(localStream); + } catch (Exception e) { + LOGGER.error("本地文件MD5计算失败: {} | 路径: {}", + tsFile.getFileName(), + filePathConfig.getValue() + tsFile.getWorkPath(), + e + ); + } + try { + // 计算MinIO文件MD5 + minioMD5 = getMinioMD5Data( + bucketConfig.getValue(), + tsFile.getBackupPath(), + tsFile.getFileName(), + storageSource + ); + } catch (Exception e) { + LOGGER.error("MinIO文件MD5获取失败: {} | 路径: {}/{}", + tsFile.getFileName(), + bucketConfig.getValue(), + tsFile.getBackupPath(), + e + ); + } + + // 设置MD5字段（无论是否成功） + tsFile.setLocatMd5(localMD5); + tsFile.setMinioMd5(minioMD5); + + // 仅当两者都成功计算且不相等时保留 + return StringUtils.isNoneEmpty(localMD5, minioMD5) + && !localMD5.equals(minioMD5); }) .collect(Collectors.toList()); +// List filteredRecords = records.stream() +// .filter(tsFile -> { +// try { +// if("FOLDER".equals(tsFile.getIsFile())){ +// return false; +// } +// // 计算本地文件MD5 +// File localFile = new File(filePathConfig.getValue() + tsFile.getWorkPath(), tsFile.getFileName()); +// String localMD5 = calculateMD5Data(new FileInputStream(localFile)); +// +// // 计算MinIO文件MD5 +// +// String minioMD5 = getMinioMD5Data(bucketConfig.getValue(), tsFile.getBackupPath(), tsFile.getFileName(),storageSource); +// +// // 路径处理 +// // tsFile.setWorkPath(processingPath(tsFile.getWorkPath(), tsFile.getNodeId())); +// // tsFile.setBackupPath(processingPath(tsFile.getBackupPath(), tsFile.getNodeId())); +// +// // 设置MD5字段（即使不满足条件也保留字段） +// tsFile.setLocatMd5(localMD5); +// tsFile.setMinioMd5(minioMD5); +// +// // 返回是否满足过滤条件 +// return StringUtils.isNoneEmpty(localMD5, minioMD5) && !localMD5.equals(minioMD5); +// } catch (Exception e) { +// LOGGER.error("MD5计算失败: {}", tsFile.getFileName(), e); +// return false; +// } +// }) +// .collect(Collectors.toList()); + return filteredRecords; } catch (Exception e) { LOGGER.error("未知异常：{}", e.getMessage(), e); @@ -4905,6 +4951,10 @@ public class TsFilesServiceImpl extends ServiceImpl impl queryWrapper.eq("task_id", taskId); } queryWrapper.isNotNull("work_path").ne("work_path", ""); + //文件还是文件夹 + queryWrapper.orderByDesc("is_file"); + //文件名称 + queryWrapper.orderByAsc("file_name"); List allNodes = tsFilesMapper.selectList(queryWrapper); @@ -4970,6 +5020,11 @@ public class TsFilesServiceImpl extends ServiceImpl impl } queryWrapper.isNotNull("backup_path").ne("backup_path", ""); + //文件还是文件夹 + queryWrapper.orderByDesc("is_file"); + //文件名称 + queryWrapper.orderByAsc("file_name"); + // queryWrapper.eq("parent_id", "00"); List allNodes = tsFilesMapper.selectList(queryWrapper); // 2. 双树构建容器