From f03a3918e8ae96d5b3896af96bb9a12ac34924f2 Mon Sep 17 00:00:00 2001 From: lilin Date: Mon, 7 Jul 2025 13:58:40 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BB=A3=E7=A0=81=E6=8F=90?= =?UTF-8?q?=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/impl/TsFilesServiceImpl.java | 113 +++++++++++++----- 1 file changed, 85 insertions(+), 28 deletions(-) diff --git a/java/src/main/java/com/yfd/platform/modules/experimentalData/service/impl/TsFilesServiceImpl.java b/java/src/main/java/com/yfd/platform/modules/experimentalData/service/impl/TsFilesServiceImpl.java index 8f6a532..47f049a 100644 --- a/java/src/main/java/com/yfd/platform/modules/experimentalData/service/impl/TsFilesServiceImpl.java +++ b/java/src/main/java/com/yfd/platform/modules/experimentalData/service/impl/TsFilesServiceImpl.java @@ -3086,37 +3086,94 @@ public class TsFilesServiceImpl extends ServiceImpl impl } // ================ 2. 过滤并处理符合条件的记录 ================ - List filteredRecords = tsFilesPage.getRecords().stream() + + // ================ 2. 优化后的MD5比较处理 ================ + List filteredRecords = tsFilesPage.getRecords().parallelStream() .filter(tsFile -> { - try { - if("FOLDER".equals(tsFile.getIsFile())){ - return false; - } - // 计算本地文件MD5 - File localFile = new File(filePathConfig.getValue() + tsFile.getWorkPath(), tsFile.getFileName()); - String localMD5 = calculateMD5Data(new FileInputStream(localFile)); - - // 计算MinIO文件MD5 - - String minioMD5 = getMinioMD5Data(bucketConfig.getValue(), tsFile.getBackupPath(), tsFile.getFileName(),storageSource); - - // 路径处理 - //tsFile.setWorkPath(processingPath(tsFile.getWorkPath(), tsFile.getNodeId())); - // tsFile.setBackupPath(processingPath(tsFile.getBackupPath(), tsFile.getNodeId())); - - // 设置MD5字段（即使不满足条件也保留字段） - tsFile.setLocatMd5(localMD5); - tsFile.setMinioMd5(minioMD5); - - // 返回是否满足过滤条件 - return StringUtils.isNoneEmpty(localMD5, minioMD5) && !localMD5.equals(minioMD5); - } catch (Exception e) { - LOGGER.error("MD5计算失败: {}", tsFile.getFileName(), e); - return false; + if ("FOLDER".equals(tsFile.getIsFile())) { + return false; // 跳过文件夹 } + + String localMD5 = null; + String minioMD5 = null; + boolean hasError = false; + + // 计算本地文件MD5（带资源自动管理） + String localFilePath = filePathConfig.getValue() + tsFile.getWorkPath(); + File localFile = new File(localFilePath, tsFile.getFileName()); + + if (localFile.exists()) { + try (InputStream is = new FileInputStream(localFile)) { + localMD5 = calculateMD5Data(is); + } catch (Exception e) { + LOGGER.error("本地文件MD5计算失败: {} | 路径: {}", + tsFile.getFileName(), localFilePath, e); + hasError = true; + } + } else { + LOGGER.warn("本地文件不存在: {} | 路径: {}", tsFile.getFileName(), localFilePath); + hasError = true; + } + + // 计算MinIO文件MD5 + try { + minioMD5 = getMinioMD5Data( + bucketConfig.getValue(), + tsFile.getBackupPath(), + tsFile.getFileName(), + storageSource + ); + } catch (Exception e) { + LOGGER.error("MinIO文件MD5获取失败: {} | Bucket: {}/{}", + tsFile.getFileName(), + bucketConfig.getValue(), + tsFile.getBackupPath(), + e); + hasError = true; + } + + // 始终设置MD5字段（成功为实际值，失败为null） + tsFile.setLocatMd5(localMD5); + tsFile.setMinioMd5(minioMD5); + + // 仅当两者都成功计算且不相等时保留 + return !hasError && + StringUtils.isNoneEmpty(localMD5, minioMD5) && + !localMD5.equals(minioMD5); }) .collect(Collectors.toList()); +// List filteredRecords = tsFilesPage.getRecords().stream() +// .filter(tsFile -> { +// try { +// if("FOLDER".equals(tsFile.getIsFile())){ +// return false; +// } +// // 计算本地文件MD5 +// File localFile = new File(filePathConfig.getValue() + tsFile.getWorkPath(), tsFile.getFileName()); +// String localMD5 = calculateMD5Data(new FileInputStream(localFile)); +// +// // 计算MinIO文件MD5 +// +// String minioMD5 = getMinioMD5Data(bucketConfig.getValue(), tsFile.getBackupPath(), tsFile.getFileName(),storageSource); +// +// // 路径处理 +// //tsFile.setWorkPath(processingPath(tsFile.getWorkPath(), tsFile.getNodeId())); +// // tsFile.setBackupPath(processingPath(tsFile.getBackupPath(), tsFile.getNodeId())); +// +// // 设置MD5字段（即使不满足条件也保留字段） +// tsFile.setLocatMd5(localMD5); +// tsFile.setMinioMd5(minioMD5); +// +// // 返回是否满足过滤条件 +// return StringUtils.isNoneEmpty(localMD5, minioMD5) && !localMD5.equals(minioMD5); +// } catch (Exception e) { +// LOGGER.error("MD5计算失败: {}", tsFile.getFileName(), e); +// return false; +// } +// }) +// .collect(Collectors.toList()); + // ================ 3. 构建新的分页结果 ================ Page resultPage = new Page<>(); @@ -3824,13 +3881,13 @@ public class TsFilesServiceImpl extends ServiceImpl impl }); totalProcessed += batchList.size(); - LOGGER.debug("已处理: {}/{}", totalProcessed, batchList.size()); +// LOGGER.debug("已处理: {}/{}", totalProcessed, batchList.size()); } // 4. 批量更新路径 long updateStart = System.currentTimeMillis(); updateBackupPaths(taskId, nodeId); - LOGGER.info("批量更新耗时: {}ms", System.currentTimeMillis() - updateStart); +// LOGGER.info("批量更新耗时: {}ms", System.currentTimeMillis() - updateStart); // 5. 性能监控 long totalTime = System.currentTimeMillis() - startTime;