提交更新代码0313

This commit is contained in:
lilin 2025-03-13 11:23:06 +08:00
parent 104572a40a
commit 6063376a9a
6 changed files with 355 additions and 194 deletions

View File

@ -171,11 +171,11 @@ public class TsFilesController {
@PostMapping("/deleteTsFilesByIds") @PostMapping("/deleteTsFilesByIds")
@ApiOperation("批量删除试验数据管理文档内容") @ApiOperation("批量删除试验数据管理文档内容")
@PreAuthorize("@el.check('del:tsFiles')") @PreAuthorize("@el.check('del:tsFiles')")
public ResponseResult deleteTsFilesByIds(@RequestParam String ids, @RequestParam String type) { public ResponseResult deleteTsFilesByIds(@RequestParam String id, @RequestParam String type) {
if (StrUtil.isBlank(ids)) { if (StrUtil.isBlank(id)) {
return ResponseResult.error("参数为空"); return ResponseResult.error("参数为空");
} }
String[] splitIds = ids.split(","); String[] splitIds = id.split(",");
// 数组转集合 // 数组转集合
List<String> dataset = Arrays.asList(splitIds); List<String> dataset = Arrays.asList(splitIds);
return ResponseResult.success(tsFilesService.deleteTsFilesByIds(dataset, type)); return ResponseResult.success(tsFilesService.deleteTsFilesByIds(dataset, type));

View File

@ -6,6 +6,9 @@ import cn.hutool.core.collection.CollUtil;
import java.nio.file.*; import java.nio.file.*;
import java.security.MessageDigest; import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException; import java.security.NoSuchAlgorithmException;
import java.text.SimpleDateFormat;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*; import java.util.*;
import java.util.regex.Pattern; import java.util.regex.Pattern;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@ -44,6 +47,7 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.annotation.Transactional;
import javax.annotation.Resource; import javax.annotation.Resource;
import javax.xml.crypto.Data;
import java.io.*; import java.io.*;
import java.sql.Timestamp; import java.sql.Timestamp;
import java.time.LocalDateTime; import java.time.LocalDateTime;
@ -176,6 +180,9 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
//准备获取文件的信息 //准备获取文件的信息
AbstractBaseFileService<?> fileService = storageSourceContext.getByStorageKey("local"); AbstractBaseFileService<?> fileService = storageSourceContext.getByStorageKey("local");
FileItemResult fileItemResult = fileService.getFileItem(path); FileItemResult fileItemResult = fileService.getFileItem(path);
if (fileItemResult == null || fileItemResult.getName() == null) {
LOGGER.error("{}文件没有上传到工作空间,请重新选择上传", fileNameData);
}
tsFiles.setUrl(fileItemResult.getUrl()); tsFiles.setUrl(fileItemResult.getUrl());
//如果是压缩文件 类型就给zip //如果是压缩文件 类型就给zip
boolean isValid = hasValidExtension(fileItemResult.getName()); boolean isValid = hasValidExtension(fileItemResult.getName());
@ -199,7 +206,7 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
return false; // 如果传入的文件名为空返回 false return false; // 如果传入的文件名为空返回 false
} }
// 判断文件名后缀是否是 .zip .tar // 判断文件名后缀是否是 .zip .tar
return name.endsWith(".zip") || name.endsWith(".tar") || name.endsWith(".rar")|| name.endsWith(".xz")|| name.endsWith(".tar.gz")|| name.endsWith(".gz")|| name.endsWith(".bz2"); return name.endsWith(".zip") || name.endsWith(".tar") || name.endsWith(".rar") || name.endsWith(".xz") || name.endsWith(".tar.gz") || name.endsWith(".gz") || name.endsWith(".bz2");
} }
// 递归获取所有子节点ID包含隔代子节点 // 递归获取所有子节点ID包含隔代子节点
@ -278,6 +285,14 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
String name = names.get(i).trim(); String name = names.get(i).trim();
String sizeStr = sizes.get(i).trim(); String sizeStr = sizes.get(i).trim();
String pathAndName = tsFiles.getWorkPath() + name;
//准备获取文件的信息
AbstractBaseFileService<?> fileService = storageSourceContext.getByStorageKey("local");
FileItemResult fileItemResult = fileService.getFileItem(pathAndName);
if (fileItemResult == null || fileItemResult.getName() == null) {
return ResponseResult.error(name + "文件没有上传到工作空间,请重新选择上传!");
}
//通过节点ID 任务ID 路径 上级ID 文件名称 查询是否重复 //通过节点ID 任务ID 路径 上级ID 文件名称 查询是否重复
QueryWrapper<TsFiles> queryWrapper = new QueryWrapper<>(); QueryWrapper<TsFiles> queryWrapper = new QueryWrapper<>();
queryWrapper.eq("node_id", tsFiles.getNodeId()); queryWrapper.eq("node_id", tsFiles.getNodeId());
@ -634,7 +649,11 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
BatchDeleteRequest.DeleteItem deleteItemData = new BatchDeleteRequest.DeleteItem(); BatchDeleteRequest.DeleteItem deleteItemData = new BatchDeleteRequest.DeleteItem();
deleteItemData.setName(files.getFileName()); deleteItemData.setName(files.getFileName());
deleteItemData.setPassword(""); deleteItemData.setPassword("");
if ("local".equals(type)) {
deleteItemData.setPath(files.getWorkPath()); deleteItemData.setPath(files.getWorkPath());
} else {
deleteItemData.setPath(files.getBackupPath());
}
deleteItemData.setType(FileTypeEnum.FILE); deleteItemData.setType(FileTypeEnum.FILE);
deleteItemList.add(deleteItemData); deleteItemList.add(deleteItemData);
//首先通过ID集合查询所有的内容 然后放到List<DeleteItem> deleteItems里面 放好以后删除数据库 然后删除minio //首先通过ID集合查询所有的内容 然后放到List<DeleteItem> deleteItems里面 放好以后删除数据库 然后删除minio
@ -767,6 +786,7 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
public int deleteFiles(String id, String type) { public int deleteFiles(String id, String type) {
int deletedCount = 0; int deletedCount = 0;
TsFiles tsFilesData = tsFilesMapper.selectById(id); TsFiles tsFilesData = tsFilesMapper.selectById(id);
if (tsFilesData != null) {
//判断是删除本地还是minio //判断是删除本地还是minio
if ("local".equals(type)) { if ("local".equals(type)) {
//删除本地的时候判断minio路径是否为空 如果为空直接删除 如果不为空把workPath修改成空 //删除本地的时候判断minio路径是否为空 如果为空直接删除 如果不为空把workPath修改成空
@ -795,6 +815,7 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
} }
} }
}
return deletedCount; return deletedCount;
} }
@ -1123,6 +1144,7 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
} }
return "解压存储成功"; return "解压存储成功";
} catch (Exception e) { } catch (Exception e) {
LOGGER.error("解压失败: " + e.getMessage());
throw new RuntimeException("操作失败: " + e.getMessage()); throw new RuntimeException("操作失败: " + e.getMessage());
} }
} }
@ -1273,14 +1295,6 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
} }
} }
/**
* 路径标准化确保以/结尾
*/
private String buildFolderPathq(String path) {
path = path.replaceAll("/+", "/"); // 合并多个斜杠
return path.startsWith("/") ? path : "/" + path; // 确保绝对路径
}
// 文件夹路径标准化确保以/结尾 // 文件夹路径标准化确保以/结尾
private String buildFolderPath(String path) { private String buildFolderPath(String path) {
return path.endsWith("/") ? path : path + "/"; return path.endsWith("/") ? path : path + "/";
@ -1489,6 +1503,7 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
try { try {
processFileLists(tsFile, fileItemListMinio, fileItemListLocal); processFileLists(tsFile, fileItemListMinio, fileItemListLocal);
} catch (Exception e) { } catch (Exception e) {
LOGGER.error("通过nodeId和taskId 获取本地和minio列表失败: {}", e.getMessage(), e);
e.printStackTrace(); e.printStackTrace();
} }
}); });
@ -1500,6 +1515,7 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
try { try {
processFileLists(tsFile, fileItemListMinio, fileItemListLocal); processFileLists(tsFile, fileItemListMinio, fileItemListLocal);
} catch (Exception e) { } catch (Exception e) {
LOGGER.error("通过ID 获取本地和minio列表失败: {}", e.getMessage(), e);
e.printStackTrace(); e.printStackTrace();
} }
}); });
@ -1537,18 +1553,29 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
} }
// 辅助方法处理文件列表并行安全 // 辅助方法处理文件列表并行安全
private void processFileLists(TsFiles tsFile, private void processFileLists(TsFiles tsFile, List<FileItemResult> minioList, List<FileItemResult> localList) throws Exception {
List<FileItemResult> minioList, processFileList(tsFile.getBackupPath(), tsFile.getFileName(), "minio", minioList, tsFile.getIsFile());
List<FileItemResult> localList) throws Exception { processFileList(tsFile.getWorkPath(), tsFile.getFileName(), "local", localList, tsFile.getIsFile());
processFileList(tsFile.getBackupPath(), tsFile.getFileName(), "minio", minioList);
processFileList(tsFile.getWorkPath(), tsFile.getFileName(), "local", localList);
} }
// 辅助方法获取文件列表并添加到集合 // 辅助方法获取文件列表并添加到集合
private void processFileList(String path, String fileName, String storageKey, private void processFileList(String path, String fileName, String storageKey, List<FileItemResult> targetList, String isFile) throws Exception {
List<FileItemResult> targetList) throws Exception {
if (StringUtils.isNotEmpty(path)) { if (StringUtils.isNotEmpty(path)) {
AbstractBaseFileService<?> service = storageSourceContext.getByStorageKey(storageKey); AbstractBaseFileService<?> service = storageSourceContext.getByStorageKey(storageKey);
if (isFile.equals("FOLDER") && storageKey.equals("minio")) {
List<FileItemResult> files = service.fileListData(path, fileName + "/");
if (files != null) {
// 对每个文件的路径进行规范化
files.forEach(file -> {
String normalizedPath = ensurePathFormat(file.getPath());
file.setPath(normalizedPath);
});
// 同步添加线程安全
synchronized (targetList) {
targetList.addAll(files);
}
}
} else {
List<FileItemResult> files = service.fileListData(path, fileName); List<FileItemResult> files = service.fileListData(path, fileName);
if (files != null) { if (files != null) {
// 对每个文件的路径进行规范化 // 对每个文件的路径进行规范化
@ -1563,6 +1590,7 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
} }
} }
} }
}
/** /**
@ -1617,10 +1645,12 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
// 检查独有文件并行过滤 // 检查独有文件并行过滤
List<FileItemResult> localOnly = localFiles.parallelStream() List<FileItemResult> localOnly = localFiles.parallelStream()
.filter(file -> !minioMap.containsKey(generateMapKey(file))) .filter(file -> !minioMap.containsKey(generateMapKey(file)))
.map(file -> formatFileTime(file)) // 格式化时间字段
.collect(Collectors.toList()); .collect(Collectors.toList());
List<FileItemResult> minioOnly = minioFiles.parallelStream() List<FileItemResult> minioOnly = minioFiles.parallelStream()
.filter(file -> !localMap.containsKey(generateMapKey(file))) .filter(file -> !localMap.containsKey(generateMapKey(file)))
.map(file -> formatFileTime(file)) // 格式化时间字段
.collect(Collectors.toList()); .collect(Collectors.toList());
// 检查不一致文件并行处理 // 检查不一致文件并行处理
@ -1646,6 +1676,20 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
return result; return result;
} }
// 格式化 FileItemResult 中的时间字段
private FileItemResult formatFileTime(FileItemResult file) {
// 假设 time 字段是 Date 类型
Date time = file.getTime();
if (time != null) {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String formattedDate = sdf.format(time);
// 设置格式化后的时间到 file
file.setFormattedTime(formattedDate);
}
return file;
}
// 辅助方法生成 Map Key // 辅助方法生成 Map Key
private String generateMapKey(FileItemResult file) { private String generateMapKey(FileItemResult file) {
return normalizePath(file.getPath()) + file.getName(); return normalizePath(file.getPath()) + file.getName();
@ -1653,6 +1697,7 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
// 辅助方法丰富文件元数据 // 辅助方法丰富文件元数据
private FileItemResult enrichFileMetadata(FileItemResult file, String pathType) { private FileItemResult enrichFileMetadata(FileItemResult file, String pathType) {
String aaaa = normalizePath(file.getPath());
TsFiles dbRecord = tsFilesMapper.selectOne( TsFiles dbRecord = tsFilesMapper.selectOne(
new QueryWrapper<TsFiles>() new QueryWrapper<TsFiles>()
.eq("file_name", file.getName()) .eq("file_name", file.getName())
@ -3035,38 +3080,48 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
***********************************/ ***********************************/
@Override @Override
public DualTreeResponse listLocalAndBackup(String taskId, String nodeId) { public DualTreeResponse listLocalAndBackup(String taskId, String nodeId) {
// 1. 查询符合条件的所有节点一次性加载避免递归查询数据库 // 记录方法入参
LOGGER.info("Starting to build dual trees for taskId={}, nodeId={}", taskId, nodeId);
// 1. 批量查询所有相关节点
QueryWrapper<TsFiles> queryWrapper = new QueryWrapper<>(); QueryWrapper<TsFiles> queryWrapper = new QueryWrapper<>();
queryWrapper.eq("task_id", taskId) queryWrapper.eq("task_id", taskId)
.eq("node_id", nodeId); .eq("node_id", nodeId);
List<TsFiles> allNodes = tsFilesMapper.selectList(queryWrapper); List<TsFiles> allNodes = tsFilesMapper.selectList(queryWrapper);
// 2. 构建内存中的父子关系映射提高查询效率 // 2. 构建内存索引提升查询效率
Map<String, List<TsFiles>> parentChildrenMap = allNodes.stream() Map<String, List<TsFiles>> parentChildrenMap = allNodes.stream()
.collect(Collectors.groupingBy(TsFiles::getParentId)); .collect(Collectors.groupingBy(TsFiles::getParentId));
LOGGER.debug("使用{}个条目构建父子映射", parentChildrenMap.size());
// 3. 构建本地树和Minio树 // 3. 双树构建容器
List<TreeDTO> localTrees = new ArrayList<>(); List<TreeDTO> localTrees = new ArrayList<>();
List<TreeDTO> minioTrees = new ArrayList<>(); List<TreeDTO> minioTrees = new ArrayList<>();
// 4. 从顶级节点parentId为00开始构建树 // 4. 从顶级节点parentId为00开始构建树
List<TsFiles> rootNodes = parentChildrenMap.get("00"); List<TsFiles> rootNodes = parentChildrenMap.get("00");
if (rootNodes == null || rootNodes.isEmpty()) {
LOGGER.warn("找不到的根节点 taskId={}, nodeId={}", taskId, nodeId);
return new DualTreeResponse(localTrees, minioTrees);
}
if (rootNodes != null) { if (rootNodes != null) {
for (TsFiles rootNode : rootNodes) { for (TsFiles rootNode : rootNodes) {
// 构建本地树 // 构建本地树
TreeDTO localTree = buildTree(rootNode, parentChildrenMap, true); TreeDTO localTree = buildTree(rootNode, parentChildrenMap, true);
if (localTree != null) { if (localTree != null) {
localTrees.add(localTree); localTrees.add(localTree);
LOGGER.debug("添加了本地树节点: {}", localTree.getId());
} }
// 构建Minio树 // 构建Minio树
TreeDTO minioTree = buildTree(rootNode, parentChildrenMap, false); TreeDTO minioTree = buildTree(rootNode, parentChildrenMap, false);
if (minioTree != null) { if (minioTree != null) {
minioTrees.add(minioTree); minioTrees.add(minioTree);
LOGGER.debug("添加了MINIO树节点: {}", minioTree.getId());
} }
} }
} }
LOGGER.info("Tree construction completed. Local nodes: {}, MinIO nodes: {}", localTrees.size(), minioTrees.size());
return new DualTreeResponse(localTrees, minioTrees); return new DualTreeResponse(localTrees, minioTrees);
} }
@ -3083,6 +3138,7 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
TreeDTO dto = convertToDTO(current, isLocal); TreeDTO dto = convertToDTO(current, isLocal);
String path = isLocal ? current.getWorkPath() : current.getBackupPath(); String path = isLocal ? current.getWorkPath() : current.getBackupPath();
if (dto.getPath() == null || dto.getPath().trim().isEmpty()) { if (dto.getPath() == null || dto.getPath().trim().isEmpty()) {
LOGGER.warn("由于路径为空,跳过节点{}", current.getId());
return null; return null;
} }
@ -3137,6 +3193,7 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
if ("FILE".equals(node.getIsFile())) { if ("FILE".equals(node.getIsFile())) {
AbstractBaseFileService<?> fileService = storageSourceContext.getByStorageKey("local"); AbstractBaseFileService<?> fileService = storageSourceContext.getByStorageKey("local");
FileItemResult fileItemResult = fileService.getFileItem(path); FileItemResult fileItemResult = fileService.getFileItem(path);
if (fileItemResult != null || fileItemResult.getName() != null) {
dto.setUrl(fileItemResult.getUrl()); dto.setUrl(fileItemResult.getUrl());
//如果是压缩文件 类型就给zip //如果是压缩文件 类型就给zip
boolean isValid = hasValidExtension(fileItemResult.getName()); boolean isValid = hasValidExtension(fileItemResult.getName());
@ -3146,6 +3203,8 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
dto.setType(fileItemResult.getType().getValue()); dto.setType(fileItemResult.getType().getValue());
} }
} }
}
} else { } else {
String backupPath = node.getBackupPath(); String backupPath = node.getBackupPath();
if (backupPath == null || StringUtils.isEmpty(backupPath)) { if (backupPath == null || StringUtils.isEmpty(backupPath)) {
@ -3159,6 +3218,7 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
if ("FILE".equals(node.getIsFile())) { if ("FILE".equals(node.getIsFile())) {
AbstractBaseFileService<?> fileService = storageSourceContext.getByStorageKey("minio"); AbstractBaseFileService<?> fileService = storageSourceContext.getByStorageKey("minio");
FileItemResult fileItemResult = fileService.getFileItem(path); FileItemResult fileItemResult = fileService.getFileItem(path);
if (fileItemResult != null || fileItemResult.getName() != null) {
dto.setUrl(fileItemResult.getUrl()); dto.setUrl(fileItemResult.getUrl());
//如果是压缩文件 类型就给zip //如果是压缩文件 类型就给zip
boolean isValid = hasValidExtension(fileItemResult.getName()); boolean isValid = hasValidExtension(fileItemResult.getName());
@ -3169,6 +3229,7 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
} }
} }
} }
}
// 设置路径字段 // 设置路径字段
dto.setPath(isLocal ? node.getWorkPath() : node.getBackupPath()); dto.setPath(isLocal ? node.getWorkPath() : node.getBackupPath());
return dto; return dto;

View File

@ -38,10 +38,11 @@ public class FileItemResult implements Serializable {
@ApiModelProperty(value = "下载地址", example = "http://www.example.com/a.mp4") @ApiModelProperty(value = "下载地址", example = "http://www.example.com/a.mp4")
private String url; private String url;
//用于对比Md5文件 //用于对比Md5文件 用于展示
private String locatMd5; private String locatMd5;
private String minioMd5; private String minioMd5;
private String id; private String id;
private String formattedTime;
/** /**
* 获取路径和名称的组合, 并移除重复的路径分隔符 /. * 获取路径和名称的组合, 并移除重复的路径分隔符 /.

View File

@ -164,106 +164,147 @@ public abstract class AbstractS3BaseFileService<P extends S3BaseParam> extends A
public List<FileItemResult> s3FileListData(String path, String name) { public List<FileItemResult> s3FileListData(String path, String name) {
List<FileItemResult> fileItemList = new ArrayList<>(); List<FileItemResult> fileItemList = new ArrayList<>();
String bucketName = param.getBucketName(); String bucketName = param.getBucketName();
if (path == null || StringUtils.isEmpty(path)) { if (path == null || StringUtils.isEmpty(path)) return fileItemList;
// 规范路径格式
path = ensurePathWithSlash(path);
// 明确判断是否为文件夹仅根据 name 是否以斜杠结尾
boolean isFolder = name.endsWith("/");
String targetPath = buildTargetPath(path, name, isFolder);
if (isFolder) {
// 处理文件夹
String fullPrefix = StringUtils.trimStartSlashes(
StringUtils.concat(param.getBasePath(), targetPath)
);
listFolderContents(bucketName, fullPrefix, targetPath, fileItemList);
} else {
// 处理文件
FileItemResult fileItem = getFileItem(bucketName, targetPath);
if (fileItem != null) fileItemList.add(fileItem);
}
return fileItemList; return fileItemList;
} }
// 确保路径格式正确并拼接目标文件夹名称name /**
path = ensurePathWithSlash(path); // 输入 path="/"处理为 "/" * 构建目标路径确保文件夹以斜杠结尾
String targetPath = path + name + ZFileConstant.PATH_SEPARATOR; // 目标路径/431/ */
String fullPath = StringUtils.trimStartSlashes( private String buildTargetPath(String path, String name, boolean isFolder) {
String targetPath = path + name;
// 如果是文件夹且未以斜杠结尾则补全
if (isFolder && !targetPath.endsWith("/")) {
targetPath += "/";
}
return targetPath;
}
/**
* 获取单个文件信息
*/
private FileItemResult getFileItem(String bucketName, String targetPath) {
String fullKey = StringUtils.trimStartSlashes(
StringUtils.concat(param.getBasePath(), targetPath) StringUtils.concat(param.getBasePath(), targetPath)
); );
// 调用时传入 name=null确保递归时不再匹配其他同名文件夹 try {
listFilesInDirectory(bucketName, fullPath, targetPath, null, fileItemList, false); // 精确匹配文件键
return fileItemList; S3ObjectSummary objectSummary = s3Client.listObjects(bucketName, fullKey)
.getObjectSummaries()
.stream()
.filter(s -> s.getKey().equals(fullKey))
.findFirst()
.orElse(null);
if (objectSummary == null) return null;
FileItemResult item = new FileItemResult();
item.setName(extractFileName(targetPath));
item.setSize(objectSummary.getSize());
item.setTime(objectSummary.getLastModified());
item.setType(FileTypeEnum.FILE);
item.setPath(extractParentPath(targetPath));
item.setUrl(getDownloadUrl(targetPath));
return item;
} catch (Exception e) {
LOGGER.error("获取文件信息失败: {}", fullKey, e);
return null;
}
} }
private void listFilesInDirectory( /**
* 列出文件夹内容包括子目录
*/
private void listFolderContents(
String bucketName, String bucketName,
String fullPath, String prefix,
String path, String currentPath,
String name, List<FileItemResult> results
List<FileItemResult> fileItemList,
boolean includeAll
) { ) {
ListObjectsRequest listObjectsRequest = new ListObjectsRequest() ListObjectsRequest request = new ListObjectsRequest()
.withBucketName(bucketName) .withBucketName(bucketName)
.withPrefix(fullPath) // 关键修复直接通过 Prefix 限定目标路径 .withPrefix(prefix)
.withMaxKeys(1000) .withDelimiter("/") // 使用分隔符获取子目录
.withDelimiter("/"); .withMaxKeys(1000);
ObjectListing objectListing = s3Client.listObjects(listObjectsRequest);
boolean isFirstWhile = true;
ObjectListing listing;
do { do {
if (!isFirstWhile) { listing = s3Client.listObjects(request);
objectListing = s3Client.listNextBatchOfObjects(objectListing);
}
// 处理文件 // 处理文件
for (S3ObjectSummary s : objectListing.getObjectSummaries()) { listing.getObjectSummaries().stream()
if (s.getKey().equals(fullPath)) continue; .filter(s -> !s.getKey().endsWith("/")) // 排除目录标记
.forEach(s -> {
String fileName = s.getKey().substring(prefix.length());
results.add(createFileItem(fileName, s, currentPath));
});
String fileName = s.getKey().substring(fullPath.length()); // 处理子文件夹
if (fileName.startsWith(ZFileConstant.PATH_SEPARATOR)) { listing.getCommonPrefixes().forEach(commonPrefix -> {
fileName = fileName.substring(1); String folderName = commonPrefix.substring(prefix.length(), commonPrefix.length()-1);
String folderPath = currentPath;
results.add(createFolderItem(folderName, folderPath));
String folderPath1 = currentPath + folderName + "/";
// 递归列出子目录内容
listFolderContents(bucketName, commonPrefix, folderPath1, results);
});
request.setMarker(listing.getNextMarker());
} while (listing.isTruncated());
} }
// 包含条件强制包含 名称匹配仅在初始调用时检查 name // 辅助方法
if (includeAll || name == null || StrUtil.isEmpty(name) || fileName.equals(name)) { private FileItemResult createFileItem(String name, S3ObjectSummary summary, String path) {
FileItemResult item = new FileItemResult(); FileItemResult item = new FileItemResult();
item.setName(fileName); item.setName(name);
item.setSize(s.getSize()); item.setSize(summary.getSize());
item.setTime(s.getLastModified()); item.setTime(summary.getLastModified());
item.setType(FileTypeEnum.FILE); item.setType(FileTypeEnum.FILE);
item.setPath(path); item.setPath(path);
item.setUrl(getDownloadUrl(ensurePathWithSlash(path) + fileName)); item.setUrl(getDownloadUrl(path + name));
fileItemList.add(item); return item;
}
} }
// 处理文件夹 private FileItemResult createFolderItem(String name, String path) {
for (String commonPrefix : objectListing.getCommonPrefixes()) { FileItemResult item = new FileItemResult();
String folderName = commonPrefix.substring(fullPath.length(), commonPrefix.length() - 1); item.setName(name);
if (StrUtil.isEmpty(folderName) || folderName.equals(StringUtils.DELIMITER_STR)) continue; item.setType(FileTypeEnum.FOLDER);
item.setPath(path);
// 匹配条件仅在初始调用时检查 name递归时 name=null 直接包含 return item;
boolean matchFolder = includeAll || name == null || StrUtil.isEmpty(name) || folderName.equals(name);
if (matchFolder) {
FileItemResult folderItem = new FileItemResult();
folderItem.setName(folderName);
folderItem.setType(FileTypeEnum.FOLDER);
folderItem.setPath(path);
fileItemList.add(folderItem);
} }
// 递归处理子文件夹时name=null强制包含所有子项 private String extractFileName(String path) {
String subPath = ensurePathWithSlash(path) + folderName + ZFileConstant.PATH_SEPARATOR; return path.contains("/") ?
String subFullPath = commonPrefix; path.substring(path.lastIndexOf('/') + 1) : path;
listFilesInDirectory(
bucketName,
subFullPath,
subPath,
null, // 关键修复递归时不再传递 name避免深层匹配
fileItemList,
matchFolder // 如果父级匹配则强制包含子项
);
} }
isFirstWhile = false; private String extractParentPath(String path) {
} while (objectListing.isTruncated()); return path.contains("/") ?
path.substring(0, path.lastIndexOf('/') + 1) : "/";
} }
// 确保路径以斜杠开头
private String ensurePathWithSlash(String path) { private String ensurePathWithSlash(String path) {
if (path.startsWith(ZFileConstant.PATH_SEPARATOR)) { return path.startsWith("/") ? path : "/" + path;
return path;
} else {
return ZFileConstant.PATH_SEPARATOR + path;
}
} }
//这个方法有用 获取指定路径下的所有的文件以及文件夹 //这个方法有用 获取指定路径下的所有的文件以及文件夹
// public List<FileItemResult> s3FileLists(String path) { // public List<FileItemResult> s3FileLists(String path) {

View File

@ -94,7 +94,8 @@ public class LocalServiceImpl extends AbstractProxyTransferService<LocalParam> {
File file = new File(fullPath); File file = new File(fullPath);
if (!file.exists()) { if (!file.exists()) {
throw ExceptionUtil.wrapRuntime(new FileNotFoundException("文件不存在")); return new FileItemResult();
//throw ExceptionUtil.wrapRuntime(new FileNotFoundException("文件不存在"));
} }
String folderPath = StringUtils.getParentPath(pathAndName); String folderPath = StringUtils.getParentPath(pathAndName);
@ -191,20 +192,22 @@ public class LocalServiceImpl extends AbstractProxyTransferService<LocalParam> {
List<FileItemResult> resultList = new ArrayList<>(); List<FileItemResult> resultList = new ArrayList<>();
String basePath = param.getFilePath(); String basePath = param.getFilePath();
// 1. 构建目标路径确保前后有 / // 1. 构建目标路径动态判断是否为目录
String targetPath = formatCombinedPath(folderPath, name); // 关键修改点 String targetPath = formatCombinedPath(folderPath, name);
// 2. 拼接完整物理路径
String fullPath = Paths.get(basePath, targetPath).toString(); String fullPath = Paths.get(basePath, targetPath).toString();
File targetDir = new File(fullPath); File target = new File(fullPath);
if (!targetDir.exists()) { if (!target.exists()) {
throw new FileNotFoundException("路径不存在: " + fullPath); throw new FileNotFoundException("路径不存在: " + fullPath);
} }
// 3. 列出目录内容 // 2. 直接处理文件或目录
if (targetDir.isDirectory()) { if (target.isDirectory()) {
listFilesInDirectory(targetDir, targetPath, resultList); listFilesInDirectory(target, targetPath, resultList);
} else {
// 如果是文件直接添加到结果列表
FileItemResult item = convertToFileItem(target, folderPath);
resultList.add(item);
} }
return resultList; return resultList;
@ -214,26 +217,58 @@ public class LocalServiceImpl extends AbstractProxyTransferService<LocalParam> {
* 构建规范化路径根路径为 /其他路径为 /path/ * 构建规范化路径根路径为 /其他路径为 /path/
*/ */
private String formatCombinedPath(String folderPath, String name) { private String formatCombinedPath(String folderPath, String name) {
StringBuilder pathBuilder = new StringBuilder(); // 确保 folderPath "/" 开头如果不是则添加 "/"
if (folderPath == null || folderPath.trim().isEmpty()) {
// 处理 folderPath folderPath = "/";
if (StringUtils.isBlank(folderPath) || folderPath.equals("/")) {
pathBuilder.append("/");
} else { } else {
pathBuilder.append(folderPath.trim()); folderPath = folderPath.trim();
if (!folderPath.endsWith("/")) { if (!folderPath.startsWith("/")) {
pathBuilder.append("/"); folderPath = "/" + folderPath;
}
// 去掉 folderPath 末尾的斜杠除非是根路径
if (!folderPath.equals("/") && folderPath.endsWith("/")) {
folderPath = folderPath.substring(0, folderPath.length() - 1);
} }
} }
// 处理 name // 如果 name 非空拼接到 folderPath
if (StringUtils.isNotBlank(name)) { if (name != null && !name.trim().isEmpty()) {
String sanitizedName = name.trim().replaceAll("/+", ""); // 防止注入额外路径 folderPath += "/" + name;
pathBuilder.append(sanitizedName).append("/");
} }
// 最终格式化替换多斜杠为单斜杠 return folderPath;
return pathBuilder.toString().replaceAll("/+", "/");
// StringBuilder pathBuilder = new StringBuilder();
//
// // 处理 folderPath
// if (StringUtils.isBlank(folderPath)) {
// pathBuilder.append("/");
// } else {
// // 确保 folderPath / 开头且不以 / 结尾除非是根路径
// String trimmedPath = folderPath.trim();
// if (!trimmedPath.startsWith("/")) {
// pathBuilder.append("/");
// }
// pathBuilder.append(trimmedPath.replaceAll("/+$", ""));
// if (!trimmedPath.endsWith("/") && !trimmedPath.equals("/")) {
// pathBuilder.append("/");
// }
// }
//
// // 处理 name如果是文件不添加 /
// if (StringUtils.isNotBlank(name)) {
// String sanitizedName = name.trim().replaceAll("/+", ""); // 防止路径注入
// pathBuilder.append(sanitizedName);
//
// // 如果是目录添加 /
// File tempFile = new File(Paths.get(param.getFilePath(), folderPath, sanitizedName).toString());
// if (tempFile.isDirectory()) {
// pathBuilder.append("/");
// }
// }
//
// // 最终格式化合并多斜杠
// return pathBuilder.toString().replaceAll("/+", "/");
} }
/** /**
@ -250,7 +285,7 @@ public class LocalServiceImpl extends AbstractProxyTransferService<LocalParam> {
// 递归处理子目录 // 递归处理子目录
if (file.isDirectory()) { if (file.isDirectory()) {
String newParentPath = parentPath + file.getName() + "/"; String newParentPath = parentPath + "/" + file.getName() + "/";
listFilesInDirectory(file, newParentPath, resultList); listFilesInDirectory(file, newParentPath, resultList);
} }
} }
@ -384,12 +419,31 @@ public class LocalServiceImpl extends AbstractProxyTransferService<LocalParam> {
FileUtil.mkdir(parentPath); FileUtil.mkdir(parentPath);
} }
File uploadToFileObj = new File(uploadPath); // File uploadToFileObj = new File(uploadPath);
BufferedOutputStream outputStream = FileUtil.getOutputStream(uploadToFileObj); // BufferedOutputStream outputStream = FileUtil.getOutputStream(uploadToFileObj);
IoUtil.copy(inputStream, outputStream); // IoUtil.copy(inputStream, outputStream);
IoUtil.close(outputStream); // IoUtil.close(outputStream);
// IoUtil.close(inputStream);
// try (BufferedOutputStream outputStream = FileUtil.getOutputStream(new File(uploadPath))) {
// IoUtil.copy(inputStream, outputStream);
// } catch (IOException e) {
// throw new RuntimeException("文件保存失败", e);
// } finally {
// IoUtil.close(inputStream);
// }
try (BufferedOutputStream outputStream = FileUtil.getOutputStream(new File(uploadPath))) {
byte[] buffer = new byte[8192]; // 8KB 缓冲区
int bytesRead;
while ((bytesRead = inputStream.read(buffer)) != -1) {
outputStream.write(buffer, 0, bytesRead);
}
} catch (IOException e) {
throw new RuntimeException("文件保存失败", e);
} finally {
IoUtil.close(inputStream); IoUtil.close(inputStream);
} }
}
@Override @Override

View File

@ -24,6 +24,10 @@ spring:
multipart: multipart:
max-file-size: 50GB max-file-size: 50GB
max-request-size: 50GB max-request-size: 50GB
tomcat:
max-swallow-size: -1
connection-timeout: 86400000
max-http-form-post-size: -1
logging: logging:
file: file:
name: logs/projectname.log name: logs/projectname.log