From 006b544b047dd34e1bfe2adfc43d33122f7be847 Mon Sep 17 00:00:00 2001 From: tangwei Date: Wed, 13 May 2026 18:12:49 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E4=BC=98=E5=8C=96=E5=88=B0=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E4=B8=8A=E4=BC=A0=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controller/FishDraftDataController.java | 87 +++-- .../platform/utils/MultipartStreamParser.java | 334 ++++++++++++++++++ .../src/main/resources/application-dev.yml | 7 +- .../src/main/resources/application-prod.yml | 3 +- .../src/main/resources/application-server.yml | 3 + backend/src/main/resources/application.yml | 2 +- 6 files changed, 402 insertions(+), 34 deletions(-) create mode 100644 backend/src/main/java/com/yfd/platform/utils/MultipartStreamParser.java diff --git a/backend/src/main/java/com/yfd/platform/data/controller/FishDraftDataController.java b/backend/src/main/java/com/yfd/platform/data/controller/FishDraftDataController.java index 539ff86..d637aed 100644 --- a/backend/src/main/java/com/yfd/platform/data/controller/FishDraftDataController.java +++ b/backend/src/main/java/com/yfd/platform/data/controller/FishDraftDataController.java @@ -34,6 +34,7 @@ import com.yfd.platform.data.service.IFishStatisticsService; import com.yfd.platform.data.service.IImportTaskService; import com.yfd.platform.data.utils.ZipFileUtil; import com.yfd.platform.utils.KendoUtil; +import com.yfd.platform.utils.MultipartStreamParser; import com.yfd.platform.utils.SecurityUtils; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.tags.Tag; @@ -48,7 +49,6 @@ import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.transaction.annotation.Transactional; import org.springframework.util.StringUtils; import org.springframework.web.bind.annotation.*; -import org.springframework.web.multipart.MultipartFile; import java.io.IOException; import java.util.*; @@ -450,16 +450,11 @@ public class FishDraftDataController { @PostMapping("/importZip") @Operation(summary = "导入ZIP过鱼数据（每个用户同时只能进行一次导入）") - public ResponseResult importZip(@RequestParam("file") MultipartFile file) { + public ResponseResult importZip(HttpServletRequest request) { log.info("开始导入ZIP文件"); - if (file == null || file.isEmpty()) { - return ResponseResult.error("请上传文件"); - } - String fileName = file.getOriginalFilename(); - if (fileName == null || (!fileName.endsWith(".zip"))) { - return ResponseResult.error("请上传ZIP文件(.zip)"); - } + String uploadUserId = SecurityUtils.getUserId(); + if (importTaskService.hasImportingTask(uploadUserId)) { return ResponseResult.error("您有正在进行的导入任务，请等待完成后重试"); } @@ -467,52 +462,72 @@ public class FishDraftDataController { String importNo = "IMP" + System.currentTimeMillis(); String taskId = UUID.randomUUID().toString(); + ImportTask task = null; + Path tempDirPath = null; + try { String baseTempDir = ZipFileUtil.getDefaultTempDir(); String taskDirName = "zip_" + UUID.randomUUID().toString().substring(0, 8); - Path tempDirPath = Paths.get(baseTempDir, taskDirName); - Files.createDirectories(tempDirPath); - File savedZipFile = new File(tempDirPath.toFile(), "upload.zip"); - file.transferTo(savedZipFile); - log.info("ZIP文件已保存到: {}", savedZipFile.getAbsolutePath()); + tempDirPath = Paths.get(baseTempDir, taskDirName); - ImportTask task = new ImportTask(); + task = new ImportTask(); task.setId(taskId); task.setImportNo(importNo); task.setBizType("FISH"); - task.setFileName(fileName); - task.setFileSize(file.getSize()); - task.setStatus("UPLOADED"); + task.setStatus("UPLOADING"); task.setUploadUserId(uploadUserId); task.setUploadTime(new Date()); task.setTempDir(tempDirPath.toString()); importTaskService.save(task); - log.info("导入任务已创建: {}", taskId); + log.info("导入任务已预创建: {}, 状态: UPLOADING", taskId); + + MultipartStreamParser.StreamedFile streamedFile = + MultipartStreamParser.streamToFile(request, tempDirPath); + + String fileName = streamedFile.getOriginalFileName(); + if (fileName == null || !fileName.toLowerCase().endsWith(".zip")) { + importTaskService.markFailed(taskId, "文件格式错误: 请上传ZIP文件"); + return ResponseResult.error("请上传ZIP文件(.zip)"); + } + + File savedZipFile = streamedFile.getTargetFile(); + long fileSize = streamedFile.getFileSize(); + log.info("ZIP文件已保存到: {} ({} bytes)", savedZipFile.getAbsolutePath(), fileSize); + + task.setFileName(fileName); + task.setFileSize(fileSize); + task.setStatus("UPLOADED"); + importTaskService.updateById(task); + log.info("导入任务已更新: {}, 状态: UPLOADED, 文件名: {}", taskId, fileName); + SecurityContext securityContext = SecurityContextHolder.getContext(); + Path finalTempDirPath = tempDirPath; CompletableFuture.runAsync(() -> { try { SecurityContextHolder.setContext(securityContext); log.info("异步开始解析ZIP文件, taskId: {}", taskId); + FishImportResult result = fishImportService.parseAndMapZipFromFile( - savedZipFile, tempDirPath.toString(), uploadUserId); + savedZipFile, finalTempDirPath.toString(), uploadUserId); result.setTaskId(taskId); - String status = "VALIDATED"; - if ("1".equals(result.getCode())) { - status = "FAILED"; - } + + String status = "1".equals(result.getCode()) ? "FAILED" : "VALIDATED"; + importTaskService.updateStatus(taskId, status, result.getTempDir(), null); importTaskService.updateProgress(taskId, result.getTotalCount(), result.getSuccessCount(), result.getFailedCount()); -// String resultJson = objectMapper.writeValueAsString(result); importTaskService.saveResultJson(taskId, result); - log.info("异步解析完成, taskId: {}, 状态: {}", taskId, status); + + log.info("异步解析完成, taskId: {}, 状态: {}, 成功: {}, 失败: {}", + taskId, status, result.getSuccessCount(), result.getFailedCount()); } catch (Exception e) { log.error("异步解析ZIP失败, taskId: {}", taskId, e); importTaskService.markFailed(taskId, "导入失败: " + e.getMessage()); } finally { SecurityContextHolder.clearContext(); if (savedZipFile.exists()) { - savedZipFile.delete(); + boolean deleted = savedZipFile.delete(); + log.debug("临时ZIP文件删除: {}, 结果: {}", savedZipFile.getAbsolutePath(), deleted); } } }); @@ -522,9 +537,23 @@ public class FishDraftDataController { response.put("importNo", importNo); response.put("status", "UPLOADED"); return ResponseResult.successData(response); + } catch (Exception e) { - log.error("创建导入任务失败", e); - importTaskService.markFailed(taskId, "导入失败: " + e.getMessage()); + log.error("导入ZIP文件失败, taskId: {}", taskId, e); + + if (task != null) { + importTaskService.markFailed(taskId, "导入失败: " + e.getMessage()); + } + + if (tempDirPath != null) { + try { + cn.hutool.core.io.FileUtil.del(tempDirPath.toFile()); + log.debug("清理临时目录: {}", tempDirPath); + } catch (Exception cleanupEx) { + log.warn("清理临时目录失败: {}", tempDirPath, cleanupEx); + } + } + return ResponseResult.error("导入失败: " + e.getMessage()); } } diff --git a/backend/src/main/java/com/yfd/platform/utils/MultipartStreamParser.java b/backend/src/main/java/com/yfd/platform/utils/MultipartStreamParser.java new file mode 100644 index 0000000..62876a9 --- /dev/null +++ b/backend/src/main/java/com/yfd/platform/utils/MultipartStreamParser.java @@ -0,0 +1,334 @@ +package com.yfd.platform.utils; + +import jakarta.servlet.http.HttpServletRequest; +import lombok.extern.slf4j.Slf4j; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * 流式 multipart 文件上传解析器。 + *

+ * 绕过 Spring 的 {@code MultipartResolver}（它会先把整个请求体解析完才调用 Controller）， + * 直接从 {@link HttpServletRequest#getInputStream()} 边读边写目标文件， + * 80MB+ 的大文件也能在 Controller 方法内立即开始接收。 + *

+ * 前提：{@code spring.servlet.multipart.resolve-lazily=true} + */ +@Slf4j +public final class MultipartStreamParser { + + private static final int BUFFER_SIZE = 8192; + + private MultipartStreamParser() { + } + + /** + * 从 multipart/form-data 请求中流式提取第一个文件并写入目标目录。 + * + * @param request HTTP 请求（Content-Type: multipart/form-data） + * @param targetDir 目标目录（文件将保存为 upload.zip） + * @return 解析结果（原始文件名、目标文件、文件大小） + */ + public static StreamedFile streamToFile(HttpServletRequest request, Path targetDir) throws IOException { + long t0 = System.currentTimeMillis(); + + String contentType = request.getContentType(); + if (contentType == null || !contentType.startsWith("multipart/form-data")) { + throw new IOException("请求不是 multipart/form-data 类型"); + } + + String boundary = extractBoundary(contentType); + byte[] closingDelim = ("\r\n--" + boundary + "--").getBytes(StandardCharsets.ISO_8859_1); + + InputStream rawInput = new BufferedInputStream(request.getInputStream(), BUFFER_SIZE); + + long t1 = System.currentTimeMillis(); + String originalFileName = readPartHeaders(rawInput, boundary); + long t2 = System.currentTimeMillis(); + + Files.createDirectories(targetDir); + File targetFile = new File(targetDir.toFile(), "upload.zip"); + long fileSize = streamBinaryToFile(rawInput, targetFile, closingDelim); + long t3 = System.currentTimeMillis(); + + log.info("流式接收完成: {} -> {} ({} bytes), 头解析{}ms, 体写入{}ms, 总耗时{}ms", + originalFileName, targetFile.getAbsolutePath(), fileSize, + t2 - t1, t3 - t2, t3 - t0); + return new StreamedFile(originalFileName, targetFile, fileSize); + } + + // ==================== 阶段1：读 part 头 ==================== + + /** + * 跳过第一个 boundary 行，读取该 part 的 HTTP 头，从中提取 filename。 + */ + private static String readPartHeaders(InputStream in, String boundary) throws IOException { + String firstBoundary = "--" + boundary; + String fileName = null; + + // 跳过第一个 boundary 行 + String line = readLine(in); + if (line == null || !line.equals(firstBoundary)) { + // 可能前面有空行或 boundary 带了 \r + if (line != null && line.startsWith(firstBoundary)) { + // ok + } else { + throw new IOException("未找到 multipart boundary，收到: " + line); + } + } + + // 读取 part 头，直到空行 + while ((line = readLine(in)) != null) { + if (line.isEmpty()) { + break; // 空行 = 头结束，接下来是二进制数据 + } + if (line.startsWith("Content-Disposition") || line.startsWith("content-disposition")) { + fileName = extractFileName(line); + } + } + + if (fileName == null) { + fileName = "unknown.zip"; + } + return fileName; + } + + /** + * 从 Content-Disposition 头中提取 filename 值。 + * 格式：form-data; name="file"; filename="xxx.zip" + */ + static String extractFileName(String headerLine) { + for (String part : headerLine.split(";")) { + part = part.trim(); + if (part.startsWith("filename")) { + int eq = part.indexOf('='); + if (eq >= 0) { + String val = part.substring(eq + 1).trim(); + if (val.startsWith("\"") && val.endsWith("\"")) { + val = val.substring(1, val.length() - 1); + } + + // 先进行 URL 解码（处理 %E4%B8%AD%E6%96%87 这种编码） + try { + val = java.net.URLDecoder.decode(val, StandardCharsets.UTF_8.name()); + } catch (Exception e) { + log.debug("URL解码失败，使用原始值: {}", val); + } + + // 再进行字符编码转换（处理 GBK/UTF-8 乱码） + String decodedFileName = decodeFileName(val); + return decodedFileName; + } + } + } + return null; + } + + /** + * 尝试修复乱码的文件名，支持 GBK 和 UTF-8 编码 + */ + private static String decodeFileName(String rawFileName) { + if (rawFileName == null || rawFileName.isEmpty()) { + return rawFileName; + } + + try { + byte[] rawBytes = rawFileName.getBytes(StandardCharsets.ISO_8859_1); + + // 优先尝试 UTF-8 解码（现代浏览器常用） + String utf8Decoded = new String(rawBytes, StandardCharsets.UTF_8); + if (isValidChinese(utf8Decoded)) { + return utf8Decoded; + } + + // 再尝试 GBK 解码（Windows/旧版浏览器常用） + String gbkDecoded = new String(rawBytes, "GBK"); + if (isValidChinese(gbkDecoded)) { + return gbkDecoded; + } + + // 都不包含中文，返回 UTF-8 解码结果 + return utf8Decoded; + } catch (Exception e) { + log.warn("文件名解码失败，使用原始值: {}", rawFileName, e); + return rawFileName; + } + } + + /** + * 简单判断字符串是否包含有效的中文字符 + */ + private static boolean isValidChinese(String str) { + if (str == null || str.isEmpty()) { + return false; + } + for (char c : str.toCharArray()) { + // 检查是否在中文 Unicode 范围内 + if (c >= '\u4e00' && c <= '\u9fa5') { + return true; + } + // 检查中文标点符号范围 + if (c >= '\u3000' && c <= '\u303f') { + return true; + } + } + return false; + } + + // ==================== 阶段2：流式写二进制 ==================== + + /** + * 从输入流读取二进制数据写入目标文件，遇到 closingDelim 时停止。 + *

+ * 使用重叠窗口策略：每次读取一个 chunk，保留末尾 (delimLen-1) 字节 + * 与下一个 chunk 拼接，防止 boundary 跨 chunk 被截断。 + */ + private static long streamBinaryToFile(InputStream in, File targetFile, byte[] closingDelim) throws IOException { + int delimLen = closingDelim.length; + int overlap = delimLen - 1; + + byte[] buf = new byte[BUFFER_SIZE]; + byte[] carry = new byte[overlap]; // 上一轮末尾的重叠字节 + int carryLen = 0; + long totalWritten = 0; + + try (OutputStream out = new BufferedOutputStream(new FileOutputStream(targetFile), BUFFER_SIZE)) { + + while (true) { + int n = in.read(buf); + if (n == -1) { + // 流结束，写出剩余的重叠字节 + if (carryLen > 0) { + out.write(carry, 0, carryLen); + totalWritten += carryLen; + } + break; + } + + // 将上一轮的重叠字节 + 本轮数据拼接，搜索 closing boundary + byte[] searchBuf = new byte[carryLen + n]; + if (carryLen > 0) { + System.arraycopy(carry, 0, searchBuf, 0, carryLen); + } + System.arraycopy(buf, 0, searchBuf, carryLen, n); + + int boundaryPos = indexOf(searchBuf, closingDelim); + if (boundaryPos >= 0) { + // 找到 closing boundary，写出之前的数据 + if (boundaryPos > 0) { + out.write(searchBuf, 0, boundaryPos); + totalWritten += boundaryPos; + } + break; + } + + // 未找到 boundary：写出安全部分（保留末尾 overlap 字节） + int safeLen = searchBuf.length - overlap; + if (safeLen > 0) { + out.write(searchBuf, 0, safeLen); + totalWritten += safeLen; + // 保留末尾 overlap 字节到 carry + System.arraycopy(searchBuf, safeLen, carry, 0, overlap); + carryLen = overlap; + } else { + // 数据量不足 overlap，全部保留 + System.arraycopy(searchBuf, 0, carry, 0, searchBuf.length); + carryLen = searchBuf.length; + } + } + } + + return totalWritten; + } + + // ==================== 工具方法 ==================== + + /** + * 从 Content-Type 头中提取 boundary 值。 + * 格式：multipart/form-data; boundary=----WebKitFormBoundary... + */ + static String extractBoundary(String contentType) { + for (String part : contentType.split(";")) { + part = part.trim(); + if (part.startsWith("boundary=")) { + return part.substring("boundary=".length()); + } + } + throw new IllegalArgumentException("无法从 Content-Type 提取 boundary: " + contentType); + } + + /** + * 按行读取（以 \r\n 或 \n 结尾），返回行内容（不含换行符）。 + */ + private static String readLine(InputStream in) throws IOException { + ByteArrayOutputStream bos = new ByteArrayOutputStream(256); + int prev = -1; + int b; + while ((b = in.read()) != -1) { + if (b == '\n') { + // 去掉末尾的 \r + byte[] bytes = bos.toByteArray(); + int len = bytes.length; + if (len > 0 && bytes[len - 1] == '\r') { + len--; + } + return new String(bytes, 0, len, StandardCharsets.ISO_8859_1); + } + bos.write(b); + prev = b; + } + // 流结束 + if (bos.size() > 0) { + return bos.toString(StandardCharsets.ISO_8859_1); + } + return null; + } + + /** + * 在 byte 数组中查找子数组的位置（Boyer-Moore 简化版）。 + */ + private static int indexOf(byte[] haystack, byte[] needle) { + if (needle.length == 0) return 0; + if (haystack.length < needle.length) return -1; + + outer: + for (int i = 0; i <= haystack.length - needle.length; i++) { + for (int j = 0; j < needle.length; j++) { + if (haystack[i + j] != needle[j]) { + continue outer; + } + } + return i; + } + return -1; + } + + // ==================== 结果 DTO ==================== + + public static class StreamedFile { + private final String originalFileName; + private final File targetFile; + private final long fileSize; + + StreamedFile(String originalFileName, File targetFile, long fileSize) { + this.originalFileName = originalFileName; + this.targetFile = targetFile; + this.fileSize = fileSize; + } + + public String getOriginalFileName() { + return originalFileName; + } + + public File getTargetFile() { + return targetFile; + } + + public long getFileSize() { + return fileSize; + } + } +} \ No newline at end of file diff --git a/backend/src/main/resources/application-dev.yml b/backend/src/main/resources/application-dev.yml index 1df2467..0b53e07 100644 --- a/backend/src/main/resources/application-dev.yml +++ b/backend/src/main/resources/application-dev.yml @@ -24,8 +24,11 @@ spring: matching-strategy: ant_path_matcher servlet: multipart: - max-file-size: 30MB - max-request-size: 100MB + max-file-size: 300MB + max-request-size: 500MB + file-size-threshold: 1KB + location: /tmp/upload + resolve-lazily: true logging: file: diff --git a/backend/src/main/resources/application-prod.yml b/backend/src/main/resources/application-prod.yml index 9c441c1..8520bc6 100644 --- a/backend/src/main/resources/application-prod.yml +++ b/backend/src/main/resources/application-prod.yml @@ -113,10 +113,9 @@ spring: multipart: max-file-size: 300MB max-request-size: 500MB - # 关键：文件超过 1KB 就写入磁盘临时文件，避免内存积压 file-size-threshold: 1KB - # 指定临时目录（确保 Docker 容器内该目录可写且有空间） location: /tmp/upload + resolve-lazily: true logging: file: name: logs/platform-dev.log diff --git a/backend/src/main/resources/application-server.yml b/backend/src/main/resources/application-server.yml index b03a5b5..6d5f3aa 100644 --- a/backend/src/main/resources/application-server.yml +++ b/backend/src/main/resources/application-server.yml @@ -29,6 +29,9 @@ spring: multipart: max-file-size: 300MB max-request-size: 500MB + file-size-threshold: 1KB + location: /tmp/upload + resolve-lazily: true logging: file: diff --git a/backend/src/main/resources/application.yml b/backend/src/main/resources/application.yml index d8e6c7e..59a6a5c 100644 --- a/backend/src/main/resources/application.yml +++ b/backend/src/main/resources/application.yml @@ -1,6 +1,6 @@ spring: profiles: - active: devtw + active: prod jasypt: encryptor: