fix: 优化到文件上传逻辑
This commit is contained in:
tangwei
parent
85434d4d33
commit
006b544b04
@ -34,6 +34,7 @@ import com.yfd.platform.data.service.IFishStatisticsService;
|
||||
import com.yfd.platform.data.service.IImportTaskService;
|
||||
import com.yfd.platform.data.utils.ZipFileUtil;
|
||||
import com.yfd.platform.utils.KendoUtil;
|
||||
import com.yfd.platform.utils.MultipartStreamParser;
|
||||
import com.yfd.platform.utils.SecurityUtils;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
@ -48,7 +49,6 @@ import org.springframework.security.core.context.SecurityContextHolder;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.util.StringUtils;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.*;
|
||||
@ -450,16 +450,11 @@ public class FishDraftDataController {
|
||||
|
||||
@PostMapping("/importZip")
|
||||
@Operation(summary = "导入ZIP过鱼数据（每个用户同时只能进行一次导入）")
|
||||
public ResponseResult importZip(@RequestParam("file") MultipartFile file) {
|
||||
public ResponseResult importZip(HttpServletRequest request) {
|
||||
log.info("开始导入ZIP文件");
|
||||
if (file == null || file.isEmpty()) {
|
||||
return ResponseResult.error("请上传文件");
|
||||
}
|
||||
String fileName = file.getOriginalFilename();
|
||||
if (fileName == null || (!fileName.endsWith(".zip"))) {
|
||||
return ResponseResult.error("请上传ZIP文件(.zip)");
|
||||
}
|
||||
|
||||
String uploadUserId = SecurityUtils.getUserId();
|
||||
|
||||
if (importTaskService.hasImportingTask(uploadUserId)) {
|
||||
return ResponseResult.error("您有正在进行的导入任务，请等待完成后重试");
|
||||
}
|
||||
@ -467,52 +462,72 @@ public class FishDraftDataController {
|
||||
String importNo = "IMP" + System.currentTimeMillis();
|
||||
String taskId = UUID.randomUUID().toString();
|
||||
|
||||
ImportTask task = null;
|
||||
Path tempDirPath = null;
|
||||
|
||||
try {
|
||||
String baseTempDir = ZipFileUtil.getDefaultTempDir();
|
||||
String taskDirName = "zip_" + UUID.randomUUID().toString().substring(0, 8);
|
||||
Path tempDirPath = Paths.get(baseTempDir, taskDirName);
|
||||
Files.createDirectories(tempDirPath);
|
||||
File savedZipFile = new File(tempDirPath.toFile(), "upload.zip");
|
||||
file.transferTo(savedZipFile);
|
||||
log.info("ZIP文件已保存到: {}", savedZipFile.getAbsolutePath());
|
||||
tempDirPath = Paths.get(baseTempDir, taskDirName);
|
||||
|
||||
ImportTask task = new ImportTask();
|
||||
task = new ImportTask();
|
||||
task.setId(taskId);
|
||||
task.setImportNo(importNo);
|
||||
task.setBizType("FISH");
|
||||
task.setFileName(fileName);
|
||||
task.setFileSize(file.getSize());
|
||||
task.setStatus("UPLOADED");
|
||||
task.setStatus("UPLOADING");
|
||||
task.setUploadUserId(uploadUserId);
|
||||
task.setUploadTime(new Date());
|
||||
task.setTempDir(tempDirPath.toString());
|
||||
importTaskService.save(task);
|
||||
log.info("导入任务已创建: {}", taskId);
|
||||
log.info("导入任务已预创建: {}, 状态: UPLOADING", taskId);
|
||||
|
||||
MultipartStreamParser.StreamedFile streamedFile =
|
||||
MultipartStreamParser.streamToFile(request, tempDirPath);
|
||||
|
||||
String fileName = streamedFile.getOriginalFileName();
|
||||
if (fileName == null || !fileName.toLowerCase().endsWith(".zip")) {
|
||||
importTaskService.markFailed(taskId, "文件格式错误: 请上传ZIP文件");
|
||||
return ResponseResult.error("请上传ZIP文件(.zip)");
|
||||
}
|
||||
|
||||
File savedZipFile = streamedFile.getTargetFile();
|
||||
long fileSize = streamedFile.getFileSize();
|
||||
log.info("ZIP文件已保存到: {} ({} bytes)", savedZipFile.getAbsolutePath(), fileSize);
|
||||
|
||||
task.setFileName(fileName);
|
||||
task.setFileSize(fileSize);
|
||||
task.setStatus("UPLOADED");
|
||||
importTaskService.updateById(task);
|
||||
log.info("导入任务已更新: {}, 状态: UPLOADED, 文件名: {}", taskId, fileName);
|
||||
|
||||
SecurityContext securityContext = SecurityContextHolder.getContext();
|
||||
Path finalTempDirPath = tempDirPath;
|
||||
CompletableFuture.runAsync(() -> {
|
||||
try {
|
||||
SecurityContextHolder.setContext(securityContext);
|
||||
log.info("异步开始解析ZIP文件, taskId: {}", taskId);
|
||||
|
||||
FishImportResult result = fishImportService.parseAndMapZipFromFile(
|
||||
savedZipFile, tempDirPath.toString(), uploadUserId);
|
||||
savedZipFile, finalTempDirPath.toString(), uploadUserId);
|
||||
result.setTaskId(taskId);
|
||||
String status = "VALIDATED";
|
||||
if ("1".equals(result.getCode())) {
|
||||
status = "FAILED";
|
||||
}
|
||||
|
||||
String status = "1".equals(result.getCode()) ? "FAILED" : "VALIDATED";
|
||||
|
||||
importTaskService.updateStatus(taskId, status, result.getTempDir(), null);
|
||||
importTaskService.updateProgress(taskId, result.getTotalCount(),
|
||||
result.getSuccessCount(), result.getFailedCount());
|
||||
// String resultJson = objectMapper.writeValueAsString(result);
|
||||
importTaskService.saveResultJson(taskId, result);
|
||||
log.info("异步解析完成, taskId: {}, 状态: {}", taskId, status);
|
||||
|
||||
log.info("异步解析完成, taskId: {}, 状态: {}, 成功: {}, 失败: {}",
|
||||
taskId, status, result.getSuccessCount(), result.getFailedCount());
|
||||
} catch (Exception e) {
|
||||
log.error("异步解析ZIP失败, taskId: {}", taskId, e);
|
||||
importTaskService.markFailed(taskId, "导入失败: " + e.getMessage());
|
||||
} finally {
|
||||
SecurityContextHolder.clearContext();
|
||||
if (savedZipFile.exists()) {
|
||||
savedZipFile.delete();
|
||||
boolean deleted = savedZipFile.delete();
|
||||
log.debug("临时ZIP文件删除: {}, 结果: {}", savedZipFile.getAbsolutePath(), deleted);
|
||||
}
|
||||
}
|
||||
});
|
||||
@ -522,9 +537,23 @@ public class FishDraftDataController {
|
||||
response.put("importNo", importNo);
|
||||
response.put("status", "UPLOADED");
|
||||
return ResponseResult.successData(response);
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error("创建导入任务失败", e);
|
||||
importTaskService.markFailed(taskId, "导入失败: " + e.getMessage());
|
||||
log.error("导入ZIP文件失败, taskId: {}", taskId, e);
|
||||
|
||||
if (task != null) {
|
||||
importTaskService.markFailed(taskId, "导入失败: " + e.getMessage());
|
||||
}
|
||||
|
||||
if (tempDirPath != null) {
|
||||
try {
|
||||
cn.hutool.core.io.FileUtil.del(tempDirPath.toFile());
|
||||
log.debug("清理临时目录: {}", tempDirPath);
|
||||
} catch (Exception cleanupEx) {
|
||||
log.warn("清理临时目录失败: {}", tempDirPath, cleanupEx);
|
||||
}
|
||||
}
|
||||
|
||||
return ResponseResult.error("导入失败: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,334 @@
|
||||
package com.yfd.platform.utils;
|
||||
|
||||
import jakarta.servlet.http.HttpServletRequest;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
|
||||
import java.io.*;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
|
||||
/**
|
||||
* 流式 multipart 文件上传解析器。
|
||||
* <p>
|
||||
* 绕过 Spring 的 {@code MultipartResolver}（它会先把整个请求体解析完才调用 Controller），
|
||||
* 直接从 {@link HttpServletRequest#getInputStream()} 边读边写目标文件，
|
||||
* 80MB+ 的大文件也能在 Controller 方法内立即开始接收。
|
||||
* <p>
|
||||
* 前提：{@code spring.servlet.multipart.resolve-lazily=true}
|
||||
*/
|
||||
@Slf4j
|
||||
public final class MultipartStreamParser {
|
||||
|
||||
private static final int BUFFER_SIZE = 8192;
|
||||
|
||||
private MultipartStreamParser() {
|
||||
}
|
||||
|
||||
/**
|
||||
* 从 multipart/form-data 请求中流式提取第一个文件并写入目标目录。
|
||||
*
|
||||
* @param request HTTP 请求（Content-Type: multipart/form-data）
|
||||
* @param targetDir 目标目录（文件将保存为 upload.zip）
|
||||
* @return 解析结果（原始文件名、目标文件、文件大小）
|
||||
*/
|
||||
public static StreamedFile streamToFile(HttpServletRequest request, Path targetDir) throws IOException {
|
||||
long t0 = System.currentTimeMillis();
|
||||
|
||||
String contentType = request.getContentType();
|
||||
if (contentType == null || !contentType.startsWith("multipart/form-data")) {
|
||||
throw new IOException("请求不是 multipart/form-data 类型");
|
||||
}
|
||||
|
||||
String boundary = extractBoundary(contentType);
|
||||
byte[] closingDelim = ("\r\n--" + boundary + "--").getBytes(StandardCharsets.ISO_8859_1);
|
||||
|
||||
InputStream rawInput = new BufferedInputStream(request.getInputStream(), BUFFER_SIZE);
|
||||
|
||||
long t1 = System.currentTimeMillis();
|
||||
String originalFileName = readPartHeaders(rawInput, boundary);
|
||||
long t2 = System.currentTimeMillis();
|
||||
|
||||
Files.createDirectories(targetDir);
|
||||
File targetFile = new File(targetDir.toFile(), "upload.zip");
|
||||
long fileSize = streamBinaryToFile(rawInput, targetFile, closingDelim);
|
||||
long t3 = System.currentTimeMillis();
|
||||
|
||||
log.info("流式接收完成: {} -> {} ({} bytes), 头解析{}ms, 体写入{}ms, 总耗时{}ms",
|
||||
originalFileName, targetFile.getAbsolutePath(), fileSize,
|
||||
t2 - t1, t3 - t2, t3 - t0);
|
||||
return new StreamedFile(originalFileName, targetFile, fileSize);
|
||||
}
|
||||
|
||||
// ==================== 阶段1：读 part 头 ====================
|
||||
|
||||
/**
|
||||
* 跳过第一个 boundary 行，读取该 part 的 HTTP 头，从中提取 filename。
|
||||
*/
|
||||
private static String readPartHeaders(InputStream in, String boundary) throws IOException {
|
||||
String firstBoundary = "--" + boundary;
|
||||
String fileName = null;
|
||||
|
||||
// 跳过第一个 boundary 行
|
||||
String line = readLine(in);
|
||||
if (line == null || !line.equals(firstBoundary)) {
|
||||
// 可能前面有空行或 boundary 带了 \r
|
||||
if (line != null && line.startsWith(firstBoundary)) {
|
||||
// ok
|
||||
} else {
|
||||
throw new IOException("未找到 multipart boundary，收到: " + line);
|
||||
}
|
||||
}
|
||||
|
||||
// 读取 part 头，直到空行
|
||||
while ((line = readLine(in)) != null) {
|
||||
if (line.isEmpty()) {
|
||||
break; // 空行 = 头结束，接下来是二进制数据
|
||||
}
|
||||
if (line.startsWith("Content-Disposition") || line.startsWith("content-disposition")) {
|
||||
fileName = extractFileName(line);
|
||||
}
|
||||
}
|
||||
|
||||
if (fileName == null) {
|
||||
fileName = "unknown.zip";
|
||||
}
|
||||
return fileName;
|
||||
}
|
||||
|
||||
/**
|
||||
* 从 Content-Disposition 头中提取 filename 值。
|
||||
* 格式：form-data; name="file"; filename="xxx.zip"
|
||||
*/
|
||||
static String extractFileName(String headerLine) {
|
||||
for (String part : headerLine.split(";")) {
|
||||
part = part.trim();
|
||||
if (part.startsWith("filename")) {
|
||||
int eq = part.indexOf('=');
|
||||
if (eq >= 0) {
|
||||
String val = part.substring(eq + 1).trim();
|
||||
if (val.startsWith("\"") && val.endsWith("\"")) {
|
||||
val = val.substring(1, val.length() - 1);
|
||||
}
|
||||
|
||||
// 先进行 URL 解码（处理 %E4%B8%AD%E6%96%87 这种编码）
|
||||
try {
|
||||
val = java.net.URLDecoder.decode(val, StandardCharsets.UTF_8.name());
|
||||
} catch (Exception e) {
|
||||
log.debug("URL解码失败，使用原始值: {}", val);
|
||||
}
|
||||
|
||||
// 再进行字符编码转换（处理 GBK/UTF-8 乱码）
|
||||
String decodedFileName = decodeFileName(val);
|
||||
return decodedFileName;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 尝试修复乱码的文件名，支持 GBK 和 UTF-8 编码
|
||||
*/
|
||||
private static String decodeFileName(String rawFileName) {
|
||||
if (rawFileName == null || rawFileName.isEmpty()) {
|
||||
return rawFileName;
|
||||
}
|
||||
|
||||
try {
|
||||
byte[] rawBytes = rawFileName.getBytes(StandardCharsets.ISO_8859_1);
|
||||
|
||||
// 优先尝试 UTF-8 解码（现代浏览器常用）
|
||||
String utf8Decoded = new String(rawBytes, StandardCharsets.UTF_8);
|
||||
if (isValidChinese(utf8Decoded)) {
|
||||
return utf8Decoded;
|
||||
}
|
||||
|
||||
// 再尝试 GBK 解码（Windows/旧版浏览器常用）
|
||||
String gbkDecoded = new String(rawBytes, "GBK");
|
||||
if (isValidChinese(gbkDecoded)) {
|
||||
return gbkDecoded;
|
||||
}
|
||||
|
||||
// 都不包含中文，返回 UTF-8 解码结果
|
||||
return utf8Decoded;
|
||||
} catch (Exception e) {
|
||||
log.warn("文件名解码失败，使用原始值: {}", rawFileName, e);
|
||||
return rawFileName;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 简单判断字符串是否包含有效的中文字符
|
||||
*/
|
||||
private static boolean isValidChinese(String str) {
|
||||
if (str == null || str.isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
for (char c : str.toCharArray()) {
|
||||
// 检查是否在中文 Unicode 范围内
|
||||
if (c >= '\u4e00' && c <= '\u9fa5') {
|
||||
return true;
|
||||
}
|
||||
// 检查中文标点符号范围
|
||||
if (c >= '\u3000' && c <= '\u303f') {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// ==================== 阶段2：流式写二进制 ====================
|
||||
|
||||
/**
|
||||
* 从输入流读取二进制数据写入目标文件，遇到 closingDelim 时停止。
|
||||
* <p>
|
||||
* 使用重叠窗口策略：每次读取一个 chunk，保留末尾 (delimLen-1) 字节
|
||||
* 与下一个 chunk 拼接，防止 boundary 跨 chunk 被截断。
|
||||
*/
|
||||
private static long streamBinaryToFile(InputStream in, File targetFile, byte[] closingDelim) throws IOException {
|
||||
int delimLen = closingDelim.length;
|
||||
int overlap = delimLen - 1;
|
||||
|
||||
byte[] buf = new byte[BUFFER_SIZE];
|
||||
byte[] carry = new byte[overlap]; // 上一轮末尾的重叠字节
|
||||
int carryLen = 0;
|
||||
long totalWritten = 0;
|
||||
|
||||
try (OutputStream out = new BufferedOutputStream(new FileOutputStream(targetFile), BUFFER_SIZE)) {
|
||||
|
||||
while (true) {
|
||||
int n = in.read(buf);
|
||||
if (n == -1) {
|
||||
// 流结束，写出剩余的重叠字节
|
||||
if (carryLen > 0) {
|
||||
out.write(carry, 0, carryLen);
|
||||
totalWritten += carryLen;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
// 将上一轮的重叠字节 + 本轮数据拼接，搜索 closing boundary
|
||||
byte[] searchBuf = new byte[carryLen + n];
|
||||
if (carryLen > 0) {
|
||||
System.arraycopy(carry, 0, searchBuf, 0, carryLen);
|
||||
}
|
||||
System.arraycopy(buf, 0, searchBuf, carryLen, n);
|
||||
|
||||
int boundaryPos = indexOf(searchBuf, closingDelim);
|
||||
if (boundaryPos >= 0) {
|
||||
// 找到 closing boundary，写出之前的数据
|
||||
if (boundaryPos > 0) {
|
||||
out.write(searchBuf, 0, boundaryPos);
|
||||
totalWritten += boundaryPos;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
// 未找到 boundary：写出安全部分（保留末尾 overlap 字节）
|
||||
int safeLen = searchBuf.length - overlap;
|
||||
if (safeLen > 0) {
|
||||
out.write(searchBuf, 0, safeLen);
|
||||
totalWritten += safeLen;
|
||||
// 保留末尾 overlap 字节到 carry
|
||||
System.arraycopy(searchBuf, safeLen, carry, 0, overlap);
|
||||
carryLen = overlap;
|
||||
} else {
|
||||
// 数据量不足 overlap，全部保留
|
||||
System.arraycopy(searchBuf, 0, carry, 0, searchBuf.length);
|
||||
carryLen = searchBuf.length;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return totalWritten;
|
||||
}
|
||||
|
||||
// ==================== 工具方法 ====================
|
||||
|
||||
/**
|
||||
* 从 Content-Type 头中提取 boundary 值。
|
||||
* 格式：multipart/form-data; boundary=----WebKitFormBoundary...
|
||||
*/
|
||||
static String extractBoundary(String contentType) {
|
||||
for (String part : contentType.split(";")) {
|
||||
part = part.trim();
|
||||
if (part.startsWith("boundary=")) {
|
||||
return part.substring("boundary=".length());
|
||||
}
|
||||
}
|
||||
throw new IllegalArgumentException("无法从 Content-Type 提取 boundary: " + contentType);
|
||||
}
|
||||
|
||||
/**
|
||||
* 按行读取（以 \r\n 或 \n 结尾），返回行内容（不含换行符）。
|
||||
*/
|
||||
private static String readLine(InputStream in) throws IOException {
|
||||
ByteArrayOutputStream bos = new ByteArrayOutputStream(256);
|
||||
int prev = -1;
|
||||
int b;
|
||||
while ((b = in.read()) != -1) {
|
||||
if (b == '\n') {
|
||||
// 去掉末尾的 \r
|
||||
byte[] bytes = bos.toByteArray();
|
||||
int len = bytes.length;
|
||||
if (len > 0 && bytes[len - 1] == '\r') {
|
||||
len--;
|
||||
}
|
||||
return new String(bytes, 0, len, StandardCharsets.ISO_8859_1);
|
||||
}
|
||||
bos.write(b);
|
||||
prev = b;
|
||||
}
|
||||
// 流结束
|
||||
if (bos.size() > 0) {
|
||||
return bos.toString(StandardCharsets.ISO_8859_1);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 在 byte 数组中查找子数组的位置（Boyer-Moore 简化版）。
|
||||
*/
|
||||
private static int indexOf(byte[] haystack, byte[] needle) {
|
||||
if (needle.length == 0) return 0;
|
||||
if (haystack.length < needle.length) return -1;
|
||||
|
||||
outer:
|
||||
for (int i = 0; i <= haystack.length - needle.length; i++) {
|
||||
for (int j = 0; j < needle.length; j++) {
|
||||
if (haystack[i + j] != needle[j]) {
|
||||
continue outer;
|
||||
}
|
||||
}
|
||||
return i;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
// ==================== 结果 DTO ====================
|
||||
|
||||
public static class StreamedFile {
|
||||
private final String originalFileName;
|
||||
private final File targetFile;
|
||||
private final long fileSize;
|
||||
|
||||
StreamedFile(String originalFileName, File targetFile, long fileSize) {
|
||||
this.originalFileName = originalFileName;
|
||||
this.targetFile = targetFile;
|
||||
this.fileSize = fileSize;
|
||||
}
|
||||
|
||||
public String getOriginalFileName() {
|
||||
return originalFileName;
|
||||
}
|
||||
|
||||
public File getTargetFile() {
|
||||
return targetFile;
|
||||
}
|
||||
|
||||
public long getFileSize() {
|
||||
return fileSize;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -24,8 +24,11 @@ spring:
|
||||
matching-strategy: ant_path_matcher
|
||||
servlet:
|
||||
multipart:
|
||||
max-file-size: 30MB
|
||||
max-request-size: 100MB
|
||||
max-file-size: 300MB
|
||||
max-request-size: 500MB
|
||||
file-size-threshold: 1KB
|
||||
location: /tmp/upload
|
||||
resolve-lazily: true
|
||||
|
||||
logging:
|
||||
file:
|
||||
|
||||
@ -113,10 +113,9 @@ spring:
|
||||
multipart:
|
||||
max-file-size: 300MB
|
||||
max-request-size: 500MB
|
||||
# 关键：文件超过 1KB 就写入磁盘临时文件，避免内存积压
|
||||
file-size-threshold: 1KB
|
||||
# 指定临时目录（确保 Docker 容器内该目录可写且有空间）
|
||||
location: /tmp/upload
|
||||
resolve-lazily: true
|
||||
logging:
|
||||
file:
|
||||
name: logs/platform-dev.log
|
||||
|
||||
@ -29,6 +29,9 @@ spring:
|
||||
multipart:
|
||||
max-file-size: 300MB
|
||||
max-request-size: 500MB
|
||||
file-size-threshold: 1KB
|
||||
location: /tmp/upload
|
||||
resolve-lazily: true
|
||||
|
||||
logging:
|
||||
file:
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
spring:
|
||||
profiles:
|
||||
active: devtw
|
||||
active: prod
|
||||
|
||||
jasypt:
|
||||
encryptor:
|
||||
|
||||
Loading…
Reference in New Issue
Block a user