zhengsl/WholeProcessPlatform
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix: 优化导入速度慢问题

Browse Source
This commit is contained in:
tangwei 2026-05-07 18:24:18 +08:00
parent 12b9190ccc
commit 3a944aaf2a
4 changed files with 126 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
backend/src/main/java/com/yfd/platform/data/controller/FishDraftDataController.java
View File

@ -5,9 +5,12 @@ import java.io.FileInputStream;
import java.io.OutputStream;
import java.net.URLEncoder;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Base64;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import cn.hutool.core.io.FileUtil;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
@ -353,22 +356,27 @@ public class FishDraftDataController {
if (file == null || file.isEmpty()) {
return ResponseResult.error("请上传文件");
}
log.info("开始上传文件");
String fileName = file.getOriginalFilename();
if (fileName == null || (!fileName.endsWith(".zip"))) {
return ResponseResult.error("请上传ZIP文件(.zip)");
}
log.info("开始处理文件");
String uploadUserId = SecurityUtils.getUserId();
if (importTaskService.hasImportingTask(uploadUserId)) {
return ResponseResult.error("您有正在进行的导入任务，请等待完成后重试");
}
log.info("开始保存导入任务");
String importNo = "IMP" + System.currentTimeMillis();
String taskId = UUID.randomUUID().toString();
try {
String baseTempDir = ZipFileUtil.getDefaultTempDir();
String taskDirName = "zip_" + UUID.randomUUID().toString().substring(0, 8);
Path tempDirPath = Paths.get(baseTempDir, taskDirName);
Files.createDirectories(tempDirPath);
File savedZipFile = new File(tempDirPath.toFile(), "upload.zip");
file.transferTo(savedZipFile);
log.info("ZIP文件已保存到: {}", savedZipFile.getAbsolutePath());
ImportTask task = new ImportTask();
task.setId(taskId);
task.setImportNo(importNo);
@ -378,33 +386,43 @@ public class FishDraftDataController {
task.setStatus("UPLOADED");
task.setUploadUserId(uploadUserId);
task.setUploadTime(new Date());
log.info("保存导入任务成功");
task.setTempDir(tempDirPath.toString());
importTaskService.save(task);
log.info("导入任务已创建: {}", taskId);
log.info("开始保存文件");
FishImportRequest request = new FishImportRequest();
request.setImportNo(importNo);
request.setUploadUserId(uploadUserId);
request.setBizType("FISH");
CompletableFuture.runAsync(() -> {
try {
log.info("异步开始解析ZIP文件, taskId: {}", taskId);
FishImportResult result = fishImportService.parseAndMapZipFromFile(
savedZipFile, tempDirPath.toString(), uploadUserId);
result.setTaskId(taskId);
String status = "VALIDATED";
if ("1".equals(result.getCode())) {
status = "FAILED";
}
importTaskService.updateStatus(taskId, status, result.getTempDir(), null);
importTaskService.updateProgress(taskId, result.getTotalCount(),
result.getSuccessCount(), result.getFailedCount());
String resultJson = objectMapper.writeValueAsString(result);
importTaskService.saveResultJson(taskId, resultJson);
log.info("异步解析完成, taskId: {}, 状态: {}", taskId, status);
} catch (Exception e) {
log.error("异步解析ZIP失败, taskId: {}", taskId, e);
importTaskService.markFailed(taskId, "导入失败: " + e.getMessage());
} finally {
if (savedZipFile.exists()) {
savedZipFile.delete();
}
}
});
FishImportResult result = fishImportService.parseAndMapZip(file, uploadUserId);
result.setTaskId(taskId);
String status = "VALIDATED";
if ("1".equals(result.getCode())) {
status = "FAILED";
}
importTaskService.updateStatus(taskId, status, result.getTempDir(), null);
importTaskService.updateProgress(taskId, result.getTotalCount(), result.getSuccessCount(), result.getFailedCount());
try {
String resultJson = objectMapper.writeValueAsString(result);
importTaskService.saveResultJson(taskId, resultJson);
} catch (Exception e) {
// 忽略JSON序列化错误，不影响主流程
e.printStackTrace();
}
return ResponseResult.successData(result);
Map<String, String> response = new HashMap<>();
response.put("taskId", taskId);
response.put("importNo", importNo);
response.put("status", "UPLOADED");
return ResponseResult.successData(response);
} catch (Exception e) {
log.error("创建导入任务失败", e);
importTaskService.markFailed(taskId, "导入失败: " + e.getMessage());
return ResponseResult.error("导入失败: " + e.getMessage());
}

7
backend/src/main/java/com/yfd/platform/data/service/IFishImportService.java
View File

@ -5,6 +5,7 @@ import com.yfd.platform.data.domain.FishImportResult;
import com.yfd.platform.data.utils.ZipFileUtil;
import org.springframework.web.multipart.MultipartFile;
import java.io.File;
import java.io.InputStream;
/**
@ -14,12 +15,14 @@ import java.io.InputStream;
*/
public interface IFishImportService {
FishImportResult parseAndMapExcel(FishImportRequest request, InputStream inputStream);
FishImportResult parseAndMapExcel(FishImportRequest request, InputStream inputStream,String userId);
FishImportResult parseAndMapExcelFromPath(FishImportRequest request);
FishImportResult parseAndMapExcelFromPath(FishImportRequest request,String userId);
FishImportResult parseAndMapZip(MultipartFile file, String uploadUserId);
FishImportResult parseAndMapZipFromFile(File zipFile, String tempDir, String uploadUserId);
String resolveStationCode(String stationName);
String resolveFpssCode(String name);

56
backend/src/main/java/com/yfd/platform/data/service/impl/FishImportServiceImpl.java
View File

@ -23,6 +23,7 @@ import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import org.springframework.web.multipart.MultipartFile;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
@ -130,12 +131,12 @@ public class FishImportServiceImpl implements IFishImportService {
}
@Override
public FishImportResult parseAndMapExcel(FishImportRequest request, InputStream inputStream) {
public FishImportResult parseAndMapExcel(FishImportRequest request, InputStream inputStream,String userId) {
FishImportResult result = new FishImportResult();
try (Workbook workbook = new XSSFWorkbook(inputStream)) {
Sheet sheet = workbook.getSheetAt(0);
return parseSheet(sheet, result);
return parseSheet(sheet, result,userId);
} catch (IOException e) {
result.setSummary("Excel文件解析失败: " + e.getMessage());
return result;
@ -143,20 +144,20 @@ public class FishImportServiceImpl implements IFishImportService {
}
@Override
public FishImportResult parseAndMapExcelFromPath(FishImportRequest request) {
public FishImportResult parseAndMapExcelFromPath(FishImportRequest request,String userId) {
FishImportResult result = new FishImportResult();
try (InputStream inputStream = request.getFilePath().startsWith("http")
? new java.net.URL(request.getFilePath()).openStream()
: new java.io.FileInputStream(request.getFilePath())) {
return parseAndMapExcel(request, inputStream);
return parseAndMapExcel(request, inputStream,userId);
} catch (IOException e) {
result.setSummary("读取文件失败: " + e.getMessage());
return result;
}
}
private FishImportResult parseSheet(Sheet sheet, FishImportResult result) {
private FishImportResult parseSheet(Sheet sheet, FishImportResult result, String uploadUserId) {
loadStationAndBaseCache();
Row headerRow = sheet.getRow(0);
@ -174,7 +175,7 @@ public class FishImportServiceImpl implements IFishImportService {
continue;
}
FishImportResult.FishImportRow importRow = parseRow(result, row, columnIndexMap, i);
FishImportResult.FishImportRow importRow = parseRow(result, row, columnIndexMap, i,uploadUserId);
result.setTotalCount(result.getTotalCount() + 1);
if (importRow.getData() != null && importRow.getWarnings().isEmpty()) {
result.addSuccessRow(importRow);
@ -190,11 +191,10 @@ public class FishImportServiceImpl implements IFishImportService {
return result;
}
private FishImportResult.FishImportRow parseRow(FishImportResult result, Row row, Map<Integer, String> columnIndexMap, int rowIndex) {
private FishImportResult.FishImportRow parseRow(FishImportResult result, Row row, Map<Integer, String> columnIndexMap, int rowIndex,String userId) {
FishImportResult.FishImportRow importRow = new FishImportResult.FishImportRow(rowIndex);
FishDraftData data = new FishDraftData();
data.setId(UUID.randomUUID().toString());
String userId = SecurityUtils.getUserId();
Set<String> allowedHbrvcdSet = new HashSet<>();
Set<String> directStcdSet = new HashSet<>();
Set<String> directBHSet = new HashSet<>();
@ -1199,7 +1199,7 @@ public class FishImportServiceImpl implements IFishImportService {
result.setVideoFiles(zipContent.videos);
try (Workbook workbook = new XSSFWorkbook(new FileInputStream(zipContent.excelFilePath))) {
Sheet sheet = workbook.getSheetAt(0);
result = parseSheet(sheet, result);
result = parseSheet(sheet, result,uploadUserId);
}
result.setExcelFileName(zipContent.excelFileName);
result.setExcelFilePath(zipContent.excelFilePath);
@ -1222,6 +1222,44 @@ public class FishImportServiceImpl implements IFishImportService {
}
}
@Override
public FishImportResult parseAndMapZipFromFile(File zipFile, String tempDir, String uploadUserId) {
FishImportResult result = new FishImportResult();
try {
log.info("开始异步解析ZIP文件...");
ZipFileUtil.ZipContent zipContent = ZipFileUtil.extractFromSavedZipFile(zipFile, tempDir);
if (zipContent.excelFilePath == null) {
result.setSummary("ZIP文件中未找到Excel文件");
result.setCode("1");
result.setMessage("ZIP文件中未找到Excel文件");
return result;
}
result.setTempDir(zipContent.tempDir);
result.setImageFiles(zipContent.images);
result.setVideoFiles(zipContent.videos);
try (Workbook workbook = new XSSFWorkbook(new FileInputStream(zipContent.excelFilePath))) {
Sheet sheet = workbook.getSheetAt(0);
result = parseSheet(sheet, result,uploadUserId);
}
result.setExcelFileName(zipContent.excelFileName);
result.setExcelFilePath(zipContent.excelFilePath);
log.info("ZIP文件异步解析完成");
result.setSummary(result.getSummary() + String.format("\nZIP内容: 发现%d张图片, %d个视频",
zipContent.images.size(), zipContent.videos.size()));
return result;
} catch (Exception e) {
log.error("ZIP文件异步解析失败", e);
result.setSummary("ZIP文件解析失败: " + e.getMessage());
result.setCode("1");
result.setMessage("ZIP文件解析失败");
return result;
}
}
@Override
public void processAttachments(FishImportResult result, ZipFileUtil.ZipContent zipContent) {
for (FishImportResult.FishImportRow importRow : result.getRows()) {

59
backend/src/main/java/com/yfd/platform/data/utils/ZipFileUtil.java
View File

@ -135,35 +135,7 @@ public class ZipFileUtil {
file.transferTo(zipFile);
try {
log.info("--------------isValidZipFile------------");
if (!isValidZipFile(zipFile)) {
log.info("--------------文件不是有效的ZIP格式或已损坏------------");
throw new IOException("文件不是有效的ZIP格式或已损坏");
}
IOException lastException = null;
Charset[] charsets = new Charset[]{
StandardCharsets.UTF_8,
Charset.forName("GBK"),
StandardCharsets.ISO_8859_1,
Charset.forName("GB18030")
};
for (Charset charset : charsets) {
try {
content = extractExcelWithRelatedFiles(zipFile, tempDirPath.toFile(), charset);
content.tempDir = tempDirPath.toString();
return content;
} catch (IOException e) {
lastException = e;
content = new ZipContent();
content.tempDir = tempDirPath.toString();
}
}
log.error("extractZipToTemp: {}", lastException.getMessage());
throw lastException != null ? lastException : new IOException("无法解析ZIP文件");
return extractFromSavedZipFile(zipFile, tempDirPath.toString());
} finally {
if (zipFile.exists()) {
zipFile.delete();
@ -171,6 +143,35 @@ public class ZipFileUtil {
}
}
public static ZipContent extractFromSavedZipFile(File zipFile, String tempDir) throws IOException {
if (!isValidZipFile(zipFile)) {
log.info("--------------文件不是有效的ZIP格式或已损坏------------");
throw new IOException("文件不是有效的ZIP格式或已损坏");
}
IOException lastException = null;
Charset[] charsets = new Charset[]{
StandardCharsets.UTF_8,
Charset.forName("GBK"),
StandardCharsets.ISO_8859_1,
Charset.forName("GB18030")
};
for (Charset charset : charsets) {
try {
ZipContent content = extractExcelWithRelatedFiles(zipFile, new File(tempDir), charset);
content.tempDir = tempDir;
return content;
} catch (IOException e) {
lastException = e;
log.warn("使用编码 {} 解析失败: {}", charset, e.getMessage());
}
}
log.error("extractFromSavedZipFile: {}", lastException != null ? lastException.getMessage() : "未知错误");
throw lastException != null ? lastException : new IOException("无法解析ZIP文件");
}
private static boolean isValidZipFile(File file) {
if (file == null || file.length() < 4) {
return false;