ThbTech/FileManage
2
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

1.配置文件增加data目录，放置通用标签文件。2.试验任务扫描，文件入库批量入库，parentid更新用Java内存计算父子关系，分批更新。3.config、templates支持jar文件同级目录放置。

Browse Source
This commit is contained in:
wanxiaoli 2026-01-12 15:20:31 +08:00
parent 2ea468c04c
commit e5c5510918
13 changed files with 180 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
java/src/main/java/com/yfd/platform/modules/common/exception/BizException.java Normal file
View File

@ -0,0 +1,12 @@
package com.yfd.platform.modules.common.exception;
public class BizException extends RuntimeException {
public BizException(String message) {
super(message);
}
public BizException(String message, Throwable cause) {
super(message, cause);
}
}

12
java/src/main/java/com/yfd/platform/modules/experimentalData/controller/CommonItemController.java
View File

@ -3,6 +3,7 @@ package com.yfd.platform.modules.experimentalData.controller;
import com.yfd.platform.modules.experimentalData.dto.ItemReq;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
@ -24,9 +25,10 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
@RestController
@RequestMapping("/api/common-items")
public class CommonItemController {
@Value("${app.data-dir}")
private String appDataDir;
private static final String FILE_NAME = "common_items.json";
private static final String DATA_DIR = "data";
//private static final String DATA_DIR = "data";
private final ObjectMapper objectMapper = new ObjectMapper();
@ -132,8 +134,7 @@ public class CommonItemController {
private Map<String, List<String>> readData() throws IOException {
// 获取项目根目录下的data文件夹路径
String userDir = System.getProperty("user.dir");
Path dataDir = Paths.get(userDir, DATA_DIR);
Path dataDir = Paths.get(appDataDir);
Path filePath = dataDir.resolve(FILE_NAME);
if (!Files.exists(filePath)) {
@ -171,8 +172,7 @@ public class CommonItemController {
}
private void writeData(Map<String, List<String>> data) throws IOException {
String userDir = System.getProperty("user.dir");
Path dataDir = Paths.get(userDir, DATA_DIR);
Path dataDir = Paths.get(appDataDir);
Path filePath = dataDir.resolve(FILE_NAME);
Files.createDirectories(dataDir);

18
java/src/main/java/com/yfd/platform/modules/experimentalData/controller/TsFilesController.java
View File

@ -10,6 +10,7 @@ import com.yfd.platform.annotation.Log;
import com.yfd.platform.component.ExtractTaskStatus;
import com.yfd.platform.component.TaskStatusHolder;
import com.yfd.platform.config.ResponseResult;
import com.yfd.platform.modules.common.exception.BizException;
import com.yfd.platform.modules.experimentalData.domain.*;
import com.yfd.platform.modules.experimentalData.service.ITsFilesService;
import com.yfd.platform.modules.experimentalData.service.ITsTaskService;
@ -826,22 +827,29 @@ public class TsFilesController {
@Log(module = "实验数据管理", value = "拆分文件接口！")
@PostMapping("/splitFile")
@ApiOperation("解压缩接口")
public ResponseResult splitFile(String id,String taskId,@RequestParam(required = false) MultipartFile jsonFile) {
public ResponseResult splitFile(@RequestParam("id") String id,@RequestParam("taskId") String taskId,@RequestParam(required = false) MultipartFile jsonFile) {
File jsonFileOnDisk = null;
try {
if (StrUtil.isBlank(id) ) {
return ResponseResult.error("参数为空");
// 参数检查
if (id == null || id.isEmpty()) {
return ResponseResult.error("id不能为空");
}
if (jsonFile != null) {
if (taskId == null || taskId.isEmpty()) {
return ResponseResult.error("taskId不能为空");
}
if (jsonFile != null && !jsonFile.isEmpty()) {
jsonFileOnDisk = File.createTempFile("convertConfFile", ".json");
jsonFile.transferTo(jsonFileOnDisk);
}
Map<String, TsFiles> mapTsfiles = tsFilesService.splitFile(id,taskId,jsonFileOnDisk);
return ResponseResult.successData(mapTsfiles);
} catch (BizException | IllegalArgumentException e) {
//System.out.print("拆分文件异常原因" + e);
return ResponseResult.error(e.getMessage());
} catch (Exception e) {
System.out.print("拆分文件异常原因" + e);
return ResponseResult.error("拆分文件失败");
return ResponseResult.error("系统错误: " + e.getMessage());
}
}

11
java/src/main/java/com/yfd/platform/modules/experimentalData/controller/TsNodesController.java
View File

@ -20,6 +20,7 @@ import io.swagger.annotations.ApiOperation;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.security.authentication.UsernamePasswordAuthenticationToken;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.web.bind.annotation.*;
@ -169,8 +170,14 @@ public class TsNodesController {
} else if ("COMPLETED".equals(existingStatus)) {
return ResponseResult.success("试验数据扫描任务已完成!");
}
UsernamePasswordAuthenticationToken authentication = (UsernamePasswordAuthenticationToken) SecurityContextHolder.getContext().getAuthentication();
LoginUser loginuser = (LoginUser) authentication.getPrincipal();
Authentication authentication = SecurityContextHolder.getContext().getAuthentication();
if (!(authentication instanceof UsernamePasswordAuthenticationToken)) {
return ResponseResult.error("未登录，无法启动试验数据扫描任务");
}
LoginUser loginuser =
(LoginUser) ((UsernamePasswordAuthenticationToken) authentication).getPrincipal();
// 原子性启动新任务
if (taskStatusHolder.startTaskIfAbsent(asyncKey)) {
// 直接异步执行并推送结果

10
java/src/main/java/com/yfd/platform/modules/experimentalData/mapper/TsFilesMapper.java
View File

@ -24,4 +24,14 @@ public interface TsFilesMapper extends BaseMapper<TsFiles> {
void updateTsFileByPath(@Param("taskId") String taskId,@Param("oldBasePath") String oldBasePath,@Param("newBasePath") String newBasePath);
/**
* 查询某任务、某节点的所有文件
*/
List<TsFiles> selectByTaskAndNode(String taskId, String nodeId);
/**
* 批量更新 parent_id
*/
int updateParentIdBatch(List<TsFiles> list);
}

5
java/src/main/java/com/yfd/platform/modules/experimentalData/service/ITsFilesService.java
View File

@ -275,4 +275,9 @@ public interface ITsFilesService extends IService<TsFiles> {
List<TsFiles> getByTaskId(String taskId);
Map<String, TsFiles> splitFile(String id, String taskId, File jsonFile) throws Exception;
/**
* 批量计算并更新 parent_id
*/
int updateParentId(String taskId, String nodeId);
}

67
java/src/main/java/com/yfd/platform/modules/experimentalData/service/InsFileConvertNewService.java
View File

@ -1,6 +1,7 @@
package com.yfd.platform.modules.experimentalData.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.yfd.platform.modules.common.exception.BizException;
import com.yfd.platform.modules.experimentalData.config.DownsampleConfig;
import com.yfd.platform.modules.experimentalData.config.InsConvertConfig;
import com.yfd.platform.modules.experimentalData.config.OutputConfig;
@ -66,7 +67,7 @@ public class InsFileConvertNewService {
} else if (insHeaderLine.contains(",")) {
insDelimiter = ",";
} else {
throw new RuntimeException("无法识别 INS 文件的分隔符");
throw new RuntimeException("未能识别源数据文件分隔符，请确认文件使用逗号（,）或制表符（\\t）分隔。");
}
// 构建列名索引
@ -146,15 +147,24 @@ public class InsFileConvertNewService {
/** 加载 JSON 配置，客户自定义优先，否则默认 */
private InsConvertConfig loadConfig(File jsonConfigFile) throws IOException {
ObjectMapper mapper = new ObjectMapper();
if (jsonConfigFile != null) {
// 1. 显式传入的配置文件（优先级最高）
if (jsonConfigFile != null && jsonConfigFile.exists()) {
log.info("加载客户自定义 JSON 配置：{}", jsonConfigFile.getAbsolutePath());
return mapper.readValue(jsonConfigFile, InsConvertConfig.class);
} else {
log.info("加载默认 JSON 配置：config/ins-convert-default.json");
ClassPathResource res = new ClassPathResource("config/ins-convert-default.json");
try (InputStreamReader in = new InputStreamReader(res.getInputStream(), UTF8)) {
return mapper.readValue(in, InsConvertConfig.class);
}
}
// 2. 尝试 jar 同级 config 目录
File external = new File("config/ins-convert-default.json");
if (external.exists()) {
log.info("加载 jar 同级 JSON 配置：{}", external.getAbsolutePath());
return mapper.readValue(external, InsConvertConfig.class);
}
// 3. 回退到 jar 内 classpath
log.info("加载内置默认 JSON 配置：classpath:config/ins-convert-default.json");
ClassPathResource res = new ClassPathResource("config/ins-convert-default.json");
try (InputStreamReader in = new InputStreamReader(res.getInputStream(), UTF8)) {
return mapper.readValue(in, InsConvertConfig.class);
}
}
@ -191,7 +201,7 @@ public class InsFileConvertNewService {
for (String col : required) {
if (!insHeader.containsKey(col)) {
throw new IllegalArgumentException("INS 缺少必需字段：" + col);
throw new BizException("源数据文件 缺少必须字段：" + col);
}
}
}
@ -204,12 +214,17 @@ public class InsFileConvertNewService {
for (OutputConfig out : outputs) {
// 1.读取模板文件表头
ClassPathResource tpl = new ClassPathResource(out.getTemplateResource());
// ClassPathResource tpl = new ClassPathResource(out.getTemplateResource());
// String headerLine;
// try (BufferedReader r = new BufferedReader(new InputStreamReader(tpl.getInputStream(), UTF8))) {
// headerLine = r.readLine();
// }
String headerLine;
try (BufferedReader r = new BufferedReader(new InputStreamReader(tpl.getInputStream(), UTF8))) {
try (BufferedReader r = openTemplateReader(out.getTemplateResource())) {
headerLine = r.readLine();
}
if (headerLine == null || headerLine.isEmpty()) {
throw new RuntimeException("模板文件为空：" + out.getTemplateResource());
}
@ -221,8 +236,12 @@ public class InsFileConvertNewService {
// ---------- 2. 输出规则列校验 ----------
for (String col : out.getRules().keySet()) {
if (!headerIndex.containsKey(col)) {
throw new IllegalStateException(
"规则列不存在于模板：" + col + " 模板文件：" + out.getTemplateResource());
throw new BizException(
String.format(
"输出模板文件 [%s] 中未找到规则列 [%s]。",
out.getTemplateResource(),
col
));
}
}
@ -255,6 +274,22 @@ public class InsFileConvertNewService {
return map;
}
private BufferedReader openTemplateReader(String templatePath) throws IOException {
// 1. jar 同级文件（相对当前工作目录）
File external = new File(templatePath);
if (external.exists()) {
log.info("加载 jar 同级模板文件：{}", external.getAbsolutePath());
return Files.newBufferedReader(external.toPath(), UTF8);
}
// 2. classpath 内模板
log.info("加载内置模板文件：classpath:{}", templatePath);
ClassPathResource res = new ClassPathResource(templatePath);
return new BufferedReader(new InputStreamReader(res.getInputStream(), UTF8));
}
// ===== 新增：初始化降采样步长 =====
private void initDownsample(Map<String, OutputContext> outputs) {
for (OutputContext ctx : outputs.values()) {
@ -262,8 +297,8 @@ public class InsFileConvertNewService {
if (ds != null && ds.isEnabled()) {
if (ds.getOutputInterval() <= 0 || ds.getInputDelta() <= 0) {
throw new IllegalArgumentException(
"Invalid downsample config for output: " + ctx.name
throw new BizException(
String.format("错误：输出 [%s] 的降采样（inputDelta/outputInterval）配置无效。", ctx.name)
);
}
@ -296,7 +331,7 @@ public class InsFileConvertNewService {
try {
InsFileConvertNewService service = new InsFileConvertNewService();
File insFile = new File("E:\\yun\\20260101_20260130_甘肃兰州_载机名称一_v1_v2\\飞行批次1\\txt\\ins_frameSimu_0.txt"); // 修改为实际路径
File insFile = new File("D:/data/test.txt"); // 修改为实际路径
File jsonFile = new File("D:/data/ins-convert-2.json");
// File jsonFile = new File("D:/data/customer-ins-convert.json"); // 客户自定义 JSON

36
java/src/main/java/com/yfd/platform/modules/experimentalData/service/impl/TsFilesServiceImpl.java
View File

@ -152,6 +152,8 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
private RedisTemplate<String, Object> redisTemplate;
private TaskMessage taskMessage;
private static final int BATCH_SIZE = 5000;
/**********************************
* 用途说明: 分页查询试验数据管理-文档内容
* 参数说明
@ -6447,6 +6449,40 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
}
/**
* 批量计算并更新 parent_id
*/
public int updateParentId(String taskId, String nodeId) {
// 1. 查询所有文件和文件夹
List<TsFiles> allFiles = tsFilesMapper.selectByTaskAndNode(taskId, nodeId);
// 2. 构建父路径 -> id 映射
Map<String, String> pathToId = allFiles.stream()
.collect(Collectors.toMap(
f -> f.getWorkPath() + f.getFileName() + "/", // 父路径
TsFiles::getId,
(existing, replacement) -> existing // 避免重复 key
));
// 3. 设置 parent_id
for (TsFiles f : allFiles) {
String parentId = pathToId.getOrDefault(f.getWorkPath(), "00"); // 顶级为 00
f.setParentId(parentId);
}
// 4. 分批更新并累加返回值
int totalUpdated = 0;
for (int i = 0; i < allFiles.size(); i += BATCH_SIZE) {
List<TsFiles> subList = allFiles.subList(i, Math.min(i + BATCH_SIZE, allFiles.size()));
int updated = tsFilesMapper.updateParentIdBatch(subList);
totalUpdated += updated;
}
// 5. 返回总更新行数
return totalUpdated;
}
}

4
java/src/main/java/com/yfd/platform/modules/experimentalData/service/impl/TsNodesServiceImpl.java
View File

@ -1136,7 +1136,9 @@ public class TsNodesServiceImpl extends ServiceImpl<TsNodesMapper, TsNodes> impl
// 记录开始时间
long startTimeFiles = System.currentTimeMillis();
// 执行更新操作 taskId, String nodeId
int affectedLevelFilesRows = tsFilesMapper.updateParentIdByPathHierarchy(taskId, nodeId);
// int affectedLevelFilesRows = tsFilesMapper.updateParentIdByPathHierarchy(taskId, nodeId);
//在Java内存中计算父子关系——避免上面方法中SQL自连接+字符串拼接
int affectedLevelFilesRows = tsFilesService.updateParentId(taskId, nodeId);
// 记录结束时间
long endTimeFiles = System.currentTimeMillis();
// 计算耗时

3
java/src/main/resources/application-dev.yml
View File

@ -96,7 +96,8 @@ ip:
file-space: #项目文档空间
system: D:\file\system\ #单独上传的文件
app:
data-dir: E:\projectJava\FileManage\data
# 文件预览大小
file-system:
preview:

3
java/src/main/resources/application-prod.yml
View File

@ -84,6 +84,9 @@ ip:
file-space: #项目文档空间
system: /data/local-data/ #单独上传的文件
app:
data-dir: E:\projectJava\FileManage\data
file-system:
preview:
text:

2
java/src/main/resources/config/ins-convert-default.json
View File

@ -7,7 +7,7 @@
"extension": ".txt",
"downsample": {
"enabled": true,
"inputDelta": 0.005,
"inputDelta": 0.05,
"outputInterval": 0.1
},
"rules": {

29
java/src/main/resources/mapper/experimentalData/TsFilesMapper.xml
View File

@ -50,4 +50,33 @@
task_id = #{taskId}
AND work_path LIKE CONCAT(#{oldBasePath}, '%')
</update>
<!-- 查询某任务某节点的所有文件 -->
<select id="selectByTaskAndNode" resultType="com.yfd.platform.modules.experimentalData.domain.TsFiles">
SELECT id, work_path, file_name
FROM ts_files
WHERE task_id = #{taskId} AND node_id = #{nodeId}
</select>
<!-- 批量更新 parent_id -->
<!-- <update id="updateParentIdBatch">-->
<!-- <foreach collection="list" item="item" separator=" ">-->
<!-- UPDATE ts_files-->
<!-- SET parent_id = #{item.parentId}-->
<!-- WHERE id = #{item.id};-->
<!-- </foreach>-->
<!-- </update>-->
<update id="updateParentIdBatch">
UPDATE ts_files
SET parent_id = CASE id
<foreach collection="list" item="item" separator=" ">
WHEN #{item.id} THEN #{item.parentId}
</foreach>
END
WHERE id IN
<foreach collection="list" item="item" open="(" separator="," close=")">
#{item.id}
</foreach>
</update>
</mapper>