ThbTech/FileManage
2
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

文件转换-支持降采样处理

Browse Source
This commit is contained in:
wanxiaoli 2026-01-11 12:51:23 +08:00
parent 26392c6047
commit 0e28431435
7 changed files with 166 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
java/src/main/java/com/yfd/platform/modules/experimentalData/config/DownsampleConfig.java Normal file
View File

@ -0,0 +1,30 @@
package com.yfd.platform.modules.experimentalData.config;
import lombok.Data;
import java.util.Map;
@Data
public class DownsampleConfig {
/** 是否启用 */
private boolean enabled = false;
/** 当前仅支持 UNIFORM（等频） */
//private String mode; // UNIFORM
/** 时间字段名（来自输入行） */
//private String timeColumn;
/** 输入时间步长（秒） */
private double inputDelta;
/** 输出频率（秒） */
private double outputInterval;
/** 是否保留首行 */
//private boolean keepFirst = true;
/** 是否保留末行 */
//private boolean keepLast = true;
}

3
java/src/main/java/com/yfd/platform/modules/experimentalData/config/OutputConfig.java
View File

@ -10,4 +10,7 @@ public class OutputConfig {
private String delimiter = ","; // 分隔符，默认逗号
private String extension = ".txt"; // 文件扩展名，默认 .txt
private Map<String, RuleConfig> rules; // 输出列规则
/** 行级处理策略：降采样 */
private DownsampleConfig downsample;
}

11
java/src/main/java/com/yfd/platform/modules/experimentalData/controller/TsFilesController.java
View File

@ -19,8 +19,10 @@ import io.swagger.annotations.ApiOperation;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import javax.annotation.Resource;
import java.io.File;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.CompletableFuture;
@ -824,12 +826,17 @@ public class TsFilesController {
@Log(module = "实验数据管理", value = "拆分文件接口！")
@PostMapping("/splitFile")
@ApiOperation("解压缩接口")
public ResponseResult splitFile(String id,String taskId) {
public ResponseResult splitFile(String id,String taskId,@RequestParam(required = false) MultipartFile jsonFile) {
File jsonFileOnDisk = null;
try {
if (StrUtil.isBlank(id) ) {
return ResponseResult.error("参数为空");
}
Map<String, TsFiles> mapTsfiles = tsFilesService.splitFile(id,taskId);
if (jsonFile != null) {
jsonFileOnDisk = File.createTempFile("convertConfFile", ".json");
jsonFile.transferTo(jsonFileOnDisk);
}
Map<String, TsFiles> mapTsfiles = tsFilesService.splitFile(id,taskId,jsonFileOnDisk);
return ResponseResult.successData(mapTsfiles);
} catch (Exception e) {

3
java/src/main/java/com/yfd/platform/modules/experimentalData/service/ITsFilesService.java
View File

@ -8,6 +8,7 @@ import com.baomidou.mybatisplus.extension.service.IService;
import com.yfd.platform.modules.storage.model.result.FileItemResult;
import org.springframework.web.bind.annotation.RequestParam;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.security.NoSuchAlgorithmException;
@ -273,5 +274,5 @@ public interface ITsFilesService extends IService<TsFiles> {
*/
List<TsFiles> getByTaskId(String taskId);
Map<String, TsFiles> splitFile(String id, String taskId) throws Exception;
Map<String, TsFiles> splitFile(String id, String taskId, File jsonFile) throws Exception;
}

49
java/src/main/java/com/yfd/platform/modules/experimentalData/service/InsFileConvertNewService.java
View File

@ -1,6 +1,7 @@
package com.yfd.platform.modules.experimentalData.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.yfd.platform.modules.experimentalData.config.DownsampleConfig;
import com.yfd.platform.modules.experimentalData.config.InsConvertConfig;
import com.yfd.platform.modules.experimentalData.config.OutputConfig;
import com.yfd.platform.modules.experimentalData.config.RuleConfig;
@ -39,7 +40,7 @@ public class InsFileConvertNewService {
* @param jsonConfigFile 客户自定义 JSON 配置（可为 null，使用默认）
* @return Map<输出文件标识, 输出文件对象>
*/
public Map<String, File> convert(File insFile, File jsonConfigFile) throws Exception {
public Map<String, String> convert(File insFile, File jsonConfigFile) throws Exception {
long start = System.currentTimeMillis();
@ -83,6 +84,9 @@ public class InsFileConvertNewService {
// 4. 初始化每个输出文件
Map<String, OutputContext> outputs = initOutputs(config.getOutputs(), insFile.getParentFile());
// ===== 新增：初始化降采样 =====
initDownsample(outputs);
// 5. 流式读取 INS 正文
long rowCount = 0;
try (BufferedReader insReader = Files.newBufferedReader(insFile.toPath(), UTF8)) {
@ -93,6 +97,11 @@ public class InsFileConvertNewService {
String[] insValues = line.split(insDelimiter, -1);
for (OutputContext ctx : outputs.values()) {
// ===== 新增：降采样判断 =====
if (ctx.sampleStep > 1 && rowCount % ctx.sampleStep != 0) {
continue;
}
String[] out = new String[ctx.headers.length];
Arrays.fill(out, "0");
@ -125,9 +134,9 @@ public class InsFileConvertNewService {
log.info("INS 转换完成，行数={}, 耗时={}ms", rowCount, System.currentTimeMillis() - start);
Map<String, File> result = new HashMap<>();
Map<String, String> result = new HashMap<>();
for (OutputContext ctx : outputs.values()) {
result.put(ctx.name, ctx.outFile);
result.put(ctx.name, ctx.outFile.getAbsolutePath());
}
return result;
}
@ -238,11 +247,35 @@ public class InsFileConvertNewService {
ctx.delimiter = out.getDelimiter();
ctx.outFile = outFile;
// ===== 新增：绑定降采样配置 =====
ctx.downsample = out.getDownsample();
map.put(ctx.name, ctx);
}
return map;
}
// ===== 新增：初始化降采样步长 =====
private void initDownsample(Map<String, OutputContext> outputs) {
for (OutputContext ctx : outputs.values()) {
DownsampleConfig ds = ctx.downsample;
if (ds != null && ds.isEnabled()) {
if (ds.getOutputInterval() <= 0 || ds.getInputDelta() <= 0) {
throw new IllegalArgumentException(
"Invalid downsample config for output: " + ctx.name
);
}
double ratio = ds.getOutputInterval() / ds.getInputDelta();
int step = (int) Math.round(ratio);
ctx.sampleStep = step <= 1 ? 1 : step;
}
}
}
/** 输出上下文 */
private static class OutputContext {
String name;
@ -252,6 +285,10 @@ public class InsFileConvertNewService {
BufferedWriter writer;
String delimiter;
File outFile;
// ===== 新增（最小）=====
DownsampleConfig downsample;
int sampleStep = 1; // 默认不降采样
}
// =================== 测试 main ===================
@ -259,13 +296,13 @@ public class InsFileConvertNewService {
try {
InsFileConvertNewService service = new InsFileConvertNewService();
File insFile = new File("D:/data/ins_frameSimu_0.txt"); // 修改为实际路径
File insFile = new File("E:\\yun\\20260101_20260130_甘肃兰州_载机名称一_v1_v2\\飞行批次1\\txt\\ins_frameSimu_0.txt"); // 修改为实际路径
File jsonFile = new File("D:/data/ins-convert-2.json");
// File jsonFile = new File("D:/data/customer-ins-convert.json"); // 客户自定义 JSON
Map<String, File> result = service.convert(insFile,jsonFile); // 使用默认配置
Map<String, String> result = service.convert(insFile,null); // 使用默认配置
result.forEach((k, f) -> System.out.println(k + " -> " + f.getAbsolutePath()));
result.forEach((k, f) -> System.out.println(k + " -> " + f));
System.out.println("转换完成！");
} catch (Exception e) {
e.printStackTrace();

75
java/src/main/java/com/yfd/platform/modules/experimentalData/service/impl/TsFilesServiceImpl.java
View File

@ -46,6 +46,7 @@ import com.yfd.platform.modules.experimentalData.mapper.TsFilesMapper;
import com.yfd.platform.modules.experimentalData.mapper.TsTaskMapper;
import com.yfd.platform.modules.experimentalData.service.ITsFilesService;
import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
import com.yfd.platform.modules.experimentalData.service.InsFileConvertNewService;
import com.yfd.platform.modules.experimentalData.service.InsFileConvertService;
import com.yfd.platform.modules.storage.context.StorageSourceContext;
import com.yfd.platform.modules.storage.mapper.StorageSourceConfigMapper;
@ -79,6 +80,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.scheduling.annotation.Async;
import org.springframework.security.authentication.UsernamePasswordAuthenticationToken;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.SecurityContextHolder;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@ -143,6 +145,9 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
@Autowired
private InsFileConvertService insFileConvertService;
@Autowired
private InsFileConvertNewService insFileConvertNewService;
@Autowired
private RedisTemplate<String, Object> redisTemplate;
private TaskMessage taskMessage;
@ -480,9 +485,23 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
List<String> names = Arrays.asList(tsFiles.getFileName().split(","));
List<String> sizes = Arrays.asList(tsFiles.getFileSize().split(","));
// 获取当前登录用户
UsernamePasswordAuthenticationToken authentication =
(UsernamePasswordAuthenticationToken) SecurityContextHolder.getContext().getAuthentication();
LoginUser loginuser = (LoginUser) authentication.getPrincipal();
// UsernamePasswordAuthenticationToken authentication =
// (UsernamePasswordAuthenticationToken) SecurityContextHolder.getContext().getAuthentication();
// LoginUser loginuser = (LoginUser) authentication.getPrincipal();
// ===== 安全获取当前登录用户 =====
Authentication authentication = SecurityContextHolder.getContext().getAuthentication();
LoginUser loginUser;
if (authentication instanceof UsernamePasswordAuthenticationToken) {
UsernamePasswordAuthenticationToken token = (UsernamePasswordAuthenticationToken) authentication;
loginUser = (LoginUser) token.getPrincipal();
// 这里继续你的业务逻辑
} else {
// 匿名访问处理：可以抛异常或记录默认值
throw new IllegalStateException("未登录用户无法新增试验数据");
}
// 数据校验
if (names.size() != sizes.size()) {
@ -528,7 +547,7 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
files1.setKeywords(tsFiles.getKeywords());
files1.setDescription(tsFiles.getDescription());
files1.setUploadTime(tsFiles.getUploadTime());
files1.setUploader(loginuser.getUsername());
files1.setUploader(loginUser.getUsername());
files1.setFileName(name);
files1.setFileSize("0.000".equals(sizeStr) ? "0.001" : sizeStr);
@ -6374,7 +6393,7 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
* @param id
*/
@Override
public Map<String, TsFiles> splitFile(String id,String taskId) throws Exception {
public Map<String, TsFiles> splitFile(String id,String taskId,File jsonFile) throws Exception {
//1:动态表名 以及通过ID查询tsfiles 然后根据id path taskId nodeid 等等条件查询所欲的集合
TsTask tsTask = tsTaskMapper.selectById(taskId);
TableNameContextHolder.setTaskCode(tsTask.getTaskCode());
@ -6394,40 +6413,30 @@ public class TsFilesServiceImpl extends ServiceImpl<TsFilesMapper, TsFiles> impl
}
// 3. 根据规则拆分文件
Map<String, String> outputPaths = insFileConvertService.convertByTemplate(sourceFile);
Map<String, String> outputPaths = insFileConvertNewService.convert(sourceFile,jsonFile);
// 3. 生成两个 TsFiles 对象
// 3. 生成TsFiles对象集合
Map<String, TsFiles> resultMap = new HashMap<>();
LocalDateTime now = LocalDateTime.now();
// VINS 文件
File vinsFile = new File(outputPaths.get("vins"));
TsFiles vinsTs = new TsFiles();
vinsTs.setNodeId(tsFile.getNodeId());
vinsTs.setTaskId(tsFile.getTaskId());
vinsTs.setIsFile("FILE");
vinsTs.setParentId(tsFile.getParentId());
vinsTs.setFileName(vinsFile.getName());
vinsTs.setFileSize(String.valueOf(vinsFile.length() / (1024.0 * 1024.0))); // 转 M
vinsTs.setWorkPath(tsFile.getWorkPath());
// 遍历所有输出文件
for (Map.Entry<String, String> entry : outputPaths.entrySet()) {
String outputName = entry.getKey(); // 如 vins、fvns
String path = entry.getValue();
File outFile = new File(path);
TsFiles ts = new TsFiles();
ts.setNodeId(tsFile.getNodeId());
ts.setTaskId(tsFile.getTaskId());
ts.setIsFile("FILE");
ts.setParentId(tsFile.getParentId());
ts.setFileName(outFile.getName());
ts.setFileSize(String.valueOf(outFile.length() / (1024.0 * 1024.0))); // 转 M
ts.setWorkPath(tsFile.getWorkPath());
this.addTsFiles(ts);
this.addTsFiles(vinsTs);
resultMap.put("vins", vinsTs);
// FVNS 文件
File fvnsFile = new File(outputPaths.get("fvns"));
TsFiles fvnsTs = new TsFiles();
fvnsTs.setNodeId(tsFile.getNodeId());
fvnsTs.setTaskId(tsFile.getTaskId());
fvnsTs.setIsFile("FILE");
fvnsTs.setParentId(tsFile.getParentId());
fvnsTs.setFileName(fvnsFile.getName());
fvnsTs.setFileSize(String.valueOf(fvnsFile.length() / (1024.0 * 1024.0)));
fvnsTs.setWorkPath(tsFile.getWorkPath());
this.addTsFiles(fvnsTs);
resultMap.put("fvns", fvnsTs);
resultMap.put(outputName, ts);
}
return resultMap;

37
java/src/main/resources/config/ins-convert-default.json Normal file
View File

@ -0,0 +1,37 @@
{
"outputs": [
{
"name": "VINS",
"templateResource": "templates/VINS_0_mode1.txt",
"delimiter": ",",
"extension": ".txt",
"downsample": {
"enabled": true,
"inputDelta": 0.005,
"outputInterval": 0.1
},
"rules": {
"UTC": { "type": "INPUT", "from": "utc" },
"LatGps": { "type": "INPUT", "from": "LatJG" },
"LonGps": { "type": "INPUT", "from": "LonJG" }
}
},
{
"name": "FVNS",
"templateResource": "templates/FVNS.csv",
"delimiter": ",",
"extension": ".csv",
"downsample": {
"enabled": true,
"inputDelta": 0.005,
"outputInterval": 10
},
"rules": {
"UTC": { "type": "INPUT", "from": "utc" },
"FvnsSts": { "type": "INPUT", "from": "posx" },
"PosxFvns": { "type": "INPUT", "from": "posy" },
"PosyFvns": { "type": "INPUT", "from": "posz" }
}
}
]
}