ThbTech/FileManage
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

minio剩余空间占比代码提交

Browse Source
This commit is contained in:
lilin 2025-07-21 10:59:41 +08:00
parent 3c75b516ef
commit 10ef87753c
4 changed files with 330 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
java/pom.xml
View File

@ -99,12 +99,6 @@
<version>30.0-jre</version> <version>30.0-jre</version>
</dependency> </dependency>
<!-- spring-内置Tomcat-->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-tomcat</artifactId>
<scope>provided</scope>
</dependency>
<!-- spring-quartz任务--> <!-- spring-quartz任务-->
<dependency> <dependency>
@ -159,6 +153,14 @@
<version>1.2.3</version> <version>1.2.3</version>
</dependency> </dependency>
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
<version>3.1.0</version>
<!-- <scope>compile</scope>-->
</dependency>
<!--数据库连接--> <!--数据库连接-->
<dependency> <dependency>
<groupId>org.springframework.boot</groupId> <groupId>org.springframework.boot</groupId>

3
java/src/main/java/com/yfd/platform/modules/storage/controller/base/StorageSourceController.java
View File

@ -1,13 +1,11 @@
package com.yfd.platform.modules.storage.controller.base; package com.yfd.platform.modules.storage.controller.base;
import cn.hutool.core.bean.BeanUtil;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import com.github.xiaoymin.knife4j.annotations.ApiOperationSupport; import com.github.xiaoymin.knife4j.annotations.ApiOperationSupport;
import com.github.xiaoymin.knife4j.annotations.ApiSort; import com.github.xiaoymin.knife4j.annotations.ApiSort;
import com.yfd.platform.config.ResponseResult; import com.yfd.platform.config.ResponseResult;
import com.yfd.platform.modules.config.model.request.SaveStorageSourceRequest; import com.yfd.platform.modules.config.model.request.SaveStorageSourceRequest;
import com.yfd.platform.modules.config.model.request.UpdateStorageSortRequest; import com.yfd.platform.modules.config.model.request.UpdateStorageSortRequest;
import com.yfd.platform.modules.experimentalData.domain.TsFiles;
import com.yfd.platform.modules.storage.convert.StorageSourceConvert; import com.yfd.platform.modules.storage.convert.StorageSourceConvert;
import com.yfd.platform.modules.storage.model.dto.StorageSourceDTO; import com.yfd.platform.modules.storage.model.dto.StorageSourceDTO;
import com.yfd.platform.modules.storage.model.entity.StorageSource; import com.yfd.platform.modules.storage.model.entity.StorageSource;
@ -22,7 +20,6 @@ import org.springframework.web.bind.annotation.*;
import javax.annotation.Resource; import javax.annotation.Resource;
import java.util.List; import java.util.List;
import java.util.stream.Collectors;
/** /**
* 存储源基础设置模块接口 * 存储源基础设置模块接口

349
java/src/main/java/com/yfd/platform/modules/storage/convert/impl/StorageSourceConvertImpl.java
View File

@ -18,21 +18,36 @@ import com.yfd.platform.modules.storage.model.result.StorageSourceAdminResult;
import com.yfd.platform.modules.storage.model.result.StorageSourceConfigResult; import com.yfd.platform.modules.storage.model.result.StorageSourceConfigResult;
import com.yfd.platform.modules.storage.model.result.StorageSourceResult; import com.yfd.platform.modules.storage.model.result.StorageSourceResult;
import com.yfd.platform.utils.StringUtils; import com.yfd.platform.utils.StringUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.json.JSONObject;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import javax.annotation.Resource; import javax.annotation.Resource;
import java.io.File; import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileStore; import java.nio.file.FileStore;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.text.DecimalFormat; import java.text.DecimalFormat;
import java.util.ArrayList; import java.time.ZoneOffset;
import java.util.Arrays; import java.time.ZonedDateTime;
import java.util.Collections; import java.time.format.DateTimeFormatter;
import java.util.List; import java.util.*;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.HashMap;
import java.util.Map;
import java.util.TreeMap;
/** /**
* @Date: 2025/1/10 12:44 * @Date: 2025/1/10 12:44
* @Description: * @Description:
@ -161,35 +176,54 @@ public class StorageSourceConvertImpl implements StorageSourceConvert {
} }
storageSource.setStoreContent(String.valueOf(result)); storageSource.setStoreContent(String.valueOf(result));
//存储路径 //存储路径
LambdaQueryWrapper<StorageSourceConfig> queryWrapperData = new LambdaQueryWrapper<>(); // 使用方式
queryWrapperData.eq(StorageSourceConfig::getStorageId, storageSource.getId()); String value = getConfigValue(storageSource.getId(), "filePath");
queryWrapperData.eq(StorageSourceConfig::getName, "filePath");
String value = storageSourceConfigMapper.selectOne(queryWrapperData).getValue();
storageSource.setValueData(value); storageSource.setValueData(value);
String accessKeyValue = getConfigValue(storageSource.getId(), "accessKey");
String secretKeyValue = getConfigValue(storageSource.getId(), "secretKey");
String endPointValue = getConfigValue(storageSource.getId(), "endPoint");
Map<String, String> minioConfig = new HashMap<>();
minioConfig.put("endPoint", endPointValue);
minioConfig.put("accessKey", accessKeyValue);
minioConfig.put("secretKey", secretKeyValue);
minioConfig.put("region", "us-east-1");
try {
Map<String, Long> diskSpace = getMinioDiskSpace(minioConfig);
if(StringUtils.isNotBlank(value)){ // 在main方法中添加计算逻辑
String[] values = value.split(";"); double freePercentage = (double)diskSpace.get("freeSpace") / diskSpace.get("totalSpace") * 100;
// 将结果存入集合 double freeSpaceGB = (double)diskSpace.get("freeSpace") / (1024 * 1024 * 1024);
List<String> valueList = Arrays.asList(values); storageSource.setSpaceOccupancyRatio(String.format("%.2f", freePercentage) + "%");
storageSource.setRemainingSpaceSize(String.format("%.1f", freeSpaceGB) + " GB");
StringBuilder spaceOccupancyRatio = new StringBuilder(); } catch (Exception e) {
StringBuilder remainingSpaceSize = new StringBuilder(); e.printStackTrace();
for (String valueData : valueList){
String percentage = calculateLocalStorageUsage(valueData);
String spaceSize = calculateFreeSpaceInGB(valueData);
if (spaceOccupancyRatio.length() > 0) {
spaceOccupancyRatio.append(";"); // 添加分号分隔
}
spaceOccupancyRatio.append(percentage);
remainingSpaceSize.append(spaceSize);
}
//空间使用率
storageSource.setSpaceOccupancyRatio(spaceOccupancyRatio.toString());
storageSource.setRemainingSpaceSize(remainingSpaceSize.toString());
} }
// if(StringUtils.isNotBlank(value)){
// String[] values = value.split(";");
// // 将结果存入集合
// List<String> valueList = Arrays.asList(values);
//
// StringBuilder spaceOccupancyRatio = new StringBuilder();
// StringBuilder remainingSpaceSize = new StringBuilder();
//
// for (String valueData : valueList){
// String percentage = calculateLocalStorageUsage(valueData);
// String spaceSize = calculateFreeSpaceInGB(valueData);
// if (spaceOccupancyRatio.length() > 0) {
// spaceOccupancyRatio.append(";"); // 添加分号分隔
// }
// spaceOccupancyRatio.append(percentage);
// remainingSpaceSize.append(spaceSize);
// }
// //空间使用率
// storageSource.setSpaceOccupancyRatio(spaceOccupancyRatio.toString());
// storageSource.setRemainingSpaceSize(remainingSpaceSize.toString());
// }
} }
} }
@ -216,6 +250,263 @@ public class StorageSourceConvertImpl implements StorageSourceConvert {
return resultPage; return resultPage;
} }
private String getConfigValue(int storageId, String configName) {
return Optional.ofNullable(storageSourceConfigMapper.selectOne(
new LambdaQueryWrapper<StorageSourceConfig>()
.eq(StorageSourceConfig::getStorageId, storageId)
.eq(StorageSourceConfig::getName, configName)
)).map(StorageSourceConfig::getValue).orElse(null);
}
/**
* 获取Minio存储服务的磁盘空间信息
*
* @param minioConfig Minio配置信息，包含以下键值：
* - endPoint: Minio服务地址 (e.g., "http://192.168.1.208:9000")
* - accessKey: 访问密钥
* - secretKey: 秘密密钥
* - region: 区域（可选，默认为"us-east-1"）
*
* @return 包含磁盘空间信息的Map:
* - totalSpace: 总空间（字节）
* - freeSpace: 可用空间（字节）
* - usedSpace: 已用空间（字节）
*
* @throws Exception 如果请求失败或解析出错
*/
public static Map<String, Long> getMinioDiskSpace(Map<String, String> minioConfig) throws Exception {
// 从配置中获取Minio连接信息
String endPoint = minioConfig.get("endPoint");
String accessKey = minioConfig.get("accessKey");
String secretKey = minioConfig.get("secretKey");
// 1. 构建Admin API URL
// 解析URI获取主机名、端口和协议
URI uri = new URI(endPoint);
String host = uri.getHost();
int port = uri.getPort();
String protocol = uri.getScheme();
// 组合完整的Admin API URL
String adminUrl = protocol + "://" + host + ":" + port + "/minio/admin/v3/info";
// 2. 生成签名所需的时间参数（使用UTC时区）
// 日期格式要求：
// amzDate: yyyyMMdd'T'HHmmss'Z' (e.g., "20230718T093000Z")
// dateStamp: yyyyMMdd (e.g., "20230718")
ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
String amzDate = now.format(DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmss'Z'"));
String dateStamp = now.format(DateTimeFormatter.ofPattern("yyyyMMdd"));
// 3. 准备请求头
// 使用TreeMap确保头字段按小写字母排序（AWS签名要求）
Map<String, String> headers = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
headers.put("Host", host + ":" + port); // 主机头（必需）
headers.put("X-Amz-Date", amzDate); // AWS格式的时间戳
headers.put("X-Amz-Content-Sha256", "UNSIGNED-PAYLOAD"); // 表示没有请求体
// 4. 生成AWS V4签名
// 关键点：服务名使用"s3"（Minio Admin API使用S3签名方案）
String region = minioConfig.getOrDefault("region", "us-east-1"); // 获取区域或使用默认值
String authorization = getV4Signature(
accessKey,
secretKey,
"s3", // 服务名称（固定为"s3"）
region, // 区域（如"us-east-1"）
"GET", // HTTP方法
"/minio/admin/v3/info", // API路径
headers, // 请求头集合
dateStamp, // 日期戳
amzDate // 完整时间戳
);
headers.put("Authorization", authorization); // 添加签名到头信息
// 5. 配置HTTP客户端超时参数
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(5000) // 连接超时5秒
.setSocketTimeout(15000) // 响应超时15秒
.build();
// 使用try-with-resources确保HttpClient正确关闭
try (CloseableHttpClient client = HttpClients.custom()
.setDefaultRequestConfig(requestConfig)
.build()) {
// 创建GET请求
HttpGet request = new HttpGet(adminUrl);
// 添加所有请求头
headers.forEach(request::addHeader);
// 执行请求并获取响应
HttpResponse response = client.execute(request);
int statusCode = response.getStatusLine().getStatusCode();
// 读取响应内容
HttpEntity entity = response.getEntity();
String result = EntityUtils.toString(entity);
// 6. 检查HTTP状态码
if (statusCode != 200) {
// 错误处理：打印详细信息
System.err.println("Minio API请求失败. 状态码: " + statusCode);
System.err.println("响应内容: " + result);
System.err.println("请求URL: " + adminUrl);
System.err.println("签名参数: dateStamp=" + dateStamp + ", amzDate=" + amzDate);
System.err.println("签名服务: s3");
// 抛出异常
throw new RuntimeException("Minio API请求失败. 状态码: " + statusCode + ", 响应: " + result);
}
// 7. 解析磁盘空间数据
return parseDiskSpace(result);
}
}
/**
* 生成AWS V4签名
*
* 参考AWS官方文档：https://docs.aws.amazon.com/zh_cn/general/latest/gr/sigv4_signing.html
*/
private static String getV4Signature(String accessKey, String secretKey, String service,
String region, String method, String canonicalUri,
Map<String, String> headers, String dateStamp, String amzDate)
throws NoSuchAlgorithmException, InvalidKeyException {
// 1. 创建规范请求 (Canonical Request)
String canonicalQueryString = ""; // 没有查询参数
// 获取签名头列表（按小写字母排序）
String signedHeaders = getSignedHeaders(headers);
// 构建规范头（按小写字段名排序）
StringBuilder canonicalHeaders = new StringBuilder();
for (Map.Entry<String, String> entry : headers.entrySet()) {
String key = entry.getKey().toLowerCase(); // 转换为小写
String value = entry.getValue().trim().replaceAll("\\s+", " "); // 标准化空格
canonicalHeaders.append(key).append(":").append(value).append("\n");
}
// 组装规范请求
String canonicalRequest = method + "\n" + // HTTP方法
canonicalUri + "\n" + // 规范路径
canonicalQueryString + "\n" + // 查询字符串
canonicalHeaders + "\n" + // 规范头
signedHeaders + "\n" + // 签名头列表
"UNSIGNED-PAYLOAD"; // 请求体哈希（无内容）
// 2. 创建待签名字符串 (String to Sign)
String credentialScope = dateStamp + "/" + region + "/" + service + "/aws4_request";
String stringToSign = "AWS4-HMAC-SHA256\n" + // 签名算法
amzDate + "\n" + // 请求时间戳
credentialScope + "\n" + // 凭证范围
sha256Hex(canonicalRequest); // 规范请求的SHA256哈希
// 3. 计算签名
// 生成签名密钥
byte[] kSecret = ("AWS4" + secretKey).getBytes(StandardCharsets.UTF_8);
byte[] kDate = hmacSha256(dateStamp, kSecret); // 日期密钥
byte[] kRegion = hmacSha256(region, kDate); // 区域密钥
byte[] kService = hmacSha256(service, kRegion); // 服务密钥
byte[] kSigning = hmacSha256("aws4_request", kService); // 最终签名密钥
// 计算签名
byte[] signature = hmacSha256(stringToSign, kSigning);
// 4. 构建Authorization头
return "AWS4-HMAC-SHA256 " +
"Credential=" + accessKey + "/" + credentialScope + ", " +
"SignedHeaders=" + signedHeaders + ", " +
"Signature=" + bytesToHex(signature);
}
/**
* 获取签名头列表（按小写字母排序）
*/
private static String getSignedHeaders(Map<String, String> headers) {
StringBuilder signedHeaders = new StringBuilder();
for (String key : headers.keySet()) {
if (signedHeaders.length() > 0) {
signedHeaders.append(";");
}
signedHeaders.append(key.toLowerCase()); // 全部转换为小写
}
return signedHeaders.toString();
}
/**
* 解析磁盘空间信息
*/
private static Map<String, Long> parseDiskSpace(String jsonResponse) {
Map<String, Long> diskSpace = new HashMap<>();
// 解析JSON响应
JSONObject json = new JSONObject(jsonResponse);
// 提取磁盘信息（假设单节点部署）
JSONObject drive = json.getJSONArray("servers") // 获取servers数组
.getJSONObject(0) // 获取第一个服务器
.getJSONArray("drives") // 获取drives数组
.getJSONObject(0); // 获取第一个驱动器
// 获取空间信息
long total = drive.getLong("totalspace"); // 总空间
long used = drive.getLong("usedspace"); // 已用空间
long free = drive.getLong("availspace"); // 可用空间
// 存入结果Map
diskSpace.put("totalSpace", total);
diskSpace.put("freeSpace", free);
diskSpace.put("usedSpace", used);
return diskSpace;
}
// ===== 辅助方法 =====
/**
* 计算HMAC-SHA256签名
*/
private static byte[] hmacSha256(String data, byte[] key)
throws NoSuchAlgorithmException, InvalidKeyException {
Mac mac = Mac.getInstance("HmacSHA256");
mac.init(new SecretKeySpec(key, "HmacSHA256"));
return mac.doFinal(data.getBytes(StandardCharsets.UTF_8));
}
/**
* 计算字符串的SHA256哈希
*/
private static String sha256Hex(String data) throws NoSuchAlgorithmException {
java.security.MessageDigest md = java.security.MessageDigest.getInstance("SHA-256");
byte[] hash = md.digest(data.getBytes(StandardCharsets.UTF_8));
return bytesToHex(hash);
}
/**
* 字节数组转十六进制字符串
*/
private static String bytesToHex(byte[] bytes) {
StringBuilder hexString = new StringBuilder();
for (byte b : bytes) {
String hex = Integer.toHexString(0xff & b); // 转换为无符号十六进制
if (hex.length() == 1) {
hexString.append('0'); // 补零
}
hexString.append(hex);
}
return hexString.toString();
}
// 计算本地路径的空间剩余率 // 计算本地路径的空间剩余率
public static String calculateLocalStorageUsage(String path) { public static String calculateLocalStorageUsage(String path) {
try { try {

4
java/src/main/java/com/yfd/platform/modules/storage/model/param/MinIOParam.java
View File

@ -20,7 +20,7 @@ public class MinIOParam extends S3BaseParam {
@StorageParamItem(name = "服务地址", order = 5, description = "为 minio 的服务地址，非 web 访问地址，一般为 http://ip:9000") @StorageParamItem(name = "服务地址", order = 5, description = "为 minio 的服务地址，非 web 访问地址，一般为 http://ip:9000")
private String endPoint; private String endPoint;
@StorageParamItem(name = "minio宿主机挂载路径", order = 0,description = "只支持绝对路径<br>Docker 方式部署的话需提前映射宿主机路径！ " ) // @StorageParamItem(name = "minio宿主机挂载路径", order = 0,description = "只支持绝对路径<br>Docker 方式部署的话需提前映射宿主机路径！ " )
private String filePath; // private String filePath;
} }