ThbTech/gis-bi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修改并提交了后端文件

Browse Source
This commit is contained in:
root 2025-06-23 10:34:29 +08:00
parent 2bcb606319
commit adabf1e604
116 changed files with 4759 additions and 2173 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
core/core-backend/pom.xml
View File

@ -3,33 +3,40 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>gisbi</artifactId>
<artifactId>core</artifactId>
<groupId>io.gisbi</groupId>
<version>1.0.0</version>
<version>2.0.0</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<packaging>jar</packaging>
<artifactId>core-backend</artifactId>
<dependencies>
<dependency>
<groupId>commons-net</groupId>
<artifactId>commons-net</artifactId>
<version>3.8.0</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
<version>33.0.0-jre</version>
</dependency>
<dependency>
<groupId>io.gisbi</groupId>
<artifactId>api-base</artifactId>
<version>1.0.0</version>
<version>2.0.0</version>
</dependency>
<dependency>
<groupId>io.gisbi</groupId>
<artifactId>common</artifactId>
<version>1.0.0</version>
<version>2.0.0</version>
</dependency>
<dependency>
<groupId>io.gisbi</groupId>
<artifactId>api-sync</artifactId>
<version>1.0.0</version>
<version>2.0.0</version>
</dependency>
<dependency>
<groupId>com.mysql</groupId>
@ -38,18 +45,18 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-dbcp2</artifactId>
<version>${commons-dbcp2.version}</version>
<version>2.6.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.antlr/antlr -->
<dependency>
<groupId>org.antlr</groupId>
<artifactId>antlr</artifactId>
<version>${antlr.version}</version>
<version>3.5.2</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>${junit.version}</version>
<version>4.13.2</version>
<scope>test</scope>
</dependency>
<dependency>
@ -73,7 +80,6 @@
<artifactId>hutool-all</artifactId>
<version>5.8.8</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-websocket</artifactId>
@ -93,15 +99,10 @@
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
<dependency>
<groupId>cn.hutool</groupId>
<artifactId>hutool-all</artifactId>
<version>5.8.38</version>
</dependency>
<dependency>
<groupId>io.gisbi</groupId>
<artifactId>api-permissions</artifactId>
<version>1.0.0</version>
<version>2.0.0</version>
</dependency>
</dependencies>
@ -125,12 +126,12 @@
<dependency>
<groupId>org.seleniumhq.selenium</groupId>
<artifactId>selenium-java</artifactId>
<version>${selenium-java.version}</version>
<version>4.19.1</version>
</dependency>
<dependency>
<groupId>org.eclipse.angus</groupId>
<artifactId>angus-mail</artifactId>
<version>${angus-mail.version}</version>
<version>2.0.3</version>
<exclusions>
<exclusion>
<groupId>org.eclipse.angus</groupId>
@ -141,13 +142,13 @@
<dependency>
<groupId>com.itextpdf</groupId>
<artifactId>itext7-core</artifactId>
<version>${itextpdf.version}</version>
<version>8.0.4</version>
<type>pom</type>
</dependency>
<dependency>
<groupId>com.vladsch.flexmark</groupId>
<artifactId>flexmark-all</artifactId>
<version>${flexmark.version}</version>
<version>0.62.2</version>
</dependency>
</dependencies>

2
core/core-backend/src/main/java/io/gisbi/MybatisPlusGenerator.java
View File

@ -14,7 +14,7 @@ public class MybatisPlusGenerator {
* 第一 我嫌麻烦
* 第二 后面配置会放到nacos读起来更麻烦了
*/
private static final String url = "jdbc:mysql://localhost:3306/dataease?autoReconnect=false&useUnicode=true&characterEncoding=UTF-8&characterSetResults=UTF-8&zeroDateTimeBehavior=convertToNull&useSSL=false";
private static final String url = "jdbc:mysql://localhost:3306/gisbi?autoReconnect=false&useUnicode=true&characterEncoding=UTF-8&characterSetResults=UTF-8&zeroDateTimeBehavior=convertToNull&useSSL=false";
private static final String username = "root";
private static final String password = "123456";

130
core/core-backend/src/main/java/io/gisbi/chart/charts/impl/DefaultChartHandler.java
View File

@ -1,12 +1,14 @@
package io.gisbi.chart.charts.impl;
import com.beust.jcommander.Strings;
import io.gisbi.api.dataset.union.DatasetGroupInfoDTO;
import io.gisbi.chart.charts.ChartHandlerManager;
import io.gisbi.chart.constant.ChartConstants;
import io.gisbi.chart.manage.ChartDataManage;
import io.gisbi.chart.manage.ChartViewManege;
import io.gisbi.chart.utils.ChartDataBuild;
import io.gisbi.dataset.manage.DatasetTableFieldManage;
import io.gisbi.constant.SQLConstants;
import io.gisbi.dataset.manage.DatasetTableFieldManage;
import io.gisbi.engine.sql.SQLProvider;
import io.gisbi.engine.trans.Dimension2SQLObj;
import io.gisbi.engine.trans.Quota2SQLObj;
@ -82,7 +84,7 @@ public class DefaultChartHandler extends AbstractChartPlugin {
var desensitizationList = (Map<String, ColumnPermissionItem>) formatResult.getContext().get("desensitizationList");
if (MapUtils.isNotEmpty(desensitizationList)) {
formatResult.getAxisMap().forEach((axis, fields) -> {
fields.removeIf(f -> desensitizationList.containsKey(f.getDataeaseName()));
fields.removeIf(f -> desensitizationList.containsKey(f.getGisbiName()));
});
}
return (T) new CustomFilterResult(filterList, formatResult.getContext());
@ -107,8 +109,9 @@ public class DefaultChartHandler extends AbstractChartPlugin {
dsList.add(next.getValue().getType());
}
boolean needOrder = Utils.isNeedOrder(dsList);
boolean crossDs = Utils.isCrossDs(dsMap);
boolean crossDs = ((DatasetGroupInfoDTO) formatResult.getContext().get("dataset")).getIsCross();
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setIsCross(crossDs);
datasourceRequest.setDsList(dsMap);
var xAxis = formatResult.getAxisMap().get(ChartAxis.xAxis);
var yAxis = formatResult.getAxisMap().get(ChartAxis.yAxis);
@ -168,8 +171,8 @@ public class DefaultChartHandler extends AbstractChartPlugin {
dataMap.putAll(calcResult.getData());
dataMap.putAll(mapTableNormal);
dataMap.put("sourceFields", allFields);
mergeAssistField(calcResult.getDynamicAssistFields(), calcResult.getAssistData());
dataMap.put("dynamicAssistLines", calcResult.getDynamicAssistFields());
List<ChartSeniorAssistDTO> chartSeniorAssistDTOS = mergeAssistField(calcResult.getDynamicAssistFields(), calcResult.getAssistData(), calcResult.getDynamicAssistFieldsOriginList(), calcResult.getAssistDataOriginList());
dataMap.put("dynamicAssistLines", chartSeniorAssistDTOS);
view.setData(dataMap);
view.setSql(Base64.getEncoder().encodeToString(calcResult.getQuerySql().getBytes()));
view.setDrill(isDrill);
@ -177,20 +180,32 @@ public class DefaultChartHandler extends AbstractChartPlugin {
return view;
}
protected void mergeAssistField(List<ChartSeniorAssistDTO> dynamicAssistFields, List<String[]> assistData) {
if (ObjectUtils.isEmpty(assistData)) {
return;
}
String[] strings = assistData.get(0);
protected List<ChartSeniorAssistDTO> mergeAssistField(List<ChartSeniorAssistDTO> dynamicAssistFields, List<String[]> assistData, List<ChartSeniorAssistDTO> dynamicAssistFieldsOriginList, List<String[]> assistDataOriginList) {
List<ChartSeniorAssistDTO> list = new ArrayList<>();
if (ObjectUtils.isNotEmpty(assistData)) {
String[] strings = assistData.getFirst();
for (int i = 0; i < dynamicAssistFields.size(); i++) {
if (i < strings.length) {
ChartSeniorAssistDTO chartSeniorAssistDTO = dynamicAssistFields.get(i);
chartSeniorAssistDTO.setValue(strings[i]);
list.add(chartSeniorAssistDTO);
}
}
}
if (ObjectUtils.isNotEmpty(assistDataOriginList)) {
String[] stringsOriginList = assistDataOriginList.getLast();// 取最后一项，如果有其他运算逻辑需要取明细数据可增加逻辑
for (int i = 0; i < dynamicAssistFieldsOriginList.size(); i++) {
if (i < stringsOriginList.length) {
ChartSeniorAssistDTO chartSeniorAssistDTO = dynamicAssistFieldsOriginList.get(i);
chartSeniorAssistDTO.setValue(stringsOriginList[i]);
list.add(chartSeniorAssistDTO);
}
}
}
return list;
}
protected List<ChartSeniorAssistDTO> getDynamicAssistFields(ChartViewDTO view) {
List<ChartSeniorAssistDTO> list = new ArrayList<>();
@ -370,7 +385,7 @@ public class DefaultChartHandler extends AbstractChartPlugin {
return conditionField;
}
protected String assistSQL(String sql, List<ChartViewFieldDTO> assistFields, Map<Long, DatasourceSchemaDTO> dsMap) {
protected String assistSQL(String sql, List<ChartViewFieldDTO> assistFields, Map<Long, DatasourceSchemaDTO> dsMap, boolean crossDs) {
// get datasource prefix and suffix
String dsType = dsMap.entrySet().iterator().next().getValue().getType();
String prefix = "";
@ -391,25 +406,54 @@ public class DefaultChartHandler extends AbstractChartPlugin {
}
}
boolean crossDs = Utils.isCrossDs(dsMap);
StringBuilder stringBuilder = new StringBuilder();
List<String> fieldList = new ArrayList<>();
for (int i = 0; i < assistFields.size(); i++) {
ChartViewFieldDTO dto = assistFields.get(i);
if (StringUtils.equalsIgnoreCase(dto.getSummary(), "last_item")) {
continue;
}
if (crossDs) {
if (i == (assistFields.size() - 1)) {
stringBuilder.append(dto.getSummary() + "(" + dto.getOriginName() + ")");
fieldList.add(dto.getSummary() + "(" + dto.getOriginName() + ")");
} else {
stringBuilder.append(dto.getSummary() + "(" + dto.getOriginName() + "),");
fieldList.add(dto.getSummary() + "(" + prefix + dto.getOriginName() + suffix + ")");
}
}
return "SELECT " + Strings.join(",", fieldList) + " FROM (" + sql + ") tmp";
}
protected String assistSQLOriginList(String sql, List<ChartViewFieldDTO> assistFields, Map<Long, DatasourceSchemaDTO> dsMap, boolean crossDs) {
// get datasource prefix and suffix
String dsType = dsMap.entrySet().iterator().next().getValue().getType();
String prefix = "";
String suffix = "";
if (Arrays.stream(DatasourceConfiguration.DatasourceType.values()).map(DatasourceConfiguration.DatasourceType::getType).toList().contains(dsType)) {
DatasourceConfiguration.DatasourceType datasourceType = DatasourceConfiguration.DatasourceType.valueOf(dsType);
prefix = datasourceType.getPrefix();
suffix = datasourceType.getSuffix();
} else {
if (i == (assistFields.size() - 1)) {
stringBuilder.append(dto.getSummary() + "(" + prefix + dto.getOriginName() + suffix + ")");
List<XpackPluginsDatasourceVO> xpackPluginsDatasourceVOS = pluginManage.queryPluginDs();
List<XpackPluginsDatasourceVO> list = xpackPluginsDatasourceVOS.stream().filter(ele -> StringUtils.equals(ele.getType(), dsType)).toList();
if (ObjectUtils.isNotEmpty(list)) {
XpackPluginsDatasourceVO first = list.getFirst();
prefix = first.getPrefix();
suffix = first.getSuffix();
} else {
stringBuilder.append(dto.getSummary() + "(" + prefix + dto.getOriginName() + suffix + "),");
DEException.throwException("当前数据源插件不存在");
}
}
List<String> fieldList = new ArrayList<>();
for (int i = 0; i < assistFields.size(); i++) {
ChartViewFieldDTO dto = assistFields.get(i);
if (StringUtils.equalsIgnoreCase(dto.getSummary(), "last_item")) {
if (crossDs) {
fieldList.add(dto.getOriginName());
} else {
fieldList.add(prefix + dto.getOriginName() + suffix);
}
}
}
return "SELECT " + stringBuilder + " FROM (" + sql + ") tmp";
return "SELECT " + Strings.join(",", fieldList) + " FROM (" + sql + ") tmp";
}
protected List<String> mergeIds(List<ChartViewFieldDTO> xAxisExt, List<ChartViewFieldDTO> extStack) {
@ -522,26 +566,29 @@ public class DefaultChartHandler extends AbstractChartPlugin {
if (CollectionUtils.isEmpty(data)) {
break;
}
if (StringUtils.containsAny(chartType, "group", "stack")) {
if (Objects.isNull(extStack)) {
extStack = Arrays.asList();
}
if (Objects.isNull(xAxisExt)) {
xAxisExt = Arrays.asList();
}
boolean isStack = StringUtils.containsIgnoreCase(chartType, "stack") && CollectionUtils.isNotEmpty(extStack);
boolean isGroup = StringUtils.containsIgnoreCase(chartType, "group")
|| (CollectionUtils.isNotEmpty(xAxisExt));
if (isStack || isGroup) {
if (CollectionUtils.isEmpty(xAxis)) {
break;
}
if (StringUtils.containsIgnoreCase(chartType, "stack") && extStack.isEmpty()) {
break;
}
if (StringUtils.containsIgnoreCase(chartType, "group") && xAxisExt.isEmpty()) {
break;
}
final Map<String, Integer> mainIndexMap = new HashMap<>();
final List<List<String[]>> mainMatrix = new ArrayList<>();
// 排除group和stack的字段
List<String> groupStackAxisIds = mergeIds(xAxisExt, extStack);
List<ChartViewFieldDTO> finalXAxisBase = xAxis.stream().filter(ele->!groupStackAxisIds.contains(String.valueOf(ele.getId()))).toList();
if (CollectionUtils.isEmpty(finalXAxisBase) && CollectionUtils.isNotEmpty(xAxis)) {
finalXAxisBase.add(xAxis.get(0));
List<ChartViewFieldDTO> xAxisBase = xAxis.stream().filter(ele -> !groupStackAxisIds.contains(String.valueOf(ele.getId()))).collect(Collectors.toList());
if (CollectionUtils.isEmpty(xAxisBase) && CollectionUtils.isNotEmpty(xAxis)) {
xAxisBase.add(xAxis.get(0));
}
data.forEach(item -> {
String[] mainAxisArr = Arrays.copyOfRange(item, 0, finalXAxisBase.size());
String[] mainAxisArr = Arrays.copyOfRange(item, 0, xAxisBase.size());
String mainAxis = StringUtils.join(mainAxisArr, '-');
Integer index = mainIndexMap.get(mainAxis);
if (index == null) {
@ -555,21 +602,24 @@ public class DefaultChartHandler extends AbstractChartPlugin {
}
});
int finalDataIndex = dataIndex;
int subEndIndex = finalXAxisBase.size();
if (StringUtils.containsIgnoreCase(chartType, "group")) {
int subEndIndex = xAxisBase.size();
if (CollectionUtils.isNotEmpty(xAxisExt)
|| StringUtils.containsIgnoreCase(chartType, "group")
|| StringUtils.containsIgnoreCase(chartType, "-mix")) {
subEndIndex += xAxisExt.size();
}
if (StringUtils.containsIgnoreCase(chartType, "stack")) {
subEndIndex += extStack.size();
}
int finalSubEndIndex = subEndIndex;
// 存储上次的值
Map<String, BigDecimal> preDataMap = new HashMap<>();
//滑动累加
for (int k = 1; k < mainMatrix.size(); k++) {
List<String[]> preDataItems = mainMatrix.get(k - 1);
List<String[]> curDataItems = mainMatrix.get(k);
Map<String, BigDecimal> preDataMap = new HashMap<>();
preDataItems.forEach(preDataItem -> {
String[] groupStackAxisArr = Arrays.copyOfRange(preDataItem, finalXAxisBase.size(), finalSubEndIndex);
String[] groupStackAxisArr = Arrays.copyOfRange(preDataItem, xAxisBase.size(), finalSubEndIndex);
String groupStackAxis = StringUtils.join(groupStackAxisArr, '-');
String preVal = preDataItem[finalDataIndex];
if (StringUtils.isBlank(preVal)) {
@ -578,13 +628,19 @@ public class DefaultChartHandler extends AbstractChartPlugin {
preDataMap.put(groupStackAxis, new BigDecimal(preVal));
});
curDataItems.forEach(curDataItem -> {
String[] groupStackAxisArr = Arrays.copyOfRange(curDataItem, finalXAxisBase.size(), finalSubEndIndex);
String[] groupStackAxisArr = Arrays.copyOfRange(curDataItem, xAxisBase.size(), finalSubEndIndex);
String groupStackAxis = StringUtils.join(groupStackAxisArr, '-');
BigDecimal preValue = preDataMap.get(groupStackAxis);
if (preValue != null) {
curDataItem[finalDataIndex] = new BigDecimal(curDataItem[finalDataIndex])
.add(preValue)
.toString();
} else {
if (preDataMap.containsKey(groupStackAxis)) {
curDataItem[finalDataIndex] = new BigDecimal(curDataItem[finalDataIndex])
.add(preDataMap.get(groupStackAxis))
.toString();
}
}
});
}

4
core/core-backend/src/main/java/io/gisbi/chart/charts/impl/YoyChartHandler.java
View File

@ -2,6 +2,7 @@ package io.gisbi.chart.charts.impl;
import com.fasterxml.jackson.core.type.TypeReference;
import io.gisbi.api.dataset.union.DatasetGroupInfoDTO;
import io.gisbi.engine.sql.SQLProvider;
import io.gisbi.engine.trans.ExtWhere2Str;
import io.gisbi.engine.utils.Utils;
@ -74,7 +75,7 @@ public class YoyChartHandler extends DefaultChartHandler {
dsList.add(next.getValue().getType());
}
boolean needOrder = Utils.isNeedOrder(dsList);
boolean crossDs = Utils.isCrossDs(dsMap);
boolean crossDs = ((DatasetGroupInfoDTO) formatResult.getContext().get("dataset")).getIsCross();
// 这里拿到的可能有一年前的数据
var expandedResult = (T) super.calcChartResult(view, formatResult, filterResult, sqlMap, sqlMeta, provider);
// 检查同环比过滤，拿到实际数据
@ -86,6 +87,7 @@ public class YoyChartHandler extends DefaultChartHandler {
var originSql = SQLProvider.createQuerySQL(sqlMeta, true, needOrder, view);
originSql = provider.rebuildSQL(originSql, sqlMeta, crossDs, dsMap);
var request = new DatasourceRequest();
request.setIsCross(crossDs);
request.setDsList(dsMap);
request.setQuery(originSql);
logger.debug("calcite yoy sql: " + originSql);

24
core/core-backend/src/main/java/io/gisbi/chart/charts/impl/bar/BarHandler.java
View File

@ -1,5 +1,6 @@
package io.gisbi.chart.charts.impl.bar;
import io.gisbi.api.dataset.union.DatasetGroupInfoDTO;
import io.gisbi.chart.charts.impl.YoyChartHandler;
import io.gisbi.engine.utils.Utils;
import io.gisbi.extensions.datasource.dto.DatasourceRequest;
@ -8,6 +9,8 @@ import io.gisbi.extensions.datasource.model.SQLMeta;
import io.gisbi.extensions.datasource.provider.Provider;
import io.gisbi.extensions.view.dto.*;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
@ -42,7 +45,7 @@ public class BarHandler extends YoyChartHandler {
dsList.add(next.getValue().getType());
}
boolean needOrder = Utils.isNeedOrder(dsList);
boolean crossDs = Utils.isCrossDs(dsMap);
boolean crossDs = ((DatasetGroupInfoDTO) formatResult.getContext().get("dataset")).getIsCross();
var result = (T) super.calcChartResult(view, formatResult, filterResult, sqlMap, sqlMeta, provider);
try {
//如果有同环比过滤,应该用原始sql
@ -52,13 +55,28 @@ public class BarHandler extends YoyChartHandler {
var assistFields = getAssistFields(dynamicAssistFields, yAxis);
if (CollectionUtils.isNotEmpty(assistFields)) {
var req = new DatasourceRequest();
req.setIsCross(crossDs);
req.setDsList(dsMap);
var assistSql = assistSQL(originSql, assistFields, dsMap);
List<ChartSeniorAssistDTO> assists = dynamicAssistFields.stream().filter(ele -> !StringUtils.equalsIgnoreCase(ele.getSummary(), "last_item")).toList();
if (ObjectUtils.isNotEmpty(assists)) {
var assistSql = assistSQL(originSql, assistFields, dsMap, crossDs);
req.setQuery(assistSql);
logger.debug("calcite assistSql sql: " + assistSql);
var assistData = (List<String[]>) provider.fetchResultField(req).get("data");
result.setAssistData(assistData);
result.setDynamicAssistFields(dynamicAssistFields);
result.setDynamicAssistFields(assists);
}
List<ChartSeniorAssistDTO> assistsOriginList = dynamicAssistFields.stream().filter(ele -> StringUtils.equalsIgnoreCase(ele.getSummary(), "last_item")).toList();
if (ObjectUtils.isNotEmpty(assistsOriginList)) {
var assistSqlOriginList = assistSQLOriginList(originSql, assistFields, dsMap, crossDs);
req.setQuery(assistSqlOriginList);
logger.debug("calcite assistSql sql origin list: " + assistSqlOriginList);
var assistDataOriginList = (List<String[]>) provider.fetchResultField(req).get("data");
result.setAssistDataOriginList(assistDataOriginList);
result.setDynamicAssistFieldsOriginList(assistsOriginList);
}
}
} catch (Exception e) {
e.printStackTrace();

23
core/core-backend/src/main/java/io/gisbi/chart/charts/impl/line/LineHandler.java
View File

@ -1,5 +1,6 @@
package io.gisbi.chart.charts.impl.line;
import io.gisbi.api.dataset.union.DatasetGroupInfoDTO;
import io.gisbi.chart.charts.impl.YoyChartHandler;
import io.gisbi.chart.utils.ChartDataBuild;
import io.gisbi.extensions.datasource.dto.DatasourceRequest;
@ -9,6 +10,8 @@ import io.gisbi.extensions.datasource.provider.Provider;
import io.gisbi.extensions.view.dto.*;
import lombok.Getter;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
@ -55,6 +58,7 @@ public class LineHandler extends YoyChartHandler {
for (Map.Entry<Long, DatasourceSchemaDTO> next : dsMap.entrySet()) {
dsList.add(next.getValue().getType());
}
boolean crossDs = ((DatasetGroupInfoDTO) formatResult.getContext().get("dataset")).getIsCross();
var result = (T) super.calcChartResult(view, formatResult, filterResult, sqlMap, sqlMeta, provider);
try {
//如果有同环比过滤,应该用原始sql
@ -64,13 +68,28 @@ public class LineHandler extends YoyChartHandler {
var assistFields = getAssistFields(dynamicAssistFields, yAxis);
if (CollectionUtils.isNotEmpty(assistFields)) {
var req = new DatasourceRequest();
req.setIsCross(crossDs);
req.setDsList(dsMap);
var assistSql = assistSQL(originSql, assistFields, dsMap);
List<ChartSeniorAssistDTO> assists = dynamicAssistFields.stream().filter(ele -> !StringUtils.equalsIgnoreCase(ele.getSummary(), "last_item")).toList();
if (ObjectUtils.isNotEmpty(assists)) {
var assistSql = assistSQL(originSql, assistFields, dsMap, crossDs);
req.setQuery(assistSql);
logger.debug("calcite assistSql sql: " + assistSql);
var assistData = (List<String[]>) provider.fetchResultField(req).get("data");
result.setAssistData(assistData);
result.setDynamicAssistFields(dynamicAssistFields);
result.setDynamicAssistFields(assists);
}
List<ChartSeniorAssistDTO> assistsOriginList = dynamicAssistFields.stream().filter(ele -> StringUtils.equalsIgnoreCase(ele.getSummary(), "last_item")).toList();
if (ObjectUtils.isNotEmpty(assistsOriginList)) {
var assistSqlOriginList = assistSQLOriginList(originSql, assistFields, dsMap, crossDs);
req.setQuery(assistSqlOriginList);
logger.debug("calcite assistSql sql origin list: " + assistSqlOriginList);
var assistDataOriginList = (List<String[]>) provider.fetchResultField(req).get("data");
result.setAssistDataOriginList(assistDataOriginList);
result.setDynamicAssistFieldsOriginList(assistsOriginList);
}
}
} catch (Exception e) {
e.printStackTrace();

25
core/core-backend/src/main/java/io/gisbi/chart/charts/impl/line/StackAreaHandler.java
View File

@ -1,5 +1,6 @@
package io.gisbi.chart.charts.impl.line;
import io.gisbi.api.dataset.union.DatasetGroupInfoDTO;
import io.gisbi.chart.charts.impl.YoyChartHandler;
import io.gisbi.chart.utils.ChartDataBuild;
import io.gisbi.engine.utils.Utils;
@ -11,6 +12,7 @@ import io.gisbi.extensions.view.dto.*;
import lombok.Getter;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
@ -89,7 +91,7 @@ public class StackAreaHandler extends YoyChartHandler {
dsList.add(next.getValue().getType());
}
boolean needOrder = Utils.isNeedOrder(dsList);
boolean crossDs = Utils.isCrossDs(dsMap);
boolean crossDs = ((DatasetGroupInfoDTO) formatResult.getContext().get("dataset")).getIsCross();
var result = (T) super.calcChartResult(view, formatResult, filterResult, sqlMap, sqlMeta, provider);
try {
//如果有同环比过滤,应该用原始sql
@ -99,13 +101,28 @@ public class StackAreaHandler extends YoyChartHandler {
var assistFields = getAssistFields(dynamicAssistFields, yAxis);
if (CollectionUtils.isNotEmpty(assistFields)) {
var req = new DatasourceRequest();
req.setIsCross(crossDs);
req.setDsList(dsMap);
var assistSql = assistSQL(originSql, assistFields, dsMap);
List<ChartSeniorAssistDTO> assists = dynamicAssistFields.stream().filter(ele -> !StringUtils.equalsIgnoreCase(ele.getSummary(), "last_item")).toList();
if (ObjectUtils.isNotEmpty(assists)) {
var assistSql = assistSQL(originSql, assistFields, dsMap, crossDs);
req.setQuery(assistSql);
logger.debug("calcite assist sql: " + assistSql);
logger.debug("calcite assistSql sql: " + assistSql);
var assistData = (List<String[]>) provider.fetchResultField(req).get("data");
result.setAssistData(assistData);
result.setDynamicAssistFields(dynamicAssistFields);
result.setDynamicAssistFields(assists);
}
List<ChartSeniorAssistDTO> assistsOriginList = dynamicAssistFields.stream().filter(ele -> StringUtils.equalsIgnoreCase(ele.getSummary(), "last_item")).toList();
if (ObjectUtils.isNotEmpty(assistsOriginList)) {
var assistSqlOriginList = assistSQLOriginList(originSql, assistFields, dsMap, crossDs);
req.setQuery(assistSqlOriginList);
logger.debug("calcite assistSql sql origin list: " + assistSqlOriginList);
var assistDataOriginList = (List<String[]>) provider.fetchResultField(req).get("data");
result.setAssistDataOriginList(assistDataOriginList);
result.setDynamicAssistFieldsOriginList(assistsOriginList);
}
}
} catch (Exception e) {
e.printStackTrace();

15
core/core-backend/src/main/java/io/gisbi/chart/charts/impl/map/SymbolicMapHandler.java
View File

@ -1,5 +1,6 @@
package io.gisbi.chart.charts.impl.map;
import io.gisbi.api.dataset.union.DatasetGroupInfoDTO;
import io.gisbi.chart.charts.impl.GroupChartHandler;
import io.gisbi.chart.utils.ChartDataBuild;
import io.gisbi.dataset.manage.DatasetDataManage;
@ -60,22 +61,23 @@ public class SymbolicMapHandler extends GroupChartHandler {
boolean needOrder = Utils.isNeedOrder(dsList);
boolean crossDs = Utils.isCrossDs(dsMap);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setIsCross(((DatasetGroupInfoDTO) formatResult.getContext().get("dataset")).getIsCross());
datasourceRequest.setDsList(dsMap);
var xAxis = formatResult.getAxisMap().get(ChartAxis.xAxis);
var extBubble = formatResult.getAxisMap().get(ChartAxis.extBubble);
var allFields = (List<ChartViewFieldDTO>) filterResult.getContext().get("allFields");
List<ChartViewFieldDTO> countField =chartViewManege.transFieldDTO(Collections.singletonList(chartViewManege.createCountField(view.getTableId())));
List<ChartViewFieldDTO> countField = chartViewManege.transFieldDTO(Collections.singletonList(chartViewManege.createCountField(view.getTableId())));
List<DatasetTableFieldDTO> datasetTableFieldDTOList = FieldUtil.transFields(allFields);
SQLMeta sqlMeta1 = new SQLMeta();
BeanUtils.copyBean(sqlMeta1, sqlMeta);
Dimension2SQLObj.dimension2sqlObj(sqlMeta, xAxis, datasetTableFieldDTOList, crossDs, dsMap, Utils.getParams(datasetTableFieldDTOList), view.getCalParams(), pluginManage);
List<ChartViewFieldDTO> yAxis = new ArrayList<>();
if(!extBubble.isEmpty() && !"*".equals(extBubble.get(0).getDataeaseName())){
if (!extBubble.isEmpty() && !"*".equals(extBubble.get(0).getGisbiName())) {
yAxis.addAll(extBubble);
}
yAxis.addAll(countField);
datasetTableFieldDTOList.addAll(FieldUtil.transFields(countField));
formatResult.getAxisMap().put(ChartAxis.yAxis,countField);
formatResult.getAxisMap().put(ChartAxis.yAxis, countField);
Quota2SQLObj.quota2sqlObj(sqlMeta, yAxis, datasetTableFieldDTOList, crossDs, dsMap, Utils.getParams(datasetTableFieldDTOList), view.getCalParams(), pluginManage);
String querySql = SQLProvider.createQuerySQL(sqlMeta, true, needOrder, view);
querySql = provider.rebuildSQL(querySql, sqlMeta, crossDs, dsMap);
@ -102,6 +104,7 @@ public class SymbolicMapHandler extends GroupChartHandler {
logger.debug("calcite data preview sql: " + querySQL);
// 调用数据源的calcite获得data
DatasourceRequest datasourceRequest1 = new DatasourceRequest();
datasourceRequest1.setIsCross(((DatasetGroupInfoDTO) formatResult.getContext().get("dataset")).getIsCross());
datasourceRequest1.setQuery(querySQL);
datasourceRequest1.setDsList(dsMap);
detailData = (List<String[]>) provider.fetchResultField(datasourceRequest1).get("data");
@ -115,7 +118,7 @@ public class SymbolicMapHandler extends GroupChartHandler {
calcResult.setContext(filterResult.getContext());
calcResult.setQuerySql(querySql);
calcResult.setOriginData(data);
formatResult.getAxisMap().put(ChartAxis.yAxis,new ArrayList<>());
formatResult.getAxisMap().put(ChartAxis.yAxis, new ArrayList<>());
return calcResult;
}
@ -142,8 +145,8 @@ public class SymbolicMapHandler extends GroupChartHandler {
dataMap.putAll(calcResult.getData());
dataMap.putAll(mapTableNormal);
dataMap.put("sourceFields", allFields);
mergeAssistField(calcResult.getDynamicAssistFields(), calcResult.getAssistData());
dataMap.put("dynamicAssistLines", calcResult.getDynamicAssistFields());
List<ChartSeniorAssistDTO> chartSeniorAssistDTOS = mergeAssistField(calcResult.getDynamicAssistFields(), calcResult.getAssistData(), calcResult.getDynamicAssistFieldsOriginList(), calcResult.getAssistDataOriginList());
dataMap.put("dynamicAssistLines", chartSeniorAssistDTOS);
view.setData(dataMap);
view.setSql(Base64.getEncoder().encodeToString(calcResult.getQuerySql().getBytes()));
view.setDrill(isDrill);

53
core/core-backend/src/main/java/io/gisbi/chart/charts/impl/mix/MixHandler.java
View File

@ -1,5 +1,6 @@
package io.gisbi.chart.charts.impl.mix;
import io.gisbi.api.dataset.union.DatasetGroupInfoDTO;
import io.gisbi.chart.charts.impl.YoyChartHandler;
import io.gisbi.chart.utils.ChartDataBuild;
import io.gisbi.engine.utils.Utils;
@ -10,6 +11,7 @@ import io.gisbi.extensions.datasource.provider.Provider;
import io.gisbi.extensions.view.dto.*;
import lombok.Getter;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
@ -80,7 +82,7 @@ public class MixHandler extends YoyChartHandler {
dsList.add(next.getValue().getType());
}
boolean needOrder = Utils.isNeedOrder(dsList);
boolean crossDs = Utils.isCrossDs(dsMap);
boolean crossDs = ((DatasetGroupInfoDTO) formatResult.getContext().get("dataset")).getIsCross();
var leftResult = (T) super.calcChartResult(view, formatResult, filterResult, sqlMap, sqlMeta, provider);
var dynamicAssistFields = getDynamicAssistFields(view);
try {
@ -91,13 +93,28 @@ public class MixHandler extends YoyChartHandler {
var assistFields = getAssistFields(leftAssistFields, yAxis);
if (CollectionUtils.isNotEmpty(assistFields)) {
var req = new DatasourceRequest();
req.setIsCross(crossDs);
req.setDsList(dsMap);
var assistSql = assistSQL(originSql, assistFields, dsMap);
List<ChartSeniorAssistDTO> assists = leftAssistFields.stream().filter(ele -> !StringUtils.equalsIgnoreCase(ele.getSummary(), "last_item")).toList();
if (ObjectUtils.isNotEmpty(assists)) {
var assistSql = assistSQL(originSql, assistFields, dsMap, crossDs);
req.setQuery(assistSql);
logger.debug("calcite assistSql sql: " + assistSql);
var assistData = (List<String[]>) provider.fetchResultField(req).get("data");
leftResult.setAssistData(assistData);
leftResult.setDynamicAssistFields(leftAssistFields);
leftResult.setDynamicAssistFields(assists);
}
List<ChartSeniorAssistDTO> assistsOriginList = leftAssistFields.stream().filter(ele -> StringUtils.equalsIgnoreCase(ele.getSummary(), "last_item")).toList();
if (ObjectUtils.isNotEmpty(assistsOriginList)) {
var assistSqlOriginList = assistSQLOriginList(originSql, assistFields, dsMap, crossDs);
req.setQuery(assistSqlOriginList);
logger.debug("calcite assistSql sql origin list: " + assistSqlOriginList);
var assistDataOriginList = (List<String[]>) provider.fetchResultField(req).get("data");
leftResult.setAssistDataOriginList(assistDataOriginList);
leftResult.setDynamicAssistFieldsOriginList(assistsOriginList);
}
}
} catch (Exception e) {
e.printStackTrace();
@ -143,12 +160,28 @@ public class MixHandler extends YoyChartHandler {
var assistFields = getAssistFields(rightAssistFields, yAxis);
if (CollectionUtils.isNotEmpty(assistFields)) {
var req = new DatasourceRequest();
req.setIsCross(crossDs);
req.setDsList(dsMap);
var assistSql = assistSQL(originSql, assistFields, dsMap);
List<ChartSeniorAssistDTO> assists = rightAssistFields.stream().filter(ele -> !StringUtils.equalsIgnoreCase(ele.getSummary(), "last_item")).toList();
if (ObjectUtils.isNotEmpty(assists)) {
var assistSql = assistSQL(originSql, assistFields, dsMap, crossDs);
req.setQuery(assistSql);
logger.debug("calcite assistSql sql: " + assistSql);
var assistData = (List<String[]>) provider.fetchResultField(req).get("data");
rightResult.setAssistData(assistData);
rightResult.setDynamicAssistFields(rightAssistFields);
rightResult.setDynamicAssistFields(assists);
}
List<ChartSeniorAssistDTO> assistsOriginList = rightAssistFields.stream().filter(ele -> StringUtils.equalsIgnoreCase(ele.getSummary(), "last_item")).toList();
if (ObjectUtils.isNotEmpty(assistsOriginList)) {
var assistSqlOriginList = assistSQLOriginList(originSql, assistFields, dsMap, crossDs);
req.setQuery(assistSqlOriginList);
logger.debug("calcite assistSql sql origin list: " + assistSqlOriginList);
var assistDataOriginList = (List<String[]>) provider.fetchResultField(req).get("data");
rightResult.setAssistDataOriginList(assistDataOriginList);
rightResult.setDynamicAssistFieldsOriginList(assistsOriginList);
}
}
} catch (Exception e) {
e.printStackTrace();
@ -169,12 +202,12 @@ public class MixHandler extends YoyChartHandler {
var leftFields = new ArrayList<ChartViewFieldDTO>();
leftFields.addAll(formatResult.getAxisMap().get(ChartAxis.xAxis));
leftFields.addAll(formatResult.getAxisMap().get(ChartAxis.yAxis));
mergeAssistField(leftCalcResult.getDynamicAssistFields(), leftCalcResult.getAssistData());
List<ChartSeniorAssistDTO> chartSeniorAssistDTOSLeft = mergeAssistField(leftCalcResult.getDynamicAssistFields(), leftCalcResult.getAssistData(), leftCalcResult.getDynamicAssistFieldsOriginList(), leftCalcResult.getAssistDataOriginList());
var leftOriginData = leftCalcResult.getOriginData();
var leftTable = ChartDataBuild.transTableNormal(leftFields, view, leftOriginData, desensitizationList);
var leftData = new HashMap<String, Object>(leftTable);
leftData.putAll(leftCalcResult.getData());
leftData.put("dynamicAssistLines", leftCalcResult.getDynamicAssistFields());
leftData.put("dynamicAssistLines", chartSeniorAssistDTOSLeft);
var rightCalcResult = (ChartCalcDataResult) calcResult.getData().get("right");
var rightFields = new ArrayList<ChartViewFieldDTO>();
@ -183,12 +216,12 @@ public class MixHandler extends YoyChartHandler {
rightFields.addAll(subAxisMap.get(ChartAxis.xAxis));
rightFields.addAll(subAxisMap.get(ChartAxis.yAxis));
mergeAssistField(rightCalcResult.getDynamicAssistFields(), rightCalcResult.getAssistData());
List<ChartSeniorAssistDTO> chartSeniorAssistDTOSRight = mergeAssistField(rightCalcResult.getDynamicAssistFields(), rightCalcResult.getAssistData(), rightCalcResult.getDynamicAssistFieldsOriginList(), rightCalcResult.getAssistDataOriginList());
var rightOriginData = rightCalcResult.getOriginData();
var rightTable = ChartDataBuild.transTableNormal(rightFields, view, rightOriginData, desensitizationList);
var rightData = new HashMap<String, Object>(rightTable);
rightData.putAll(rightCalcResult.getData());
rightData.put("dynamicAssistLines", rightCalcResult.getDynamicAssistFields());
rightData.put("dynamicAssistLines", chartSeniorAssistDTOSRight);
// 构建结果
Map<String, Object> chartData = new TreeMap<>();
@ -205,7 +238,7 @@ public class MixHandler extends YoyChartHandler {
var isDrill = CollectionUtils.isNotEmpty(drillFilters);
view.setDrillFilters(drillFilters);
view.setDrill(isDrill);
view.setSql(leftCalcResult.getQuerySql());
view.setSql(Base64.getEncoder().encodeToString(leftCalcResult.getQuerySql().getBytes()));
view.setData(chartData);
return view;
}

9
core/core-backend/src/main/java/io/gisbi/chart/charts/impl/numeric/NumericalChartHandler.java
View File

@ -1,5 +1,6 @@
package io.gisbi.chart.charts.impl.numeric;
import io.gisbi.api.dataset.union.DatasetGroupInfoDTO;
import io.gisbi.chart.charts.impl.DefaultChartHandler;
import io.gisbi.chart.utils.ChartDataBuild;
import io.gisbi.engine.sql.SQLProvider;
@ -33,6 +34,7 @@ public class NumericalChartHandler extends DefaultChartHandler {
boolean needOrder = Utils.isNeedOrder(dsList);
boolean crossDs = Utils.isCrossDs(dsMap);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setIsCross(((DatasetGroupInfoDTO) formatResult.getContext().get("dataset")).getIsCross());
datasourceRequest.setDsList(dsMap);
var xAxis = formatResult.getAxisMap().get(ChartAxis.xAxis);
var yAxis = formatResult.getAxisMap().get(ChartAxis.yAxis);
@ -67,14 +69,11 @@ public class NumericalChartHandler extends DefaultChartHandler {
String summary = (String) maxField.get("summary");
DatasetTableFieldDTO datasetTableField = datasetTableFieldManage.selectById(id);
if (ObjectUtils.isNotEmpty(datasetTableField)) {
if (datasetTableField.getDeType() == 0 || datasetTableField.getDeType() == 1 || datasetTableField.getDeType() == 5) {
if (!StringUtils.containsIgnoreCase(summary, "count")) {
DEException.throwException(Translator.get("i18n_gauge_field_change"));
}
}
ChartViewFieldDTO dto = new ChartViewFieldDTO();
BeanUtils.copyBean(dto, datasetTableField);
if (StringUtils.isEmpty(dto.getSummary())) {
dto.setSummary(summary);
}
return dto;
} else {
DEException.throwException(Translator.get("i18n_gauge_field_delete"));

80
core/core-backend/src/main/java/io/gisbi/chart/charts/impl/pie/PieHandler.java
View File

@ -1,11 +1,15 @@
package io.gisbi.chart.charts.impl.pie;
import io.gisbi.chart.charts.impl.YoyChartHandler;
import io.gisbi.extensions.view.dto.AxisFormatResult;
import io.gisbi.extensions.view.dto.ChartAxis;
import io.gisbi.extensions.view.dto.ChartViewDTO;
import io.gisbi.extensions.view.dto.*;
import org.springframework.stereotype.Component;
import java.math.BigDecimal;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
@Component
public class PieHandler extends YoyChartHandler {
@Override
@ -26,4 +30,74 @@ public class PieHandler extends YoyChartHandler {
result.getAxisMap().put(ChartAxis.extTooltip, view.getExtTooltip());
return result;
}
@Override
public ChartViewDTO buildChart(ChartViewDTO view, ChartCalcDataResult calcResult, AxisFormatResult formatResult, CustomFilterResult filterResult) {
ChartViewDTO result = super.buildChart(view, calcResult, formatResult, filterResult);
filterPositiveData(result, "data", AxisChartDataAntVDTO.class);
filterPositiveData(result, "tableRow", Map.class, view.getYAxis().get(0).getGisbiName());
return result;
}
/**
* 过滤正数数据根据data
* @param result
* @param key
* @param clazz
* @param <T>
*/
private <T> void filterPositiveData(ChartViewDTO result, String key, Class<T> clazz) {
if (result.getData().containsKey(key)) {
List<T> list = ((List<T>) result.getData().get(key))
.stream()
.filter(item -> {
if (clazz == AxisChartDataAntVDTO.class) {
if (Objects.isNull(((AxisChartDataAntVDTO) item).getValue())) return false;
return ((AxisChartDataAntVDTO) item).getValue().compareTo(BigDecimal.ZERO) >= 0;
} else if (clazz == Map.class) {
return isPositive(((Map<String, Object>) item).get("value"));
}
return false;
})
.collect(Collectors.toList());
result.getData().put(key, list);
}
}
/**
* 过滤正数数据根据tableRow
* @param result
* @param key
* @param clazz
* @param yAxisName
* @param <T>
*/
private <T> void filterPositiveData(ChartViewDTO result, String key, Class<T> clazz, String yAxisName) {
if (result.getData().containsKey(key)) {
List<T> list = ((List<T>) result.getData().get(key))
.stream()
.filter(item -> {
if (clazz == Map.class) {
Object value = ((Map<String, Object>) item).get(yAxisName);
return isPositive(value);
}
return false;
})
.collect(Collectors.toList());
result.getData().put(key, list);
}
}
private boolean isPositive(Object value) {
if (value instanceof String) {
try {
return new BigDecimal((String) value).compareTo(BigDecimal.ZERO) >= 0;
} catch (NumberFormatException e) {
return false;
}
} else if (value instanceof BigDecimal) {
return ((BigDecimal) value).compareTo(BigDecimal.ZERO) >= 0;
}
return false;
}
}

2
core/core-backend/src/main/java/io/gisbi/chart/charts/impl/table/TableHeatmapHandler.java
View File

@ -1,5 +1,6 @@
package io.gisbi.chart.charts.impl.table;
import io.gisbi.api.dataset.union.DatasetGroupInfoDTO;
import io.gisbi.chart.charts.impl.DefaultChartHandler;
import io.gisbi.chart.utils.ChartDataBuild;
import io.gisbi.engine.sql.SQLProvider;
@ -55,6 +56,7 @@ public class TableHeatmapHandler extends DefaultChartHandler {
boolean needOrder = Utils.isNeedOrder(dsList);
boolean crossDs = Utils.isCrossDs(dsMap);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setIsCross(((DatasetGroupInfoDTO) formatResult.getContext().get("dataset")).getIsCross());
datasourceRequest.setDsList(dsMap);
var xAxis = formatResult.getAxisMap().get(ChartAxis.xAxis);
var yAxis = formatResult.getAxisMap().get(ChartAxis.yAxis);

33
core/core-backend/src/main/java/io/gisbi/chart/charts/impl/table/TableInfoHandler.java
View File

@ -1,6 +1,7 @@
package io.gisbi.chart.charts.impl.table;
import io.gisbi.api.chart.dto.PageInfo;
import io.gisbi.api.dataset.union.DatasetGroupInfoDTO;
import io.gisbi.chart.charts.impl.DefaultChartHandler;
import io.gisbi.engine.sql.SQLProvider;
import io.gisbi.engine.trans.Dimension2SQLObj;
@ -65,6 +66,11 @@ public class TableInfoHandler extends DefaultChartHandler {
return (T) new CustomFilterResult(filterList, formatResult.getContext());
}
@Override
public Map<String, Object> buildResult(ChartViewDTO view, AxisFormatResult formatResult, CustomFilterResult filterResult, List<String[]> data) {
return new HashMap<>();
}
@Override
public <T extends ChartCalcDataResult> T calcChartResult(ChartViewDTO view, AxisFormatResult formatResult, CustomFilterResult filterResult, Map<String, Object> sqlMap, SQLMeta sqlMeta, Provider provider) {
var chartExtRequest = view.getChartExtRequest();
@ -73,8 +79,9 @@ public class TableInfoHandler extends DefaultChartHandler {
for (Map.Entry<Long, DatasourceSchemaDTO> next : dsMap.entrySet()) {
dsList.add(next.getValue().getType());
}
boolean crossDs = Utils.isCrossDs(dsMap);
boolean crossDs = ((DatasetGroupInfoDTO) formatResult.getContext().get("dataset")).getIsCross();
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setIsCross(crossDs);
datasourceRequest.setDsList(dsMap);
var xAxis = formatResult.getAxisMap().get(ChartAxis.xAxis);
var allFields = (List<ChartViewFieldDTO>) filterResult.getContext().get("allFields");
@ -150,12 +157,28 @@ public class TableInfoHandler extends DefaultChartHandler {
if (CollectionUtils.isNotEmpty(assistFields)) {
var req = new DatasourceRequest();
req.setDsList(dsMap);
var assistSql = assistSQL(querySql, assistFields, dsMap);
req.setQuery(assistSql);
logger.debug("calcite assistSql sql: " + assistSql);
List<ChartSeniorAssistDTO> assists = dynamicAssistFields.stream().filter(ele -> !StringUtils.equalsIgnoreCase(ele.getSummary(), "last_item")).toList();
if (ObjectUtils.isNotEmpty(assists)) {
var assistSql = assistSQL(originSql, assistFields, dsMap, crossDs);
var tmpSql = provider.rebuildSQL(assistSql, sqlMeta, crossDs, dsMap);
req.setQuery(tmpSql);
logger.debug("calcite assistSql sql: " + tmpSql);
var assistData = (List<String[]>) provider.fetchResultField(req).get("data");
calcResult.setAssistData(assistData);
calcResult.setDynamicAssistFields(dynamicAssistFields);
calcResult.setDynamicAssistFields(assists);
}
List<ChartSeniorAssistDTO> assistsOriginList = dynamicAssistFields.stream().filter(ele -> StringUtils.equalsIgnoreCase(ele.getSummary(), "last_item")).toList();
if (ObjectUtils.isNotEmpty(assistsOriginList)) {
var assistSqlOriginList = assistSQLOriginList(originSql, assistFields, dsMap, crossDs);
var tmpSql = provider.rebuildSQL(assistSqlOriginList, sqlMeta, crossDs, dsMap);
req.setQuery(tmpSql);
logger.debug("calcite assistSql sql origin list: " + tmpSql);
var assistDataOriginList = (List<String[]>) provider.fetchResultField(req).get("data");
calcResult.setAssistDataOriginList(assistDataOriginList);
calcResult.setDynamicAssistFieldsOriginList(assistsOriginList);
}
}
} catch (Exception e) {
e.printStackTrace();

196
core/core-backend/src/main/java/io/gisbi/chart/charts/impl/table/TableNormalHandler.java
View File

@ -1,49 +1,213 @@
package io.gisbi.chart.charts.impl.table;
import io.gisbi.chart.charts.impl.YoyChartHandler;
import com.fasterxml.jackson.core.type.TypeReference;
import io.gisbi.api.chart.dto.PageInfo;
import io.gisbi.api.dataset.union.DatasetGroupInfoDTO;
import io.gisbi.chart.charts.impl.DefaultChartHandler;
import io.gisbi.engine.sql.SQLProvider;
import io.gisbi.engine.trans.Dimension2SQLObj;
import io.gisbi.engine.trans.ExtWhere2Str;
import io.gisbi.engine.trans.Quota2SQLObj;
import io.gisbi.engine.utils.Utils;
import io.gisbi.extensions.datasource.dto.DatasourceRequest;
import io.gisbi.extensions.datasource.dto.DatasourceSchemaDTO;
import io.gisbi.extensions.datasource.model.SQLMeta;
import io.gisbi.extensions.datasource.provider.Provider;
import io.gisbi.extensions.view.dto.*;
import io.gisbi.extensions.view.util.ChartDataUtil;
import io.gisbi.extensions.view.util.FieldUtil;
import io.gisbi.utils.JsonUtil;
import lombok.Getter;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
import java.util.List;
import java.util.Map;
import java.util.*;
/**
* @author jianneng
* @date 2024/9/11 11:37
**/
@Component
public class TableNormalHandler extends YoyChartHandler {
public class TableNormalHandler extends DefaultChartHandler {
@Getter
private String type = "table-normal";
private final String type = "table-normal";
@Override
public <T extends CustomFilterResult> T customFilter(ChartViewDTO view, List<ChartExtFilterDTO> filterList, AxisFormatResult formatResult) {
var chartExtRequest = view.getChartExtRequest();
Map<String, Object> mapAttr = view.getCustomAttr();
Map<String, Object> basicStyle = (Map<String, Object>) mapAttr.get("basicStyle");
var tablePageMode = (String) basicStyle.get("tablePageMode");
formatResult.getContext().put("tablePageMode", tablePageMode);
if (StringUtils.equalsIgnoreCase(tablePageMode, "page")) {
if (chartExtRequest.getGoPage() == null) {
chartExtRequest.setGoPage(1L);
}
if (chartExtRequest.getPageSize() == null) {
int pageSize = (int) basicStyle.get("tablePageSize");
if (StringUtils.equalsIgnoreCase(view.getResultMode(), "custom")) {
chartExtRequest.setPageSize(Math.min(pageSize, view.getResultCount().longValue()));
} else {
chartExtRequest.setPageSize((long) pageSize);
}
}
} else {
if (StringUtils.equalsIgnoreCase(view.getResultMode(), "custom")) {
chartExtRequest.setGoPage(1L);
chartExtRequest.setPageSize(view.getResultCount().longValue());
} else if (!view.getIsExcelExport()) {
chartExtRequest.setGoPage(null);
chartExtRequest.setPageSize(null);
}
}
var yAxis = formatResult.getAxisMap().get(ChartAxis.yAxis);
String originFilterJson = (String) JsonUtil.toJSONString(filterList);
List<ChartExtFilterDTO> originFilter = JsonUtil.parseList(originFilterJson, new TypeReference<>() {
});
boolean yoyFiltered = checkYoyFilter(originFilter, yAxis);
if (yoyFiltered) {
formatResult.getContext().put("expandedFilter", originFilter);
formatResult.getContext().put("yoyFiltered", true);
}
return (T) new CustomFilterResult(filterList, formatResult.getContext());
}
@Override
public Map<String, Object> buildResult(ChartViewDTO view, AxisFormatResult formatResult, CustomFilterResult filterResult, List<String[]> data) {
return new HashMap<>();
}
@Override
public <T extends ChartCalcDataResult> T calcChartResult(ChartViewDTO view, AxisFormatResult formatResult, CustomFilterResult filterResult, Map<String, Object> sqlMap, SQLMeta sqlMeta, Provider provider) {
var chartExtRequest = view.getChartExtRequest();
var dsMap = (Map<Long, DatasourceSchemaDTO>) sqlMap.get("dsMap");
var result = (T) super.calcChartResult(view, formatResult, filterResult, sqlMap, sqlMeta, provider);
try {
var originSql = result.getQuerySql();
var dynamicAssistFields = getDynamicThresholdFields(view);
List<String> dsList = new ArrayList<>();
for (Map.Entry<Long, DatasourceSchemaDTO> next : dsMap.entrySet()) {
dsList.add(next.getValue().getType());
}
boolean crossDs = ((DatasetGroupInfoDTO) formatResult.getContext().get("dataset")).getIsCross();
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setIsCross(crossDs);
datasourceRequest.setDsList(dsMap);
var xAxis = formatResult.getAxisMap().get(ChartAxis.xAxis);
var yAxis = formatResult.getAxisMap().get(ChartAxis.yAxis);
var assistFields = getAssistFields(dynamicAssistFields, yAxis);
var allFields = (List<ChartViewFieldDTO>) filterResult.getContext().get("allFields");
PageInfo pageInfo = new PageInfo();
pageInfo.setGoPage(chartExtRequest.getGoPage());
if (StringUtils.equalsIgnoreCase(view.getResultMode(), "custom")) {
pageInfo.setPageSize(Math.min(view.getResultCount() - (chartExtRequest.getGoPage() - 1) * chartExtRequest.getPageSize(), chartExtRequest.getPageSize()));
} else {
pageInfo.setPageSize(chartExtRequest.getPageSize());
}
Dimension2SQLObj.dimension2sqlObj(sqlMeta, xAxis, FieldUtil.transFields(allFields), crossDs, dsMap, Utils.getParams(FieldUtil.transFields(allFields)), view.getCalParams(), pluginManage);
Quota2SQLObj.quota2sqlObj(sqlMeta, yAxis, FieldUtil.transFields(allFields), crossDs, dsMap, Utils.getParams(FieldUtil.transFields(allFields)), view.getCalParams(), pluginManage);
String originSql = SQLProvider.createQuerySQL(sqlMeta, true, !StringUtils.equalsIgnoreCase(dsMap.values().iterator().next().getType(), "es"), view);// 分页强制加排序
String limit = ((pageInfo.getGoPage() != null && pageInfo.getPageSize() != null) ? " LIMIT " + pageInfo.getPageSize() + " OFFSET " + (pageInfo.getGoPage() - 1) * chartExtRequest.getPageSize() : "");
var querySql = originSql + limit;
var tablePageMode = (String) filterResult.getContext().get("tablePageMode");
var totalPageSql = "SELECT COUNT(*) FROM (" + SQLProvider.createQuerySQLNoSort(sqlMeta, true, view) + ") COUNT_TEMP";
if (StringUtils.isNotEmpty(totalPageSql) && StringUtils.equalsIgnoreCase(tablePageMode, "page")) {
totalPageSql = provider.rebuildSQL(totalPageSql, sqlMeta, crossDs, dsMap);
datasourceRequest.setQuery(totalPageSql);
datasourceRequest.setTotalPageFlag(true);
logger.debug("calcite total sql: " + totalPageSql);
List<String[]> tmpData = (List<String[]>) provider.fetchResultField(datasourceRequest).get("data");
var totalItems = ObjectUtils.isEmpty(tmpData) ? 0 : Long.valueOf(tmpData.get(0)[0]);
if (StringUtils.equalsIgnoreCase(view.getResultMode(), "custom")) {
totalItems = totalItems <= view.getResultCount() ? totalItems : view.getResultCount();
}
var totalPage = (totalItems / pageInfo.getPageSize()) + (totalItems % pageInfo.getPageSize() > 0 ? 1 : 0);
view.setTotalItems(totalItems);
view.setTotalPage(totalPage);
}
querySql = provider.rebuildSQL(querySql, sqlMeta, crossDs, dsMap);
datasourceRequest.setQuery(querySql);
logger.debug("calcite chart sql: " + querySql);
List<String[]> data = (List<String[]>) provider.fetchResultField(datasourceRequest).get("data");
//自定义排序
data = ChartDataUtil.resultCustomSort(xAxis, yAxis, view.getSortPriority(), data);
var yoyFiltered = filterResult.getContext().get("yoyFiltered") != null;
if (yoyFiltered) {
// 这里没加分页，因为加了分页参数可能会把原始数据挤出去
var expandedFilter = (List<ChartExtFilterDTO>) filterResult.getContext().get("expandedFilter");
ExtWhere2Str.extWhere2sqlOjb(sqlMeta, expandedFilter, FieldUtil.transFields(allFields), crossDs, dsMap, Utils.getParams(FieldUtil.transFields(allFields)), view.getCalParams(), pluginManage);
var expandedSql = SQLProvider.createQuerySQL(sqlMeta, true, !StringUtils.equalsIgnoreCase(dsMap.values().iterator().next().getType(), "es"), view);
expandedSql = provider.rebuildSQL(expandedSql, sqlMeta, crossDs, dsMap);
var expandedReq = new DatasourceRequest();
expandedReq.setIsCross(crossDs);
expandedReq.setDsList(dsMap);
expandedReq.setQuery(expandedSql);
logger.debug("expanded sql: " + expandedSql);
var expandedData = (List<String[]>) provider.fetchResultField(expandedReq).get("data");
logger.debug("expanded data: " + expandedData);
quickCalc(xAxis, yAxis, Collections.emptyList(), Collections.emptyList(), view.getType(), expandedData);
var resultData = new ArrayList<String[]>();
for (String[] originDataLine : data) {
var originDim = new StringBuilder();
for (int i = 0; i < xAxis.size(); i++) {
originDim.append(originDataLine[i]);
}
for (String[] expandedDataLine : expandedData) {
var expandedDim = new StringBuilder();
for (int i = 0; i < xAxis.size(); i++) {
expandedDim.append(expandedDataLine[i]);
}
if (StringUtils.equals(originDim, expandedDim)) {
resultData.add(expandedDataLine);
break;
}
}
}
data = resultData;
} else {
quickCalc(xAxis, yAxis, Collections.emptyList(), Collections.emptyList(), view.getType(), data);
}
//数据重组逻辑可重载
var result = this.buildResult(view, formatResult, filterResult, data);
T calcResult = (T) new ChartCalcDataResult();
calcResult.setData(result);
calcResult.setContext(filterResult.getContext());
calcResult.setQuerySql(querySql);
calcResult.setOriginData(data);
try {
var dynamicAssistFields = getDynamicThresholdFields(view);
var assistFields = getAssistFields(dynamicAssistFields, yAxis, xAxis);
if (CollectionUtils.isNotEmpty(assistFields)) {
var req = new DatasourceRequest();
req.setIsCross(crossDs);
req.setDsList(dsMap);
var assistSql = assistSQL(originSql, assistFields, dsMap);
req.setQuery(assistSql);
logger.debug("calcite assistSql sql: " + assistSql);
List<ChartSeniorAssistDTO> assists = dynamicAssistFields.stream().filter(ele -> !StringUtils.equalsIgnoreCase(ele.getSummary(), "last_item")).toList();
if (ObjectUtils.isNotEmpty(assists)) {
var assistSql = assistSQL(originSql, assistFields, dsMap, crossDs);
var tmpSql = provider.rebuildSQL(assistSql, sqlMeta, crossDs, dsMap);
req.setQuery(tmpSql);
logger.debug("calcite assistSql sql: " + tmpSql);
var assistData = (List<String[]>) provider.fetchResultField(req).get("data");
result.setAssistData(assistData);
result.setDynamicAssistFields(dynamicAssistFields);
calcResult.setAssistData(assistData);
calcResult.setDynamicAssistFields(assists);
}
List<ChartSeniorAssistDTO> assistsOriginList = dynamicAssistFields.stream().filter(ele -> StringUtils.equalsIgnoreCase(ele.getSummary(), "last_item")).toList();
if (ObjectUtils.isNotEmpty(assistsOriginList)) {
var assistSqlOriginList = assistSQLOriginList(originSql, assistFields, dsMap, crossDs);
var tmpSql = provider.rebuildSQL(assistSqlOriginList, sqlMeta, crossDs, dsMap);
req.setQuery(tmpSql);
logger.debug("calcite assistSql sql origin list: " + tmpSql);
var assistDataOriginList = (List<String[]>) provider.fetchResultField(req).get("data");
calcResult.setAssistDataOriginList(assistDataOriginList);
calcResult.setDynamicAssistFieldsOriginList(assistsOriginList);
}
}
} catch (Exception e) {
e.printStackTrace();
}
return result;
return calcResult;
}
}

56
core/core-backend/src/main/java/io/gisbi/chart/charts/impl/table/TablePivotHandler.java
View File

@ -1,5 +1,6 @@
package io.gisbi.chart.charts.impl.table;
import io.gisbi.api.dataset.union.DatasetGroupInfoDTO;
import io.gisbi.chart.charts.impl.GroupChartHandler;
import io.gisbi.constant.DeTypeConstants;
import io.gisbi.engine.constant.ExtFieldConstant;
@ -18,6 +19,7 @@ import io.gisbi.utils.IDUtils;
import io.gisbi.utils.JsonUtil;
import lombok.Getter;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
import reactor.util.function.Tuple2;
@ -34,7 +36,8 @@ public class TablePivotHandler extends GroupChartHandler {
@Override
public <T extends ChartCalcDataResult> T calcChartResult(ChartViewDTO view, AxisFormatResult formatResult, CustomFilterResult filterResult, Map<String, Object> sqlMap, SQLMeta sqlMeta, Provider provider) {
T result = super.calcChartResult(view, formatResult, filterResult, sqlMap, sqlMeta, provider);
Map<String, Object> customCalc = calcCustomExpr(view, filterResult, sqlMap, sqlMeta, provider);
Map<String, Object> customCalc = calcCustomExpr(view, formatResult, filterResult, sqlMap, sqlMeta, provider);
boolean crossDs = ((DatasetGroupInfoDTO) formatResult.getContext().get("dataset")).getIsCross();
result.getData().put("customCalc", customCalc);
try {
var dsMap = (Map<Long, DatasourceSchemaDTO>) sqlMap.get("dsMap");
@ -44,13 +47,28 @@ public class TablePivotHandler extends GroupChartHandler {
var assistFields = getAssistFields(dynamicAssistFields, yAxis);
if (CollectionUtils.isNotEmpty(assistFields)) {
var req = new DatasourceRequest();
req.setIsCross(crossDs);
req.setDsList(dsMap);
var assistSql = assistSQL(originSql, assistFields, dsMap);
List<ChartSeniorAssistDTO> assists = dynamicAssistFields.stream().filter(ele -> !StringUtils.equalsIgnoreCase(ele.getSummary(), "last_item")).toList();
if (ObjectUtils.isNotEmpty(assists)) {
var assistSql = assistSQL(originSql, assistFields, dsMap, crossDs);
req.setQuery(assistSql);
logger.debug("calcite assistSql sql: " + assistSql);
var assistData = (List<String[]>) provider.fetchResultField(req).get("data");
result.setAssistData(assistData);
result.setDynamicAssistFields(dynamicAssistFields);
result.setDynamicAssistFields(assists);
}
List<ChartSeniorAssistDTO> assistsOriginList = dynamicAssistFields.stream().filter(ele -> StringUtils.equalsIgnoreCase(ele.getSummary(), "last_item")).toList();
if (ObjectUtils.isNotEmpty(assistsOriginList)) {
var assistSqlOriginList = assistSQLOriginList(originSql, assistFields, dsMap, crossDs);
req.setQuery(assistSqlOriginList);
logger.debug("calcite assistSql sql origin list: " + assistSqlOriginList);
var assistDataOriginList = (List<String[]>) provider.fetchResultField(req).get("data");
result.setAssistDataOriginList(assistDataOriginList);
result.setDynamicAssistFieldsOriginList(assistsOriginList);
}
}
} catch (Exception e) {
e.printStackTrace();
@ -58,7 +76,18 @@ public class TablePivotHandler extends GroupChartHandler {
return result;
}
private Map<String, Object> calcCustomExpr(ChartViewDTO view, CustomFilterResult filterResult, Map<String, Object> sqlMap, SQLMeta sqlMeta, Provider provider) {
@Override
public Map<String, Object> buildResult(ChartViewDTO view, AxisFormatResult formatResult, CustomFilterResult filterResult, List<String[]> data) {
var result = new HashMap<String, Object>();
var yoyFiltered = filterResult.getContext().get("yoyFiltered") != null;
// 带过滤同环比直接返回原始数据,再由视图重新组装
if (yoyFiltered) {
result.put("data", data);
}
return result;
}
private Map<String, Object> calcCustomExpr(ChartViewDTO view, AxisFormatResult formatResult, CustomFilterResult filterResult, Map<String, Object> sqlMap, SQLMeta sqlMeta, Provider provider) {
Object totalStr = JsonUtil.toJSONString(view.getCustomAttr().get("tableTotal"));
TableTotal tableTotal = JsonUtil.parseObject((String) totalStr, TableTotal.class);
var dsMap = (Map<Long, DatasourceSchemaDTO>) sqlMap.get("dsMap");
@ -67,7 +96,7 @@ public class TablePivotHandler extends GroupChartHandler {
dsList.add(next.getValue().getType());
}
boolean needOrder = Utils.isNeedOrder(dsList);
boolean crossDs = Utils.isCrossDs(dsMap);
boolean crossDs = ((DatasetGroupInfoDTO) formatResult.getContext().get("dataset")).getIsCross();
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDsList(dsMap);
var allFields = (List<ChartViewFieldDTO>) filterResult.getContext().get("allFields");
@ -178,7 +207,7 @@ public class TablePivotHandler extends GroupChartHandler {
var tmpData = new HashMap<String, String>();
for (int i = 0; i < yAxis.size(); i++) {
var a = yAxis.get(i);
tmpData.put(a.getDataeaseName(), data.getFirst()[i]);
tmpData.put(a.getGisbiName(), data.getFirst()[i]);
}
tmp.put("data", tmpData);
tmp.put("sql", Base64.getEncoder().encodeToString(querySql.getBytes()));
@ -208,8 +237,8 @@ public class TablePivotHandler extends GroupChartHandler {
}
}
// 列总计里面的行小计
if (col.isShowGrandTotals() && row.isShowGrandTotals() && rowAxis.size() >= 2) {
var yAxis = getCustomFields(view, row.getCalcTotals().getCfg());
if (col.isShowGrandTotals() && row.isShowSubTotals() && rowAxis.size() >= 2) {
var yAxis = getCustomFields(view, row.getCalcSubTotals().getCfg());
if (!yAxis.isEmpty()) {
var tmpData = new ArrayList<Map<String, Object>>();
dataMap.put("rowSubInColTotal", tmpData);
@ -243,11 +272,11 @@ public class TablePivotHandler extends GroupChartHandler {
var tmpList = new ArrayList<Map<String, Object>>();
tmpData.add(tmpList);
var subRow = rowAxis.subList(0, i + 1);
var xAxis = new ArrayList<>(subRow);
for (int j = 0; j < colAxis.size(); j++) {
if (j == colAxis.size() - 1) {
break;
}
var xAxis = new ArrayList<>(subRow);
var subCol = colAxis.subList(0, j + 1);
xAxis.addAll(subCol);
var tmpAllList = new ArrayList<>(allFields);
@ -272,7 +301,7 @@ public class TablePivotHandler extends GroupChartHandler {
var rowData = data.getFirst();
for (int i = 0; i < rowData.length; i++) {
var qAxis = quotaAxis.get(i);
rootResult.put(qAxis.getDataeaseName(), rowData[i]);
rootResult.put(qAxis.getGisbiName(), rowData[i]);
}
return rootResult;
}
@ -291,7 +320,7 @@ public class TablePivotHandler extends GroupChartHandler {
if (j == dimAxis.size() - 1) {
for (int k = 0; k < quotaAxis.size(); k++) {
var qAxis = quotaAxis.get(k);
curSubMap.put(qAxis.getDataeaseName(), rowData[j + k + 1]);
curSubMap.put(qAxis.getGisbiName(), rowData[j + k + 1]);
}
}
}
@ -303,6 +332,7 @@ public class TablePivotHandler extends GroupChartHandler {
List<ChartViewFieldDTO> allFields, boolean crossDs, Map<Long, DatasourceSchemaDTO> dsMap,
ChartViewDTO view, Provider provider, boolean needOrder) {
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setIsCross(crossDs);
datasourceRequest.setDsList(dsMap);
Dimension2SQLObj.dimension2sqlObj(sqlMeta, xAxis, FieldUtil.transFields(allFields), crossDs, dsMap, Utils.getParams(FieldUtil.transFields(allFields)), view.getCalParams(), pluginManage);
Quota2SQLObj.quota2sqlObj(sqlMeta, yAxis, FieldUtil.transFields(allFields), crossDs, dsMap, Utils.getParams(FieldUtil.transFields(allFields)), view.getCalParams(), pluginManage);
@ -326,10 +356,10 @@ public class TablePivotHandler extends GroupChartHandler {
}
private List<ChartViewFieldDTO> getCustomFields(ChartViewDTO view, List<TableCalcTotalCfg> cfgList) {
var quotaIds = view.getYAxis().stream().map(ChartViewFieldDTO::getDataeaseName).collect(Collectors.toSet());
var quotaIds = view.getYAxis().stream().map(ChartViewFieldDTO::getGisbiName).collect(Collectors.toSet());
var customFields = new ArrayList<ChartViewFieldDTO>();
for (TableCalcTotalCfg totalCfg : cfgList) {
if (!quotaIds.contains(totalCfg.getDataeaseName())) {
if (!quotaIds.contains(totalCfg.getGisbiName())) {
continue;
}
if (StringUtils.equalsIgnoreCase(totalCfg.getAggregation(), "CUSTOM")) {

40
core/core-backend/src/main/java/io/gisbi/chart/dao/ext/mapper/ExtChartViewMapper.java
View File

@ -1,7 +1,9 @@
package io.gisbi.chart.dao.ext.mapper;
import io.gisbi.api.chart.vo.ViewSelectorVO;
import io.gisbi.chart.dao.auto.entity.CoreChartView;
import io.gisbi.chart.dao.ext.entity.ChartBasePO;
import io.gisbi.extensions.view.dto.ChartViewDTO;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Select;
@ -16,29 +18,21 @@ public interface ExtChartViewMapper {
""")
List<ViewSelectorVO> queryViewOption(@Param("resourceId") Long resourceId);
ChartBasePO queryChart(@Param("id") Long id, @Param("resourceTable")String resourceTable);
List<CoreChartView> selectListCustom(@Param("sceneId") Long sceneId, @Param("resourceTable") String resourceTable);
void deleteViewsBySceneId(@Param("sceneId") Long sceneId, @Param("resourceTable") String resourceTable);
@Select("""
select
ccv.id as chart_id,
ccv.title as chart_name,
ccv.type as chart_type,
ccv.table_id,
dvi.id as resource_id,
dvi.name as resource_name,
dvi.type as resource_type,
ccv.x_axis,
ccv.x_axis_ext,
ccv.y_axis,
ccv.y_axis_ext,
ccv.ext_stack,
ccv.ext_bubble,
ccv.ext_label,
ccv.ext_tooltip,
ccv.flow_map_start_name,
ccv.flow_map_end_name,
ccv.ext_color
from core_chart_view ccv
left join data_visualization_info dvi on dvi.id = ccv.scene_id
where ccv.id = #{id}
SELECT id, scene_id as pid, title, type FROM (
SELECT id, scene_id, title, type FROM core_chart_view
WHERE id = #{viewId}
UNION ALL
SELECT id, scene_id, title, type FROM snapshot_core_chart_view
WHERE id = #{viewId}
) combined_views
LIMIT 1
""")
ChartBasePO queryChart(@Param("id") Long id);
ChartViewDTO findChartViewAround(@Param("viewId") String viewId);
}

91
core/core-backend/src/main/java/io/gisbi/chart/manage/ChartDataManage.java
View File

@ -1,14 +1,16 @@
package io.gisbi.chart.manage;
import io.gisbi.api.dataset.union.DatasetGroupInfoDTO;
import io.gisbi.api.permissions.auth.dto.BusiPerCheckDTO;
import io.gisbi.api.permissions.dataset.dto.DataSetRowPermissionsTreeDTO;
import io.gisbi.chart.charts.ChartHandlerManager;
import io.gisbi.chart.constant.ChartConstants;
import io.gisbi.constant.AuthEnum;
import io.gisbi.dataset.manage.DatasetGroupManage;
import io.gisbi.dataset.manage.DatasetSQLManage;
import io.gisbi.dataset.manage.DatasetTableFieldManage;
import io.gisbi.dataset.manage.PermissionManage;
import io.gisbi.dataset.utils.DatasetUtils;
import io.gisbi.engine.sql.SQLProvider;
import io.gisbi.engine.trans.*;
import io.gisbi.engine.utils.SQLUtils;
@ -28,6 +30,7 @@ import io.gisbi.extensions.view.plugin.AbstractChartPlugin;
import io.gisbi.extensions.view.util.FieldUtil;
import io.gisbi.i18n.Translator;
import io.gisbi.result.ResultCode;
import io.gisbi.system.manage.CorePermissionManage;
import io.gisbi.utils.AuthUtils;
import io.gisbi.utils.BeanUtils;
import jakarta.annotation.Resource;
@ -58,12 +61,14 @@ public class ChartDataManage {
@Resource
private ChartViewManege chartViewManege;
@Resource
private PermissionManage permissionManage;
@Resource
private ChartFilterTreeService chartFilterTreeService;
@Resource
private ChartHandlerManager chartHandlerManager;
@Resource
private PermissionManage permissionManage;
@Resource
private CorePermissionManage corePermissionManage;
@Autowired(required = false)
private PluginManageApi pluginManage;
@ -132,17 +137,18 @@ public class ChartDataManage {
// row permission
List<DataSetRowPermissionsTreeDTO> rowPermissionsTree = permissionManage.getRowPermissionsTree(table.getId(), chartExtRequest.getUser());
//将没有权限的列删掉
List<String> dataeaseNames = columnPermissionFields.stream().map(DatasetTableFieldDTO::getDataeaseName).collect(Collectors.toList());
List<String> gisbiNames = columnPermissionFields.stream().map(DatasetTableFieldDTO::getGisbiName).collect(Collectors.toList());
//计数字段
dataeaseNames.add("*");
gisbiNames.add("*");
AxisFormatResult formatResult = chartHandler.formatAxis(view);
formatResult.getContext().put("dataset", table);
formatResult.getContext().put("desensitizationList", desensitizationList);
var xAxis = formatResult.getAxisMap().get(ChartAxis.xAxis);
var yAxis = formatResult.getAxisMap().get(ChartAxis.yAxis);
formatResult.getContext().put("allFields", allFields);
var axisMap = formatResult.getAxisMap();
axisMap.forEach((axis, fields) -> {
fields.removeIf(fieldDTO -> !dataeaseNames.contains(fieldDTO.getDataeaseName()));
fields.removeIf(fieldDTO -> !gisbiNames.contains(fieldDTO.getGisbiName()));
});
// 过滤来自仪表板的条件
@ -225,11 +231,17 @@ public class ChartDataManage {
}
List<ChartExtFilterDTO> filters = new ArrayList<>();
FilterTreeObj customLinkageFilter = null;
// 联动条件
if (ObjectUtils.isNotEmpty(chartExtRequest.getLinkageFilters())) {
filters.addAll(chartExtRequest.getLinkageFilters());
for (ChartExtFilterDTO linkageFilter : chartExtRequest.getLinkageFilters()) {
if (3 == linkageFilter.getFilterType()) {
customLinkageFilter = linkageFilter.getCustomFilter();
} else {
filters.add(linkageFilter);
}
}
}
// 外部参数条件
if (ObjectUtils.isNotEmpty(chartExtRequest.getOuterParamsFilters())) {
filters.addAll(chartExtRequest.getOuterParamsFilters());
@ -357,13 +369,17 @@ public class ChartDataManage {
}
// 字段过滤器
FilterTreeObj fieldCustomFilter = view.getCustomFilter();
// 指标表联动时 使用的CustomFilter
if (customLinkageFilter != null) {
fieldCustomFilter = customLinkageFilter;
}
chartFilterTreeService.searchFieldAndSet(fieldCustomFilter);
fieldCustomFilter = chartFilterTreeService.charReplace(fieldCustomFilter);
// 获取dsMap,union sql
Map<String, Object> sqlMap = datasetSQLManage.getUnionSQLForEdit(table, chartExtRequest);
String sql = (String) sqlMap.get("sql");
Map<Long, DatasourceSchemaDTO> dsMap = (Map<Long, DatasourceSchemaDTO>) sqlMap.get("dsMap");
boolean crossDs = Utils.isCrossDs(dsMap);
boolean crossDs = table.getIsCross();
if (!crossDs) {
sql = Utils.replaceSchemaAlias(sql, dsMap);
}
@ -592,11 +608,11 @@ public class ChartDataManage {
int getIndex = 0;
for (int i = 0; i < fieldList.size(); i++) {
ChartViewFieldDTO item = fieldList.get(i);
if (StringUtils.equalsIgnoreCase(item.getSort(), "custom_sort")) {
if (StringUtils.equalsIgnoreCase(item.getSort(), "custom_sort")) {// 此处与已有的自定义字段对比
chartViewFieldDTO = item;
index = i;
}
if (Objects.equals(item.getId(), field.getId())) {
if (Objects.equals(item.getId(), field.getId())) {// 获得当前自定义的字段
getIndex = i;
}
}
@ -648,7 +664,7 @@ public class ChartDataManage {
// 获取图表所有字段配置
List<ChartViewFieldDTO> allFields = getAllChartFields(view);
// 处理堆叠/分组图表的特殊x轴配置
// 针对分组切换堆叠时会遇到的问题
if (StringUtils.equalsIgnoreCase(view.getType(), "bar-stack") || StringUtils.equalsIgnoreCase(view.getType(), "chart-mix-stack")) {
view.setXAxisExt(new ArrayList<>());
}
@ -668,7 +684,6 @@ public class ChartDataManage {
xAxis.addAll(xAxisExt);
}
List<ChartViewFieldDTO> yAxis = new ArrayList<>(view.getYAxis());
// 合并扩展y轴配置（针对混合图表类型）
if (StringUtils.containsIgnoreCase(view.getType(), "chart-mix")) {
List<ChartViewFieldDTO> yAxisExt = new ArrayList<>(view.getYAxisExt());
yAxis.addAll(yAxisExt);
@ -678,6 +693,10 @@ public class ChartDataManage {
List<ChartViewFieldDTO> sizeField = getSizeField(view);
yAxis.addAll(sizeField);
}
List<ChartViewFieldDTO> extStack = new ArrayList<>(view.getExtStack());
List<ChartViewFieldDTO> extBubble = new ArrayList<>(view.getExtBubble());
FilterTreeObj fieldCustomFilter = view.getCustomFilter();
List<ChartViewFieldDTO> drill = new ArrayList<>(view.getDrillFields());
// 获取数据集权限信息
DatasetGroupInfoDTO table = datasetGroupManage.getDatasetGroupInfoDTO(view.getTableId(), null);
@ -715,6 +734,7 @@ public class ChartDataManage {
}
break;
case "table-normal":
break;
case "bar-group":
case "bar-group-stack":
case "flow-map":
@ -739,6 +759,7 @@ public class ChartDataManage {
// 初始化数据源请求对象
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDsList(dsMap);
datasourceRequest.setIsCross(crossDs);
// 根据数据源类型选择查询提供器
Provider provider;
@ -750,8 +771,10 @@ public class ChartDataManage {
// 执行数据查询
List<String[]> data = new ArrayList<>();
String querySql = null;
if (table.getMode() == 0) { // 直连模式处理
//如果不是插件图表 走原生逻辑
if (table.getMode() == 0) {// 直连
if (ObjectUtils.isEmpty(dsMap)) {
DEException.throwException(Translator.get("i18n_datasource_delete"));
}
@ -768,21 +791,20 @@ public class ChartDataManage {
Table2SQLObj.table2sqlobj(sqlMeta, null, "(" + sql + ")", crossDs);
WhereTree2Str.transFilterTrees(sqlMeta, rowPermissionsTree, transFields(allFields), crossDs, dsMap, Utils.getParams(transFields(allFields)), view.getCalParams(), pluginManage);
// 根据图表类型生成不同SQL逻辑
if (StringUtils.equalsAnyIgnoreCase(view.getType(), "indicator", "gauge", "liquid")) {
Quota2SQLObj.quota2sqlObj(sqlMeta, yAxis, transFields(allFields), crossDs, dsMap, Utils.getParams(transFields(allFields)), view.getCalParams(), pluginManage);
querySql = SQLProvider.createQuerySQL(sqlMeta, true, needOrder, view);
} else if (StringUtils.containsIgnoreCase(view.getType(), "stack")) {
List<ChartViewFieldDTO> xFields = new ArrayList<>();
xFields.addAll(xAxis);
xFields.addAll(view.getExtStack());
xFields.addAll(extStack);
Dimension2SQLObj.dimension2sqlObj(sqlMeta, xFields, transFields(allFields), crossDs, dsMap, Utils.getParams(transFields(allFields)), view.getCalParams(), pluginManage);
Quota2SQLObj.quota2sqlObj(sqlMeta, yAxis, transFields(allFields), crossDs, dsMap, Utils.getParams(transFields(allFields)), view.getCalParams(), pluginManage);
querySql = SQLProvider.createQuerySQL(sqlMeta, true, needOrder, view);
} else if (StringUtils.containsIgnoreCase(view.getType(), "scatter")) {
List<ChartViewFieldDTO> yFields = new ArrayList<>();
yFields.addAll(yAxis);
yFields.addAll(view.getExtBubble());
yFields.addAll(extBubble);
Dimension2SQLObj.dimension2sqlObj(sqlMeta, xAxis, transFields(allFields), crossDs, dsMap, Utils.getParams(transFields(allFields)), view.getCalParams(), pluginManage);
Quota2SQLObj.quota2sqlObj(sqlMeta, yFields, transFields(allFields), crossDs, dsMap, Utils.getParams(transFields(allFields)), view.getCalParams(), pluginManage);
querySql = SQLProvider.createQuerySQL(sqlMeta, true, needOrder, view);
@ -804,7 +826,6 @@ public class ChartDataManage {
return data;
}
private List<ChartViewFieldDTO> getAllChartFields(ChartViewDTO view) {
// get all fields
Map<String, List<ChartViewFieldDTO>> stringListMap = chartViewManege.listByDQ(view.getTableId(), view.getId(), view);
@ -837,14 +858,13 @@ public class ChartDataManage {
DEException.throwException(e);
}
} else {
// 如果当前图表ID未存在于检查数据中，则记录为废弃ID
disuseChartIdList.add(chartViewDTO.getId());
}
});
// 如果存在需要废弃的图表ID列表，则批量标记为废弃
if (CollectionUtils.isNotEmpty(disuseChartIdList)) {
chartViewManege.disuse(disuseChartIdList);
}
// 阈值告警处理 统一在发布时处理
// if (CollectionUtils.isNotEmpty(disuseChartIdList)) {
// chartViewManege.disuse(disuseChartIdList);
// }
}
}
@ -879,19 +899,18 @@ public class ChartDataManage {
// 执行SQL查询获取原始数据
List<String[]> sqlData = sqlData(view, view.getChartExtRequest(), fieldId);
// 根据自定义排序规则对数据进行排序
List<String[]> result = customSort(
Optional.ofNullable(targetField.getCustomSort()).orElse(new ArrayList<>()),
sqlData,
0
);
// 提取第一列数据并去重生成最终结果列表
return result.stream()
.map(i -> i[0])
.distinct()
.collect(Collectors.toList());
List<String[]> result = customSort(Optional.ofNullable(targetField.getCustomSort()).orElse(new ArrayList<>()), sqlData, 0);
return result.stream().map(i -> i[0]).distinct().collect(Collectors.toList());
}
public void encodeData(ChartViewDTO chartViewDTO) {
if (chartViewDTO.getData() != null) {
if (chartViewDTO.getType().startsWith("chart-mix")) {
DatasetUtils.listEncode((List<ChartViewFieldDTO>) ((Map<String, Object>) chartViewDTO.getData().get("left")).get("sourceFields"));
DatasetUtils.listEncode((List<ChartViewFieldDTO>) ((Map<String, Object>) chartViewDTO.getData().get("right")).get("sourceFields"));
} else {
DatasetUtils.listEncode((List<ChartViewFieldDTO>) chartViewDTO.getData().get("sourceFields"));
}
}
}
}

67
core/core-backend/src/main/java/io/gisbi/chart/manage/ChartViewManege.java
View File

@ -10,6 +10,7 @@ import io.gisbi.chart.dao.auto.entity.CoreChartView;
import io.gisbi.chart.dao.auto.mapper.CoreChartViewMapper;
import io.gisbi.chart.dao.ext.entity.ChartBasePO;
import io.gisbi.chart.dao.ext.mapper.ExtChartViewMapper;
import io.gisbi.constant.CommonConstants;
import io.gisbi.dataset.dao.auto.entity.CoreDatasetTableField;
import io.gisbi.dataset.dao.auto.mapper.CoreDatasetTableFieldMapper;
import io.gisbi.dataset.manage.DatasetTableFieldManage;
@ -25,13 +26,16 @@ import io.gisbi.extensions.datasource.dto.FieldGroupDTO;
import io.gisbi.extensions.datasource.model.SQLObj;
import io.gisbi.extensions.view.dto.*;
import io.gisbi.extensions.view.filter.FilterTreeObj;
import io.gisbi.i18n.Lang;
import io.gisbi.i18n.Translator;
import io.gisbi.utils.BeanUtils;
import io.gisbi.utils.IDUtils;
import io.gisbi.utils.JsonUtil;
import io.gisbi.utils.LogUtil;
import io.gisbi.visualization.dao.auto.entity.DataVisualizationInfo;
import io.gisbi.visualization.dao.auto.entity.SnapshotCoreChartView;
import io.gisbi.visualization.dao.auto.mapper.DataVisualizationInfoMapper;
import io.gisbi.visualization.dao.auto.mapper.SnapshotCoreChartViewMapper;
import jakarta.annotation.Resource;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@ -55,6 +59,8 @@ public class ChartViewManege {
@Resource
private CoreChartViewMapper coreChartViewMapper;
@Resource
private SnapshotCoreChartViewMapper snapshotCoreChartViewMapper;
@Resource
private ChartDataManage chartDataManage;
@Resource
private CoreDatasetTableFieldMapper coreDatasetTableFieldMapper;
@ -82,18 +88,19 @@ public class ChartViewManege {
if (id == null) {
DEException.throwException(Translator.get("i18n_no_id"));
}
CoreChartView coreChartView = coreChartViewMapper.selectById(id);
CoreChartView record = transDTO2Record(chartViewDTO);
SnapshotCoreChartView coreChartView = snapshotCoreChartViewMapper.selectById(id);
SnapshotCoreChartView record = transDTO2Record(chartViewDTO);
if (ObjectUtils.isEmpty(coreChartView)) {
coreChartViewMapper.insert(record);
snapshotCoreChartViewMapper.deleteById(record.getId());
snapshotCoreChartViewMapper.insert(record);
} else {
UpdateWrapper<CoreChartView> updateWrapper = new UpdateWrapper<>();
UpdateWrapper<SnapshotCoreChartView> updateWrapper = new UpdateWrapper<>();
updateWrapper.eq("id", record.getId());
//富文本允许设置空的tableId 这里额外更新一下
if (record.getTableId() == null) {
updateWrapper.set("table_id", null);
}
coreChartViewMapper.update(record, updateWrapper);
snapshotCoreChartViewMapper.update(record, updateWrapper);
}
return chartViewDTO;
}
@ -113,11 +120,21 @@ public class ChartViewManege {
coreChartViewMapper.delete(wrapper);
}
public ChartViewDTO getDetails(Long id) {
CoreChartView coreChartView = coreChartViewMapper.selectById(id);
public ChartViewDTO getDetails(Long id, String resourceTable) {
CoreChartView coreChartView = null;
if (CommonConstants.RESOURCE_TABLE.SNAPSHOT.equals(resourceTable)) {
SnapshotCoreChartView snapshotCoreChartView = snapshotCoreChartViewMapper.selectById(id);
if (ObjectUtils.isEmpty(snapshotCoreChartView)) {
return null;
}
coreChartView = new CoreChartView();
BeanUtils.copyBean(coreChartView, snapshotCoreChartView);
} else {
coreChartView = coreChartViewMapper.selectById(id);
if (ObjectUtils.isEmpty(coreChartView)) {
return null;
}
}
ChartViewDTO dto = transRecord2DTO(coreChartView);
return dto;
}
@ -125,10 +142,10 @@ public class ChartViewManege {
/**
* sceneId 为仪表板或者数据大屏id
*/
public List<ChartViewDTO> listBySceneId(Long sceneId) {
public List<ChartViewDTO> listBySceneId(Long sceneId, String resourceTable) {
QueryWrapper<CoreChartView> wrapper = new QueryWrapper<>();
wrapper.eq("scene_id", sceneId);
List<ChartViewDTO> chartViewDTOS = transChart(coreChartViewMapper.selectList(wrapper));
List<ChartViewDTO> chartViewDTOS = transChart(extChartViewMapper.selectListCustom(sceneId, resourceTable));
if (!CollectionUtils.isEmpty(chartViewDTOS)) {
List<Long> tableIds = chartViewDTOS.stream()
.map(ChartViewDTO::getTableId)
@ -192,8 +209,8 @@ public class ChartViewManege {
}).collect(Collectors.toList());
}
public ChartViewDTO getChart(Long id) throws Exception {
ChartViewDTO details = getDetails(id);
public ChartViewDTO getChart(Long id, String resourceTable) throws Exception {
ChartViewDTO details = getDetails(id, resourceTable);
if (details == null) {
return null;
}
@ -220,9 +237,8 @@ public class ChartViewManege {
}).collect(Collectors.toList());
// filter column disable field
Map<String, ColumnPermissionItem> desensitizationList = new HashMap<>();
// List<DatasetTableFieldDTO> datasetTableFieldDTOS = permissionManage.filterColumnPermissions(collect, desensitizationList, id, null);
List<DatasetTableFieldDTO> datasetTableFieldDTOS = collect;
datasetTableFieldDTOS.forEach(ele -> ele.setDesensitized(desensitizationList.containsKey(ele.getDataeaseName())));
datasetTableFieldDTOS.forEach(ele -> ele.setDesensitized(desensitizationList.containsKey(ele.getGisbiName())));
datasetTableFieldDTOS.add(createCountField(id));
List<ChartViewFieldDTO> list = transFieldDTO(datasetTableFieldDTOS);
@ -282,8 +298,8 @@ public class ChartViewManege {
coreDatasetTableField.setExtField(2);
coreDatasetTableField.setOriginName("[" + id + "]");
coreDatasetTableField.setId(IDUtils.snowID());
coreDatasetTableField.setDataeaseName(TableUtils.fieldNameShort(coreDatasetTableField.getId() + "_" + coreDatasetTableField.getOriginName()));
coreDatasetTableField.setFieldShortName(coreDatasetTableField.getDataeaseName());
coreDatasetTableField.setGisbiName(TableUtils.fieldNameShort(coreDatasetTableField.getId() + "_" + coreDatasetTableField.getOriginName()));
coreDatasetTableField.setFieldShortName(coreDatasetTableField.getGisbiName());
coreDatasetTableFieldMapper.insert(coreDatasetTableField);
}
@ -306,8 +322,8 @@ public class ChartViewManege {
coreDatasetTableFieldMapper.delete(queryWrapper);
}
public ChartBaseVO chartBaseInfo(Long id) {
ChartBasePO po = extChartViewMapper.queryChart(id);
public ChartBaseVO chartBaseInfo(Long id, String resourceTable) {
ChartBasePO po = extChartViewMapper.queryChart(id, resourceTable);
if (ObjectUtils.isEmpty(po)) return null;
ChartBaseVO vo = BeanUtils.copyBean(new ChartBaseVO(), po);
TypeReference<List<ChartViewFieldDTO>> tokenType = new TypeReference<>() {
@ -336,7 +352,7 @@ public class ChartViewManege {
dto.setDatasetGroupId(id);
dto.setOriginName("*");
dto.setName("记录数*");
dto.setDataeaseName("*");
dto.setGisbiName("*");
dto.setType("INT");
dto.setChecked(true);
dto.setColumnIndex(999);
@ -353,6 +369,7 @@ public class ChartViewManege {
BeanUtils.copyBean(dto, ele);
dto.setDateStyle("y_M_d");
dto.setDatePattern("date_sub");
dto.setDateShowFormat("y_M_d");
dto.setChartType("bar");
if (dto.getId() == -1L || dto.getDeType() == 0 || dto.getDeType() == 1 || dto.getDeType() == 7) {
@ -365,7 +382,7 @@ public class ChartViewManege {
chartFieldCompareDTO.setType("none");
dto.setCompareCalc(chartFieldCompareDTO);
dto.setFormatterCfg(new FormatterCfgDTO());
dto.setFormatterCfg(new FormatterCfgDTO().setUnitLanguage(Lang.isChinese() ? "ch" : "en"));
dto.setSort("none");
dto.setFilter(Collections.emptyList());
@ -373,8 +390,8 @@ public class ChartViewManege {
}).collect(Collectors.toList());
}
public CoreChartView transDTO2Record(ChartViewDTO dto) throws Exception {
CoreChartView record = new CoreChartView();
public SnapshotCoreChartView transDTO2Record(ChartViewDTO dto) throws Exception {
SnapshotCoreChartView record = new SnapshotCoreChartView();
BeanUtils.copyBean(record, dto);
record.setxAxis(objectMapper.writeValueAsString(dto.getXAxis()));
@ -400,7 +417,7 @@ public class ChartViewManege {
record.setFlowMapStartName(objectMapper.writeValueAsString(dto.getFlowMapStartName()));
record.setFlowMapEndName(objectMapper.writeValueAsString(dto.getFlowMapEndName()));
record.setExtColor(objectMapper.writeValueAsString(dto.getExtColor()));
record.setSortPriority(objectMapper.writeValueAsString(dto.getSortPriority()));
return record;
}
@ -434,6 +451,8 @@ public class ChartViewManege {
dto.setFlowMapStartName(JsonUtil.parseList(record.getFlowMapStartName(), tokenType));
dto.setFlowMapEndName(JsonUtil.parseList(record.getFlowMapEndName(), tokenType));
dto.setExtColor(JsonUtil.parseList(record.getExtColor(), tokenType));
dto.setSortPriority(JsonUtil.parseList(record.getSortPriority(), new TypeReference<List<SortAxis>>() {
}));
return dto;
@ -462,4 +481,8 @@ public class ChartViewManege {
return result;
}
}
public ChartViewDTO findChartViewAround(String viewId) {
return extChartViewMapper.findChartViewAround(viewId);
}
}

36
core/core-backend/src/main/java/io/gisbi/chart/manage/ChartViewThresholdManage.java
View File

@ -20,6 +20,7 @@ import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
import java.math.BigDecimal;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
@ -34,8 +35,8 @@ public class ChartViewThresholdManage {
@Resource
private ChartViewManege chartViewManege;
public String convertThresholdRules(Long chartId, String thresholdRules) {
ChartViewDTO details = chartViewManege.getDetails(chartId);
public String convertThresholdRules(Long chartId, String thresholdRules, String resourceTable) {
ChartViewDTO details = chartViewManege.getDetails(chartId, resourceTable);
return convertThresholdRules(details, thresholdRules);
}
@ -251,7 +252,7 @@ public class ChartViewThresholdManage {
String thresholdRules = request.getThresholdRules();
Long chartId = request.getChartId();
try {
ChartViewDTO chart = chartViewManege.getChart(chartId);
ChartViewDTO chart = chartViewManege.getChart(chartId, request.getResourceTable());
Map<String, Object> data = null;
if (ObjectUtils.isEmpty(chart) || MapUtils.isEmpty(data = chart.getData())) {
return new ThresholdCheckVO(false, null, "查询图表异常！", null);
@ -280,9 +281,16 @@ public class ChartViewThresholdManage {
DatasetTableFieldDTO fieldDTO = fieldMap.get(id);
if (ObjectUtils.isEmpty(fieldDTO)) continue;
String fieldDTOName = fieldDTO.getName();
String dataeaseName = fieldDTO.getDataeaseName();
List<String> valueList = rows.stream().map(row -> ObjectUtils.isEmpty(row.get(dataeaseName)) ? null : row.get(dataeaseName).toString()).collect(Collectors.toList());
String replacement = fieldDTOName + ": " + JsonUtil.toJSONString(valueList);
String gisbiName = fieldDTO.getGisbiName();
String replacement = null;
if (fieldDTO.getDeType().equals(DeTypeConstants.DE_FLOAT) || fieldDTO.getDeType().equals(DeTypeConstants.DE_INT)) {
List<String> valueList = rows.stream().map(row -> ObjectUtils.isEmpty(row.get(gisbiName)) ? null : stripTrailingZeros2String(row.get(gisbiName))).collect(Collectors.toList());
replacement = fieldDTOName + ": " + JsonUtil.toJSONString(valueList);
} else {
List<String> valueList = rows.stream().map(row -> ObjectUtils.isEmpty(row.get(gisbiName)) ? null : row.get(gisbiName).toString()).collect(Collectors.toList());
replacement = fieldDTOName + ": " + JsonUtil.toJSONString(valueList);
}
// 替换文本
matcher.appendReplacement(sb, replacement);
}
@ -297,6 +305,14 @@ public class ChartViewThresholdManage {
}
}
private String stripTrailingZeros2String(Object value) {
if (ObjectUtils.isEmpty(value)) {
return null;
}
if (!(value instanceof BigDecimal)) return value.toString();
return ((BigDecimal) value).stripTrailingZeros().toPlainString();
}
private void chartDynamicMap(List<Map<String, Object>> rows, FilterTreeObj conditionTree, Map<Long, DatasetTableFieldDTO> fieldMap) {
List<FilterTreeItem> items = conditionTree.getItems();
items.forEach(item -> {
@ -404,13 +420,13 @@ public class ChartViewThresholdManage {
private String formatValue(List<Map<String, Object>> rows, FilterTreeItem item) {
DatasetTableFieldDTO field = item.getField();
String dataeaseName = field.getDataeaseName();
String gisbiName = field.getGisbiName();
String value = item.getValue();
Float tempFVal = StringUtils.equalsAny(value, "min", "max") ? null : 0f;
int validLen = 0;
for (Map<String, Object> row : rows) {
Object o = row.get(dataeaseName);
Object o = row.get(gisbiName);
if (ObjectUtils.isEmpty(o)) continue;
float fvalue = Float.parseFloat(o.toString());
if (StringUtils.equals("min", value)) {
@ -487,10 +503,10 @@ public class ChartViewThresholdManage {
}
private boolean rowMatch(Map<String, Object> row, FilterTreeItem item, DatasetTableFieldDTO fieldDTO) {
String dataeaseName = fieldDTO.getDataeaseName();
String gisbiName = fieldDTO.getGisbiName();
String filterType = item.getFilterType();
Integer deType = fieldDTO.getDeType();
Object valueObj = row.get(dataeaseName);
Object valueObj = row.get(gisbiName);
if (StringUtils.equals(filterType, "enum")) {
List<String> enumValue = item.getEnumValue();
return ObjectUtils.isNotEmpty(valueObj) && enumValue.contains(valueObj);

244
core/core-backend/src/main/java/io/gisbi/chart/server/ChartDataServer.java
View File

@ -5,7 +5,6 @@ import io.gisbi.api.chart.ChartDataApi;
import io.gisbi.api.chart.dto.ViewDetailField;
import io.gisbi.api.chart.request.ChartExcelRequest;
import io.gisbi.api.chart.request.ChartExcelRequestInner;
import io.gisbi.auth.DeLinkPermit;
import io.gisbi.chart.constant.ChartConstants;
import io.gisbi.chart.manage.ChartDataManage;
import io.gisbi.constant.AuthConstant;
@ -14,11 +13,13 @@ import io.gisbi.constant.CommonConstants;
import io.gisbi.dataset.server.DatasetFieldServer;
import io.gisbi.dataset.manage.PermissionManage;
import io.gisbi.constant.DeTypeConstants;
import io.gisbi.dataset.utils.DatasetUtils;
import io.gisbi.exception.DEException;
import io.gisbi.exportCenter.manage.ExportCenterManage;
import io.gisbi.exportCenter.util.ExportCenterUtils;
import io.gisbi.extensions.datasource.dto.DatasetTableFieldDTO;
import io.gisbi.extensions.view.dto.*;
import io.gisbi.i18n.Lang;
import io.gisbi.result.ResultCode;
import io.gisbi.utils.JsonUtil;
import io.gisbi.utils.LogUtil;
@ -69,12 +70,12 @@ public class ChartDataServer implements ChartDataApi {
@Resource
private DatasetFieldServer datasetFieldServer;
@Value("${dataease.export.page.size:50000}")
@Value("${gisbi.export.page.size:50000}")
private Integer extractPageSize;
private final Long sheetLimit = 1000000L;
@DeLinkPermit("#p0.sceneId")
@Override
public ChartViewDTO getData(ChartViewDTO chartViewDTO) throws Exception {
try {
@ -82,7 +83,11 @@ public class ChartDataServer implements ChartDataApi {
if (CommonConstants.VIEW_DATA_FROM.TEMPLATE.equalsIgnoreCase(chartViewDTO.getDataFrom())) {
return extendDataManage.getChartDataInfo(chartViewDTO.getId(), chartViewDTO);
} else {
return chartDataManage.calcData(chartViewDTO);
DatasetUtils.viewDecode(chartViewDTO);
ChartViewDTO dto = chartDataManage.calcData(chartViewDTO);
DatasetUtils.viewEncode(dto);
chartDataManage.encodeData(dto);
return dto;
}
} catch (Exception e) {
DEException.throwException(ResultCode.DATA_IS_WRONG.code(), e.getMessage() + "\n\n" + ExceptionUtils.getStackTrace(e));
@ -103,8 +108,8 @@ public class ChartDataServer implements ChartDataApi {
viewDTO.setType("table-info");
viewDTO.setRender("antv");
List<DatasetTableFieldDTO> sourceFields = datasetFieldServer.listByDatasetGroup(viewDTO.getTableId());
List<String> fileNames = permissionManage.filterColumnPermissions(sourceFields, new HashMap<>(), viewDTO.getTableId(), null).stream().map(DatasetTableFieldDTO::getDataeaseName).collect(Collectors.toList());
sourceFields = sourceFields.stream().filter(datasetTableFieldDTO -> fileNames.contains(datasetTableFieldDTO.getDataeaseName())).collect(Collectors.toList());
List<String> fileNames = permissionManage.filterColumnPermissions(sourceFields, new HashMap<>(), viewDTO.getTableId(), null).stream().map(DatasetTableFieldDTO::getGisbiName).collect(Collectors.toList());
sourceFields = sourceFields.stream().filter(datasetTableFieldDTO -> fileNames.contains(datasetTableFieldDTO.getGisbiName())).collect(Collectors.toList());
dsHeader = sourceFields.stream().map(DatasetTableFieldDTO::getName).toArray(String[]::new);
dsTypes = sourceFields.stream().map(DatasetTableFieldDTO::getDeType).toArray(Integer[]::new);
TypeReference<List<ChartViewFieldDTO>> listTypeReference = new TypeReference<List<ChartViewFieldDTO>>() {
@ -120,7 +125,11 @@ public class ChartDataServer implements ChartDataApi {
} else {
viewDTO.setResultCount(viewLimit);
}
chartViewInfo = getData(viewDTO);
if (CommonConstants.VIEW_DATA_FROM.TEMPLATE.equalsIgnoreCase(viewDTO.getDataFrom())) {
chartViewInfo = extendDataManage.getChartDataInfo(viewDTO.getId(), viewDTO);
} else {
chartViewInfo = chartDataManage.calcData(viewDTO);
}
List<Object[]> tableRow = (List) chartViewInfo.getData().get("sourceData");
if ("dataset".equals(request.getDownloadType())) {
request.setHeader(dsHeader);
@ -214,13 +223,12 @@ public class ChartDataServer implements ChartDataApi {
}
@DeLinkPermit("#p0.dvId")
@Override
public void innerExportDetails(ChartExcelRequest request, HttpServletResponse response) throws Exception {
HttpServletRequest httpServletRequest = ((ServletRequestAttributes) RequestContextHolder.getRequestAttributes()).getRequest();
String linkToken = httpServletRequest.getHeader(AuthConstant.LINK_TOKEN_KEY);
LogUtil.info(request.getViewInfo().getId() + " " + StringUtils.isNotEmpty(linkToken) + " " + request.isDataEaseBi());
if ((StringUtils.isNotEmpty(linkToken) && !request.isDataEaseBi()) || (request.isDataEaseBi() && StringUtils.isEmpty(linkToken))) {
LogUtil.info(request.getViewInfo().getId() + " " + StringUtils.isNotEmpty(linkToken) + " " + request.isGisbiBi());
if ((StringUtils.isNotEmpty(linkToken) && !request.isGisbiBi()) || (request.isGisbiBi() && StringUtils.isEmpty(linkToken))) {
OutputStream outputStream = response.getOutputStream();
try {
Workbook wb = new SXSSFWorkbook();
@ -238,7 +246,7 @@ public class ChartDataServer implements ChartDataApi {
//设置单元格填充样式(使用纯色背景颜色填充)
cellStyle.setFillPattern(FillPatternType.SOLID_FOREGROUND);
if ("dataset".equals(request.getDownloadType()) || request.getViewInfo().getType().equalsIgnoreCase("table-info")) {
if ("dataset".equals(request.getDownloadType()) || request.getViewInfo().getType().equalsIgnoreCase("table-info") || request.getViewInfo().getType().equalsIgnoreCase("table-normal")) {
List<Object[]> details = new ArrayList<>();
Sheet detailsSheet;
Integer sheetIndex = 1;
@ -251,9 +259,21 @@ public class ChartDataServer implements ChartDataApi {
if ((details.size() + extractPageSize) > sheetLimit || i == chartViewDTO.getTotalPage()) {
detailsSheet = wb.createSheet("数据" + sheetIndex);
Integer[] excelTypes = request.getExcelTypes();
details.add(0, request.getHeader());
ViewDetailField[] detailFields = request.getDetailFields();
Object[] header = request.getHeader();
request.getViewInfo().setXAxis(request.getViewInfo().getXAxis().stream().filter(ele -> !ele.isHide()).collect(Collectors.toList()));
request.getViewInfo().setYAxis(request.getViewInfo().getYAxis().stream().filter(ele -> !ele.isHide()).collect(Collectors.toList()));
request.getViewInfo().setXAxisExt(request.getViewInfo().getXAxisExt().stream().filter(ele -> !ele.isHide()).collect(Collectors.toList()));
request.getViewInfo().setYAxisExt(request.getViewInfo().getYAxisExt().stream().filter(ele -> !ele.isHide()).collect(Collectors.toList()));
request.getViewInfo().setExtStack(request.getViewInfo().getExtStack().stream().filter(ele -> !ele.isHide()).collect(Collectors.toList()));
List<ChartViewFieldDTO> xAxis = new ArrayList<>();
xAxis.addAll(request.getViewInfo().getXAxis());
xAxis.addAll(request.getViewInfo().getYAxis());
xAxis.addAll(request.getViewInfo().getXAxisExt());
xAxis.addAll(request.getViewInfo().getYAxisExt());
xAxis.addAll(request.getViewInfo().getExtStack());
header = Arrays.stream(request.getHeader()).filter(item -> xAxis.stream().map(DatasetTableFieldDTO::getName).collect(Collectors.toList()).contains(item)).collect(Collectors.toList()).toArray();
details.add(0, header);
ViewDetailField[] detailFields = request.getDetailFields();
ChartDataServer.setExcelData(detailsSheet, cellStyle, header, details, detailFields, excelTypes, request.getViewInfo(), wb);
sheetIndex++;
details.clear();
@ -268,7 +288,11 @@ public class ChartDataServer implements ChartDataApi {
ViewDetailField[] detailFields = request.getDetailFields();
Object[] header = request.getHeader();
Sheet detailsSheet = wb.createSheet("数据");
if (request.getViewInfo().getType().equalsIgnoreCase("table-normal")) {
setExcelData(detailsSheet, cellStyle, header, details, detailFields, excelTypes, request.getViewInfo(), wb);
} else {
setExcelData(detailsSheet, cellStyle, header, details, detailFields, excelTypes, request.getViewInfo(), null);
}
} else {
for (int i = 0; i < request.getMultiInfo().size(); i++) {
ChartExcelRequestInner requestInner = request.getMultiInfo().get(i);
@ -293,12 +317,10 @@ public class ChartDataServer implements ChartDataApi {
DEException.throwException(e);
}
} else {
exportCenterManage.addTask(request.getViewId(), "chart", request);
return;
exportCenterManage.addTask(request.getViewId(), "chart", request, request.getBusiFlag());
}
}
@DeLinkPermit("#p0.dvId")
@Override
public void innerExportDataSetDetails(ChartExcelRequest request, HttpServletResponse response) throws Exception {
this.innerExportDetails(request, response);
@ -318,7 +340,9 @@ public class ChartDataServer implements ChartDataApi {
xAxis.addAll(viewInfo.getXAxisExt());
xAxis.addAll(viewInfo.getYAxisExt());
xAxis.addAll(viewInfo.getExtStack());
xAxis.addAll(viewInfo.getDrillFields());
TableHeader tableHeader = null;
Integer totalDepth = 0;
if (viewInfo.getType().equalsIgnoreCase("table-normal") || viewInfo.getType().equalsIgnoreCase("table-info")) {
for (ChartViewFieldDTO xAxi : xAxis) {
if (xAxi.getDeType().equals(DeTypeConstants.DE_INT) || xAxi.getDeType().equals(DeTypeConstants.DE_FLOAT)) {
@ -328,6 +352,22 @@ public class ChartDataServer implements ChartDataApi {
styles.add(null);
}
}
Map<String, Object> customAttr = viewInfo.getCustomAttr();
Map<String, Object> tableHeaderMap = (Map<String, Object>) customAttr.get("tableHeader");
if (tableHeaderMap.get("headerGroup") != null && Boolean.parseBoolean(tableHeaderMap.get("headerGroup").toString())) {
var tmpHeader = JsonUtil.parseObject((String) JsonUtil.toJSONString(customAttr.get("tableHeader")), TableHeader.class);
// 校验字段数量和顺序
if (validateHeaderGroup(tmpHeader, viewInfo.getXAxis())) {
tableHeader = tmpHeader;
for (TableHeader.ColumnInfo column : tableHeader.getHeaderGroupConfig().getColumns()) {
totalDepth = Math.max(totalDepth, getDepth(column, 1));
}
for (TableHeader.ColumnInfo column : tableHeader.getHeaderGroupConfig().getColumns()) {
setWidth(column, 1);
}
}
}
}
boolean mergeHead = false;
@ -374,9 +414,31 @@ public class ChartDataServer implements ChartDataApi {
mergeHead = true;
}
if (CollectionUtils.isNotEmpty(details) && (!mergeHead || details.size() > 2)) {
int realDetailRowIndex = 2;
int realDetailRowIndex = tableHeader == null ? 2 : totalDepth;
if (tableHeader != null) {
cellStyle.setBorderTop(BorderStyle.THIN);
cellStyle.setBorderRight(BorderStyle.THIN);
cellStyle.setBorderBottom(BorderStyle.THIN);
cellStyle.setBorderLeft(BorderStyle.THIN);
Map<String, Row> rowMap = new HashMap<>();
for (Integer i = 0; i < totalDepth; i++) {
rowMap.put("row" + i, detailsSheet.createRow(i));
}
int width = 0;
Integer depth = 0;
for (TableHeader.ColumnInfo column : tableHeader.getHeaderGroupConfig().getColumns()) {
createCell(tableHeader, column, width, depth, detailsSheet, cellStyle, totalDepth, rowMap, xAxis);
width = width + column.getWidth();
}
}
for (int i = (mergeHead ? 2 : 0); i < details.size(); i++) {
Row row = detailsSheet.createRow(realDetailRowIndex > 2 ? realDetailRowIndex : i);
int rowIndex = i;
if (tableHeader != null) {
rowIndex = realDetailRowIndex - 1 + i;
} else {
rowIndex = realDetailRowIndex > 2 ? realDetailRowIndex : i;
}
Row row = detailsSheet.createRow(rowIndex);
Object[] rowData = details.get(i);
if (rowData != null) {
for (int j = 0; j < rowData.length; j++) {
@ -417,7 +479,7 @@ public class ChartDataServer implements ChartDataApi {
try {
if ((viewInfo.getType().equalsIgnoreCase("table-normal") || viewInfo.getType().equalsIgnoreCase("table-info")) && (xAxis.get(j).getDeType().equals(DeTypeConstants.DE_INT) || xAxis.get(j).getDeType().equals(DeTypeConstants.DE_FLOAT))) {
try {
FormatterCfgDTO formatterCfgDTO = xAxis.get(j).getFormatterCfg() == null ? new FormatterCfgDTO() : xAxis.get(j).getFormatterCfg();
FormatterCfgDTO formatterCfgDTO = xAxis.get(j).getFormatterCfg() == null ? new FormatterCfgDTO().setUnitLanguage(Lang.isChinese() ? "ch" : "en") : xAxis.get(j).getFormatterCfg();
row.getCell(j).setCellStyle(styles.get(j));
row.getCell(j).setCellValue(Double.valueOf(cellValue(formatterCfgDTO, new BigDecimal(cellValObj.toString()))));
} catch (Exception e) {
@ -436,10 +498,11 @@ public class ChartDataServer implements ChartDataApi {
} else {
if (!viewInfo.getType().equalsIgnoreCase("circle-packing")) {
Map<String, Object> senior = viewInfo.getSenior();
viewInfo.getCustomAttr().get("");
ChartSeniorFunctionCfgDTO functionCfgDTO = JsonUtil.parseObject((String) JsonUtil.toJSONString(senior.get("functionCfg")), ChartSeniorFunctionCfgDTO.class);
if (functionCfgDTO != null && StringUtils.isNotEmpty(functionCfgDTO.getEmptyDataStrategy()) && functionCfgDTO.getEmptyDataStrategy().equalsIgnoreCase("setZero")) {
if ((viewInfo.getType().equalsIgnoreCase("table-normal") || viewInfo.getType().equalsIgnoreCase("table-info"))) {
if (functionCfgDTO.getEmptyDataFieldCtrl().contains(xAxis.get(j).getDataeaseName())) {
if (functionCfgDTO.getEmptyDataFieldCtrl().contains(xAxis.get(j).getGisbiName())) {
cell.setCellValue(0);
}
} else {
@ -454,6 +517,123 @@ public class ChartDataServer implements ChartDataApi {
}
}
private static boolean validateHeaderGroup(TableHeader header, List<ChartViewFieldDTO> fields) {
if (header == null) {
return false;
}
var columns = header.getHeaderGroupConfig().getColumns();
if (CollectionUtils.isEmpty(columns)) {
return false;
}
var leafColumn = getHeaderLeafColumn(columns);
if (CollectionUtils.isEmpty(leafColumn) || leafColumn.size() != fields.size()) {
return false;
}
for (int i = 0; i < leafColumn.size(); i++) {
var a = leafColumn.get(i);
var b = fields.get(i).getGisbiName();
if (!StringUtils.equals(a, b)) {
return false;
}
}
return true;
}
private static List<String> getHeaderLeafColumn(List<TableHeader.ColumnInfo> columns) {
var result = new ArrayList<String>();
for (TableHeader.ColumnInfo column : columns) {
if (CollectionUtils.isEmpty(column.getChildren())) {
result.add(column.getKey());
} else {
result.addAll(getHeaderLeafColumn(column.getChildren()));
}
}
return result;
}
private static Integer getDepth(TableHeader.ColumnInfo column, Integer parentDepth) {
if (org.springframework.util.CollectionUtils.isEmpty(column.getChildren())) {
return parentDepth;
} else {
Integer depth = 0;
for (TableHeader.ColumnInfo child : column.getChildren()) {
depth = Math.max(depth, getDepth(child, parentDepth + 1));
}
return depth;
}
}
private static void createCell(TableHeader tableHeader, TableHeader.ColumnInfo column, Integer width, Integer depth, Sheet sheet, CellStyle cellStyle, Integer totaalDepth, Map<String, Row> rowMap, List<ChartViewFieldDTO> xAxis) {
if (org.springframework.util.CollectionUtils.isEmpty(column.getChildren())) {
Integer toDepth = totaalDepth - 1 > depth ? totaalDepth - 1 : depth;
if (depth.equals(toDepth)) {
Cell cell = rowMap.get("row" + depth).createCell(width);
cell.setCellStyle(cellStyle);
cell.setCellValue(getDeFieldName(xAxis, column.getKey()));
} else {
for (int i = depth; i <= toDepth; i++) {
Cell cell1 = rowMap.get("row" + i).createCell(width);
cell1.setCellValue(getDeFieldName(xAxis, column.getKey()));
cell1.setCellStyle(cellStyle);
}
CellRangeAddress region = new CellRangeAddress(depth, toDepth, width, width);
sheet.addMergedRegion(region);
Cell mergedCell = rowMap.get("row" + depth).getCell(width);
mergedCell.setCellStyle(cellStyle);
}
} else {
Cell cell1 = rowMap.get("row" + depth).createCell(width);
cell1.setCellValue(getGroupName(tableHeader, column.getKey()));
cell1.setCellStyle(cellStyle);
Cell cell2 = rowMap.get("row" + depth).createCell(width + column.getWidth() - 1);
cell2.setCellValue(getGroupName(tableHeader, column.getKey()));
cell2.setCellStyle(cellStyle);
CellRangeAddress region = new CellRangeAddress(depth, depth, width, width + column.getWidth() - 1);
sheet.addMergedRegion(region);
Cell mergedCell = rowMap.get("row" + depth).getCell(width);
mergedCell.setCellStyle(cellStyle);
int subWith = width;
for (TableHeader.ColumnInfo child : column.getChildren()) {
createCell(tableHeader, child, subWith, depth + 1, sheet, cellStyle, totaalDepth, rowMap, xAxis);
subWith = subWith + child.getWidth();
}
}
}
private static String getGroupName(TableHeader tableHeader, String key) {
for (TableHeader.MetaInfo metaInfo : tableHeader.getHeaderGroupConfig().getMeta()) {
if (metaInfo.getField().equals(key)) {
return metaInfo.getName();
}
}
return "";
}
private static String getDeFieldName(List<ChartViewFieldDTO> xAxis, String key) {
for (ChartViewFieldDTO xAxi : xAxis) {
if (xAxi.getGisbiName().equals(key)) {
return StringUtils.isNotBlank(xAxi.getChartShowName()) ? xAxi.getChartShowName() : xAxi.getName();
}
}
return "";
}
private static Integer setWidth(TableHeader.ColumnInfo column, Integer parentWidth) {
if (org.springframework.util.CollectionUtils.isEmpty(column.getChildren())) {
column.setWidth(parentWidth);
return parentWidth;
} else {
Integer depth = 0;
for (TableHeader.ColumnInfo child : column.getChildren()) {
depth = depth + setWidth(child, 1);
}
column.setWidth(depth);
return depth;
}
}
private static String cellValue(FormatterCfgDTO formatterCfgDTO, BigDecimal value) {
if (formatterCfgDTO.getType().equalsIgnoreCase("percent")) {
return value.toString();
@ -474,22 +654,22 @@ public class ChartDataServer implements ChartDataApi {
if (formatter.getType().equals("auto")) {
String[] valueSplit = String.valueOf(value).split(".");
if (StringUtils.isEmpty(value) || !value.contains(".")) {
formatStr = "0";
formatStr = "General";
} else {
formatStr = "0." + new String(new char[valueSplit.length]).replace('\0', '0');
}
switch (formatter.getUnit()) {
case 1000:
formatStr = formatStr + "千";
formatStr = formatStr + "\"千\"";
break;
case 10000:
formatStr = formatStr + "万";
formatStr = formatStr + "\"万\"";
break;
case 1000000:
formatStr = formatStr + "百万";
formatStr = formatStr + "\"百万\"";
break;
case 100000000:
formatStr = formatStr + "'亿'";
formatStr = formatStr + "\"亿\"";
break;
default:
break;
@ -501,7 +681,7 @@ public class ChartDataServer implements ChartDataApi {
if (formatter.getSuffix().equals("%")) {
formatStr = formatStr + "\"%\"";
} else {
formatStr = formatStr + formatter.getSuffix();
formatStr = formatStr + "\"" + formatter.getSuffix() + "\"";
}
}
}
@ -513,16 +693,16 @@ public class ChartDataServer implements ChartDataApi {
}
switch (formatter.getUnit()) {
case 1000:
formatStr = formatStr + "千";
formatStr = formatStr + "\"千\"";
break;
case 10000:
formatStr = formatStr + "万";
formatStr = formatStr + "\"万\"";
break;
case 1000000:
formatStr = formatStr + "百万";
formatStr = formatStr + "\"百万\"";
break;
case 100000000:
formatStr = formatStr + "'亿'";
formatStr = formatStr + "\"亿\"";
break;
default:
break;
@ -534,7 +714,7 @@ public class ChartDataServer implements ChartDataApi {
if (formatter.getSuffix().equals("%")) {
formatStr = formatStr + "\"%\"";
} else {
formatStr = formatStr + formatter.getSuffix();
formatStr = formatStr + "\"" + formatter.getSuffix() + "\"";
}
}
} else if (formatter.getType().equals("percent")) {

21
core/core-backend/src/main/java/io/gisbi/chart/server/ChartViewServer.java
View File

@ -2,11 +2,13 @@ package io.gisbi.chart.server;
import io.gisbi.api.chart.ChartViewApi;
import io.gisbi.api.chart.vo.ChartBaseVO;
import io.gisbi.extensions.view.dto.ChartViewDTO;
import io.gisbi.extensions.view.dto.ChartViewFieldDTO;
import io.gisbi.api.chart.vo.ViewSelectorVO;
import io.gisbi.chart.manage.ChartViewManege;
import io.gisbi.constant.CommonConstants;
import io.gisbi.dataset.utils.DatasetUtils;
import io.gisbi.exception.DEException;
import io.gisbi.extensions.view.dto.ChartViewDTO;
import io.gisbi.extensions.view.dto.ChartViewFieldDTO;
import io.gisbi.result.ResultCode;
import jakarta.annotation.Resource;
import org.springframework.web.bind.annotation.RequestMapping;
@ -27,7 +29,7 @@ public class ChartViewServer implements ChartViewApi {
@Override
public ChartViewDTO getData(Long id) throws Exception {
try {
return chartViewManege.getChart(id);
return chartViewManege.getChart(id, CommonConstants.RESOURCE_TABLE.CORE);
} catch (Exception e) {
DEException.throwException(ResultCode.DATA_IS_WRONG.code(), e.getMessage());
}
@ -36,7 +38,10 @@ public class ChartViewServer implements ChartViewApi {
@Override
public Map<String, List<ChartViewFieldDTO>> listByDQ(Long id, Long chartId, ChartViewDTO dto) {
return chartViewManege.listByDQ(id, chartId, dto);
Map<String, List<ChartViewFieldDTO>> stringListMap = chartViewManege.listByDQ(id, chartId, dto);
DatasetUtils.listEncode(stringListMap.get("dimensionList"));
DatasetUtils.listEncode(stringListMap.get("quotaList"));
return stringListMap;
}
@Override
@ -50,8 +55,8 @@ public class ChartViewServer implements ChartViewApi {
}
@Override
public ChartViewDTO getDetail(Long id) {
return chartViewManege.getDetails(id);
public ChartViewDTO getDetail(Long id, String resourceTable) {
return chartViewManege.getDetails(id, resourceTable);
}
@Override
@ -75,7 +80,7 @@ public class ChartViewServer implements ChartViewApi {
}
@Override
public ChartBaseVO chartBaseInfo(Long id) {
return chartViewManege.chartBaseInfo(id);
public ChartBaseVO chartBaseInfo(Long id, String resourceTable) {
return chartViewManege.chartBaseInfo(id, resourceTable);
}
}

39
core/core-backend/src/main/java/io/gisbi/chart/utils/ChartDataBuild.java
View File

@ -1180,13 +1180,13 @@ public class ChartDataBuild {
Map<String, List<String[]>> groupDataList = detailData.stream().collect(Collectors.groupingBy(item -> "(" + StringUtils.join(ArrayUtils.subarray(item, 0, xEndIndex), ")-de-(") + ")"));
tableRow.forEach(row -> {
String key = xAxis.stream().map(x -> String.format(format, row.get(x.getDataeaseName()).toString())).collect(Collectors.joining("-de-"));
String key = xAxis.stream().map(x -> String.format(format, row.get(x.getGisbiName()).toString())).collect(Collectors.joining("-de-"));
List<String[]> detailFieldValueList = groupDataList.get(key);
List<Map<String, Object>> detailValueMapList = detailFieldValueList.stream().map((detailArr -> {
Map<String, Object> temp = new HashMap<>();
for (int i = 0; i < realDetailFields.size(); i++) {
ChartViewFieldDTO realDetailField = realDetailFields.get(i);
temp.put(realDetailField.getDataeaseName(), detailArr[detailIndex + i]);
temp.put(realDetailField.getGisbiName(), detailArr[detailIndex + i]);
}
return temp;
})).collect(Collectors.toList());
@ -1196,7 +1196,7 @@ public class ChartDataBuild {
ChartViewFieldDTO detailFieldDTO = new ChartViewFieldDTO();
detailFieldDTO.setId(IDUtils.snowID());
detailFieldDTO.setName("detail");
detailFieldDTO.setDataeaseName("detail");
detailFieldDTO.setGisbiName("detail");
fields.add(detailFieldDTO);
map.put("fields", fields);
map.put("detailFields", realDetailFields);
@ -1299,21 +1299,21 @@ public class ChartDataBuild {
data.forEach(ele -> {
Map<String, Object> d = new HashMap<>();
for (int i = 0; i < fields.size(); i++) {
if (ObjectUtils.isNotEmpty(desensitizationList.keySet()) && desensitizationList.containsKey(fields.get(i).getDataeaseName())) {
String desensitizationValue = desensitizationValue(desensitizationList.get(fields.get(i).getDataeaseName()), String.valueOf(ele[i]));
if (ObjectUtils.isNotEmpty(desensitizationList.keySet()) && desensitizationList.containsKey(fields.get(i).getGisbiName())) {
String desensitizationValue = desensitizationValue(desensitizationList.get(fields.get(i).getGisbiName()), String.valueOf(ele[i]));
ele[i] = desensitizationValue;
d.put(fields.get(i).getDataeaseName(), desensitizationValue);
d.put(fields.get(i).getGisbiName(), desensitizationValue);
continue;
}
if (i == ele.length) break;
ChartViewFieldDTO chartViewFieldDTO = fields.get(i);
if (chartViewFieldDTO.getDeType() == 0 || chartViewFieldDTO.getDeType() == 1 || chartViewFieldDTO.getDeType() == 5 || chartViewFieldDTO.getDeType() == 7) {
d.put(fields.get(i).getDataeaseName(), StringUtils.isEmpty(ele[i]) ? "" : ele[i]);
d.put(fields.get(i).getGisbiName(), StringUtils.isEmpty(ele[i]) ? "" : ele[i]);
} else if (chartViewFieldDTO.getDeType() == 2 || chartViewFieldDTO.getDeType() == 3 || chartViewFieldDTO.getDeType() == 4) {
if (view.getIsExcelExport()) {
d.put(fields.get(i).getDataeaseName(), StringUtils.isEmpty(ele[i]) ? "" : ele[i]);
d.put(fields.get(i).getGisbiName(), StringUtils.isEmpty(ele[i]) ? "" : ele[i]);
} else {
d.put(fields.get(i).getDataeaseName(), StringUtils.isEmpty(ele[i]) ? null : new BigDecimal(ele[i]).setScale(8, RoundingMode.HALF_UP));
d.put(fields.get(i).getGisbiName(), StringUtils.isEmpty(ele[i]) ? null : new BigDecimal(ele[i]).setScale(8, RoundingMode.HALF_UP));
}
}
}
@ -1383,10 +1383,7 @@ public class ChartDataBuild {
chartDimensionDTO.setValue(row[j]);
dimensionList.add(chartDimensionDTO);
}
ChartDimensionDTO chartDimensionDTO = new ChartDimensionDTO();
chartDimensionDTO.setId(extStack.get(0).getId());
chartDimensionDTO.setValue(row[xAxis.size()]);
dimensionList.add(chartDimensionDTO);
axisChartDataDTO.setDimensionList(dimensionList);
if (ObjectUtils.isNotEmpty(yAxis)) {
@ -1785,19 +1782,19 @@ public class ChartDataBuild {
List<Map<String, Object>> tableRow = (List<Map<String, Object>>) map.get("tableRow");
final int xEndIndex = detailIndex;
Map<String, List<String[]>> groupDataList = detailData.stream().collect(Collectors.groupingBy(item -> "(" + StringUtils.join(ArrayUtils.subarray(item, 0, xEndIndex), ")-de-(") + ")"));
String extBubbleDataeaseName = ObjectUtils.isNotEmpty(extBubble) ? extBubble.get(0).getDataeaseName() : "";
String extBubblegisbiName = ObjectUtils.isNotEmpty(extBubble) ? extBubble.get(0).getGisbiName() : "";
tableRow.forEach(row -> {
BigDecimal rowValue = row.get(extBubbleDataeaseName) == null ? BigDecimal.ZERO : new BigDecimal(row.get(extBubbleDataeaseName).toString());
String key = xAxis.stream().map(x -> String.format(format, row.get(x.getDataeaseName()).toString())).collect(Collectors.joining("-de-"));
BigDecimal rowValue = row.get(extBubblegisbiName) == null ? BigDecimal.ZERO : new BigDecimal(row.get(extBubblegisbiName).toString());
String key = xAxis.stream().map(x -> String.format(format, row.get(x.getGisbiName()).toString())).collect(Collectors.joining("-de-"));
List<String[]> detailFieldValueList = groupDataList.get(key);
List<Map<String, Object>> detailValueMapList = Optional.ofNullable(detailFieldValueList).orElse(new ArrayList<>()).stream().map((detailArr -> {
Map<String, Object> temp = new HashMap<>();
for (int i = 0; i < realDetailFields.size(); i++) {
ChartViewFieldDTO realDetailField = realDetailFields.get(i);
if (StringUtils.equalsIgnoreCase(extBubbleDataeaseName, realDetailField.getDataeaseName())) {
temp.put(realDetailField.getDataeaseName(), rowValue);
if (StringUtils.equalsIgnoreCase(extBubblegisbiName, realDetailField.getGisbiName())) {
temp.put(realDetailField.getGisbiName(), rowValue);
} else {
temp.put(realDetailField.getDataeaseName(), detailArr[detailIndex + i]);
temp.put(realDetailField.getGisbiName(), detailArr[detailIndex + i]);
}
}
return temp;
@ -1807,10 +1804,10 @@ public class ChartDataBuild {
});
// 先过滤掉所有记录数字段
List<ChartViewFieldDTO> filterCountAxis = fields.stream()
.filter(item -> !StringUtils.equalsIgnoreCase(item.getDataeaseName(), "*"))
.filter(item -> !StringUtils.equalsIgnoreCase(item.getGisbiName(), "*"))
.collect(Collectors.toList());
// 如果气泡大小是记录数，添加到字段列表中
if (ObjectUtils.isNotEmpty(extBubble) && "*".equals(extBubble.get(0).getDataeaseName())) {
if (ObjectUtils.isNotEmpty(extBubble) && "*".equals(extBubble.get(0).getGisbiName())) {
filterCountAxis.addAll(yAxis);
}
map.put("fields", filterCountAxis);

3
core/core-backend/src/main/java/io/gisbi/commons/constants/DataVisualizationConstants.java
View File

@ -10,6 +10,9 @@ public class DataVisualizationConstants {
// 主工程
public static final String MAIN = "main";
// 主工程编辑区
public static final String MAIN_EDIT = "main-edit";
}
//新建仪表板来源

1
core/core-backend/src/main/java/io/gisbi/commons/utils/EncryptUtils.java
View File

@ -10,6 +10,7 @@ public class EncryptUtils extends CodingUtil {
private static final String secretKey = "www.fit2cloud.co";
private static final String iv = "1234567890123456";
public static Object aesEncrypt(Object o) {
if (o == null) {
return null;

41
core/core-backend/src/main/java/io/gisbi/commons/utils/ExcelWatermarkUtils.java
View File

@ -1,5 +1,6 @@
package io.gisbi.commons.utils;
import io.gisbi.api.permissions.user.vo.UserFormVO;
import io.gisbi.i18n.Translator;
import io.gisbi.utils.IPUtils;
import io.gisbi.visualization.dto.WatermarkContentDTO;
@ -20,31 +21,31 @@ public class ExcelWatermarkUtils {
private static SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
// public static String transContent(WatermarkContentDTO watermarkContent, UserFormVO userInfo) {
// String content = "";
// switch (watermarkContent.getType()) {
// case "custom" -> content = watermarkContent.getContent();
// case "nickName" -> content = "${nickName}";
// case "ip" -> content = "${ip}";
// case "time" -> content = "${time}";
// default -> content = "${username}";
// }
// String nickName = userInfo.getName().contains("i18n_") ?Translator.get(userInfo.getName()):userInfo.getName();
// content = content.replaceAll("\\$\\{ip}", IPUtils.get() == null ? "127.0.0.1" : IPUtils.get());
// content = content.replaceAll("\\$\\{username}", userInfo.getAccount());
// content = content.replaceAll("\\$\\{nickName}", nickName);
// content = content.replaceAll("\\$\\{time}", sdf.format(new Date()));
// return content;
// }
public static String transContent(WatermarkContentDTO watermarkContent, UserFormVO userInfo) {
String content = "";
switch (watermarkContent.getType()) {
case "custom" -> content = watermarkContent.getContent();
case "nickName" -> content = "${nickName}";
case "ip" -> content = "${ip}";
case "time" -> content = "${time}";
default -> content = "${username}";
}
String nickName = userInfo.getName().contains("i18n_") ?Translator.get(userInfo.getName()):userInfo.getName();
content = content.replaceAll("\\$\\{ip}", IPUtils.get() == null ? "127.0.0.1" : IPUtils.get());
content = content.replaceAll("\\$\\{username}", userInfo.getAccount());
content = content.replaceAll("\\$\\{nickName}", nickName);
content = content.replaceAll("\\$\\{time}", sdf.format(new Date()));
return content;
}
/**
* 添加水印图片到工作簿并返回图片 ID
*/
// public static int addWatermarkImage(Workbook wb, WatermarkContentDTO watermarkContent, UserFormVO userInfo) {
// byte[] imageBytes = createTextImage(transContent(watermarkContent, userInfo), watermarkContent); // 生成文字水印图片
// return wb.addPicture(imageBytes, Workbook.PICTURE_TYPE_PNG); // 添加到工作簿并返回 ID
// }
public static int addWatermarkImage(Workbook wb, WatermarkContentDTO watermarkContent, UserFormVO userInfo) {
byte[] imageBytes = createTextImage(transContent(watermarkContent, userInfo), watermarkContent); // 生成文字水印图片
return wb.addPicture(imageBytes, Workbook.PICTURE_TYPE_PNG); // 添加到工作簿并返回 ID
}
public static void addWatermarkToSheet(Sheet sheet, Integer pictureIdx) {
Drawing<?> drawing = sheet.createDrawingPatriarch();

61
core/core-backend/src/main/java/io/gisbi/commons/utils/SqlparserUtils.java
View File

@ -1,6 +1,9 @@
package io.gisbi.commons.utils;
import com.fasterxml.jackson.core.type.TypeReference;
import io.gisbi.api.permissions.user.vo.UserFormVO;
import io.gisbi.api.permissions.variable.dto.SysVariableValueDto;
import io.gisbi.api.permissions.variable.dto.SysVariableValueItem;
import io.gisbi.exception.DEException;
import io.gisbi.extensions.datasource.api.PluginManageApi;
import io.gisbi.extensions.datasource.dto.DatasourceSchemaDTO;
@ -18,7 +21,6 @@ import net.sf.jsqlparser.parser.CCJSqlParserUtil;
import net.sf.jsqlparser.statement.Statement;
import net.sf.jsqlparser.statement.select.*;
import net.sf.jsqlparser.util.deparser.ExpressionDeParser;
import net.sf.jsqlparser.util.deparser.SelectDeParser;
import org.apache.calcite.sql.*;
import org.apache.calcite.sql.parser.SqlParser;
import org.apache.calcite.sql.util.SqlShuttle;
@ -37,24 +39,29 @@ import static org.apache.calcite.sql.SqlKind.*;
public class SqlparserUtils {
public static final String regex = "\\$\\{(.*?)\\}";
public static final String regex2 = "\\[(.*?)\\]";
private static final String SubstitutedParams = "DATAEASE_PATAMS_BI";
public static final String regex2 = "\\$f2cde\\[(.*?)\\]";
private static final String SubstitutedParams = "gisbi_PATAMS_BI";
private static final String SysParamsSubstitutedParams = "DeSysParams_";
private static final String SubstitutedSql = " 'DE-BI' = 'DE-BI' ";
private boolean removeSysParams;
boolean hasVariables = false;
private UserFormVO userEntity;
private final List<Map<String, String>> sysParams = new ArrayList<>();
private static final String deVariablePattern = "\\$DE_PARAM\\{(.*?)\\}";
public String handleVariableDefaultValue(String sql, String sqlVariableDetails, boolean isEdit, boolean isFromDataSet, List<SqlVariableDetails> parameters, boolean isCross, Map<Long, DatasourceSchemaDTO> dsMap, PluginManageApi pluginManage, UserFormVO userEntity) {
Pattern r = Pattern.compile(deVariablePattern);
Matcher m = r.matcher(sql);
if (m.find()) {
return new DeSqlparserUtils().handleVariableDefaultValue(sql, sqlVariableDetails, isEdit, isFromDataSet, parameters, isCross, dsMap, pluginManage, userEntity);
}
public String handleVariableDefaultValue(String sql, String sqlVariableDetails, boolean isEdit, boolean isFromDataSet, List<SqlVariableDetails> parameters, boolean isCross, Map<Long, DatasourceSchemaDTO> dsMap, PluginManageApi pluginManage) {
DatasourceSchemaDTO ds = dsMap.entrySet().iterator().next().getValue();
if (StringUtils.isEmpty(sql)) {
DEException.throwException(Translator.get("i18n_sql_not_empty"));
}
try {
this.removeSysParams = true;
removeVariables(sql, ds.getType());
} catch (Exception e) {
DEException.throwException(e);
}
this.userEntity = userEntity;
hasVariables = false;
sql = sql.trim();
if (sql.endsWith(";")) {
sql = sql.substring(0, sql.length() - 1);
@ -139,40 +146,62 @@ public class SqlparserUtils {
return sql;
}
private static boolean isParams(String paramId) {
if (Arrays.asList("sysParams.userId", "sysParams.userEmail", "sysParams.userName").contains(paramId)) {
return true;
}
boolean isLong = false;
try {
Long.valueOf(paramId);
isLong = true;
} catch (Exception e) {
isLong = false;
}
if (paramId.length() >= 18 && isLong) {
return true;
}
return false;
}
private String removeVariables(final String sql, String dsType) throws Exception {
String tmpSql = sql.replaceAll("(?m)^\\s*$[\n\r]{0,}", "");
Pattern pattern = Pattern.compile(regex);
Matcher matcher = pattern.matcher(tmpSql);
boolean hasVariables = false;
while (matcher.find()) {
hasVariables = true;
tmpSql = tmpSql.replace(matcher.group(), SubstitutedParams);
}
if (removeSysParams) {
for (Map<String, String> sysParam : sysParams) {
tmpSql = tmpSql.replaceAll(sysParam.get("replace"), sysParam.get("origin"));
tmpSql = tmpSql.replace(sysParam.get("replace"), sysParam.get("origin"));
}
pattern = Pattern.compile(regex2);
matcher = pattern.matcher(tmpSql);
while (matcher.find()) {
String paramId = matcher.group().substring(7, matcher.group().length() - 1);
if (!isParams(paramId)) {
continue;
}
hasVariables = true;
tmpSql = tmpSql.replace(matcher.group(), SubstitutedParams);
}
} else {
pattern = Pattern.compile(regex2);
matcher = pattern.matcher(tmpSql);
sysParams.clear();
while (matcher.find()) {
String paramId = matcher.group().substring(7, matcher.group().length() - 1);
if (!isParams(paramId)) {
continue;
}
hasVariables = true;
tmpSql = tmpSql.replace(matcher.group(), SysParamsSubstitutedParams + matcher.group().substring(1, matcher.group().length() - 1));
tmpSql = tmpSql.replace(matcher.group(), SysParamsSubstitutedParams + matcher.group().substring(7, matcher.group().length() - 1));
Map<String, String> sysParam = new HashMap<>();
sysParam.put("origin", matcher.group());
sysParam.put("replace", SysParamsSubstitutedParams + matcher.group().substring(1, matcher.group().length() - 1));
sysParam.put("replace", SysParamsSubstitutedParams + matcher.group().substring(7, matcher.group().length() - 1));
sysParams.add(sysParam);
}
}
if(!hasVariables && !sql.contains(SubstitutedParams)){
if (!hasVariables && !sql.contains(SubstitutedParams)) {
return sql;
}
Statement statement = CCJSqlParserUtil.parse(tmpSql);

8
core/core-backend/src/main/java/io/gisbi/config/DeMvcConfig.java
View File

@ -25,20 +25,20 @@ public class DeMvcConfig implements WebMvcConfigurer {
@Override
public void addResourceHandlers(ResourceHandlerRegistry registry) {
String workDir = FILE_PROTOCOL + ensureSuffix(WORK_DIR, FILE_SEPARATOR);
String uploadUrlPattern = ensureBoth(URL_SEPARATOR + UPLOAD_URL_PREFIX, URL_SEPARATOR) + "**";
String uploadUrlPattern = ensureBoth(URL_SEPARATOR + UPLOAD_URL_PREFIX, AuthConstant.DE_API_PREFIX, URL_SEPARATOR) + "**";
registry.addResourceHandler(uploadUrlPattern).addResourceLocations(workDir);
// map
String mapDir = FILE_PROTOCOL + ensureSuffix(MAP_DIR, FILE_SEPARATOR);
String mapUrlPattern = ensureBoth(MAP_URL, URL_SEPARATOR) + "**";
String mapUrlPattern = ensureBoth(MAP_URL, AuthConstant.DE_API_PREFIX, URL_SEPARATOR) + "**";
registry.addResourceHandler(mapUrlPattern).addResourceLocations(mapDir);
String geoDir = FILE_PROTOCOL + ensureSuffix(CUSTOM_MAP_DIR, FILE_SEPARATOR);
String geoUrlPattern = ensureBoth(GEO_URL, URL_SEPARATOR) + "**";
String geoUrlPattern = ensureBoth(GEO_URL, AuthConstant.DE_API_PREFIX, URL_SEPARATOR) + "**";
registry.addResourceHandler(geoUrlPattern).addResourceLocations(geoDir);
String i18nDir = FILE_PROTOCOL + ensureSuffix(I18N_DIR, FILE_SEPARATOR);
String i18nUrlPattern = ensureBoth(I18N_URL, URL_SEPARATOR) + "**";
String i18nUrlPattern = ensureBoth(I18N_URL, AuthConstant.DE_API_PREFIX, URL_SEPARATOR) + "**";
registry.addResourceHandler(i18nUrlPattern).addResourceLocations(i18nDir);
}

28
core/core-backend/src/main/java/io/gisbi/dataset/dao/auto/entity/CoreDatasetGroup.java
View File

@ -5,7 +5,7 @@ import java.io.Serializable;
/**
* <p>
*
* 数据集分组表
* </p>
*
* @Author bi-coder
@ -71,6 +71,9 @@ public class CoreDatasetGroup implements Serializable {
*/
private Long createTime;
/**
* Quartz 实例 ID
*/
private String qrtzInstance;
/**
@ -93,6 +96,11 @@ public class CoreDatasetGroup implements Serializable {
*/
private String unionSql;
/**
* 是否跨源
*/
private Boolean isCross;
public Long getId() {
return id;
}
@ -213,11 +221,18 @@ public class CoreDatasetGroup implements Serializable {
this.unionSql = unionSql;
}
public Boolean getIsCross() {
return isCross;
}
public void setIsCross(Boolean isCross) {
this.isCross = isCross;
}
@Override
public String toString() {
return "CoreDatasetGroup{" +
"id = " + id +
", appId = " + appId +
", name = " + name +
", pid = " + pid +
", level = " + level +
@ -232,14 +247,7 @@ public class CoreDatasetGroup implements Serializable {
", updateBy = " + updateBy +
", lastUpdateTime = " + lastUpdateTime +
", unionSql = " + unionSql +
", isCross = " + isCross +
"}";
}
public String getAppId() {
return appId;
}
public void setAppId(String appId) {
this.appId = appId;
}
}

16
core/core-backend/src/main/java/io/gisbi/dataset/dao/auto/entity/CoreDatasetTableField.java
View File

@ -57,9 +57,9 @@ public class CoreDatasetTableField implements Serializable {
private String description;
/**
* de字段名用作唯一标识
* 字段名用作唯一标识
*/
private String dataeaseName;
private String gisbiName;
/**
* de字段别名
@ -92,7 +92,7 @@ public class CoreDatasetTableField implements Serializable {
private Integer size;
/**
* dataease字段类型：0-文本，1-时间，2-整型数值，3-浮点数值，4-布尔，5-地理位置，6-二进制
* gisbi字段类型：0-文本，1-时间，2-整型数值，3-浮点数值，4-布尔，5-地理位置，6-二进制
*/
private Integer deType;
@ -205,12 +205,12 @@ public class CoreDatasetTableField implements Serializable {
this.description = description;
}
public String getDataeaseName() {
return dataeaseName;
public String getGisbiName() {
return gisbiName;
}
public void setDataeaseName(String dataeaseName) {
this.dataeaseName = dataeaseName;
public void setGisbiName(String gisbiName) {
this.gisbiName = gisbiName;
}
public String getFieldShortName() {
@ -352,7 +352,7 @@ public class CoreDatasetTableField implements Serializable {
", originName = " + originName +
", name = " + name +
", description = " + description +
", dataeaseName = " + dataeaseName +
", gisbiName = " + gisbiName +
", fieldShortName = " + fieldShortName +
", groupList = " + groupList +
", otherGroup = " + otherGroup +

4
core/core-backend/src/main/java/io/gisbi/dataset/dao/auto/mapper/CoreDatasetTableMapper.java
View File

@ -1,10 +1,8 @@
package io.gisbi.dataset.dao.auto.mapper;
import io.gisbi.application.system.domain.Role;
import io.gisbi.dataset.dao.auto.entity.CoreDatasetTable;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import io.gisbi.dataset.dao.auto.entity.CoreDatasetTable;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Select;
import java.util.List;

2
core/core-backend/src/main/java/io/gisbi/dataset/dao/ext/mapper/CoreDataSetExtMapper.java
View File

@ -19,6 +19,6 @@ public interface CoreDataSetExtMapper {
""")
List<DataSetNodePO> query(@Param("ew") QueryWrapper queryWrapper);
@Select("select id, name, node_type, create_by, create_time, update_by, last_update_time from core_dataset_group where id = #{id}")
@Select("select id, name, node_type, create_by, create_time, update_by, last_update_time, is_cross from core_dataset_group where id = #{id}")
DataSetBarVO queryBarInfo(@Param("id") Long id);
}

195
core/core-backend/src/main/java/io/gisbi/dataset/manage/DatasetDataManage.java
View File

@ -1,20 +1,23 @@
package io.gisbi.dataset.manage;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import io.gisbi.api.chart.dto.DeSortField;
import io.gisbi.api.dataset.dto.*;
import io.gisbi.api.dataset.union.DatasetGroupInfoDTO;
import io.gisbi.api.dataset.union.DatasetTableInfoDTO;
import io.gisbi.api.permissions.auth.dto.BusiPerCheckDTO;
import io.gisbi.api.permissions.dataset.api.RowPermissionsApi;
import io.gisbi.api.permissions.user.vo.UserFormVO;
import io.gisbi.chart.utils.ChartDataBuild;
import io.gisbi.commons.utils.SqlparserUtils;
import io.gisbi.constant.AuthEnum;
import io.gisbi.constant.SQLConstants;
import io.gisbi.dataset.constant.DatasetTableType;
import io.gisbi.dataset.dao.auto.entity.CoreDatasetTable;
import io.gisbi.dataset.dao.auto.entity.CoreDatasetTableField;
import io.gisbi.dataset.dao.auto.mapper.CoreDatasetTableFieldMapper;
import io.gisbi.dataset.dao.auto.mapper.CoreDatasetTableMapper;
import io.gisbi.dataset.utils.DatasetUtils;
import io.gisbi.dataset.utils.FieldUtils;
import io.gisbi.dataset.utils.SqlUtils;
import io.gisbi.dataset.utils.TableUtils;
import io.gisbi.datasource.dao.auto.entity.CoreDatasource;
import io.gisbi.datasource.dao.auto.mapper.CoreDatasourceMapper;
@ -22,7 +25,6 @@ import io.gisbi.datasource.manage.DataSourceManage;
import io.gisbi.datasource.manage.EngineManage;
import io.gisbi.datasource.utils.DatasourceUtils;
import io.gisbi.engine.constant.ExtFieldConstant;
import io.gisbi.constant.SQLConstants;
import io.gisbi.engine.sql.SQLProvider;
import io.gisbi.engine.trans.*;
import io.gisbi.engine.utils.SQLUtils;
@ -40,10 +42,8 @@ import io.gisbi.extensions.view.dto.ChartExtRequest;
import io.gisbi.extensions.view.dto.ColumnPermissionItem;
import io.gisbi.extensions.view.dto.SqlVariableDetails;
import io.gisbi.i18n.Translator;
import io.gisbi.utils.BeanUtils;
import io.gisbi.utils.IDUtils;
import io.gisbi.utils.JsonUtil;
import io.gisbi.utils.TreeUtils;
import io.gisbi.system.manage.CorePermissionManage;
import io.gisbi.utils.*;
import jakarta.annotation.Resource;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@ -58,6 +58,7 @@ import java.util.*;
import java.util.stream.Collectors;
import static io.gisbi.chart.manage.ChartDataManage.START_END_SEPARATOR;
import static io.gisbi.dataset.utils.TableUtils.format;
/**
* @Author bi-coder
@ -79,6 +80,10 @@ public class DatasetDataManage {
@Autowired(required = false)
private PluginManageApi pluginManage;
@Resource
private CorePermissionManage corePermissionManage;
@Autowired(required = false)
private RowPermissionsApi rowPermissionsApi;
@Resource
private CoreDatasetTableFieldMapper coreDatasetTableFieldMapper;
@Resource
private CoreDatasetTableMapper coreDatasetTableMapper;
@ -87,7 +92,11 @@ public class DatasetDataManage {
private static Logger logger = LoggerFactory.getLogger(DatasetDataManage.class);
public static final List<String> notFullDs = List.of("mysql", "mariadb", "Excel", "API");
private RowPermissionsApi getRowPermissionsApi() {
return rowPermissionsApi;
}
public static final List<String> notFullDs = List.of("mysql", "mariadb", "Excel", "API", "H2", "h2");
public List<DatasetTableFieldDTO> getTableFields(DatasetTableDTO datasetTableDTO) throws Exception {
List<DatasetTableFieldDTO> list = null;
@ -97,7 +106,7 @@ public class DatasetDataManage {
if (StringUtils.equalsIgnoreCase(type, DatasetTableType.DB) || StringUtils.equalsIgnoreCase(type, DatasetTableType.SQL)) {
CoreDatasource coreDatasource = dataSourceManage.getCoreDatasource(datasetTableDTO.getDatasourceId());
DatasourceSchemaDTO datasourceSchemaDTO = new DatasourceSchemaDTO();
if (StringUtils.equalsIgnoreCase("excel", coreDatasource.getType()) || coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.name())) {
if (coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.Excel.name()) || coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.name())) {
coreDatasource = engineManage.getDeEngine();
}
if (StringUtils.isNotEmpty(coreDatasource.getStatus()) && "Error".equalsIgnoreCase(coreDatasource.getStatus())) {
@ -108,6 +117,7 @@ public class DatasetDataManage {
Provider provider = ProviderFactory.getProvider(coreDatasource.getType());
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setIsCross(datasetTableDTO.getIsCross());
datasourceRequest.setDsList(Map.of(datasourceSchemaDTO.getId(), datasourceSchemaDTO));
String sql;
if (StringUtils.equalsIgnoreCase(type, DatasetTableType.DB)) {
@ -115,23 +125,34 @@ public class DatasetDataManage {
sql = TableUtils.tableName2Sql(datasourceSchemaDTO, tableInfoDTO.getTable()) + " LIMIT 0 OFFSET 0";
// replace schema alias, trans dialect
Map map = JsonUtil.parseObject(datasourceSchemaDTO.getConfiguration(), Map.class);
if (!datasourceRequest.getIsCross()) {
if (ObjectUtils.isNotEmpty(map.get("schema"))) {
sql = sql.replaceAll(SqlPlaceholderConstants.KEYWORD_PREFIX_REGEX + datasourceSchemaDTO.getSchemaAlias() + SqlPlaceholderConstants.KEYWORD_SUFFIX_REGEX, map.get("schema").toString());
sql = sql.replaceAll(SqlPlaceholderConstants.KEYWORD_PREFIX_REGEX + datasourceSchemaDTO.getSchemaAlias() + SqlPlaceholderConstants.KEYWORD_SUFFIX_REGEX, String.format(format, map.get("schema").toString()) );
} else {
sql = sql.replaceAll(SqlPlaceholderConstants.KEYWORD_PREFIX_REGEX + datasourceSchemaDTO.getSchemaAlias() + SqlPlaceholderConstants.KEYWORD_SUFFIX_REGEX + "\\.", "");
}
sql = provider.transSqlDialect(sql, datasourceRequest.getDsList());
} else {
sql = sql.replaceAll(SqlPlaceholderConstants.KEYWORD_PREFIX_REGEX + datasourceSchemaDTO.getSchemaAlias() + SqlPlaceholderConstants.KEYWORD_SUFFIX_REGEX + "\\.", "");
String tableSchema = datasetSQLManage.putObj2Map(datasourceRequest.getDsList(), datasetTableDTO, datasourceRequest.getIsCross());
sql = SqlUtils.addSchema(sql, tableSchema);
}
} else {
// parser sql params and replace default value
String s = new String(Base64.getDecoder().decode(tableInfoDTO.getSql()));
String originSql = new SqlparserUtils().handleVariableDefaultValue(s, datasetTableDTO.getSqlVariableDetails(), false, false, null, false, datasourceRequest.getDsList(), pluginManage);
String originSql = new SqlparserUtils().handleVariableDefaultValue(s, datasetTableDTO.getSqlVariableDetails(), true, false, null, datasourceRequest.getIsCross(), datasourceRequest.getDsList(), pluginManage, getUserEntity());
originSql = provider.replaceComment(originSql);
// add sql table schema
if (!datasourceRequest.getIsCross()) {
sql = SQLUtils.buildOriginPreviewSql(SqlPlaceholderConstants.TABLE_PLACEHOLDER, 0, 0);
sql = provider.transSqlDialect(sql, datasourceRequest.getDsList());
// replace placeholder
sql = provider.replaceTablePlaceHolder(sql, originSql);
} else {
String tableSchema = datasetSQLManage.putObj2Map(datasourceRequest.getDsList(), datasetTableDTO, datasourceRequest.getIsCross());
sql = SqlUtils.addSchema(originSql, tableSchema);
}
}
datasourceRequest.setQuery(sql.replaceAll("\r\n", " ")
.replaceAll("\n", " "));
@ -189,7 +210,11 @@ public class DatasetDataManage {
}).collect(Collectors.toList());
}
public Map<String, Object> previewDataWithLimit(DatasetGroupInfoDTO datasetGroupInfoDTO, Integer start, Integer count, boolean checkPermission) throws Exception {
public Map<String, Object> previewDataWithLimit(DatasetGroupInfoDTO datasetGroupInfoDTO, Integer start, Integer count, boolean checkPermission, boolean encode) throws Exception {
if (encode) {
DatasetUtils.dsDecode(datasetGroupInfoDTO);
}
Map<String, Object> sqlMap = datasetSQLManage.getUnionSQLForEdit(datasetGroupInfoDTO, null);
String sql = (String) sqlMap.get("sql");
@ -214,7 +239,7 @@ public class DatasetDataManage {
dsList.add(next.getValue().getType());
}
boolean needOrder = Utils.isNeedOrder(dsList);
boolean crossDs = Utils.isCrossDs(dsMap);
boolean crossDs = datasetGroupInfoDTO.getIsCross();
if (!crossDs) {
if (notFullDs.contains(dsMap.entrySet().iterator().next().getValue().getType()) && (boolean) sqlMap.get("isFullJoin")) {
DEException.throwException(Translator.get("i18n_not_full"));
@ -253,16 +278,20 @@ public class DatasetDataManage {
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setQuery(querySQL);
datasourceRequest.setDsList(dsMap);
datasourceRequest.setIsCross(crossDs);
Map<String, Object> data = provider.fetchResultField(datasourceRequest);
Map<String, Object> map = new LinkedHashMap<>();
// 重新构造data
Map<String, Object> previewData = buildPreviewData(data, fields, desensitizationList);
Map<String, Object> previewData = buildPreviewData(data, fields, desensitizationList, encode);
map.put("data", previewData);
if (ObjectUtils.isEmpty(datasetGroupInfoDTO.getId())) {
map.put("allFields", fields);
} else {
List<DatasetTableFieldDTO> fieldList = datasetTableFieldManage.selectByDatasetGroupId(datasetGroupInfoDTO.getId());
if (encode) {
DatasetUtils.listEncode(fieldList);
}
map.put("allFields", fieldList);
}
map.put("sql", Base64.getEncoder().encodeToString(querySQL.getBytes()));
@ -278,7 +307,8 @@ public class DatasetDataManage {
return 0L;
}
public Long getDatasetCountWithWhere(Long datasetGroupId) throws Exception {
public Long getDatasetCountWithWhere(Long datasetGroupId) {
try {
DatasetGroupInfoDTO datasetGroupInfoDTO = datasetGroupManage.getForCount(datasetGroupId);
Map<String, Object> sqlMap = datasetSQLManage.getUnionSQLForEdit(datasetGroupInfoDTO, null);
String sql = (String) sqlMap.get("sql");
@ -297,7 +327,7 @@ public class DatasetDataManage {
for (Map.Entry<Long, DatasourceSchemaDTO> next : dsMap.entrySet()) {
dsList.add(next.getValue().getType());
}
boolean crossDs = Utils.isCrossDs(dsMap);
boolean crossDs = datasetGroupInfoDTO.getIsCross();
if (!crossDs) {
if (notFullDs.contains(dsMap.entrySet().iterator().next().getValue().getType()) && (boolean) sqlMap.get("isFullJoin")) {
DEException.throwException(Translator.get("i18n_not_full"));
@ -326,12 +356,15 @@ public class DatasetDataManage {
Order2SQLObj.getOrders(sqlMeta, datasetGroupInfoDTO.getSortFields(), fields, crossDs, dsMap, Utils.getParams(fields), null, pluginManage);
String replaceSql = provider.rebuildSQL(SQLProvider.createQuerySQL(sqlMeta, false, false, false), sqlMeta, crossDs, dsMap);
return getDatasetTotal(datasetGroupInfoDTO, replaceSql, null);
} catch (Exception e) {
return null;
}
}
public Long getDatasetTotal(DatasetGroupInfoDTO datasetGroupInfoDTO, String s, ChartExtRequest request) throws Exception {
Map<String, Object> sqlMap = datasetSQLManage.getUnionSQLForEdit(datasetGroupInfoDTO, request);
Map<Long, DatasourceSchemaDTO> dsMap = (Map<Long, DatasourceSchemaDTO>) sqlMap.get("dsMap");
boolean crossDs = Utils.isCrossDs(dsMap);
boolean crossDs = datasetGroupInfoDTO.getIsCross();
String sql;
if (StringUtils.isEmpty(s)) {
sql = (String) sqlMap.get("sql");
@ -349,6 +382,7 @@ public class DatasetDataManage {
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setQuery(querySQL);
datasourceRequest.setDsList(dsMap);
datasourceRequest.setIsCross(crossDs);
Provider provider;
if (crossDs) {
@ -390,17 +424,17 @@ public class DatasetDataManage {
return map;
}
// private UserFormVO getUserEntity() {
// if (getRowPermissionsApi() == null) {
// return null;
// }
// return getRowPermissionsApi().getUserById(AuthUtils.getUser().getUserId());
// }
private UserFormVO getUserEntity() {
if (getRowPermissionsApi() == null) {
return null;
}
return getRowPermissionsApi().getUserById(AuthUtils.getUser().getUserId());
}
public Map<String, Object> previewSql(PreviewSqlDTO dto) throws DEException {
CoreDatasource coreDatasource = dataSourceManage.getCoreDatasource(dto.getDatasourceId());
DatasourceSchemaDTO datasourceSchemaDTO = new DatasourceSchemaDTO();
if (coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.name()) || coreDatasource.getType().equalsIgnoreCase("Excel")) {
if (coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.name()) || coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.Excel.name())) {
BeanUtils.copyBean(datasourceSchemaDTO, engineManage.getDeEngine());
} else {
BeanUtils.copyBean(datasourceSchemaDTO, coreDatasource);
@ -417,17 +451,38 @@ public class DatasetDataManage {
dsMap.put(datasourceSchemaDTO.getId(), datasourceSchemaDTO);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDsList(dsMap);
datasourceRequest.setIsCross(dto.getIsCross());
Provider provider = ProviderFactory.getProvider(datasourceSchemaDTO.getType());
// parser sql params and replace default value
String s = new String(Base64.getDecoder().decode(dto.getSql()));
String originSql = new SqlparserUtils().handleVariableDefaultValue(datasetSQLManage.subPrefixSuffixChar(s), dto.getSqlVariableDetails(), true, true, null, false, dsMap, pluginManage);
String originSql = new SqlparserUtils().handleVariableDefaultValue(datasetSQLManage.subPrefixSuffixChar(s), dto.getSqlVariableDetails(), true, true, null, dto.getIsCross(), dsMap, pluginManage, getUserEntity());
originSql = provider.replaceComment(originSql);
// sql 作为临时表，外层加上limit
String sql;
if (dto.getIsCross()) {
DatasetTableDTO currentDs = new DatasetTableDTO();
BeanUtils.copyBean(currentDs, dto);
currentDs.setType("sql");
String tableSchema = datasetSQLManage.putObj2Map(dsMap, currentDs, dto.getIsCross());
sql = SqlUtils.addSchema(originSql, tableSchema);
if (Utils.isNeedOrder(List.of(datasourceSchemaDTO.getType()))) {
// 先根据sql获取表字段
String sqlField = SQLUtils.buildOriginPreviewSql(sql, 0, 0);
datasourceRequest.setQuery(sqlField);
// 获取数据源表的原始字段
List<TableField> list = provider.fetchTableField(datasourceRequest);
if (ObjectUtils.isEmpty(list)) {
return null;
}
sql = SQLUtils.buildOriginPreviewSqlWithOrderBy(sql, 100, 0, String.format(SQLConstants.FIELD_DOT_FIX, list.get(0).getOriginName()) + " ASC ");
} else {
sql = SQLUtils.buildOriginPreviewSql(sql, 100, 0);
}
} else {
if (Utils.isNeedOrder(List.of(datasourceSchemaDTO.getType()))) {
// 先根据sql获取表字段
String sqlField = SQLUtils.buildOriginPreviewSql(SqlPlaceholderConstants.TABLE_PLACEHOLDER, 0, 0);
@ -449,6 +504,7 @@ public class DatasetDataManage {
sql = provider.transSqlDialect(sql, datasourceRequest.getDsList());
// replace placeholder
sql = provider.replaceTablePlaceHolder(sql, originSql);
}
logger.debug("calcite data preview sql: " + sql);
datasourceRequest.setQuery(sql);
@ -456,14 +512,14 @@ public class DatasetDataManage {
// 重新构造data
List<TableField> fList = (List<TableField>) data.get("fields");
List<DatasetTableFieldDTO> fields = transFields(fList, false);
Map<String, Object> previewData = buildPreviewData(data, fields, new HashMap<>());
Map<String, Object> previewData = buildPreviewData(data, fields, new HashMap<>(), false);
Map<String, Object> map = new LinkedHashMap<>();
map.put("data", previewData);
map.put("sql", Base64.getEncoder().encodeToString(sql.getBytes()));
return map;
}
public Map<String, Object> buildPreviewData(Map<String, Object> data, List<DatasetTableFieldDTO> fields, Map<String, ColumnPermissionItem> desensitizationList) {
public Map<String, Object> buildPreviewData(Map<String, Object> data, List<DatasetTableFieldDTO> fields, Map<String, ColumnPermissionItem> desensitizationList, boolean isEncode) {
Map<String, Object> map = new LinkedHashMap<>();
List<String[]> dataList = (List<String[]>) data.get("data");
List<LinkedHashMap<String, Object>> dataObjectList = new ArrayList<>();
@ -479,11 +535,11 @@ public class DatasetDataManage {
BigDecimal bigDecimal = new BigDecimal(res);
res = String.format("%.8f", bigDecimal);
}
if (desensitizationList.keySet().contains(fields.get(j).getDataeaseName())) {
obj.put(fields.get(j).getDataeaseName(), ChartDataBuild.desensitizationValue(desensitizationList.get(fields.get(j).getDataeaseName()), String.valueOf(res)));
if (desensitizationList.keySet().contains(fields.get(j).getGisbiName())) {
obj.put(fields.get(j).getGisbiName(), ChartDataBuild.desensitizationValue(desensitizationList.get(fields.get(j).getGisbiName()), String.valueOf(res)));
} else {
obj.put(ObjectUtils.isNotEmpty(fields.get(j).getDataeaseName()) ?
fields.get(j).getDataeaseName() : fields.get(j).getOriginName(), res);
obj.put(ObjectUtils.isNotEmpty(fields.get(j).getGisbiName()) ?
fields.get(j).getGisbiName() : fields.get(j).getOriginName(), res);
}
}
}
@ -491,6 +547,10 @@ public class DatasetDataManage {
}
}
if (isEncode) {
DatasetUtils.listEncode(fields);
}
map.put("fields", fields);
map.put("data", dataObjectList);
return map;
@ -506,27 +566,27 @@ public class DatasetDataManage {
for (DatasetTableFieldDTO fieldDTO : unionFields) {
if (Objects.equals(datasetTableFieldDTO.getDatasetTableId(), fieldDTO.getDatasetTableId())
&& Objects.equals(datasetTableFieldDTO.getOriginName(), fieldDTO.getOriginName())) {
datasetTableFieldDTO.setDataeaseName(fieldDTO.getDataeaseName());
datasetTableFieldDTO.setGisbiName(fieldDTO.getGisbiName());
datasetTableFieldDTO.setFieldShortName(fieldDTO.getFieldShortName());
}
}
}
if (Objects.equals(datasetTableFieldDTO.getExtField(), ExtFieldConstant.EXT_CALC)) {
String dataeaseName = TableUtils.fieldNameShort(datasetTableFieldDTO.getId() + "_" + datasetTableFieldDTO.getOriginName());
datasetTableFieldDTO.setDataeaseName(dataeaseName);
datasetTableFieldDTO.setFieldShortName(dataeaseName);
String gisbiName = TableUtils.fieldNameShort(datasetTableFieldDTO.getId() + "_" + datasetTableFieldDTO.getOriginName());
datasetTableFieldDTO.setGisbiName(gisbiName);
datasetTableFieldDTO.setFieldShortName(gisbiName);
datasetTableFieldDTO.setDeExtractType(datasetTableFieldDTO.getDeType());
}
if (Objects.equals(datasetTableFieldDTO.getExtField(), ExtFieldConstant.EXT_GROUP)) {
String dataeaseName = TableUtils.fieldNameShort(datasetTableFieldDTO.getId() + "_" + datasetTableFieldDTO.getOriginName());
datasetTableFieldDTO.setDataeaseName(dataeaseName);
datasetTableFieldDTO.setFieldShortName(dataeaseName);
String gisbiName = TableUtils.fieldNameShort(datasetTableFieldDTO.getId() + "_" + datasetTableFieldDTO.getOriginName());
datasetTableFieldDTO.setGisbiName(gisbiName);
datasetTableFieldDTO.setFieldShortName(gisbiName);
datasetTableFieldDTO.setDeExtractType(0);
datasetTableFieldDTO.setDeType(0);
datasetTableFieldDTO.setGroupType("d");
}
} else {
datasetTableFieldDTO.setDataeaseName(dto.getDataeaseName());
datasetTableFieldDTO.setGisbiName(dto.getGisbiName());
datasetTableFieldDTO.setFieldShortName(dto.getFieldShortName());
}
}
@ -546,7 +606,7 @@ public class DatasetDataManage {
allFields.addAll(datasetGroupInfoDTO.getAllFields());
Map<Long, DatasourceSchemaDTO> dsMap = (Map<Long, DatasourceSchemaDTO>) sqlMap.get("dsMap");
boolean crossDs = Utils.isCrossDs(dsMap);
boolean crossDs = datasetGroupInfoDTO.getIsCross();
if (!crossDs) {
sql = Utils.replaceSchemaAlias(sql, dsMap);
}
@ -579,7 +639,7 @@ public class DatasetDataManage {
}
Field2SQLObj.field2sqlObj(sqlMeta, fields, allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage);
// WhereTree2Str.transFilterTrees(sqlMeta, null, allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage);
WhereTree2Str.transFilterTrees(sqlMeta, null, allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage);
Order2SQLObj.getOrders(sqlMeta, datasetGroupInfoDTO.getSortFields(), allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage);
String querySQL;
querySQL = SQLProvider.createQuerySQL(sqlMeta, false, needOrder, !StringUtils.equalsIgnoreCase(dsType, "es"));
@ -591,6 +651,7 @@ public class DatasetDataManage {
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setQuery(querySQL);
datasourceRequest.setDsList(dsMap);
datasourceRequest.setIsCross(crossDs);
Map<String, Object> data = provider.fetchResultField(datasourceRequest);
List<String[]> dataList = (List<String[]>) data.get("data");
@ -635,13 +696,13 @@ public class DatasetDataManage {
Long datasetGroupId = field.getDatasetGroupId();
// check permission
// BusiPerCheckDTO dto = new BusiPerCheckDTO();
// dto.setId(datasetGroupId);
// dto.setAuthEnum(AuthEnum.READ);
// boolean checked = corePermissionManage.checkAuth(dto);
// if (!checked) {
// DEException.throwException(Translator.get("i18n_no_dataset_permission"));
// }
BusiPerCheckDTO dto = new BusiPerCheckDTO();
dto.setId(datasetGroupId);
dto.setAuthEnum(AuthEnum.READ);
boolean checked = corePermissionManage.checkAuth(dto);
if (!checked) {
DEException.throwException(Translator.get("i18n_no_dataset_permission"));
}
if (field.getChartId() != null) {
allFields.addAll(datasetTableFieldManage.getChartCalcFields(field.getChartId()));
}
@ -653,7 +714,7 @@ public class DatasetDataManage {
allFields.addAll(datasetGroupInfoDTO.getAllFields());
Map<Long, DatasourceSchemaDTO> dsMap = (Map<Long, DatasourceSchemaDTO>) sqlMap.get("dsMap");
boolean crossDs = Utils.isCrossDs(dsMap);
boolean crossDs = datasetGroupInfoDTO.getIsCross();
if (!crossDs) {
sql = Utils.replaceSchemaAlias(sql, dsMap);
}
@ -713,6 +774,7 @@ public class DatasetDataManage {
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setQuery(querySQL);
datasourceRequest.setDsList(dsMap);
datasourceRequest.setIsCross(crossDs);
Map<String, Object> data = provider.fetchResultField(datasourceRequest);
List<String[]> dataList = (List<String[]>) data.get("data");
@ -738,9 +800,9 @@ public class DatasetDataManage {
tmpData.set(i, val);
}
}
if (desensitizationList.keySet().contains(field.getDataeaseName())) {
if (desensitizationList.keySet().contains(field.getGisbiName())) {
for (int i = 0; i < tmpData.size(); i++) {
previewData.add(ChartDataBuild.desensitizationValue(desensitizationList.get(field.getDataeaseName()), tmpData.get(i)));
previewData.add(ChartDataBuild.desensitizationValue(desensitizationList.get(field.getGisbiName()), tmpData.get(i)));
}
} else {
previewData = tmpData;
@ -819,7 +881,7 @@ public class DatasetDataManage {
allFields.addAll(datasetGroupInfoDTO.getAllFields());
dsMap = (Map<Long, DatasourceSchemaDTO>) sqlMap.get("dsMap");
crossDs = Utils.isCrossDs(dsMap);
crossDs = datasetGroupInfoDTO.getIsCross();
if (!crossDs) {
sql = Utils.replaceSchemaAlias(sql, dsMap);
}
@ -975,6 +1037,7 @@ public class DatasetDataManage {
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setQuery(querySQL);
datasourceRequest.setDsList(dsMap);
datasourceRequest.setIsCross(crossDs);
Map<String, Object> data = provider.fetchResultField(datasourceRequest);
List<String[]> dataList = (List<String[]>) data.get("data");
@ -1012,8 +1075,8 @@ public class DatasetDataManage {
BigDecimal bigDecimal = new BigDecimal(val);
val = String.format("%.8f", bigDecimal);
}
if (desensitizationList.containsKey(field.getDataeaseName())) {
String str = ChartDataBuild.desensitizationValue(desensitizationList.get(field.getDataeaseName()), val);
if (desensitizationList.containsKey(field.getGisbiName())) {
String str = ChartDataBuild.desensitizationValue(desensitizationList.get(field.getGisbiName()), val);
map.put(field.getId() + "", str);
} else {
map.put(field.getId() + "", val);
@ -1049,7 +1112,7 @@ public class DatasetDataManage {
allFields.addAll(datasetGroupInfoDTO.getAllFields());
Map<Long, DatasourceSchemaDTO> dsMap = (Map<Long, DatasourceSchemaDTO>) sqlMap.get("dsMap");
boolean crossDs = Utils.isCrossDs(dsMap);
boolean crossDs = datasetGroupInfoDTO.getIsCross();
if (!crossDs) {
sql = Utils.replaceSchemaAlias(sql, dsMap);
}
@ -1110,6 +1173,7 @@ public class DatasetDataManage {
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setQuery(querySQL);
datasourceRequest.setDsList(dsMap);
datasourceRequest.setIsCross(crossDs);
Map<String, Object> data = provider.fetchResultField(datasourceRequest);
List<String[]> rows = (List<String[]>) data.get("data");
@ -1118,13 +1182,16 @@ public class DatasetDataManage {
Set<String> pkSet = new HashSet<>();
rows = rows.stream().filter(row -> {
boolean hasEmpty = false;
int emptyCount = 0;
for (String s : row) {
if (StringUtils.isBlank(s)) {
hasEmpty = true;
break;
emptyCount++;
hasEmpty = true; // 标记已遇到第一个null
} else if (hasEmpty) {
return false; // 在null后出现非null元素，不符合要求
}
}
return !hasEmpty;
return emptyCount != row.length;
}).toList();
List<BaseTreeNodeDTO> treeNodes = rows.stream().map(row -> buildTreeNode(row, pkSet)).flatMap(Collection::stream).collect(Collectors.toList());
List<BaseTreeNodeDTO> tree = DatasetUtils.mergeDuplicateTree(treeNodes, "root");
@ -1136,7 +1203,9 @@ public class DatasetDataManage {
List<String> parentPkList = new ArrayList<>();
for (int i = 0; i < row.length; i++) {
String text = row[i];
if (StringUtils.isEmpty(text)) {
continue;
}
parentPkList.add(text);
String val = String.join(TreeUtils.SEPARATOR, parentPkList);
String parentVal = i == 0 ? TreeUtils.DEFAULT_ROOT : row[i - 1];
@ -1358,7 +1427,7 @@ private String buildPrimaryKeyQuerySQL(String dbType, String tablename) {
logger.debug("执行插入数据的SQL: {}", sql);
// 执行插入操作
int result= provider.executeUpdate(datasourceRequest);
int result= provider.executeUpdate(datasourceRequest,null).getCount();
if (result==1) {
return true;
// process result set
@ -1501,7 +1570,7 @@ private String buildPrimaryKeyQuerySQL(String dbType, String tablename) {
logger.debug("执行更新数据的SQL: {}", sql);
// 执行更新操作
int result= provider.executeUpdate(datasourceRequest);
int result= provider.executeUpdate(datasourceRequest,null).getCount();
if (result==1) {
return true;
// process result set
@ -1540,7 +1609,7 @@ private String buildPrimaryKeyQuerySQL(String dbType, String tablename) {
logger.debug("执行删除数据的SQL: {}", sql);
// 执行删除操作
int result= provider.executeUpdate(datasourceRequest);
int result= provider.executeUpdate(datasourceRequest,null).getCount();
if (result==1) {
return true;
// process result set

82
core/core-backend/src/main/java/io/gisbi/dataset/manage/DatasetGroupManage.java
View File

@ -33,10 +33,9 @@ import jakarta.annotation.Resource;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Transactional;
import io.gisbi.dataset.utils.DatasetUtils;
import java.util.*;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
@ -80,8 +79,7 @@ public class DatasetGroupManage {
@Transactional
public DatasetGroupInfoDTO save(DatasetGroupInfoDTO datasetGroupInfoDTO, boolean rename) throws Exception {
lock.lock();
public DatasetGroupInfoDTO save(DatasetGroupInfoDTO datasetGroupInfoDTO, boolean rename, boolean encode) throws Exception {
try {
boolean isCreate;
// 用于重命名获取pid
@ -123,6 +121,9 @@ public class DatasetGroupManage {
}
// node_type=dataset需要创建dataset_table和field
if (StringUtils.equalsIgnoreCase(datasetGroupInfoDTO.getNodeType(), "dataset")) {
if (encode) {
DatasetUtils.dsDecode(datasetGroupInfoDTO);
}
List<Long> tableIds = new ArrayList<>();
List<Long> fieldIds = new ArrayList<>();
// 解析tree，保存
@ -131,12 +132,16 @@ public class DatasetGroupManage {
// 删除不要的table和field
datasetTableManage.deleteByDatasetGroupUpdate(datasetGroupInfoDTO.getId(), tableIds);
datasetTableFieldManage.deleteByDatasetGroupUpdate(datasetGroupInfoDTO.getId(), fieldIds);
if (encode) {
DatasetUtils.dsEncode(datasetGroupInfoDTO);
}
}
if (StringUtils.isNotEmpty(datasetGroupInfoDTO.getUnionSql())) {
datasetGroupInfoDTO.setUnionSql(DatasetUtils.getEncode(datasetGroupInfoDTO.getUnionSql()));
}
return datasetGroupInfoDTO;
} catch (Exception e) {
DEException.throwException(e.getMessage());
} finally {
lock.unlock();
}
return null;
}
@ -214,6 +219,7 @@ public class DatasetGroupManage {
public List<BusiNodeVO> tree(BusiNodeRequest request) {
QueryWrapper<Object> queryWrapper = new QueryWrapper<>();
if (ObjectUtils.isNotEmpty(request.getLeaf())) {
queryWrapper.eq("node_type", request.getLeaf() ? "dataset" : "folder");
@ -329,7 +335,7 @@ public class DatasetGroupManage {
if (ObjectUtils.isEmpty(datasetGroupInfoDTO.getUnion())) {
return;
}
datasetDataManage.previewDataWithLimit(datasetGroupInfoDTO, 0, 1, false);
datasetDataManage.previewDataWithLimit(datasetGroupInfoDTO, 0, 1, false, false);
// table和field均由前端生成id（如果没有id）
Long datasetGroupId = datasetGroupInfoDTO.getId();
List<DatasetTableFieldDTO> allFields = datasetGroupInfoDTO.getAllFields();
@ -345,28 +351,28 @@ public class DatasetGroupManage {
for (DatasetTableFieldDTO fieldDTO : unionFields) {
if (Objects.equals(datasetTableFieldDTO.getDatasetTableId(), fieldDTO.getDatasetTableId())
&& Objects.equals(datasetTableFieldDTO.getOriginName(), fieldDTO.getOriginName())) {
datasetTableFieldDTO.setDataeaseName(fieldDTO.getDataeaseName());
datasetTableFieldDTO.setGisbiName(fieldDTO.getGisbiName());
datasetTableFieldDTO.setFieldShortName(fieldDTO.getFieldShortName());
}
}
}
if (Objects.equals(datasetTableFieldDTO.getExtField(), ExtFieldConstant.EXT_CALC)) {
String dataeaseName = TableUtils.fieldNameShort(datasetTableFieldDTO.getId() + "_" + datasetTableFieldDTO.getOriginName());
datasetTableFieldDTO.setDataeaseName(dataeaseName);
datasetTableFieldDTO.setFieldShortName(dataeaseName);
String gisbiName = TableUtils.fieldNameShort(datasetTableFieldDTO.getId() + "_" + datasetTableFieldDTO.getOriginName());
datasetTableFieldDTO.setGisbiName(gisbiName);
datasetTableFieldDTO.setFieldShortName(gisbiName);
datasetTableFieldDTO.setDeExtractType(datasetTableFieldDTO.getDeType());
}
if (Objects.equals(datasetTableFieldDTO.getExtField(), ExtFieldConstant.EXT_GROUP)) {
String dataeaseName = TableUtils.fieldNameShort(datasetTableFieldDTO.getId() + "_" + datasetTableFieldDTO.getOriginName());
datasetTableFieldDTO.setDataeaseName(dataeaseName);
datasetTableFieldDTO.setFieldShortName(dataeaseName);
String gisbiName = TableUtils.fieldNameShort(datasetTableFieldDTO.getId() + "_" + datasetTableFieldDTO.getOriginName());
datasetTableFieldDTO.setGisbiName(gisbiName);
datasetTableFieldDTO.setFieldShortName(gisbiName);
datasetTableFieldDTO.setDeExtractType(0);
datasetTableFieldDTO.setDeType(0);
datasetTableFieldDTO.setGroupType("d");
}
datasetTableFieldDTO.setDatasetGroupId(datasetGroupId);
} else {
datasetTableFieldDTO.setDataeaseName(dto.getDataeaseName());
datasetTableFieldDTO.setGisbiName(dto.getGisbiName());
datasetTableFieldDTO.setFieldShortName(dto.getFieldShortName());
}
datasetTableFieldDTO = datasetTableFieldManage.save(datasetTableFieldDTO);
@ -390,7 +396,7 @@ public class DatasetGroupManage {
List<DatasetTableFieldDTO> allFields = dsFields.stream().map(ele -> {
DatasetTableFieldDTO datasetTableFieldDTO = new DatasetTableFieldDTO();
BeanUtils.copyBean(datasetTableFieldDTO, ele);
datasetTableFieldDTO.setFieldShortName(ele.getDataeaseName());
datasetTableFieldDTO.setFieldShortName(ele.getGisbiName());
return datasetTableFieldDTO;
}).collect(Collectors.toList());
@ -426,10 +432,12 @@ public class DatasetGroupManage {
List<DatasetTableFieldDTO> allFields = dsFields.stream().map(ele -> {
DatasetTableFieldDTO datasetTableFieldDTO = new DatasetTableFieldDTO();
BeanUtils.copyBean(datasetTableFieldDTO, ele);
datasetTableFieldDTO.setFieldShortName(ele.getDataeaseName());
datasetTableFieldDTO.setFieldShortName(ele.getGisbiName());
return datasetTableFieldDTO;
}).collect(Collectors.toList());
DatasetUtils.listEncode(allFields);
dto.setAllFields(allFields);
}
return dto;
@ -462,7 +470,7 @@ public class DatasetGroupManage {
List<DatasetTableFieldDTO> allFields = dsFields.stream().map(ele -> {
DatasetTableFieldDTO datasetTableFieldDTO = new DatasetTableFieldDTO();
BeanUtils.copyBean(datasetTableFieldDTO, ele);
datasetTableFieldDTO.setFieldShortName(ele.getDataeaseName());
datasetTableFieldDTO.setFieldShortName(ele.getGisbiName());
return datasetTableFieldDTO;
}).collect(Collectors.toList());
@ -470,7 +478,7 @@ public class DatasetGroupManage {
if ("preview".equalsIgnoreCase(type)) {
// 请求数据
Map<String, Object> map = datasetDataManage.previewDataWithLimit(dto, 0, 100, true);
Map<String, Object> map = datasetDataManage.previewDataWithLimit(dto, 0, 100, true, false);
// 获取data,sql
Map<String, List> data = (Map<String, List>) map.get("data");
String sql = (String) map.get("sql");
@ -583,6 +591,8 @@ public class DatasetGroupManage {
List<DatasetTableFieldDTO> dimensionList = fields.stream().filter(ele -> StringUtils.equalsIgnoreCase(ele.getGroupType(), "d")).toList();
List<DatasetTableFieldDTO> quotaList = fields.stream().filter(ele -> StringUtils.equalsIgnoreCase(ele.getGroupType(), "q")).toList();
Map<String, List<DatasetTableFieldDTO>> map = new LinkedHashMap<>();
DatasetUtils.listEncode(dimensionList);
DatasetUtils.listEncode(quotaList);
map.put("dimensionList", dimensionList);
map.put("quotaList", quotaList);
dto.setFields(map);
@ -592,4 +602,38 @@ public class DatasetGroupManage {
}
return result;
}
public List<DatasetGroupInfoDTO> getAllList() {
List<CoreDatasetGroup> coreDatasetGroupList = coreDatasetGroupMapper.selectList(new QueryWrapper<>());
if (CollectionUtils.isEmpty(coreDatasetGroupList)) {
return new ArrayList<>();
}
List<DatasetGroupInfoDTO> list = new ArrayList<>();
for (CoreDatasetGroup coreDatasetGroup : coreDatasetGroupList) {
DatasetGroupInfoDTO dto = new DatasetGroupInfoDTO();
BeanUtils.copyBean(dto, coreDatasetGroup);
dto.setUnionSql(null);
if (StringUtils.equalsIgnoreCase(dto.getNodeType(), "dataset")) {
List<UnionDTO> unionDTOList = JsonUtil.parseList(coreDatasetGroup.getInfo(), new TypeReference<>() {
});
dto.setUnion(unionDTOList);
// 获取field
List<DatasetTableFieldDTO> dsFields = datasetTableFieldManage.selectByDatasetGroupId(coreDatasetGroup.getId());
List<DatasetTableFieldDTO> allFields = dsFields.stream().map(ele -> {
DatasetTableFieldDTO datasetTableFieldDTO = new DatasetTableFieldDTO();
BeanUtils.copyBean(datasetTableFieldDTO, ele);
datasetTableFieldDTO.setFieldShortName(ele.getGisbiName());
return datasetTableFieldDTO;
}).collect(Collectors.toList());
DatasetUtils.listEncode(allFields);
dto.setAllFields(allFields);
list.add(dto);
}
}
return list;
}
}

99
core/core-backend/src/main/java/io/gisbi/dataset/manage/DatasetSQLManage.java
View File

@ -1,8 +1,14 @@
package io.gisbi.dataset.manage;
import io.gisbi.api.dataset.union.*;
import io.gisbi.api.permissions.auth.dto.BusiPerCheckDTO;
import io.gisbi.api.permissions.dataset.api.RowPermissionsApi;
import io.gisbi.api.permissions.user.vo.UserFormVO;
import io.gisbi.commons.utils.SqlparserUtils;
import io.gisbi.constant.AuthEnum;
import io.gisbi.dataset.constant.DatasetTableType;
import io.gisbi.dataset.dao.auto.entity.CoreDatasetGroup;
import io.gisbi.dataset.dao.auto.mapper.CoreDatasetGroupMapper;
import io.gisbi.dataset.utils.DatasetTableTypeConstants;
import io.gisbi.dataset.utils.SqlUtils;
import io.gisbi.dataset.utils.TableUtils;
@ -27,6 +33,9 @@ import io.gisbi.extensions.view.dto.ChartExtFilterDTO;
import io.gisbi.extensions.view.dto.ChartExtRequest;
import io.gisbi.extensions.view.dto.SqlVariableDetails;
import io.gisbi.i18n.Translator;
import io.gisbi.license.utils.LicenseUtil;
import io.gisbi.system.manage.CorePermissionManage;
import io.gisbi.utils.AuthUtils;
import io.gisbi.utils.BeanUtils;
import io.gisbi.utils.JsonUtil;
import jakarta.annotation.Resource;
@ -53,10 +62,23 @@ public class DatasetSQLManage {
@Resource
private EngineManage engineManage;
@Resource
private CorePermissionManage corePermissionManage;
@Autowired(required = false)
private PluginManageApi pluginManage;
@Autowired(required = false)
private RowPermissionsApi rowPermissionsApi;
@Resource
private DataSourceManage dataSourceManage;
@Resource
private DatasetGroupManage datasetGroupManage;
@Resource
private CoreDatasetGroupMapper coreDatasetGroupMapper;
private RowPermissionsApi getRowPermissionsApi() {
return rowPermissionsApi;
}
private static Logger logger = LoggerFactory.getLogger(DatasetSQLManage.class);
@ -109,22 +131,12 @@ public class DatasetSQLManage {
List<UnionParamDTO> unionList = new ArrayList<>();
List<DatasetTableFieldDTO> checkedFields = new ArrayList<>();
String sql = "";
if (ObjectUtils.isEmpty(union)) {
return null;
}
Set<Long> allDs = getAllDs(union);
boolean isCross = allDs.size() > 1;
boolean isCross = dataTableInfoDTO.getIsCross();
DatasetTableDTO currentDs = union.get(0).getCurrentDs();
// get datasource and schema,put map
String tableSchema = putObj2Map(dsMap, currentDs, isCross);
// get table
DatasetTableInfoDTO infoDTO = JsonUtil.parseObject(currentDs.getInfo(), DatasetTableInfoDTO.class);
SQLObj tableName = getUnionTable(currentDs, infoDTO, tableSchema, 0, filterParameters(chartExtRequest, currentDs.getId()), chartExtRequest == null, isCross, dsMap);
SQLObj tableName = null;
for (int i = 0; i < union.size(); i++) {
UnionDTO unionDTO = union.get(i);
DatasetTableDTO datasetTable = unionDTO.getCurrentDs();
@ -137,7 +149,9 @@ public class DatasetSQLManage {
schema = putObj2Map(dsMap, datasetTable, isCross);
}
SQLObj table = getUnionTable(datasetTable, tableInfo, schema, i, filterParameters(chartExtRequest, currentDs.getId()), chartExtRequest == null, isCross, dsMap);
if (i == 0) {
tableName = table;
}
// 获取前端传过来选中的字段
List<DatasetTableFieldDTO> fields = unionDTO.getCurrentDsFields();
fields = fields.stream().filter(DatasetTableFieldDTO::getChecked).collect(Collectors.toList());
@ -145,14 +159,13 @@ public class DatasetSQLManage {
String[] array = fields.stream()
.map(f -> {
String alias;
if (StringUtils.isEmpty(f.getDataeaseName())) {
if (StringUtils.isEmpty(f.getGisbiName())) {
alias = TableUtils.fieldNameShort(table.getTableAlias() + "_" + f.getOriginName());
} else {
alias = f.getDataeaseName();
alias = f.getGisbiName();
}
f.setFieldShortName(alias);
f.setDataeaseName(f.getFieldShortName());
f.setgisbiName(f.getFieldShortName());
f.setDatasetTableId(datasetTable.getId());
String prefix = "";
String suffix = "";
@ -320,14 +333,14 @@ public class DatasetSQLManage {
String[] array = fields.stream()
.map(f -> {
String alias;
if (StringUtils.isEmpty(f.getDataeaseName())) {
if (StringUtils.isEmpty(f.getGisbiName())) {
alias = TableUtils.fieldNameShort(table.getTableAlias() + "_" + f.getOriginName());
} else {
alias = f.getDataeaseName();
alias = f.getGisbiName();
}
f.setFieldShortName(alias);
f.setDataeaseName(f.getFieldShortName());
f.setgisbiName(f.getFieldShortName());
f.setDatasetTableId(datasetTable.getId());
String prefix = "";
String suffix = "";
@ -398,6 +411,7 @@ public class DatasetSQLManage {
BeanUtils.copyBean(dto, datasourceType);
return dto;
} else {
if (LicenseUtil.licenseValid()) {
List<XpackPluginsDatasourceVO> xpackPluginsDatasourceVOS = pluginManage.queryPluginDs();
List<XpackPluginsDatasourceVO> list = xpackPluginsDatasourceVOS.stream().filter(ele -> StringUtils.equals(ele.getType(), type)).toList();
if (ObjectUtils.isNotEmpty(list)) {
@ -412,6 +426,7 @@ public class DatasetSQLManage {
} else {
DEException.throwException(Translator.get("i18n_dataset_plugin_error"));
}
}
return null;
}
}
@ -445,6 +460,12 @@ public class DatasetSQLManage {
}
}
private UserFormVO getUserEntity() {
if (getRowPermissionsApi() == null) {
return null;
}
return getRowPermissionsApi().getUserById(AuthUtils.getUser().getUserId());
}
private SQLObj getUnionTable(DatasetTableDTO currentDs, DatasetTableInfoDTO infoDTO, String tableSchema, int index, List<SqlVariableDetails> parameters, boolean isFromDataSet, boolean isCross, Map<Long, DatasourceSchemaDTO> dsMap) {
SQLObj tableObj;
@ -455,7 +476,7 @@ public class DatasetSQLManage {
Provider provider = ProviderFactory.getProvider(dsMap.entrySet().iterator().next().getValue().getType());
// parser sql params and replace default value
String s = new String(Base64.getDecoder().decode(infoDTO.getSql()));
String sql = new SqlparserUtils().handleVariableDefaultValue(s, currentDs.getSqlVariableDetails(), false, isFromDataSet, parameters, isCross, dsMap, pluginManage);
String sql = new SqlparserUtils().handleVariableDefaultValue(s, currentDs.getSqlVariableDetails(), false, isFromDataSet, parameters, isCross, dsMap, pluginManage, getUserEntity());
sql = provider.replaceComment(sql);
// add table schema
if (isCross) {
@ -469,22 +490,24 @@ public class DatasetSQLManage {
return tableObj;
}
private String putObj2Map(Map<Long, DatasourceSchemaDTO> dsMap, DatasetTableDTO ds, boolean isCross) throws Exception {
public String putObj2Map(Map<Long, DatasourceSchemaDTO> dsMap, DatasetTableDTO ds, boolean isCross) {
// 通过datasource id校验数据源权限
// BusiPerCheckDTO dto = new BusiPerCheckDTO();
// dto.setId(ds.getDatasourceId());
// dto.setAuthEnum(AuthEnum.READ);
// boolean checked = corePermissionManage.checkAuth(dto);
// if (!checked) {
// DEException.throwException(Translator.get("i18n_no_datasource_permission"));
// }
BusiPerCheckDTO dto = new BusiPerCheckDTO();
dto.setId(ds.getDatasourceId());
dto.setAuthEnum(AuthEnum.READ);
boolean checked = corePermissionManage.checkAuth(dto);
if (!checked) {
DEException.throwException(Translator.get("i18n_no_datasource_permission"));
}
String schemaAlias;
if (StringUtils.equalsIgnoreCase(ds.getType(), DatasetTableType.DB) || StringUtils.equalsIgnoreCase(ds.getType(), DatasetTableType.SQL)) {
CoreDatasource coreDatasource = dataSourceManage.getCoreDatasource(ds.getDatasourceId());
if (coreDatasource == null) {
DEException.throwException(Translator.get("i18n_dataset_ds_error") + ",ID:" + ds.getDatasourceId());
}
if (StringUtils.equalsIgnoreCase("excel", coreDatasource.getType()) || coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.name())) {
if (coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.Excel.name()) || coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.name())) {
coreDatasource = engineManage.getDeEngine();
}
@ -522,4 +545,20 @@ public class DatasetSQLManage {
}
return schemaAlias;
}
public void datasetCrossDefault() {
List<DatasetGroupInfoDTO> allList = datasetGroupManage.getAllList();
for (DatasetGroupInfoDTO ele : allList) {
mergeDatasetCrossDefault(ele);
CoreDatasetGroup record = new CoreDatasetGroup();
BeanUtils.copyBean(record, ele);
coreDatasetGroupMapper.updateById(record);
}
}
public void mergeDatasetCrossDefault(DatasetGroupInfoDTO ele) {
Set<Long> allDs = getAllDs(ele.getUnion());
boolean isCross = allDs.size() > 1;
ele.setIsCross(isCross);
}
}

77
core/core-backend/src/main/java/io/gisbi/dataset/manage/DatasetTableFieldManage.java
View File

@ -5,6 +5,7 @@ import com.fasterxml.jackson.core.type.TypeReference;
import io.gisbi.api.dataset.union.DatasetGroupInfoDTO;
import io.gisbi.dataset.dao.auto.entity.CoreDatasetTableField;
import io.gisbi.dataset.dao.auto.mapper.CoreDatasetTableFieldMapper;
import io.gisbi.dataset.utils.DatasetUtils;
import io.gisbi.dataset.utils.TableUtils;
import io.gisbi.engine.constant.ExtFieldConstant;
import io.gisbi.engine.func.FunctionConstant;
@ -41,7 +42,8 @@ import java.util.stream.Collectors;
public class DatasetTableFieldManage {
@Resource
private CoreDatasetTableFieldMapper coreDatasetTableFieldMapper;
@Resource
private PermissionManage permissionManage;
@Resource
private DatasetSQLManage datasetSQLManage;
@Resource
@ -86,10 +88,10 @@ public class DatasetTableFieldManage {
checkNameLength(datasetTableFieldDTO.getName());
CoreDatasetTableField coreDatasetTableField = coreDatasetTableFieldMapper.selectById(datasetTableFieldDTO.getId());
CoreDatasetTableField record = transDTO2Record(datasetTableFieldDTO);
if (ObjectUtils.isEmpty(record.getDataeaseName())) {
if (ObjectUtils.isEmpty(record.getGisbiName())) {
String n = TableUtils.fieldNameShort(record.getId() + "");
record.setFieldShortName(n);
record.setDataeaseName(n);
record.setgisbiName(n);
}
if (ObjectUtils.isEmpty(coreDatasetTableField)) {
coreDatasetTableFieldMapper.insert(record);
@ -228,7 +230,7 @@ public class DatasetTableFieldManage {
List<DatasetTableFieldDTO> list = transDTO(coreDatasetTableFieldMapper.selectList(wrapper));
Map<String, ColumnPermissionItem> desensitizationList = new HashMap<>();
// list = permissionManage.filterColumnPermissions(list, desensitizationList, id, null);
list = permissionManage.filterColumnPermissions(list, desensitizationList, id, null);
List<DatasetTableFieldDTO> dimensionList = list.stream().filter(ele -> StringUtils.equalsIgnoreCase(ele.getGroupType(), "d")).collect(Collectors.toList());
List<DatasetTableFieldDTO> quotaList = list.stream().filter(ele -> StringUtils.equalsIgnoreCase(ele.getGroupType(), "q")).collect(Collectors.toList());
@ -240,43 +242,44 @@ public class DatasetTableFieldManage {
public List<DatasetTableFieldDTO> listFieldsWithPermissions(Long id) {
List<DatasetTableFieldDTO> fields = selectByDatasetGroupId(id);
// Map<String, ColumnPermissionItem> desensitizationList = new HashMap<>();
// Long userId = AuthUtils.getUser() == null ? null : AuthUtils.getUser().getUserId();
// List<DatasetTableFieldDTO> tmp = permissionManage
// .filterColumnPermissions(fields, desensitizationList, id, userId)
// .stream()
// .sorted(Comparator.comparing(DatasetTableFieldDTO::getGroupType))
// .toList();
// tmp.forEach(ele -> ele.setDesensitized(desensitizationList.containsKey(ele.getDataeaseName())));
return fields;
Map<String, ColumnPermissionItem> desensitizationList = new HashMap<>();
Long userId = AuthUtils.getUser() == null ? null : AuthUtils.getUser().getUserId();
List<DatasetTableFieldDTO> tmp = permissionManage
.filterColumnPermissions(fields, desensitizationList, id, userId)
.stream()
.sorted(Comparator.comparing(DatasetTableFieldDTO::getGroupType))
.toList();
tmp.forEach(ele -> ele.setDesensitized(desensitizationList.containsKey(ele.getGisbiName())));
return tmp;
}
public List<DatasetTableFieldDTO> listFieldsWithPermissionsRemoveAgg(Long id) {
List<DatasetTableFieldDTO> fields = selectByDatasetGroupId(id);
// Map<String, ColumnPermissionItem> desensitizationList = new HashMap<>();
// Long userId = AuthUtils.getUser() == null ? null : AuthUtils.getUser().getUserId();
// SQLObj tableObj = new SQLObj();
// tableObj.setTableAlias("");
// List<DatasetTableFieldDTO> tmp = permissionManage
// .filterColumnPermissions(fields, desensitizationList, id, userId)
// .stream()
// .filter(ele -> {
// boolean flag = true;
// if (Objects.equals(ele.getExtField(), ExtFieldConstant.EXT_CALC)) {
// String originField = Utils.calcFieldRegex(ele, tableObj, fields, true, null, Utils.mergeParam(Utils.getParams(fields), null), pluginManage);
// for (String func : FunctionConstant.AGG_FUNC) {
// if (Utils.matchFunction(func, originField)) {
// flag = false;
// break;
// }
// }
// }
// return flag;
// })
// .sorted(Comparator.comparing(DatasetTableFieldDTO::getGroupType))
// .toList();
// tmp.forEach(ele -> ele.setDesensitized(desensitizationList.containsKey(ele.getDataeaseName())));
return fields;
Map<String, ColumnPermissionItem> desensitizationList = new HashMap<>();
Long userId = AuthUtils.getUser() == null ? null : AuthUtils.getUser().getUserId();
SQLObj tableObj = new SQLObj();
tableObj.setTableAlias("");
List<DatasetTableFieldDTO> tmp = permissionManage
.filterColumnPermissions(fields, desensitizationList, id, userId)
.stream()
.filter(ele -> {
boolean flag = true;
if (Objects.equals(ele.getExtField(), ExtFieldConstant.EXT_CALC)) {
String originField = Utils.calcFieldRegex(ele, tableObj, fields, true, null, Utils.mergeParam(Utils.getParams(fields), null), pluginManage);
for (String func : FunctionConstant.AGG_FUNC) {
if (Utils.matchFunction(func, originField)) {
flag = false;
break;
}
}
}
return flag;
})
.sorted(Comparator.comparing(DatasetTableFieldDTO::getGroupType))
.toList();
tmp.forEach(ele -> ele.setDesensitized(desensitizationList.containsKey(ele.getGisbiName())));
DatasetUtils.listEncode(tmp);
return tmp;
}
public DatasetTableFieldDTO transObj(CoreDatasetTableField ele) {

6
core/core-backend/src/main/java/io/gisbi/dataset/manage/PermissionManage.java
View File

@ -29,8 +29,10 @@ import java.util.stream.Collectors;
@Service
public class PermissionManage {
@Autowired(required = false)
private RowPermissionsApi rowPermissionsApi;
@Autowired(required = false)
private ColumnPermissionsApi columnPermissionsApi = null;
@Resource
@ -71,14 +73,14 @@ public class PermissionManage {
if (CollectionUtils.isNotEmpty(fieldUserColumnPermissionItems)) {
if (fieldUserColumnPermissionItems.stream().map(ColumnPermissionItem::getOpt).collect(Collectors.toList()).contains(ColumnPermissionConstants.Desensitization)) {
desensitizationList.put(field.getDataeaseName(), fieldUserColumnPermissionItems.get(0));
desensitizationList.put(field.getGisbiName(), fieldUserColumnPermissionItems.get(0));
result.add(field);
}
return;
}
if (CollectionUtils.isNotEmpty(fieldRoleColumnPermissionItems)) {
if (fieldRoleColumnPermissionItems.stream().map(ColumnPermissionItem::getOpt).collect(Collectors.toList()).contains(ColumnPermissionConstants.Desensitization)) {
desensitizationList.put(field.getDataeaseName(), fieldRoleColumnPermissionItems.get(0));
desensitizationList.put(field.getGisbiName(), fieldRoleColumnPermissionItems.get(0));
result.add(field);
}
return;

73
core/core-backend/src/main/java/io/gisbi/dataset/server/DatasetDataServer.java
View File

@ -1,21 +1,15 @@
package io.gisbi.dataset.server;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import io.gisbi.api.dataset.DatasetDataApi;
import io.gisbi.api.dataset.dto.BaseTreeNodeDTO;
import io.gisbi.api.dataset.dto.EnumValueRequest;
import io.gisbi.api.dataset.dto.MultFieldValuesRequest;
import io.gisbi.api.dataset.dto.PreviewSqlDTO;
import io.gisbi.api.dataset.dto.*;
import io.gisbi.api.dataset.union.DatasetGroupInfoDTO;
import io.gisbi.api.dataset.dto.EnumObj;
import io.gisbi.dataset.dao.auto.entity.CoreDatasetTable;
import io.gisbi.dataset.dao.auto.entity.CoreDatasetTableField;
import io.gisbi.dataset.manage.DatasetDataManage;
import io.gisbi.extensions.datasource.dto.DatasetTableDTO;
import io.gisbi.extensions.datasource.dto.DatasetTableFieldDTO;
import io.gisbi.utils.LogUtil;
import jakarta.annotation.Resource;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.util.List;
import java.util.Map;
@ -31,10 +25,9 @@ public class DatasetDataServer implements DatasetDataApi {
@Override
public Map<String, Object> previewData(DatasetGroupInfoDTO datasetGroupInfoDTO) throws Exception {
return datasetDataManage.previewDataWithLimit(datasetGroupInfoDTO, 0, 100, false);
return datasetDataManage.previewDataWithLimit(datasetGroupInfoDTO, 0, 100, false, true);
}
@Override
public List<DatasetTableFieldDTO> tableField(DatasetTableDTO datasetTableDTO) throws Exception {
return datasetDataManage.getTableFields(datasetTableDTO);
@ -103,62 +96,4 @@ public class DatasetDataServer implements DatasetDataApi {
return null;
}
}
// @GetMapping("getTablesByAppId")
// public List<Map<String,Object>> getTablesByAppId(String appid) throws Exception {
// List<Map<String,Object>> result = datasetDataManage.getTablesByAppId(appid);
// return result;
// }
@GetMapping("getTablesByAppId")
public List<Map<String,Object>> getTablesByAppId(String appid) throws Exception {
List<Map<String,Object>> result = datasetDataManage.getTablesByAppId(appid);
return result;
}
@GetMapping("getFieldsByTableId")
public Map<String, Object> getFieldsByTableId(Long datasourceId,String tablename) throws Exception {
Map<String, Object> result = datasetDataManage.getFieldsByTableId(datasourceId,tablename);
return result;
}
@PostMapping("addTableData")
public boolean addTableData(Long datasourceId, @RequestParam("tableData") String tableData) throws Exception {
boolean result = datasetDataManage.addTableData(
datasourceId,
tableData
);
return result;
}
@PostMapping("getTableDataByPk")
public Map<String, Object> getTableDataByPk(Long datasourceId,@RequestParam("whereJson") String whereJson) throws Exception {
Map<String, Object> result=datasetDataManage.getTableDataByPk(
datasourceId,
whereJson
);
return result;
}
@PostMapping("updateTableData")
public boolean updateTableData(Long datasourceId, @RequestParam("tableData") String tableData) throws Exception {
boolean result = datasetDataManage.updateTableData(
datasourceId,
tableData
);
return result;
}
@PostMapping("deleteTableData")
public boolean deleteTableData(Long datasourceId, @RequestParam("whereJson") String whereJson) throws Exception {
boolean result = datasetDataManage.deleteTableData(
datasourceId,
whereJson
);
return result;
}
@PostMapping("queryTableDataPaged")
public Page<Map<String, Object>> queryTableDataPaged(Long datasourceId, @RequestParam("queryJson") String queryJson) throws Exception {
Page<Map<String, Object>> result = datasetDataManage.queryTableDataPaged(
datasourceId,
queryJson
);
return result;
}
}

5
core/core-backend/src/main/java/io/gisbi/dataset/server/DatasetFieldServer.java
View File

@ -6,6 +6,7 @@ import io.gisbi.api.dataset.engine.SQLFunctionDTO;
import io.gisbi.api.dataset.engine.SQLFunctionsEnum;
import io.gisbi.dataset.manage.DatasetDataManage;
import io.gisbi.dataset.manage.DatasetTableFieldManage;
import io.gisbi.dataset.utils.DatasetUtils;
import io.gisbi.extensions.datasource.dto.DatasetTableFieldDTO;
import jakarta.annotation.Resource;
import org.springframework.web.bind.annotation.RequestBody;
@ -40,7 +41,9 @@ public class DatasetFieldServer implements DatasetTableApi {
@Override
public List<DatasetTableFieldDTO> listByDatasetGroup(Long id) {
return datasetTableFieldManage.selectByDatasetGroupId(id);
List<DatasetTableFieldDTO> datasetTableFieldDTOS = datasetTableFieldManage.selectByDatasetGroupId(id);
DatasetUtils.listEncode(datasetTableFieldDTOS);
return datasetTableFieldDTOS;
}
@Override

23
core/core-backend/src/main/java/io/gisbi/dataset/server/DatasetTreeServer.java
View File

@ -1,5 +1,6 @@
package io.gisbi.dataset.server;
import io.gisbi.api.dataset.DatasetTreeApi;
import io.gisbi.api.dataset.dto.DataSetExportRequest;
import io.gisbi.api.dataset.dto.DatasetNodeDTO;
@ -8,17 +9,21 @@ import io.gisbi.api.dataset.vo.DataSetBarVO;
import io.gisbi.constant.LogOT;
import io.gisbi.constant.LogST;
import io.gisbi.dataset.manage.DatasetGroupManage;
import io.gisbi.exportCenter.manage.ExportCenterDownLoadManage;
import io.gisbi.exportCenter.manage.ExportCenterManage;
import io.gisbi.extensions.datasource.dto.DatasetTableDTO;
import io.gisbi.extensions.view.dto.SqlVariableDetails;
import io.gisbi.extensions.view.dto.*;
import io.gisbi.log.DeLog;
import io.gisbi.model.BusiNodeRequest;
import io.gisbi.model.BusiNodeVO;
import jakarta.annotation.Resource;
import jakarta.servlet.http.HttpServletResponse;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.util.List;
import java.util.*;
@RestController
@RequestMapping("datasetTree")
@ -27,24 +32,26 @@ public class DatasetTreeServer implements DatasetTreeApi {
private DatasetGroupManage datasetGroupManage;
@Resource
private ExportCenterManage exportCenterManage;
@Resource
private ExportCenterDownLoadManage exportCenterDownLoadManage;
@DeLog(id = "#p0.id", ot = LogOT.MODIFY, st = LogST.DATASET)
@Override
public DatasetGroupInfoDTO save(DatasetGroupInfoDTO datasetNodeDTO) throws Exception {
return datasetGroupManage.save(datasetNodeDTO, false);
return datasetGroupManage.save(datasetNodeDTO, false, true);
}
@DeLog(id = "#p0.id", ot = LogOT.MODIFY, st = LogST.DATASET)
@Override
public DatasetNodeDTO rename(DatasetGroupInfoDTO dto) throws Exception {
return datasetGroupManage.save(dto, true);
return datasetGroupManage.save(dto, true, false);
}
@DeLog(id = "#p0.id", pid = "#p0.pid", ot = LogOT.CREATE, st = LogST.DATASET)
@Override
public DatasetNodeDTO create(DatasetGroupInfoDTO dto) throws Exception {
return datasetGroupManage.save(dto, false);
return datasetGroupManage.save(dto, false, true);
}
@DeLog(id = "#p0.id", ot = LogOT.MODIFY, st = LogST.DATASET)
@ -100,8 +107,12 @@ public class DatasetTreeServer implements DatasetTreeApi {
}
@Override
public void exportDataset(DataSetExportRequest request) throws Exception {
public void exportDataset(DataSetExportRequest request, HttpServletResponse response) throws Exception {
if (request.isGisbiBi()) {
exportCenterDownLoadManage.downloadDataset(request, response);
} else {
exportCenterManage.addTask(request.getId(), "dataset", request);
}
}
}

99
core/core-backend/src/main/java/io/gisbi/dataset/utils/DatasetUtils.java
View File

@ -1,15 +1,16 @@
package io.gisbi.dataset.utils;
import io.gisbi.api.dataset.dto.BaseTreeNodeDTO;
import io.gisbi.api.dataset.union.DatasetGroupInfoDTO;
import io.gisbi.engine.constant.ExtFieldConstant;
import io.gisbi.extensions.datasource.dto.DatasetTableFieldDTO;
import io.gisbi.extensions.view.dto.ChartViewDTO;
import io.gisbi.utils.TreeUtils;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.springframework.util.Assert;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.stream.Collectors;
/**
@ -53,4 +54,94 @@ public class DatasetUtils {
});
return result;
}
public static String getEncode(String str) {
return Base64.getEncoder().encodeToString(str.getBytes());
}
public static String getDecode(String str) {
return new String(Base64.getDecoder().decode(str));
}
/**
* 计算字段表达式base64加密
*
* @param obj
*/
public static void dsEncode(DatasetGroupInfoDTO obj) {
for (DatasetTableFieldDTO dto : obj.getAllFields()) {
if (dto.getExtField().equals(ExtFieldConstant.EXT_CALC)) {
dto.setOriginName(getEncode(dto.getOriginName()));
}
}
}
/**
* 计算字段表达式base64解密
*
* @param obj
*/
public static void dsDecode(DatasetGroupInfoDTO obj) {
for (DatasetTableFieldDTO dto : obj.getAllFields()) {
if (dto.getExtField().equals(ExtFieldConstant.EXT_CALC)) {
dto.setOriginName(getDecode(dto.getOriginName()));
}
}
}
/**
* 计算字段表达式base64加密
*
* @param fields
*/
public static void listEncode(List<? extends DatasetTableFieldDTO> fields) {
if (CollectionUtils.isEmpty(fields)) {
return;
}
for (DatasetTableFieldDTO dto : fields) {
if (dto.getExtField().equals(ExtFieldConstant.EXT_CALC)) {
dto.setOriginName(getEncode(dto.getOriginName()));
}
}
}
/**
* 计算字段表达式base64解密
*
* @param fields
*/
public static void listDecode(List<? extends DatasetTableFieldDTO> fields) {
if (CollectionUtils.isEmpty(fields)) {
return;
}
for (DatasetTableFieldDTO dto : fields) {
if (dto.getExtField().equals(ExtFieldConstant.EXT_CALC)) {
dto.setOriginName(getDecode(dto.getOriginName()));
}
}
}
public static void viewDecode(ChartViewDTO view) {
DatasetUtils.listDecode(view.getXAxis());
DatasetUtils.listDecode(view.getXAxisExt());
DatasetUtils.listDecode(view.getYAxis());
DatasetUtils.listDecode(view.getYAxisExt());
DatasetUtils.listDecode(view.getExtStack());
DatasetUtils.listDecode(view.getExtBubble());
DatasetUtils.listDecode(view.getExtLabel());
DatasetUtils.listDecode(view.getExtTooltip());
DatasetUtils.listDecode(view.getExtColor());
}
public static void viewEncode(ChartViewDTO view) {
DatasetUtils.listEncode(view.getXAxis());
DatasetUtils.listEncode(view.getXAxisExt());
DatasetUtils.listEncode(view.getYAxis());
DatasetUtils.listEncode(view.getYAxisExt());
DatasetUtils.listEncode(view.getExtStack());
DatasetUtils.listEncode(view.getExtBubble());
DatasetUtils.listEncode(view.getExtLabel());
DatasetUtils.listEncode(view.getExtTooltip());
DatasetUtils.listEncode(view.getExtColor());
}
}

28
core/core-backend/src/main/java/io/gisbi/datasource/manage/DataSourceManage.java
View File

@ -6,11 +6,14 @@ import io.gisbi.commons.constants.OptConstants;
import io.gisbi.commons.constants.TaskStatus;
import io.gisbi.constant.DataSourceType;
import io.gisbi.datasource.dao.auto.entity.CoreDatasource;
import io.gisbi.datasource.dao.auto.mapper.CoreDatasourceMapper;
import io.gisbi.datasource.dao.ext.mapper.CoreDatasourceExtMapper;
import io.gisbi.datasource.dao.ext.mapper.DataSourceExtMapper;
import io.gisbi.datasource.dao.ext.po.DataSourceNodePO;
import io.gisbi.datasource.dao.ext.po.DsItem;
import io.gisbi.datasource.dto.DatasourceNodeBO;
import io.gisbi.exception.DEException;
import io.gisbi.extensions.datasource.api.PluginManageApi;
import io.gisbi.extensions.datasource.dto.DatasourceDTO;
import io.gisbi.i18n.Translator;
import io.gisbi.model.BusiNodeRequest;
@ -24,23 +27,34 @@ import jakarta.annotation.Resource;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import io.gisbi.datasource.dao.ext.mapper.DataSourceExtMapper;
import io.gisbi.datasource.dao.auto.mapper.CoreDatasourceMapper;
import java.util.ArrayList;
import java.util.List;
import java.util.Stack;
@Component
public class DataSourceManage {
@Resource
private DataSourceExtMapper dataSourceExtMapper;
@Resource
private CoreDatasourceMapper coreDatasourceMapper;
@Resource
private CoreDatasourceExtMapper coreDatasourceExtMapper;
@Resource
private CoreOptRecentManage coreOptRecentManage;
@Resource
private CoreDatasourceExtMapper coreDatasourceExtMapper;
@Resource
private EngineManage engineManage;
@Autowired(required = false)
private PluginManageApi pluginManage;
private DatasourceNodeBO rootNode(String appId) {
return new DatasourceNodeBO(0L,appId, "root", false, 7, -1L, 0, "mysql");
}
@ -178,9 +192,9 @@ public class DataSourceManage {
public CoreDatasource getCoreDatasource(Long id) {
// if (id == -1L) {
// return engineManage.getDeEngine();
// }
if (id == -1L) {
return engineManage.getDeEngine();
}
return coreDatasourceMapper.selectById(id);
}

44
core/core-backend/src/main/java/io/gisbi/datasource/manage/DatasourceSyncManage.java
View File

@ -127,7 +127,11 @@ public class DatasourceSyncManage {
record.setQrtzInstance(context.getFireInstanceId());
datasourceMapper.update(record, updateWrapper);
}
if (coreDatasource.getType().equalsIgnoreCase("ExcelRemote")) {
extractedExcelData(taskId, coreDatasource, updateType, coreDatasourceTask.getSyncRate());
} else {
extractedData(taskId, coreDatasource, updateType, coreDatasourceTask.getSyncRate());
}
} catch (Exception e) {
LogUtil.error(e);
} finally {
@ -173,6 +177,46 @@ public class DatasourceSyncManage {
}
}
public void extractedExcelData(Long taskId, CoreDatasource coreDatasource, DatasourceServer.UpdateType updateType, String scheduleType) {
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(transDTO(coreDatasource));
List<DatasetTableDTO> tables = ExcelUtils.getTables(datasourceRequest);
for (DatasetTableDTO tableDTO : tables) {
CoreDatasourceTaskLog datasetTableTaskLog = datasourceTaskServer.initTaskLog(coreDatasource.getId(), taskId, tableDTO.getTableName(), scheduleType);
datasourceRequest.setTable(tableDTO.getTableName());
ExcelUtils.getTableFields(datasourceRequest);
List<TableField> tableFields = ExcelUtils.getTableFields(datasourceRequest);
try {
datasetTableTaskLog.setInfo(datasetTableTaskLog.getInfo() + "/n Begin to sync datatable: " + datasourceRequest.getTable());
createEngineTable(datasourceRequest.getTable(), tableFields);
if (updateType.equals(DatasourceServer.UpdateType.all_scope)) {
createEngineTable(TableUtils.tmpName(datasourceRequest.getTable()), tableFields);
}
extractExcelData(datasourceRequest, updateType, tableFields);
if (updateType.equals(DatasourceServer.UpdateType.all_scope)) {
replaceTable(datasourceRequest.getTable());
}
datasetTableTaskLog.setInfo(datasetTableTaskLog.getInfo() + "/n End to sync datatable: " + datasourceRequest.getTable());
datasetTableTaskLog.setTaskStatus(TaskStatus.Completed.toString());
datasetTableTaskLog.setEndTime(System.currentTimeMillis());
} catch (Exception e) {
try {
if (updateType.equals(DatasourceServer.UpdateType.all_scope)) {
dropEngineTable(TableUtils.tmpName(datasourceRequest.getTable()));
}
} catch (Exception ignore) {
}
datasetTableTaskLog.setInfo(datasetTableTaskLog.getInfo() + "/n Failed to sync datatable: " + datasourceRequest.getTable() + ", " + e.getMessage());
datasetTableTaskLog.setTaskStatus(TaskStatus.Error.toString());
datasetTableTaskLog.setEndTime(System.currentTimeMillis());
e.printStackTrace();
} finally {
datasourceTaskServer.saveLog(datasetTableTaskLog);
}
}
}
private void updateDsTaskStatus(Long datasourceId) {
UpdateWrapper<CoreDatasource> updateWrapper = new UpdateWrapper<>();
updateWrapper.eq("id", datasourceId);

21
core/core-backend/src/main/java/io/gisbi/datasource/manage/EngineManage.java
View File

@ -12,6 +12,8 @@ import io.gisbi.extensions.datasource.dto.DatasourceDTO;
import io.gisbi.extensions.datasource.dto.DatasourceRequest;
import io.gisbi.extensions.datasource.factory.ProviderFactory;
import io.gisbi.result.ResultMessage;
import io.gisbi.template.dao.auto.entity.DeTemplateVersion;
import io.gisbi.template.dao.auto.mapper.DeTemplateVersionMapper;
import io.gisbi.utils.BeanUtils;
import io.gisbi.utils.JsonUtil;
import io.gisbi.utils.ModelUtils;
@ -23,6 +25,8 @@ import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.CollectionUtils;
import java.time.LocalDateTime;
import java.time.temporal.ChronoUnit;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -40,9 +44,12 @@ public class EngineManage {
@Resource
private CoreDatasourceMapper datasourceMapper;
@Value("${gisbi.path.engine:jdbc:h2:/opt/gisbi/desktop_data;AUTO_SERVER=TRUE;AUTO_RECONNECT=TRUE;MODE=MySQL;CASE_INSENSITIVE_IDENTIFIERS=TRUE;DATABASE_TO_UPPER=FALSE}")
@Value("${gisbi.path.engine:jdbc:h2:/opt/gisbi2.0/desktop_data;AUTO_SERVER=TRUE;AUTO_RECONNECT=TRUE;MODE=MySQL;CASE_INSENSITIVE_IDENTIFIERS=TRUE;DATABASE_TO_UPPER=FALSE}")
private String engineUrl;
@Resource
private DeTemplateVersionMapper deTemplateVersionMapper;
public CoreDeEngine info() throws DEException {
List<CoreDeEngine> deEngines = deEngineMapper.selectList(null);
@ -163,7 +170,10 @@ public class EngineManage {
QueryWrapper<CoreDatasource> queryWrapper = new QueryWrapper<>();
queryWrapper.eq("id", 985188400292302848L);
queryWrapper.ne("create_time", 1715053684176L);
if (!datasourceMapper.exists(queryWrapper) && !ModelUtils.isDesktop()) {
// 版本检查
QueryWrapper<DeTemplateVersion> queryVersionWrapper = new QueryWrapper<>();
queryVersionWrapper.eq("version", "985188400292302848");
if (!datasourceMapper.exists(queryWrapper) && !deTemplateVersionMapper.exists(queryVersionWrapper) && !ModelUtils.isDesktop()) {
Pattern WITH_SQL_FRAGMENT = Pattern.compile("jdbc:mysql://(.*):(\\d+)/(.*)\\?(.*)");
Matcher matcher = WITH_SQL_FRAGMENT.matcher(env.getProperty("spring.datasource.url"));
if (!matcher.find()) {
@ -191,6 +201,13 @@ public class EngineManage {
initDatasource.setTaskStatus("WaitingForExecution");
datasourceMapper.deleteById(985188400292302848L);
datasourceMapper.insert(initDatasource);
DeTemplateVersion version = new DeTemplateVersion();
version.setVersion("985188400292302848");
version.setScript("Demo");
version.setInstalledOn(LocalDateTime.now().truncatedTo(ChronoUnit.MINUTES));
version.setSuccess(true);
deTemplateVersionMapper.insert(version);
}
}

85
core/core-backend/src/main/java/io/gisbi/datasource/provider/ApiUtils.java
View File

@ -5,7 +5,9 @@ import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.jayway.jsonpath.Configuration;
import com.jayway.jsonpath.JsonPath;
import com.jayway.jsonpath.Option;
import io.gisbi.extensions.datasource.dto.ApiDefinition;
import io.gisbi.extensions.datasource.dto.ApiDefinitionRequest;
import io.gisbi.exception.DEException;
@ -26,7 +28,9 @@ import java.util.regex.Pattern;
import java.util.stream.Collectors;
public class ApiUtils {
private static Configuration jsonPathConf = Configuration.builder()
.options(Option.DEFAULT_PATH_LEAF_TO_NULL, Option.ALWAYS_RETURN_LIST)
.build();
private static String path = "['%s']";
public static ObjectMapper objectMapper = CommonBeanFactory.getBean(ObjectMapper.class);
@ -35,32 +39,18 @@ public class ApiUtils {
private static TypeReference<List<Map<String, Object>>> listForMapTypeReference = new TypeReference<List<Map<String, Object>>>() {
};
/**
* 从数据源配置中解析并获取API数据表信息列表
*
* @param datasourceRequest 数据源请求对象，包含数据源配置信息
* @return 数据集表DTO列表，包含表名称、显示名称及所属数据源ID
* @throws DEException 当数据处理过程中出现异常时抛出
*/
public static List<DatasetTableDTO> getApiTables(DatasourceRequest datasourceRequest) throws DEException {
List<DatasetTableDTO> tableDescs = new ArrayList<>();
// 定义API配置的反序列化类型引用
TypeReference<List<ApiDefinition>> listTypeReference = new TypeReference<List<ApiDefinition>>() {
};
// 从数据源配置中解析API定义列表
List<ApiDefinition> apiDefinitionList = JsonUtil.parseList(datasourceRequest.getDatasource().getConfiguration(), listTypeReference);
for (ApiDefinition apiDefinition : apiDefinitionList) {
// 过滤空对象和参数类型配置
if (apiDefinition == null) {
continue;
}
if (StringUtils.isNotEmpty(apiDefinition.getType()) && apiDefinition.getType().equalsIgnoreCase("params")) {
continue;
}
// 构建数据集表信息对象
DatasetTableDTO datasetTableDTO = new DatasetTableDTO();
datasetTableDTO.setTableName(apiDefinition.getDeTableName());
datasetTableDTO.setName(apiDefinition.getName());
@ -70,25 +60,14 @@ public class ApiUtils {
return tableDescs;
}
/**
* 将JSON配置字符串解析为表名映射关系
*
* @param configration 输入的JSON配置字符串，应包含包含"name"和"deTableName"字段的数组对象
* @return 返回键值对映射，key为配置中的name字段值，value为对应的deTableName字段值
* @throws DEException 当JSON解析失败时抛出异常
*/
public static Map<String, String> getTableNamesMap(String configration) throws DEException {
Map<String, String> result = new HashMap<>();
try {
// 解析JSON根节点并遍历数组元素
JsonNode rootNode = objectMapper.readTree(configration);
for (int i = 0; i < rootNode.size(); i++) {
// 提取每个元素的name和deTableName字段构建映射关系
result.put(rootNode.get(i).get("name").asText(), rootNode.get(i).get("deTableName").asText());
}
} catch (Exception e) {
// 统一将底层异常转换为领域异常抛出
DEException.throwException(e);
}
@ -96,7 +75,6 @@ public class ApiUtils {
}
public static Map<String, Object> fetchApiResultField(DatasourceRequest datasourceRequest) throws DEException {
Map<String, Object> result = new HashMap<>();
List<String[]> dataList = new ArrayList<>();
@ -468,23 +446,34 @@ public class ApiUtils {
return response;
}
private static void previewNum(List<Map<String, Object>> field) {
for (Map<String, Object> stringObjectMap : field) {
private static void previewNum(List<Map<String, Object>> fields, String response) {
int previewNum = 100;
for (Map<String, Object> field : fields) {
JSONArray newArray = new JSONArray();
if (stringObjectMap.get("value") != null) {
try {
TypeReference<JSONArray> listTypeReference = new TypeReference<JSONArray>() {
};
JSONArray array = objectMapper.readValue(stringObjectMap.get("value").toString(), listTypeReference);
if (array.size() > 100) {
for (int i = 0; i < Math.min(100, array.size()); i++) {
newArray.add(array.get(i));
if (field.get("value") != null) {
Object object = JsonPath.using(jsonPathConf).parse(response).read(field.get("jsonPath").toString());
int i = 0;
if (object instanceof List) {
for (Object o : (List<String>) object) {
if (Objects.isNull(o)) {
newArray.add("");
} else {
newArray.add(o.toString());
}
stringObjectMap.put("value", newArray);
i++;
if (i >= previewNum) {
break;
}
} catch (Exception e) {
}
} else {
if (object != null) {
newArray.add(object.toString());
}
}
field.put("value", newArray);
} else {
List<Map<String, Object>> childrenFields = (List<Map<String, Object>>) field.get("children");
previewNum(childrenFields, response);
}
}
}
@ -527,7 +516,7 @@ public class ApiUtils {
rootPath = "$";
handleStr(apiDefinition, response, fields, rootPath);
}
previewNum(fields);
previewNum(fields, response);
apiDefinition.setJsonFields(fields);
return apiDefinition;
} else {
@ -574,7 +563,6 @@ public class ApiUtils {
};
array = objectMapper.readValue(field.get("value").toString(), listTypeReference);
} catch (Exception e) {
e.printStackTrace();
DEException.throwException(e);
}
array.add(Optional.ofNullable(data.get(field.get("originName"))).orElse("").toString().replaceAll("\n", " ").replaceAll("\r", " "));
@ -745,6 +733,7 @@ public class ApiUtils {
fieldChildren.add(itemChild);
}
}
field.put("children", fieldChildren);
}
}
@ -807,9 +796,15 @@ public class ApiUtils {
List<List<String>> columnDataList = new ArrayList<>();
for (int i = 0; i < jsonPaths.size(); i++) {
List<String> data = new ArrayList<>();
Object object = JsonPath.read(result, jsonPaths.get(i));
if (object instanceof List && jsonPaths.get(i).contains("[*]")) {
data = (List<String>) object;
Object object = JsonPath.using(jsonPathConf).parse(result).read(jsonPaths.get(i));
if (object instanceof List) {
for (Object o : (List<String>) object) {
if (Objects.isNull(o)) {
data.add("");
} else {
data.add(o.toString());
}
}
} else {
if (object != null) {
data.add(object.toString());

408
core/core-backend/src/main/java/io/gisbi/datasource/provider/CalciteProvider.java
View File

@ -2,6 +2,7 @@ package io.gisbi.datasource.provider;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.jcraft.jsch.Session;
import io.gisbi.constant.SQLConstants;
import io.gisbi.dataset.utils.FieldUtils;
import io.gisbi.datasource.dao.auto.entity.CoreDatasource;
import io.gisbi.datasource.dao.auto.entity.CoreDriver;
@ -9,7 +10,6 @@ import io.gisbi.datasource.dao.auto.mapper.CoreDatasourceMapper;
import io.gisbi.datasource.manage.EngineManage;
import io.gisbi.datasource.request.EngineRequest;
import io.gisbi.datasource.type.*;
import io.gisbi.constant.SQLConstants;
import io.gisbi.exception.DEException;
import io.gisbi.extensions.datasource.dto.*;
import io.gisbi.extensions.datasource.provider.DriverShim;
@ -31,12 +31,15 @@ import org.apache.calcite.sql.SqlNode;
import org.apache.calcite.sql.parser.SqlParser;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.dbcp2.BasicDataSource;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import java.io.File;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.math.BigDecimal;
import java.net.URL;
import java.sql.*;
@ -55,9 +58,9 @@ public class CalciteProvider extends Provider {
private EngineManage engineManage;
protected ExtendedJdbcClassLoader extendedJdbcClassLoader;
private Map<Long, ExtendedJdbcClassLoader> customJdbcClassLoaders = new HashMap<>();
@Value("${gisbi.path.driver:/opt/gisbi/drivers}")
@Value("${gisbi.path.driver:/opt/gisbi2.0/drivers}")
private String FILE_PATH;
@Value("${gisbi.path.custom-drivers:/opt/gisbi/custom-drivers/}")
@Value("${gisbi.path.custom-drivers:/opt/gisbi2.0/custom-drivers/}")
private String CUSTOM_PATH;
private static String split = "DE";
@ -155,7 +158,7 @@ public class CalciteProvider extends Provider {
@Override
public Map<String, Object> fetchResultField(DatasourceRequest datasourceRequest) throws DEException {
// 不跨数据源
if (datasourceRequest.getDsList().size() == 1) {
if (datasourceRequest.getIsCross() == null || !datasourceRequest.getIsCross()) {
return jdbcFetchResultField(datasourceRequest);
}
@ -240,8 +243,65 @@ public class CalciteProvider extends Provider {
return fieldList;
}
private Map<String, Integer> getTableTypeMap(DatasourceRequest datasourceRequest, DatasourceConfiguration datasourceConfiguration, String tableName) throws DEException {
Map<String, Integer> map = new HashMap<>();
String schemaTable = (ObjectUtils.isNotEmpty(datasourceConfiguration.getSchema()) ? (datasourceConfiguration.getSchema() + "`.`") : "") + tableName;
String sql = "SELECT * FROM `$TABLE_NAME$` LIMIT 0 OFFSET 0".replace("$TABLE_NAME$", schemaTable);
sql = transSqlDialect(sql, datasourceRequest.getDsList());
ResultSet resultSet = null;
try (Connection con = getConnectionFromPool(datasourceRequest.getDatasource().getId()); Statement statement = getStatement(con, 30)) {
resultSet = statement.executeQuery(sql);
ResultSetMetaData metaData = resultSet.getMetaData();
int columnCount = metaData.getColumnCount();
for (int j = 0; j < columnCount; j++) {
String name = StringUtils.lowerCase(metaData.getColumnName(j + 1));
Integer type = metaData.getColumnType(j + 1);
map.put(name, type);
}
} catch (Exception e) {
e.printStackTrace();
} finally {
if (resultSet != null) {
try {
resultSet.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
}
return map;
}
@Override
public List<TableField> fetchTableField(DatasourceRequest datasourceRequest) throws DEException {
if (datasourceRequest.getIsCross() != null && datasourceRequest.getIsCross()) {
List<TableField> datasetTableFields = new ArrayList<>();
PreparedStatement statement = null;
ResultSet resultSet = null;
Connection connection = take();
try {
CalciteConnection calciteConnection = connection.unwrap(CalciteConnection.class);
statement = calciteConnection.prepareStatement(datasourceRequest.getQuery());
resultSet = statement.executeQuery();
ResultSetMetaData metaData = resultSet.getMetaData();
int columnCount = metaData.getColumnCount();
for (int i = 1; i <= columnCount; i++) {
TableField tableField = new TableField();
tableField.setOriginName(metaData.getColumnLabel(i));
tableField.setType(metaData.getColumnTypeName(i));
tableField.setPrecision(metaData.getPrecision(i));
int deType = FieldUtils.transType2DeType(tableField.getType());
tableField.setDeExtractType(deType);
tableField.setDeType(deType);
tableField.setScale(metaData.getScale(i));
datasetTableFields.add(tableField);
}
} catch (Exception e) {
throw DEException.getException(e.getMessage());
}
return datasetTableFields;
}
List<TableField> datasetTableFields = new ArrayList<>();
DatasourceSchemaDTO datasourceSchemaDTO = datasourceRequest.getDsList().entrySet().iterator().next().getValue();
datasourceRequest.setDatasource(datasourceSchemaDTO);
@ -281,8 +341,11 @@ public class CalciteProvider extends Provider {
} else {
resultSet = statement.executeQuery(getTableFiledSql(datasourceRequest));
}
Map<String, Integer> tableTypeMap = getTableTypeMap(datasourceRequest, datasourceConfiguration, table);
while (resultSet.next()) {
TableField tableFieldDesc = getTableFieldDesc(datasourceRequest, resultSet, 3);
TableField tableFieldDesc = getTableFieldDesc(datasourceRequest, resultSet, 3, tableTypeMap);
boolean repeat = false;
for (TableField ele : datasetTableFields) {
if (StringUtils.equalsIgnoreCase(ele.getOriginName(), tableFieldDesc.getOriginName())) {
@ -423,16 +486,36 @@ public class CalciteProvider extends Provider {
// schema
ResultSet resultSet = null;
try (Connection con = getConnectionFromPool(datasourceRequest.getDatasource().getId()); Statement statement = getPreparedStatement(con, datasourceConfiguration.getQueryTimeout(), datasourceRequest.getQuery(), datasourceRequest.getTableFieldWithValues())) {
if (DatasourceConfiguration.DatasourceType.valueOf(value.getType()) == DatasourceConfiguration.DatasourceType.oracle) {
statement.executeUpdate("ALTER SESSION SET CURRENT_SCHEMA = " + datasourceConfiguration.getSchema());
}
try (Connection con = getConnectionFromPool(datasourceRequest.getDatasource().getId())) {
Statement statement = getStatement(value, con, datasourceRequest, datasourceConfiguration, null);
if (CollectionUtils.isNotEmpty(datasourceRequest.getTableFieldWithValues())) {
LogUtil.info("execWithPreparedStatement sql: " + datasourceRequest.getQuery());
for (int i = 0; i < datasourceRequest.getTableFieldWithValues().size(); i++) {
((PreparedStatement) statement).setObject(i + 1, datasourceRequest.getTableFieldWithValues().get(i).getValue(), datasourceRequest.getTableFieldWithValues().get(i).getType());
LogUtil.info("execWithPreparedStatement param[" + (i + 1) + "]: " + datasourceRequest.getTableFieldWithValues().get(i).getValue());
try {
Object valueObject = datasourceRequest.getTableFieldWithValues().get(i).getValue();
if (valueObject instanceof String
&& DatasourceConfiguration.DatasourceType.valueOf(value.getType()) == DatasourceConfiguration.DatasourceType.oracle) {
if (StringUtils.isNotEmpty(datasourceConfiguration.getCharset()) && StringUtils.isNotEmpty(datasourceConfiguration.getTargetCharset())) {
//转换为数据库的字符集
valueObject = new String(((String) valueObject).getBytes(datasourceConfiguration.getTargetCharset()), datasourceConfiguration.getCharset());
}
if (datasourceRequest.getTableFieldWithValues().get(i).getType().equals(Types.CLOB)) {
Reader reader = new StringReader((String) valueObject);
((PreparedStatement) statement).setCharacterStream(i + 1, reader, ((String) valueObject).length());
} else {
((PreparedStatement) statement).setObject(i + 1, valueObject, datasourceRequest.getTableFieldWithValues().get(i).getType());
}
} else {
((PreparedStatement) statement).setObject(i + 1, valueObject, datasourceRequest.getTableFieldWithValues().get(i).getType());
}
LogUtil.info("execWithPreparedStatement param[" + (i + 1) + "](" + datasourceRequest.getTableFieldWithValues().get(i).getColumnTypeName() + "): " + datasourceRequest.getTableFieldWithValues().get(i).getValue());
} catch (SQLException e) {
throw new SQLException(e.getMessage() + ". VALUE: " + datasourceRequest.getTableFieldWithValues().get(i).getValue().toString() + " , TARGET TYPE: " + datasourceRequest.getTableFieldWithValues().get(i).getColumnTypeName());
}
}
resultSet = ((PreparedStatement) statement).executeQuery();
} else {
@ -466,15 +549,35 @@ public class CalciteProvider extends Provider {
DatasourceConfiguration datasourceConfiguration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), DatasourceConfiguration.class);
// schema
ResultSet resultSet = null;
try (Connection con = getConnectionFromPool(datasourceRequest.getDatasource().getId()); Statement statement = getPreparedStatement(con, datasourceConfiguration.getQueryTimeout(), datasourceRequest.getQuery(), datasourceRequest.getTableFieldWithValues())) {
if (DatasourceConfiguration.DatasourceType.valueOf(value.getType()) == DatasourceConfiguration.DatasourceType.oracle) {
statement.executeUpdate("ALTER SESSION SET CURRENT_SCHEMA = " + datasourceConfiguration.getSchema());
}
try (Connection con = getConnectionFromPool(datasourceRequest.getDatasource().getId())) {
Statement statement = getStatement(value, con, datasourceRequest, datasourceConfiguration, null);
if (CollectionUtils.isNotEmpty(datasourceRequest.getTableFieldWithValues())) {
LogUtil.info("execWithPreparedStatement sql: " + datasourceRequest.getQuery());
for (int i = 0; i < datasourceRequest.getTableFieldWithValues().size(); i++) {
((PreparedStatement) statement).setObject(i + 1, datasourceRequest.getTableFieldWithValues().get(i).getValue(), datasourceRequest.getTableFieldWithValues().get(i).getType());
LogUtil.info("execWithPreparedStatement param[" + (i + 1) + "]: " + datasourceRequest.getTableFieldWithValues().get(i).getValue());
try {
Object valueObject = datasourceRequest.getTableFieldWithValues().get(i).getValue();
if (valueObject instanceof String
&& DatasourceConfiguration.DatasourceType.valueOf(value.getType()) == DatasourceConfiguration.DatasourceType.oracle) {
if (StringUtils.isNotEmpty(datasourceConfiguration.getCharset()) && StringUtils.isNotEmpty(datasourceConfiguration.getTargetCharset())) {
//转换为数据库的字符集
valueObject = new String(((String) valueObject).getBytes(datasourceConfiguration.getTargetCharset()), datasourceConfiguration.getCharset());
}
if (datasourceRequest.getTableFieldWithValues().get(i).getType().equals(Types.CLOB)) {
Reader reader = new StringReader((String) valueObject);
((PreparedStatement) statement).setCharacterStream(i + 1, reader, ((String) valueObject).length());
} else {
((PreparedStatement) statement).setObject(i + 1, valueObject, datasourceRequest.getTableFieldWithValues().get(i).getType());
}
} else {
((PreparedStatement) statement).setObject(i + 1, valueObject, datasourceRequest.getTableFieldWithValues().get(i).getType());
}
LogUtil.info("execWithPreparedStatement param[" + (i + 1) + "](" + datasourceRequest.getTableFieldWithValues().get(i).getColumnTypeName() + "): " + datasourceRequest.getTableFieldWithValues().get(i).getValue());
} catch (SQLException e) {
throw new SQLException(e.getMessage() + ". VALUE: " + datasourceRequest.getTableFieldWithValues().get(i).getValue().toString() + " , TARGET TYPE: " + datasourceRequest.getTableFieldWithValues().get(i).getColumnTypeName());
}
}
((PreparedStatement) statement).execute();
} else {
@ -496,29 +599,80 @@ public class CalciteProvider extends Provider {
}
}
/**
* 针对Oracle特殊处理
*/
private Statement getStatement(DatasourceSchemaDTO value, Connection con, DatasourceRequest datasourceRequest, DatasourceConfiguration datasourceConfiguration, String autoIncrementPkName) throws Exception {
Statement statement;
if (DatasourceConfiguration.DatasourceType.valueOf(value.getType()) == DatasourceConfiguration.DatasourceType.oracle) {
statement = getStatement(con, datasourceConfiguration.getQueryTimeout());
statement.executeUpdate("ALTER SESSION SET CURRENT_SCHEMA = " + datasourceConfiguration.getSchema());
statement.executeUpdate("ALTER SESSION SET NLS_TIMESTAMP_FORMAT = 'YYYY-MM-DD HH24:MI:SS'");
//调整字符集
if (StringUtils.isNotEmpty(datasourceConfiguration.getCharset()) && StringUtils.isNotEmpty(datasourceConfiguration.getTargetCharset())) {
datasourceRequest.setQuery(new String(datasourceRequest.getQuery().getBytes(datasourceConfiguration.getTargetCharset()), datasourceConfiguration.getCharset()));
}
}
statement = getPreparedStatement(con, datasourceConfiguration.getQueryTimeout(), datasourceRequest.getQuery(), datasourceRequest.getTableFieldWithValues(), autoIncrementPkName, datasourceConfiguration);
return statement;
}
@Override
public int executeUpdate(DatasourceRequest datasourceRequest) throws DEException {
public ExecuteResult executeUpdate(DatasourceRequest datasourceRequest, String autoIncrementPkName) throws DEException {
DatasourceSchemaDTO value = datasourceRequest.getDsList().entrySet().iterator().next().getValue();
datasourceRequest.setDatasource(value);
DatasourceConfiguration datasourceConfiguration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), DatasourceConfiguration.class);
// schema
ResultSet resultSet = null;
try (Connection con = getConnectionFromPool(datasourceRequest.getDatasource().getId()); Statement statement = getPreparedStatement(con, datasourceConfiguration.getQueryTimeout(), datasourceRequest.getQuery(), datasourceRequest.getTableFieldWithValues())) {
if (DatasourceConfiguration.DatasourceType.valueOf(value.getType()) == DatasourceConfiguration.DatasourceType.oracle) {
statement.executeUpdate("ALTER SESSION SET CURRENT_SCHEMA = " + datasourceConfiguration.getSchema());
}
try (Connection con = getConnectionFromPool(datasourceRequest.getDatasource().getId())) {
Statement statement = getStatement(value, con, datasourceRequest, datasourceConfiguration, autoIncrementPkName);
int count = 0;
if (CollectionUtils.isNotEmpty(datasourceRequest.getTableFieldWithValues())) {
LogUtil.info("execWithPreparedStatement sql: " + datasourceRequest.getQuery());
for (int i = 0; i < datasourceRequest.getTableFieldWithValues().size(); i++) {
((PreparedStatement) statement).setObject(i + 1, datasourceRequest.getTableFieldWithValues().get(i).getValue(), datasourceRequest.getTableFieldWithValues().get(i).getType());
LogUtil.info("execWithPreparedStatement param[" + (i + 1) + "]: " + datasourceRequest.getTableFieldWithValues().get(i).getValue());
try {
Object valueObject = datasourceRequest.getTableFieldWithValues().get(i).getValue();
if (valueObject instanceof String
&& DatasourceConfiguration.DatasourceType.valueOf(value.getType()) == DatasourceConfiguration.DatasourceType.oracle) {
if (StringUtils.isNotEmpty(datasourceConfiguration.getCharset()) && StringUtils.isNotEmpty(datasourceConfiguration.getTargetCharset())) {
//转换为数据库的字符集
valueObject = new String(((String) valueObject).getBytes(datasourceConfiguration.getTargetCharset()), datasourceConfiguration.getCharset());
}
return ((PreparedStatement) statement).executeUpdate();
if (datasourceRequest.getTableFieldWithValues().get(i).getType().equals(Types.CLOB)) {
Reader reader = new StringReader((String) valueObject);
((PreparedStatement) statement).setCharacterStream(i + 1, reader, ((String) valueObject).length());
} else {
return statement.executeUpdate(datasourceRequest.getQuery());
((PreparedStatement) statement).setObject(i + 1, valueObject, datasourceRequest.getTableFieldWithValues().get(i).getType());
}
} else {
((PreparedStatement) statement).setObject(i + 1, valueObject, datasourceRequest.getTableFieldWithValues().get(i).getType());
}
LogUtil.info("execWithPreparedStatement param[" + (i + 1) + "](" + datasourceRequest.getTableFieldWithValues().get(i).getColumnTypeName() + "): " + datasourceRequest.getTableFieldWithValues().get(i).getValue());
} catch (SQLException e) {
throw new SQLException(e.getMessage() + ". VALUE: " + datasourceRequest.getTableFieldWithValues().get(i).getValue().toString() + " , TARGET TYPE: " + datasourceRequest.getTableFieldWithValues().get(i).getColumnTypeName());
}
}
count = ((PreparedStatement) statement).executeUpdate();
} else {
count = statement.executeUpdate(datasourceRequest.getQuery());
}
ExecuteResult result = new ExecuteResult();
result.setCount(count);
if (StringUtils.isNotBlank(autoIncrementPkName)) {
List<String> generatedKeys = new ArrayList<>();
ResultSet keys = statement.getGeneratedKeys();
while (keys.next()) {
generatedKeys.add(keys.getObject(1).toString());
}
result.setGeneratedKeys(generatedKeys);
}
return result;
} catch (SQLException e) {
DEException.throwException("SQL ERROR: " + e.getMessage());
} catch (Exception e) {
@ -533,7 +687,7 @@ public class CalciteProvider extends Provider {
}
}
return 0;
return new ExecuteResult();
}
private List<TableField> getField(ResultSet rs, DatasourceRequest datasourceRequest) throws Exception {
@ -542,7 +696,7 @@ public class CalciteProvider extends Provider {
int columnCount = metaData.getColumnCount();
for (int j = 0; j < columnCount; j++) {
String f = metaData.getColumnName(j + 1);
if (StringUtils.equalsIgnoreCase(f, "DE_ROWNUM")) {
if (StringUtils.containsIgnoreCase(f, "ROWNUM")) {
continue;
}
String l = StringUtils.isNotEmpty(metaData.getColumnLabel(j + 1)) ? metaData.getColumnLabel(j + 1) : f;
@ -558,15 +712,15 @@ public class CalciteProvider extends Provider {
}
private List<String[]> getData(ResultSet rs, DatasourceRequest datasourceRequest) throws Exception {
String charset = null;
String targetCharset = "UTF-8";
String targetCharset = null;
String originCharset = null;
if (datasourceRequest != null && datasourceRequest.getDatasource().getType().equalsIgnoreCase("oracle")) {
DatasourceConfiguration jdbcConfiguration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), DatasourceConfiguration.class);
if (StringUtils.isNotEmpty(jdbcConfiguration.getCharset()) && !jdbcConfiguration.getCharset().equalsIgnoreCase("Default")) {
charset = jdbcConfiguration.getCharset();
if (StringUtils.isNotEmpty(jdbcConfiguration.getCharset())) {
originCharset = jdbcConfiguration.getCharset();
}
if (StringUtils.isNotEmpty(jdbcConfiguration.getTargetCharset()) && !jdbcConfiguration.getTargetCharset().equalsIgnoreCase("Default")) {
if (StringUtils.isNotEmpty(jdbcConfiguration.getTargetCharset())) {
targetCharset = jdbcConfiguration.getTargetCharset();
}
}
@ -583,6 +737,11 @@ public class CalciteProvider extends Provider {
row[j] = rs.getDate(j + 1).toString();
}
break;
case Types.TIMESTAMP:
if (rs.getTimestamp(j + 1) != null) {
row[j] = rs.getTimestamp(j + 1).toString();
}
break;
case Types.BOOLEAN:
row[j] = rs.getBoolean(j + 1) ? "1" : "0";
break;
@ -593,14 +752,18 @@ public class CalciteProvider extends Provider {
default:
if (metaData.getColumnTypeName(j + 1).toLowerCase().equalsIgnoreCase("blob")) {
row[j] = rs.getBlob(j + 1) == null ? "" : rs.getBlob(j + 1).toString();
}
if (targetCharset != null && StringUtils.isNotEmpty(rs.getString(j + 1)) && columnType == Types.CLOB) {
if (originCharset == null) {
row[j] = new String(rs.getString(j + 1).getBytes(), targetCharset);
} else {
if (charset != null && StringUtils.isNotEmpty(rs.getString(j + 1))) {
String originStr = new String(rs.getString(j + 1).getBytes(charset), targetCharset);
row[j] = new String(originStr.getBytes("UTF-8"), "UTF-8");
row[j] = new String(rs.getString(j + 1).getBytes(originCharset), targetCharset);
}
} else if (targetCharset != null && StringUtils.isNotEmpty(rs.getString(j + 1)) && (columnType != Types.NVARCHAR && columnType != Types.NCHAR)) {
row[j] = new String(rs.getBytes(j + 1), targetCharset);
} else {
row[j] = rs.getString(j + 1);
}
}
break;
}
@ -705,7 +868,7 @@ public class CalciteProvider extends Provider {
}
}
private TableField getTableFieldDesc(DatasourceRequest datasourceRequest, ResultSet resultSet, int commentIndex) throws SQLException {
private TableField getTableFieldDesc(DatasourceRequest datasourceRequest, ResultSet resultSet, int commentIndex, Map<String, Integer> tableTypeMap) throws SQLException {
TableField tableField = new TableField();
tableField.setOriginName(resultSet.getString(1));
tableField.setType(resultSet.getString(2).toUpperCase());
@ -718,6 +881,20 @@ public class CalciteProvider extends Provider {
tableField.setPrimary(resultSet.getInt(4) > 0);
} catch (Exception e) {
}
try {
if (StringUtils.endsWithIgnoreCase(datasourceRequest.getDatasource().getType(), "oracle")) {
if (StringUtils.contains(resultSet.getString(5), "nextval") || StringUtils.equalsIgnoreCase(resultSet.getString(5), "GENERATED ALWAYS AS IDENTITY")) {
tableField.setAutoIncrement(true);
}
} else {
tableField.setAutoIncrement(resultSet.getInt(5) > 0);
}
} catch (Exception e) {
}
try {
tableField.setTypeNumber(tableTypeMap.get(StringUtils.lowerCase(tableField.getOriginName())));
} catch (Exception e) {
}
return tableField;
}
@ -770,6 +947,13 @@ public class CalciteProvider extends Provider {
commonThreadPool.addTask(() -> {
try {
BasicDataSource dataSource = new BasicDataSource();
dataSource.setMaxWaitMillis(5 * 1000);
dataSource.setTestWhileIdle(true);
dataSource.setTestOnBorrow(true);
dataSource.setTestOnReturn(true);
dataSource.setTimeBetweenEvictionRunsMillis(60 * 1000);
dataSource.setValidationQuery("select 1");
dataSource.setValidationQueryTimeout(5);
Schema schema = null;
DatasourceConfiguration configuration = null;
DatasourceConfiguration.DatasourceType datasourceType = DatasourceConfiguration.DatasourceType.valueOf(ds.getType());
@ -838,6 +1022,7 @@ public class CalciteProvider extends Provider {
rootSchema.add(ds.getSchemaAlias(), schema);
break;
case oracle:
dataSource.setValidationQuery("SELECT 1 FROM DUAL");
configuration = JsonUtil.parseObject(ds.getConfiguration(), Oracle.class);
if (StringUtils.isNotBlank(configuration.getUsername())) {
dataSource.setUsername(configuration.getUsername());
@ -856,6 +1041,7 @@ public class CalciteProvider extends Provider {
break;
case db2:
configuration = JsonUtil.parseObject(ds.getConfiguration(), Db2.class);
dataSource.setValidationQuery("select 1 from syscat.tables WHERE TABSCHEMA ='DE_SCHEMA' AND \"TYPE\" = 'T'".replace("DE_SCHEMA", configuration.getSchema()));
if (StringUtils.isNotBlank(configuration.getUsername())) {
dataSource.setUsername(configuration.getUsername());
}
@ -1022,7 +1208,7 @@ public class CalciteProvider extends Provider {
if (database.contains(".")) {
sql = "select * from " + datasourceRequest.getTable() + " limit 0 offset 0 ";
} else {
sql = String.format("SELECT COLUMN_NAME,DATA_TYPE,COLUMN_COMMENT FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s'", database, datasourceRequest.getTable());
sql = String.format("SELECT COLUMN_NAME,DATA_TYPE,COLUMN_COMMENT,IF(COLUMN_KEY='PRI',1,0),IF(EXTRA LIKE '%%auto_increment%%',1,0) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s'", database, datasourceRequest.getTable());
}
break;
case mysql:
@ -1039,21 +1225,49 @@ public class CalciteProvider extends Provider {
String[] databasePrams = matcher.group(3).split("\\?");
database = databasePrams[0];
}
sql = String.format("SELECT COLUMN_NAME,DATA_TYPE,COLUMN_COMMENT,IF(COLUMN_KEY='PRI',1,0) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s'", database, datasourceRequest.getTable());
sql = String.format("SELECT COLUMN_NAME,DATA_TYPE,COLUMN_COMMENT,IF(COLUMN_KEY='PRI',1,0),IF(EXTRA LIKE '%%auto_increment%%',1,0) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s'", database, datasourceRequest.getTable());
break;
case oracle:
configuration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), Oracle.class);
if (StringUtils.isEmpty(configuration.getSchema())) {
DEException.throwException(Translator.get("i18n_schema_is_empty"));
}
sql = String.format("SELECT a.COLUMN_NAME , a.DATA_TYPE , b.COMMENTS ,0 FROM all_tab_columns a LEFT JOIN all_col_comments b ON a.owner = b.owner AND a.table_name = b.table_name AND a.column_name = b.column_name WHERE a.owner = '%s' AND a.table_name = '%s' ORDER BY a.table_name, a.column_id", configuration.getSchema(), datasourceRequest.getTable());
sql = String.format("""
SELECT tc.COLUMN_NAME AS ColumnName,
tc.DATA_TYPE,
cc.COMMENTS,
CASE
WHEN ac.COLUMN_NAME IS NOT NULL THEN 1
ELSE 0
END,
tc.DATA_DEFAULT
FROM ALL_TAB_COLUMNS tc
LEFT JOIN (SELECT cols.OWNER,
cols.TABLE_NAME,
cols.COLUMN_NAME
FROM ALL_CONSTRAINTS cons
JOIN
ALL_CONS_COLUMNS cols
ON cons.OWNER = cols.OWNER
AND cons.CONSTRAINT_NAME = cols.CONSTRAINT_NAME
WHERE cons.TABLE_NAME = '%s'
AND cons.CONSTRAINT_TYPE = 'P') ac
ON tc.OWNER = ac.OWNER
AND tc.TABLE_NAME = ac.TABLE_NAME
AND tc.COLUMN_NAME = ac.COLUMN_NAME
LEFT JOIN ALL_COL_COMMENTS cc
ON tc.owner = cc.owner AND tc.table_name = cc.table_name AND tc.column_name = cc.column_name
WHERE tc.TABLE_NAME = '%s'
AND tc.OWNER = '%s'
ORDER BY tc.TABLE_NAME, tc.COLUMN_ID
""", datasourceRequest.getTable(), datasourceRequest.getTable(), configuration.getSchema());
break;
case db2:
configuration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), Db2.class);
if (StringUtils.isEmpty(configuration.getSchema())) {
DEException.throwException(Translator.get("i18n_schema_is_empty"));
}
sql = String.format("SELECT COLNAME , TYPENAME , REMARKS FROM SYSCAT.COLUMNS WHERE TABSCHEMA = '%s' AND TABNAME = '%s' ", configuration.getSchema(), datasourceRequest.getTable());
sql = String.format("SELECT COLNAME, TYPENAME, REMARKS, 0, 0 FROM SYSCAT.COLUMNS WHERE TABSCHEMA = '%s' AND TABNAME = '%s' ", configuration.getSchema(), datasourceRequest.getTable());
break;
case sqlServer:
configuration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), Sqlserver.class);
@ -1061,38 +1275,46 @@ public class CalciteProvider extends Provider {
DEException.throwException(Translator.get("i18n_schema_is_empty"));
}
sql = String.format("SELECT \n" + " c.name ,t.name ,ep.value, 0 \n" + "FROM \n" + " sys.columns AS c\n" + "LEFT JOIN sys.extended_properties AS ep ON c.object_id = ep.major_id AND c.column_id = ep.minor_id\n" + "LEFT JOIN sys.types AS t ON c.user_type_id = t.user_type_id\n" + "LEFT JOIN sys.objects AS o ON c.object_id = o.object_id\n" + "WHERE o.name = '%s'", datasourceRequest.getTable());
sql = String.format("SELECT \n" + " c.name ,t.name ,ep.value, 0, 0 \n" + "FROM \n" + " sys.columns AS c\n" + "LEFT JOIN sys.extended_properties AS ep ON c.object_id = ep.major_id AND c.column_id = ep.minor_id\n" + "LEFT JOIN sys.types AS t ON c.user_type_id = t.user_type_id\n" + "LEFT JOIN sys.objects AS o ON c.object_id = o.object_id\n" + "WHERE o.name = '%s'", datasourceRequest.getTable());
break;
case pg:
configuration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), Pg.class);
if (StringUtils.isEmpty(configuration.getSchema())) {
DEException.throwException(Translator.get("i18n_schema_is_empty"));
}
sql = String.format("SELECT\n" +
" a.attname AS ColumnName,\n" +
" t.typname,\n" +
" b.description AS ColumnDescription,\n" +
" CASE\n" +
" WHEN d.indisprimary THEN 1\n" +
" ELSE 0\n" +
" END\n" +
"FROM\n" +
" pg_class c\n" +
" JOIN pg_attribute a ON a.attrelid = c.oid\n" +
" LEFT JOIN pg_description b ON a.attrelid = b.objoid AND a.attnum = b.objsubid\n" +
" JOIN pg_type t ON a.atttypid = t.oid\n" +
" LEFT JOIN pg_index d ON d.indrelid = a.attrelid AND d.indisprimary AND a.attnum = ANY(d.indkey)\n" +
"where\n" +
" c.relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = '%s')\n" +
" AND c.relname = '%s'\n" +
" AND a.attnum > 0\n" +
" AND NOT a.attisdropped\n" +
"ORDER BY\n" +
" a.attnum;", configuration.getSchema(), datasourceRequest.getTable());
sql = String.format("""
SELECT a.attname AS ColumnName,
t.typname,
b.description AS ColumnDescription,
CASE
WHEN d.indisprimary THEN 1
ELSE 0
END,
CASE
WHEN pg_get_expr(ad.adbin, ad.adrelid) LIKE 'nextval%%' THEN 1
""" + (
datasourceRequest.getDsVersion() > 9 ? """
WHEN a.attidentity = 'd' THEN 1
WHEN a.attidentity = 'a' THEN 1
""" : "") + """
ELSE 0
END
FROM pg_class c
JOIN pg_attribute a ON a.attrelid = c.oid
LEFT JOIN pg_attrdef ad ON a.attrelid = ad.adrelid AND a.attnum = ad.adnum
LEFT JOIN pg_description b ON a.attrelid = b.objoid AND a.attnum = b.objsubid
JOIN pg_type t ON a.atttypid = t.oid
LEFT JOIN pg_index d ON d.indrelid = a.attrelid AND d.indisprimary AND a.attnum = ANY (d.indkey)
where c.relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = '%s')
AND c.relname = '%s'
AND a.attnum > 0
AND NOT a.attisdropped
ORDER BY a.attnum;
""", configuration.getSchema(), datasourceRequest.getTable());
break;
case redshift:
configuration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), CK.class);
sql = String.format("SELECT\n" + " a.attname AS ColumnName,\n" + " t.typname,\n" + " b.description AS ColumnDescription,\n" + " 0\n" + "FROM\n" + " pg_class c\n" + " JOIN pg_attribute a ON a.attrelid = c.oid\n" + " LEFT JOIN pg_description b ON a.attrelid = b.objoid AND a.attnum = b.objsubid\n" + " JOIN pg_type t ON a.atttypid = t.oid\n" + "WHERE\n" + " c.relname = '%s'\n" + " AND a.attnum > 0\n" + " AND NOT a.attisdropped\n" + "ORDER BY\n" + " a.attnum\n" + " ", datasourceRequest.getTable());
sql = String.format("SELECT\n" + " a.attname AS ColumnName,\n" + " t.typname,\n" + " b.description AS ColumnDescription,\n" + " 0, 0\n" + "FROM\n" + " pg_class c\n" + " JOIN pg_attribute a ON a.attrelid = c.oid\n" + " LEFT JOIN pg_description b ON a.attrelid = b.objoid AND a.attnum = b.objsubid\n" + " JOIN pg_type t ON a.atttypid = t.oid\n" + "WHERE\n" + " c.relname = '%s'\n" + " AND a.attnum > 0\n" + " AND NOT a.attisdropped\n" + "ORDER BY\n" + " a.attnum\n" + " ", datasourceRequest.getTable());
break;
case ck:
configuration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), CK.class);
@ -1106,13 +1328,13 @@ public class CalciteProvider extends Provider {
String[] databasePrams = matcher.group(3).split("\\?");
database = databasePrams[0];
}
sql = String.format(" SELECT\n" + " name,\n" + " type,\n" + " comment,\n" + " 0\n" + "FROM\n" + " system.columns\n" + "WHERE\n" + " database = '%s' \n" + " AND table = '%s' ", database, datasourceRequest.getTable());
sql = String.format(" SELECT\n" + " name,\n" + " type,\n" + " comment,\n" + " 0, 0\n" + "FROM\n" + " system.columns\n" + "WHERE\n" + " database = '%s' \n" + " AND table = '%s' ", database, datasourceRequest.getTable());
break;
case impala:
sql = String.format("DESCRIBE `%s`", datasourceRequest.getTable());
break;
case h2:
sql = String.format("SELECT COLUMN_NAME, DATA_TYPE, REMARKS FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = '%s'", datasourceRequest.getTable());
sql = String.format("SELECT COLUMN_NAME, DATA_TYPE, REMARKS, 0, 0 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = '%s'", datasourceRequest.getTable());
break;
default:
break;
@ -1170,6 +1392,18 @@ public class CalciteProvider extends Provider {
}
tableSqls.add("select table_name, comments, owner from all_tab_comments where owner='" + configuration.getSchema() + "' AND table_type = 'TABLE'");
tableSqls.add("select table_name, comments, owner from all_tab_comments where owner='" + configuration.getSchema() + "' AND table_type = 'VIEW'");
tableSqls.add("SELECT \n" +
" m.mview_name,\n" +
" c.comments\n" +
"FROM \n" +
" ALL_MVIEWS m\n" +
"LEFT JOIN \n" +
" ALL_TAB_COMMENTS c \n" +
"ON \n" +
" m.owner = c.owner \n" +
" AND m.mview_name = c.table_name\n" +
" AND c.table_type = 'MATERIALIZED VIEW'\n" +
"WHERE m.OWNER ='DE_SCHEMA'".replace("DE_SCHEMA", configuration.getSchema()));
break;
case db2:
configuration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), Db2.class);
@ -1192,6 +1426,29 @@ public class CalciteProvider extends Provider {
DEException.throwException(Translator.get("i18n_schema_is_empty"));
}
tableSqls.add("SELECT \n" + " relname AS TableName, \n" + " obj_description(relfilenode::regclass, 'pg_class') AS TableDescription \n" + "FROM \n" + " pg_class \n" + "WHERE \n" + " relkind in ('r','p', 'f') \n" + " AND relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'SCHEMA') ".replace("SCHEMA", configuration.getSchema()));
tableSqls.add("SELECT \n" +
" c.relname AS view_name,\n" +
" COALESCE(d.description, 'No description provided') AS view_description\n" +
"FROM \n" +
" pg_class c\n" +
"JOIN \n" +
" pg_namespace n ON c.relnamespace = n.oid\n" +
"LEFT JOIN \n" +
" pg_description d ON c.oid = d.objoid\n" +
"WHERE \n" +
" c.relkind = 'v' \n" +
" AND n.nspname = 'SCHEMA'".replace("SCHEMA", configuration.getSchema()));
tableSqls.add("SELECT \n" +
" c.relname AS materialized_view_name,\n" +
" COALESCE(d.description, '') AS view_description\n" +
"FROM \n" +
" pg_class c\n" +
"JOIN \n" +
" pg_namespace n ON c.relnamespace = n.oid\n" +
"LEFT JOIN \n" +
" pg_description d ON c.oid = d.objoid\n" +
"WHERE \n" +
" c.relkind = 'm' and n.nspname ='SCHEMA'; ".replace("SCHEMA", configuration.getSchema()));
break;
case redshift:
configuration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), CK.class);
@ -1257,13 +1514,23 @@ public class CalciteProvider extends Provider {
}
public Statement getPreparedStatement(Connection connection, int queryTimeout, String sql, List<TableFieldWithValue> values) throws Exception {
return getPreparedStatement(connection, queryTimeout, sql, values, null, null);
}
public Statement getPreparedStatement(Connection connection, int queryTimeout, String sql, List<TableFieldWithValue> values, String autoIncrementPkName, DatasourceConfiguration datasourceConfiguration) throws Exception {
if (connection == null) {
throw new Exception("Failed to get connection!");
}
if (CollectionUtils.isNotEmpty(values)) {
PreparedStatement stat = null;
String pkName = autoIncrementPkName;
try {
if (StringUtils.isNotBlank(autoIncrementPkName)) {
String[] generatedColumns = {pkName};
stat = connection.prepareStatement(sql, generatedColumns);
} else {
stat = connection.prepareStatement(sql);
}
stat.setQueryTimeout(queryTimeout);
} catch (Exception e) {
DEException.throwException(e.getMessage());
@ -1332,7 +1599,7 @@ public class CalciteProvider extends Provider {
public void initConnectionPool() {
LogUtil.info("Begin to init datasource pool...");
QueryWrapper<CoreDatasource> datasourceQueryWrapper = new QueryWrapper();
List<CoreDatasource> coreDatasources = coreDatasourceMapper.selectList(datasourceQueryWrapper).stream().filter(coreDatasource -> !Arrays.asList("folder", "API", "Excel").contains(coreDatasource.getType())).collect(Collectors.toList());
List<CoreDatasource> coreDatasources = coreDatasourceMapper.selectList(datasourceQueryWrapper).stream().filter(coreDatasource -> !Arrays.asList("folder", "API", "Excel", "ExcelRemote").contains(coreDatasource.getType())).collect(Collectors.toList());
CoreDatasource engine = engineManage.deEngine();
if (engine != null) {
coreDatasources.add(engine);
@ -1382,8 +1649,8 @@ public class CalciteProvider extends Provider {
buildSchema(datasourceRequest, calciteConnection);
}
DatasourceConfiguration configuration = JsonUtil.parseObject(datasourceDTO.getConfiguration(), DatasourceConfiguration.class);
if(configuration.isUseSSH()){
Session session =Provider.getSessions().get(datasourceDTO.getId());
if (configuration.isUseSSH()) {
Session session = Provider.getSessions().get(datasourceDTO.getId());
session.disconnect();
Provider.getSessions().remove(datasourceDTO.getId());
startSshSession(configuration, null, datasourceDTO.getId());
@ -1437,6 +1704,7 @@ public class CalciteProvider extends Provider {
}
JdbcSchema jdbcSchema = rootSchema.getSubSchema(String.format(SQLConstants.SCHEMA, dsId)).unwrap(JdbcSchema.class);
BasicDataSource basicDataSource = (BasicDataSource) jdbcSchema.getDataSource();
basicDataSource.setMaxWaitMillis(5 * 1000);
return basicDataSource.getConnection();
} catch (Exception e) {
DEException.throwException(Translator.get("i18n_invalid_connection") + e.getMessage());

298
core/core-backend/src/main/java/io/gisbi/datasource/provider/ExcelUtils.java
View File

@ -4,6 +4,7 @@ package io.gisbi.datasource.provider;
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelReader;
import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.enums.CellDataTypeEnum;
import com.alibaba.excel.event.AnalysisEventListener;
import com.alibaba.excel.metadata.data.ReadCellData;
import com.alibaba.excel.read.metadata.ReadSheet;
@ -12,6 +13,7 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.gisbi.api.ds.vo.ExcelFileData;
import io.gisbi.api.ds.vo.ExcelSheetData;
import io.gisbi.api.ds.vo.RemoteExcelRequest;
import io.gisbi.commons.utils.EncryptUtils;
import io.gisbi.datasource.dao.auto.entity.CoreDatasource;
import io.gisbi.exception.DEException;
@ -19,6 +21,8 @@ import io.gisbi.extensions.datasource.dto.DatasetTableDTO;
import io.gisbi.extensions.datasource.dto.DatasourceDTO;
import io.gisbi.extensions.datasource.dto.DatasourceRequest;
import io.gisbi.extensions.datasource.dto.TableField;
import io.gisbi.api.ds.vo.ExcelConfiguration;
import io.gisbi.i18n.Translator;
import io.gisbi.utils.*;
import lombok.Data;
import org.apache.commons.lang3.StringUtils;
@ -26,6 +30,9 @@ import org.springframework.util.CollectionUtils;
import org.springframework.web.multipart.MultipartFile;
import java.io.*;
import java.net.URL;
import java.net.URLConnection;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.regex.Matcher;
@ -40,9 +47,9 @@ public class ExcelUtils {
public static String getExcelPath() {
if (ModelUtils.isDesktop()) {
return ConfigUtils.getConfig("gisbi.path.excel", "/opt/gisbi/data/excel/");
return ConfigUtils.getConfig("gisbi.path.excel", "/opt/gisbi2.0/data/excel/");
} else {
return "/opt/gisbi/data/excel/";
return "/opt/gisbi2.0/data/excel/";
}
}
@ -53,6 +60,7 @@ public class ExcelUtils {
};
public static void mergeSheets(CoreDatasource requestDatasource, DatasourceDTO sourceData) {
if (requestDatasource.getType().equalsIgnoreCase("Excel")) {
List<ExcelSheetData> newSheets = JsonUtil.parseList(requestDatasource.getConfiguration(), sheets);
List<String> tableNames = newSheets.stream().map(ExcelSheetData::getDeTableName).collect(Collectors.toList());
List<ExcelSheetData> oldSheets = JsonUtil.parseList(sourceData.getConfiguration(), sheets);
@ -62,12 +70,32 @@ public class ExcelUtils {
}
}
requestDatasource.setConfiguration(JsonUtil.toJSONString(newSheets).toString());
} else {
ExcelConfiguration excelConfiguration = JsonUtil.parseObject(requestDatasource.getConfiguration(), ExcelConfiguration.class);
List<ExcelSheetData> newSheets = excelConfiguration.getSheets();
List<String> tableNames = newSheets.stream().map(ExcelSheetData::getDeTableName).collect(Collectors.toList());
List<ExcelSheetData> oldSheets = JsonUtil.parseObject(sourceData.getConfiguration(), ExcelConfiguration.class).getSheets();
for (ExcelSheetData oldSheet : oldSheets) {
if (!tableNames.contains(oldSheet.getDeTableName())) {
newSheets.add(oldSheet);
}
}
excelConfiguration.setSheets(newSheets);
requestDatasource.setConfiguration(JsonUtil.toJSONString(excelConfiguration).toString());
}
}
public static List<DatasetTableDTO> getTables(DatasourceRequest datasourceRequest) throws DEException {
List<DatasetTableDTO> tableDescs = new ArrayList<>();
try {
JsonNode rootNode = objectMapper.readTree(datasourceRequest.getDatasource().getConfiguration());
String sheets = "";
if (datasourceRequest.getDatasource().getType().equalsIgnoreCase("Excel")) {
sheets = datasourceRequest.getDatasource().getConfiguration();
} else {
sheets = objectMapper.readTree(datasourceRequest.getDatasource().getConfiguration()).get("sheets").toString();
}
JsonNode rootNode = objectMapper.readTree(sheets);
for (int i = 0; i < rootNode.size(); i++) {
DatasetTableDTO datasetTableDTO = new DatasetTableDTO();
datasetTableDTO.setTableName(rootNode.get(i).get("deTableName").asText());
@ -83,20 +111,24 @@ public class ExcelUtils {
return tableDescs;
}
public static Map<String, String> getTableNamesMap(String configration) throws DEException {
public static Map<String, String> getTableNamesMap(String type, String configuration) throws DEException {
Map<String, String> result = new HashMap<>();
JsonNode rootNode = null;
// 兼容历史未加密信息
String sheets = configuration;
try {
rootNode = objectMapper.readTree((String) EncryptUtils.aesDecrypt(configration));
if (type.equalsIgnoreCase("ExcelRemote")) {
sheets = objectMapper.readTree(configuration).get("sheets").toString();
}
rootNode = objectMapper.readTree((String) EncryptUtils.aesDecrypt(sheets));
} catch (Exception e) {
try {
rootNode = objectMapper.readTree(configration);
rootNode = objectMapper.readTree(sheets);
} catch (Exception ex) {
DEException.throwException(ex);
}
}
if(rootNode != null) {
if (rootNode != null) {
for (int i = 0; i < rootNode.size(); i++) {
result.put(rootNode.get(i).get("tableName").asText(), rootNode.get(i).get("deTableName").asText());
}
@ -105,6 +137,12 @@ public class ExcelUtils {
}
public static String getFileName(CoreDatasource datasource) throws DEException {
if (datasource.getType().equalsIgnoreCase("ExcelRemote")) {
ExcelConfiguration excelConfiguration = JsonUtil.parseObject(datasource.getConfiguration(), ExcelConfiguration.class);
for (ExcelSheetData sheet : excelConfiguration.getSheets()) {
return sheet.getFileName();
}
}
JsonNode rootNode = null;
try {
rootNode = objectMapper.readTree((String) EncryptUtils.aesDecrypt(datasource.getConfiguration()));
@ -120,12 +158,16 @@ public class ExcelUtils {
return rootNode.get(i).get("fileName").asText();
}
}
return "";
}
public static String getSize(CoreDatasource datasource) throws DEException {
if (datasource.getType().equalsIgnoreCase("ExcelRemote")) {
ExcelConfiguration excelConfiguration = JsonUtil.parseObject(datasource.getConfiguration(), ExcelConfiguration.class);
for (ExcelSheetData sheet : excelConfiguration.getSheets()) {
return sheet.getSize();
}
}
try {
JsonNode rootNode = objectMapper.readTree(datasource.getConfiguration());
for (int i = 0; i < rootNode.size(); i++) {
@ -134,12 +176,32 @@ public class ExcelUtils {
} catch (Exception e) {
DEException.throwException(e);
}
return "0 B";
}
public List<String[]> fetchDataList(DatasourceRequest datasourceRequest) throws DEException {
public List<String[]> fetchDataList(DatasourceRequest datasourceRequest) throws DEException, IOException {
List<String[]> dataList = new ArrayList<>();
if (datasourceRequest.getDatasource().getType().equalsIgnoreCase("ExcelRemote")) {
ExcelConfiguration excelConfiguration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), ExcelConfiguration.class);
Map<String, String> fileNames = downLoadRemoteExcel(excelConfiguration);
for (ExcelSheetData sheet : excelConfiguration.getSheets()) {
if (sheet.getDeTableName().equalsIgnoreCase(datasourceRequest.getTable())) {
List<TableField> tableFields = sheet.getFields();
String suffix = fileNames.get("fileName").substring(fileNames.get("fileName").lastIndexOf(".") + 1);
InputStream inputStream = new FileInputStream(path + fileNames.get("tranName"));
if (StringUtils.equalsIgnoreCase(suffix, "csv")) {
BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8));
reader.readLine();//去掉表头
dataList = csvData(reader, false, tableFields.size());
} else {
dataList = fetchExcelDataList(sheet.getTableName(), inputStream);
}
}
}
if (StringUtils.isNotEmpty(fileNames.get("tranName"))) {
FileUtils.deleteFile(path + fileNames.get("tranName"));
}
} else {
try {
JsonNode rootNode = objectMapper.readTree(datasourceRequest.getDatasource().getConfiguration());
for (int i = 0; i < rootNode.size(); i++) {
@ -159,10 +221,11 @@ public class ExcelUtils {
} catch (Exception e) {
DEException.throwException(e);
}
}
return dataList;
}
public List<String[]> fetchExcelDataList(String sheetName, InputStream inputStream) {
private List<String[]> fetchExcelDataList(String sheetName, InputStream inputStream) {
NoModelDataListener noModelDataListener = new NoModelDataListener();
ExcelReader excelReader = EasyExcel.read(inputStream, noModelDataListener).build();
List<ReadSheet> sheets = excelReader.excelExecutor().sheetList();
@ -189,7 +252,13 @@ public class ExcelUtils {
TypeReference<List<TableField>> listTypeReference = new TypeReference<List<TableField>>() {
};
try {
JsonNode rootNode = objectMapper.readTree(datasourceRequest.getDatasource().getConfiguration());
String sheets = "";
if (datasourceRequest.getDatasource().getType().equalsIgnoreCase("Excel")) {
sheets = datasourceRequest.getDatasource().getConfiguration();
} else {
sheets = objectMapper.readTree(datasourceRequest.getDatasource().getConfiguration()).get("sheets").toString();
}
JsonNode rootNode = objectMapper.readTree(sheets);
for (int i = 0; i < rootNode.size(); i++) {
if (rootNode.get(i).get("deTableName").asText().equalsIgnoreCase(datasourceRequest.getTable())) {
tableFields = JsonUtil.parseList(rootNode.get(i).get("fields").toString(), listTypeReference);
@ -201,19 +270,17 @@ public class ExcelUtils {
return tableFields;
}
public ExcelFileData excelSaveAndParse(MultipartFile file) throws DEException {
public ExcelFileData excelSaveAndParse(MultipartFile file, String createBy) throws DEException {
String filename = file.getOriginalFilename();
List<ExcelSheetData> excelSheetDataList = null;
try {
excelSheetDataList = parseExcel(filename, file.getInputStream(), true);
excelSheetDataList = parseExcel(filename, file.getInputStream(), true, filename);
} catch (Exception e) {
DEException.throwException(e);
}
List<ExcelSheetData> returnSheetDataList = new ArrayList<>();
returnSheetDataList = excelSheetDataList;
returnSheetDataList = returnSheetDataList.stream()
.filter(excelSheetData -> !CollectionUtils.isEmpty(excelSheetData.getFields()))
.collect(Collectors.toList());
returnSheetDataList = returnSheetDataList.stream().filter(excelSheetData -> !CollectionUtils.isEmpty(excelSheetData.getFields())).collect(Collectors.toList());
// save file
String excelId = UUID.randomUUID().toString();
String filePath = saveFile(file, excelId);
@ -227,7 +294,7 @@ public class ExcelUtils {
excelSheetData.setSheetExcelId(excelId);
excelSheetData.setFileName(filename);
/**
* dataease字段类型：0-文本，1-时间，2-整型数值，3-浮点数值，4-布尔，5-地理位置，6-二进制
* gisbi字段类型：0-文本，1-时间，2-整型数值，3-浮点数值，4-布尔，5-地理位置，6-二进制
*/
for (TableField field : excelSheetData.getFields()) {
//TEXT LONG DATETIME DOUBLE
@ -272,17 +339,119 @@ public class ExcelUtils {
return excelFileData;
}
public static String checkStatus(DatasourceRequest datasourceRequest) throws FileNotFoundException {
ExcelConfiguration excelConfiguration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), ExcelConfiguration.class);
Map<String, String> fileNames = new HashMap<>();
try {
fileNames = downLoadRemoteExcel(excelConfiguration);
return "Success";
} catch (Exception e) {
throw e;
} finally {
if (StringUtils.isNotEmpty(fileNames.get("tranName"))) {
FileUtils.deleteFile(path + fileNames.get("tranName"));
}
}
}
public ExcelFileData parseRemoteExcel(RemoteExcelRequest remoteExcelRequest) throws DEException, FileNotFoundException {
Map<String, String> fileNames = downLoadRemoteExcel(remoteExcelRequest);
FileInputStream fileInputStream = new FileInputStream(path + fileNames.get("tranName"));
List<ExcelSheetData> returnSheetDataList = new ArrayList<>();
try {
returnSheetDataList = parseExcel(fileNames.get("tranName"), fileInputStream, true, fileNames.get("fileName")).stream().filter(excelSheetData -> !CollectionUtils.isEmpty(excelSheetData.getFields())).collect(Collectors.toList());
} catch (Exception e) {
DEException.throwException(e);
}
for (ExcelSheetData excelSheetData : returnSheetDataList) {
excelSheetData.setLastUpdateTime(System.currentTimeMillis());
excelSheetData.setTableName(excelSheetData.getExcelLabel());
excelSheetData.setDeTableName("excel_" + excelSheetData.getExcelLabel() + "_" + UUID.randomUUID().toString().replace("-", "").substring(0, 10));
excelSheetData.setPath(path + fileNames.get("tranName"));
excelSheetData.setSheetId(UUID.randomUUID().toString());
excelSheetData.setSheetExcelId(fileNames.get("tranName").split("\\.")[0]);
excelSheetData.setFileName(fileNames.get("fileName"));
/**
* gisbi字段类型：0-文本，1-时间，2-整型数值，3-浮点数值，4-布尔，5-地理位置，6-二进制
*/
for (TableField field : excelSheetData.getFields()) {
//TEXT LONG DATETIME DOUBLE
if (field.getFieldType().equalsIgnoreCase("TEXT")) {
field.setDeType(0);
field.setDeExtractType(0);
}
if (field.getFieldType().equalsIgnoreCase("DATETIME")) {
field.setDeType(1);
field.setDeExtractType(1);
}
if (field.getFieldType().equalsIgnoreCase("LONG")) {
field.setDeType(2);
field.setDeExtractType(2);
}
if (field.getFieldType().equalsIgnoreCase("DOUBLE")) {
field.setDeType(3);
field.setDeExtractType(3);
}
}
long size = 0;
File file = new File(path + fileNames.get("tranName"));
String unit = "B";
if (file.length() / 1024 == 0) {
size = file.length();
}
if (0 < file.length() / 1024 && file.length() / 1024 < 1024) {
size = file.length() / 1024;
unit = "KB";
}
if (1024 <= file.length() / 1024) {
size = file.length() / 1024 / 1024;
unit = "MB";
}
excelSheetData.setSize(size + " " + unit);
}
ExcelFileData excelFileData = new ExcelFileData();
excelFileData.setExcelLabel(fileNames.get("fileName").split("\\.")[0]);
excelFileData.setId(fileNames.get("tranName").split("\\.")[0]);
excelFileData.setPath(path + fileNames.get("tranName"));
excelFileData.setSheets(returnSheetDataList);
if (StringUtils.isNotEmpty(fileNames.get("tranName"))) {
FileUtils.deleteFile(path + fileNames.get("tranName"));
}
return excelFileData;
}
private static Map<String, String> downLoadRemoteExcel(ExcelConfiguration remoteExcelRequest) throws DEException, FileNotFoundException {
Map<String, String> fileNames = new HashMap<>();
if (remoteExcelRequest.getUrl().trim().startsWith("http")) {
HttpClientConfig httpClientConfig = new HttpClientConfig();
if (StringUtils.isNotEmpty(remoteExcelRequest.getUserName()) && StringUtils.isNotEmpty(remoteExcelRequest.getPasswd())) {
String authValue = "Basic " + Base64.getUrlEncoder().encodeToString((remoteExcelRequest.getUserName() + ":" + remoteExcelRequest.getPasswd()).getBytes());
httpClientConfig.addHeader("Authorization", authValue);
}
File p = new File(path);
if (!p.exists()) {
p.mkdirs();
}
fileNames = HttpClientUtil.downloadFile(remoteExcelRequest.getUrl(), httpClientConfig, path);
} else if (remoteExcelRequest.getUrl().trim().startsWith("ftp")) {
fileNames = downLoadFromFtp(remoteExcelRequest);
} else {
DEException.throwException(Translator.get("i18n_unsupported_protocol"));
}
return fileNames;
}
private static String saveFile(MultipartFile file, String fileNameUUID) throws DEException {
String filePath = null;
try {
String filename = file.getOriginalFilename();
String suffix = filename.substring(filename.lastIndexOf(".") + 1);
String dirPath = path + AuthUtils.getUser().getUserId() + "/";
File p = new File(dirPath);
File p = new File(path);
if (!p.exists()) {
p.mkdirs();
}
filePath = dirPath + fileNameUUID + "." + suffix;
filePath = path + fileNameUUID + "." + suffix;
File f = new File(filePath);
FileOutputStream fileOutputStream = new FileOutputStream(f);
fileOutputStream.write(file.getBytes());
@ -398,7 +567,13 @@ public class ExcelUtils {
super.invokeHead(headMap, context);
for (Integer key : headMap.keySet()) {
ReadCellData<?> cellData = headMap.get(key);
String value = cellData.getStringValue();
String value = null;
if (cellData.getType().equals(CellDataTypeEnum.STRING)) {
value = cellData.getStringValue();
}
if (cellData.getType().equals(CellDataTypeEnum.NUMBER)) {
value = cellData.getNumberValue().toString();
}
if (StringUtils.isEmpty(value)) {
continue;
}
@ -439,7 +614,7 @@ public class ExcelUtils {
}
public List<ExcelSheetData> parseExcel(String filename, InputStream inputStream, boolean isPreview) throws IOException {
private List<ExcelSheetData> parseExcel(String filename, InputStream inputStream, boolean isPreview, String originFilename) throws IOException {
List<ExcelSheetData> excelSheetDataList = new ArrayList<>();
String suffix = filename.substring(filename.lastIndexOf(".") + 1);
if (StringUtils.equalsIgnoreCase(suffix, "xlsx") || StringUtils.equalsIgnoreCase(suffix, "xls")) {
@ -494,11 +669,12 @@ public class ExcelUtils {
List<TableField> fields = new ArrayList<>();
BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8));
String s = reader.readLine();// first line
if (StringUtils.isNotEmpty(s)) {
String[] split = s.split(",");
for (int i = 0; i < split.length; i++) {
String filedName = split[i];
if (StringUtils.isEmpty(filedName)) {
DEException.throwException("首行行中不允许有空单元格！");
DEException.throwException(Translator.get("i18n_excel_error_first_row"));
}
if (filedName.startsWith(UFEFF)) {
filedName = filedName.replace(UFEFF, "");
@ -510,6 +686,7 @@ public class ExcelUtils {
tableFiled.setChecked(true);
fields.add(tableFiled);
}
}
List<String[]> data = csvData(reader, isPreview, fields.size());
if (isPreview) {
@ -535,7 +712,7 @@ public class ExcelUtils {
excelSheetData.setFields(fields);
excelSheetData.setData(data);
excelSheetData.setFileName(filename);
excelSheetData.setExcelLabel(filename.substring(0, filename.lastIndexOf('.')));
excelSheetData.setExcelLabel(originFilename.substring(0, originFilename.lastIndexOf('.')));
excelSheetDataList.add(excelSheetData);
}
inputStream.close();
@ -560,5 +737,74 @@ public class ExcelUtils {
return excelSheetDataList;
}
public static Map<String, String> downLoadFromFtp(ExcelConfiguration remoteExcelRequest) {
Map<String, String> fileNames = new HashMap<>();
String username = "";
String password = "";
String serverAddress = "";
String filePath = "";
if (remoteExcelRequest.getUrl().contains("@")) {
String regex = "ftp://([^:]+):([^@]+)@([^/]+)(.*)";
Pattern pattern = Pattern.compile(regex);
Matcher matcher = pattern.matcher(remoteExcelRequest.getUrl());
if (matcher.find()) {
username = matcher.group(1);
password = matcher.group(2);
serverAddress = matcher.group(3);
filePath = matcher.group(4);
} else {
DEException.throwException(Translator.get("i18n_invalid_address"));
}
} else {
String regex = "ftp://([^/]+)(.*)";
Pattern pattern = Pattern.compile(regex);
Matcher matcher = pattern.matcher(remoteExcelRequest.getUrl());
if (matcher.find()) {
serverAddress = matcher.group(1);
filePath = matcher.group(2);
} else {
DEException.throwException(Translator.get("i18n_invalid_address"));
}
}
if (StringUtils.isNotEmpty(remoteExcelRequest.getUserName())) {
username = remoteExcelRequest.getUserName();
}
if (StringUtils.isNotEmpty(remoteExcelRequest.getPasswd())) {
password = remoteExcelRequest.getPasswd();
}
filePath = filePath.startsWith("/") ? filePath.substring(1) : filePath;
String suffix = filePath.substring(filePath.lastIndexOf(".") + 1);
if (!Arrays.asList("csv", "xlsx", "xls").contains(suffix)) {
DEException.throwException(Translator.get("i18n_unsupported_file_format"));
}
String tranName = UUID.randomUUID().toString() + "." + suffix;
String localFilePath = path + tranName;
fileNames.put("fileName", filePath);
fileNames.put("tranName", tranName);
try {
URL url;
if (StringUtils.isNotEmpty(username) && StringUtils.isNotEmpty(password)) {
url = new URL("ftp://" + username + ":" + URLEncoder.encode(password, StandardCharsets.UTF_8) + "@" + serverAddress + "/" + filePath);
} else {
url = new URL("ftp://" + serverAddress + "/" + filePath);
}
URLConnection conn = url.openConnection();
InputStream inputStream = conn.getInputStream();
FileOutputStream outputStream = new FileOutputStream(localFilePath);
byte[] buffer = new byte[4096];
int bytesRead;
while ((bytesRead = inputStream.read(buffer)) != -1) {
outputStream.write(buffer, 0, bytesRead);
}
inputStream.close();
outputStream.close();
} catch (IOException e) {
DEException.throwException(Translator.get("i18n_file_download_failed") + ", " + e.getMessage());
}
return fileNames;
}
}

7
core/core-backend/src/main/java/io/gisbi/datasource/provider/H2EngineProvider.java
View File

@ -46,12 +46,7 @@ public class H2EngineProvider extends EngineProvider {
if (tableFields.get(i).isChecked()) {
if (StringUtils.isEmpty(strings[i])) {
String type = tableFields.get(i).getType() == null ? tableFields.get(i).getFieldType() : tableFields.get(i).getType();
if (type.equals("LONG") || type.equals("DOUBLE")) {
strings1[length] = "0";
} else {
strings1[length] = null;
}
} else {
strings1[length] = strings[i].replace("\\", "\\\\").replace("'", "''");
}
@ -61,7 +56,7 @@ public class H2EngineProvider extends EngineProvider {
values.append("('").append(String.join("','", Arrays.asList(strings1)))
.append("'),");
}
return (insertSql + values.substring(0, values.length() - 1));
return (insertSql + values.substring(0, values.length() - 1)).replaceAll("'null'", "null");
}

4
core/core-backend/src/main/java/io/gisbi/datasource/provider/MysqlEngineProvider.java
View File

@ -54,11 +54,7 @@ public class MysqlEngineProvider extends EngineProvider {
if (tableFields.get(i).isChecked()) {
if (StringUtils.isEmpty(strings[i])) {
String type = tableFields.get(i).getType() == null ? tableFields.get(i).getFieldType() : tableFields.get(i).getType();
if (type.equals("LONG") || type.equals("DOUBLE")) {
strings1[length] = "0";
} else {
strings1[length] = null;
}
} else {
strings1[length] = strings[i].replace("\\", "\\\\").replace("'", "\\'");
}

2
core/core-backend/src/main/java/io/gisbi/datasource/server/DatasourceDriverServer.java
View File

@ -33,7 +33,7 @@ import java.util.Map;
@RequestMapping("/datasourceDriver")
public class DatasourceDriverServer implements DatasourceDriverApi {
@Value("${gisbi.path.custom-drivers:/opt/gisbi/custom-drivers/}")
@Value("${gisbi.path.custom-drivers:/opt/gisbi2.0/custom-drivers/}")
private String DRIVER_PATH;
@Resource

238
core/core-backend/src/main/java/io/gisbi/datasource/server/DatasourceServer.java
View File

@ -10,8 +10,8 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import io.gisbi.api.dataset.dto.PreviewSqlDTO;
import io.gisbi.api.ds.DatasourceApi;
import io.gisbi.api.ds.vo.*;
import io.gisbi.api.permissions.relation.api.RelationApi;
import io.gisbi.commons.constants.TaskStatus;
import io.gisbi.constant.DataSourceType;
import io.gisbi.constant.LogOT;
import io.gisbi.constant.LogST;
import io.gisbi.constant.SQLConstants;
@ -59,10 +59,8 @@ import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.*;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.*;
import java.util.stream.Collectors;
@ -103,11 +101,15 @@ public class DatasourceServer implements DatasourceApi {
private CoreUserManage coreUserManage;
@Autowired(required = false)
private PluginManageApi pluginManage;
@Autowired(required = false)
private RelationApi relationManage;
public enum UpdateType {
all_scope, add_scope
}
public static final List<String> notFullDs = List.of("folder", "Excel", "API");
private TypeReference<List<ApiDefinition>> listTypeReference = new TypeReference<List<ApiDefinition>>() {
};
@Resource
@ -122,7 +124,7 @@ public class DatasourceServer implements DatasourceApi {
}
public boolean checkRepeat(@RequestBody BusiDsRequest dataSourceDTO) {
if (Arrays.asList("Excel", "folder", "es").contains(dataSourceDTO.getType()) || dataSourceDTO.getType().contains("API")) {
if (Arrays.asList("folder", "es").contains(dataSourceDTO.getType()) || dataSourceDTO.getType().contains("API") || dataSourceDTO.getType().contains("Excel")) {
return false;
}
BusiNodeRequest request = new BusiNodeRequest();
@ -145,7 +147,7 @@ public class DatasourceServer implements DatasourceApi {
DatasourceConfiguration configuration = JsonUtil.parseObject(dataSourceDTO.getConfiguration(), DatasourceConfiguration.class);
boolean hasRepeat = false;
for (CoreDatasource datasource : datasources) {
if (Arrays.asList("API", "Excel", "folder").contains(datasource.getType())) {
if (notFullDs.stream().anyMatch(e -> datasource.getType().contains(e))) {
continue;
}
if (StringUtils.isEmpty(datasource.getConfiguration())) {
@ -207,7 +209,7 @@ public class DatasourceServer implements DatasourceApi {
if (StringUtils.isEmpty(dataSourceDTO.getName())) {
DEException.throwException("名称不能为空！");
}
CoreDatasource datasource = dataSourceManage.getCoreDatasource(dataSourceDTO.getId());
CoreDatasource datasource = dataSourceManage.getDatasource(dataSourceDTO.getId());
datasource.setName(dataSourceDTO.getName());
dataSourceDTO.setPid(datasource.getPid());
dataSourceManage.checkName(dataSourceDTO);
@ -243,7 +245,6 @@ public class DatasourceServer implements DatasourceApi {
}
preCheckDs(dataSourceDTO);
dataSourceDTO.setId(IDUtils.snowID());
dataSourceDTO.setAppId(busiDsRequest.getAppId()); //补充了项目ID
dataSourceDTO.setCreateTime(System.currentTimeMillis());
dataSourceDTO.setUpdateTime(System.currentTimeMillis());
try {
@ -278,6 +279,20 @@ public class DatasourceServer implements DatasourceApi {
}
datasourceSyncManage.extractExcelData(coreDatasource, "all_scope");
} else if (dataSourceDTO.getType().contains(DatasourceConfiguration.DatasourceType.API.name())) {
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(dataSourceDTO);
List<DatasetTableDTO> tables = (List<DatasetTableDTO>) invokeMethod(coreDatasource.getType(), "getApiTables", DatasourceRequest.class, datasourceRequest);
checkName(tables.stream().map(DatasetTableDTO::getName).collect(Collectors.toList()));
for (DatasetTableDTO api : tables) {
datasourceRequest.setTable(api.getTableName());
List<TableField> tableFields = (List<TableField>) invokeMethod(coreDatasource.getType(), "getTableFields", DatasourceRequest.class, datasourceRequest);
try {
datasourceSyncManage.createEngineTable(datasourceRequest.getTable(), tableFields);
} catch (Exception e) {
DEException.throwException("Failed to create table " + datasourceRequest.getTable() + ": " + e.getMessage());
}
}
CoreDatasourceTask coreDatasourceTask = new CoreDatasourceTask();
BeanUtils.copyBean(coreDatasourceTask, dataSourceDTO.getSyncSetting());
coreDatasourceTask.setName(coreDatasource.getName() + "-task");
@ -295,19 +310,40 @@ public class DatasourceServer implements DatasourceApi {
coreDatasourceTask.setTaskStatus(TaskStatus.WaitingForExecution.name());
datasourceTaskServer.insert(coreDatasourceTask);
datasourceSyncManage.addSchedule(coreDatasourceTask);
} else if (dataSourceDTO.getType().contains(DatasourceConfiguration.DatasourceType.ExcelRemote.name())) {
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(dataSourceDTO);
List<DatasetTableDTO> tables = (List<DatasetTableDTO>) invokeMethod(coreDatasource.getType(), "getApiTables", DatasourceRequest.class, datasourceRequest);
checkName(tables.stream().map(DatasetTableDTO::getName).collect(Collectors.toList()));
for (DatasetTableDTO api : tables) {
datasourceRequest.setTable(api.getTableName());
List<TableField> tableFields = (List<TableField>) invokeMethod(coreDatasource.getType(), "getTableFields", DatasourceRequest.class, datasourceRequest);
List<DatasetTableDTO> tables = ExcelUtils.getTables(datasourceRequest);
for (DatasetTableDTO table : tables) {
datasourceRequest.setTable(table.getTableName());
List<TableField> tableFields = ExcelUtils.getTableFields(datasourceRequest);
try {
datasourceSyncManage.createEngineTable(datasourceRequest.getTable(), tableFields);
} catch (Exception e) {
DEException.throwException("Failed to create table " + datasourceRequest.getTable() + ": " + e.getMessage());
if (e.getMessage().toLowerCase().contains("Row size too large".toLowerCase())) {
DEException.throwException("文本内容超出最大支持范围： " + datasourceRequest.getTable() + ", " + e.getMessage());
} else {
DEException.throwException("Failed to create table " + datasourceRequest.getTable() + ", " + e.getMessage());
}
}
}
CoreDatasourceTask coreDatasourceTask = new CoreDatasourceTask();
BeanUtils.copyBean(coreDatasourceTask, dataSourceDTO.getSyncSetting());
coreDatasourceTask.setName(coreDatasource.getName() + "-task");
coreDatasourceTask.setDsId(coreDatasource.getId());
if (coreDatasourceTask.getStartTime() == null) {
coreDatasourceTask.setStartTime(System.currentTimeMillis() - 20 * 1000);
}
if (StringUtils.equalsIgnoreCase(coreDatasourceTask.getSyncRate(), RIGHTNOW.toString())) {
coreDatasourceTask.setCron(null);
} else {
if (coreDatasourceTask.getEndTime() != null && coreDatasourceTask.getEndTime() > 0 && coreDatasourceTask.getStartTime() > coreDatasourceTask.getEndTime()) {
DEException.throwException("结束时间不能小于开始时间！");
}
}
coreDatasourceTask.setTaskStatus(TaskStatus.WaitingForExecution.name());
datasourceTaskServer.insert(coreDatasourceTask);
datasourceSyncManage.addSchedule(coreDatasourceTask);
} else {
checkParams(dataSourceDTO.getConfiguration());
calciteProvider.update(dataSourceDTO);
@ -347,13 +383,14 @@ public class DatasourceServer implements DatasourceApi {
datasourceRequest.setDatasource(dataSourceDTO);
List<String> toCreateTables = new ArrayList<>();
List<String> toDeleteTables = new ArrayList<>();
if (dataSourceDTO.getType().contains(DatasourceConfiguration.DatasourceType.API.name())) {
if (dataSourceDTO.getType().contains(DatasourceConfiguration.DatasourceType.API.name()) || dataSourceDTO.getType().equals(DatasourceConfiguration.DatasourceType.ExcelRemote.name())) {
requestDatasource.setEnableDataFill(null);
List<DatasetTableDTO> sourceTableDTOs = (List<DatasetTableDTO>) invokeMethod(sourceData.getType(), "getApiTables", DatasourceRequest.class, sourceTableRequest);
List<DatasetTableDTO> sourceTableDTOs = dataSourceDTO.getType().contains(DatasourceConfiguration.DatasourceType.API.name()) ? (List<DatasetTableDTO>) invokeMethod(sourceData.getType(), "getApiTables", DatasourceRequest.class, sourceTableRequest) : ExcelUtils.getTables(sourceTableRequest);
List<String> sourceTables = sourceTableDTOs.stream().map(DatasetTableDTO::getTableName).toList();
List<DatasetTableDTO> datasetTableDTOS = (List<DatasetTableDTO>) invokeMethod(sourceData.getType(), "getApiTables", DatasourceRequest.class, datasourceRequest);
List<DatasetTableDTO> datasetTableDTOS = dataSourceDTO.getType().contains(DatasourceConfiguration.DatasourceType.API.name()) ? (List<DatasetTableDTO>) invokeMethod(sourceData.getType(), "getApiTables", DatasourceRequest.class, datasourceRequest) : ExcelUtils.getTables(datasourceRequest);
List<String> tables = datasetTableDTOS.stream().map(DatasetTableDTO::getTableName).collect(Collectors.toList());
checkName(datasetTableDTOS.stream().map(DatasetTableDTO::getName).collect(Collectors.toList()));
toCreateTables = tables.stream().filter(table -> !sourceTables.contains(table)).collect(Collectors.toList());
toDeleteTables = sourceTables.stream().filter(table -> !tables.contains(table)).collect(Collectors.toList());
@ -361,10 +398,10 @@ public class DatasourceServer implements DatasourceApi {
for (String sourceTable : sourceTables) {
if (table.equals(sourceTable)) {
datasourceRequest.setTable(table);
List<TableField> tableFieldList = (List<TableField>) invokeMethod(datasourceRequest.getDatasource().getType(), "getTableFields", DatasourceRequest.class, datasourceRequest);
List<TableField> tableFieldList = dataSourceDTO.getType().contains(DatasourceConfiguration.DatasourceType.API.name()) ? (List<TableField>) invokeMethod(datasourceRequest.getDatasource().getType(), "getTableFields", DatasourceRequest.class, datasourceRequest) : ExcelUtils.getTableFields(datasourceRequest);
List<String> tableFields = tableFieldList.stream().map(TableField::getName).sorted().collect(Collectors.toList());
sourceTableRequest.setTable(sourceTable);
List<TableField> sourceTableFieldList = (List<TableField>) invokeMethod(sourceTableRequest.getDatasource().getType(), "getTableFields", DatasourceRequest.class, sourceTableRequest);
List<TableField> sourceTableFieldList = dataSourceDTO.getType().contains(DatasourceConfiguration.DatasourceType.API.name()) ? (List<TableField>) invokeMethod(sourceTableRequest.getDatasource().getType(), "getTableFields", DatasourceRequest.class, sourceTableRequest) : ExcelUtils.getTableFields(sourceTableRequest);
List<String> sourceTableFields = sourceTableFieldList.stream().map(TableField::getName).sorted().collect(Collectors.toList());
if (!String.join(",", tableFields).equals(String.join(",", sourceTableFields))) {
toDeleteTables.add(table);
@ -397,7 +434,7 @@ public class DatasourceServer implements DatasourceApi {
for (String toCreateTable : toCreateTables) {
datasourceRequest.setTable(toCreateTable);
try {
datasourceSyncManage.createEngineTable(toCreateTable, (List<TableField>) invokeMethod(sourceTableRequest.getDatasource().getType(), "getTableFields", DatasourceRequest.class, datasourceRequest));
datasourceSyncManage.createEngineTable(toCreateTable, dataSourceDTO.getType().contains(DatasourceConfiguration.DatasourceType.API.name()) ? (List<TableField>) invokeMethod(sourceTableRequest.getDatasource().getType(), "getTableFields", DatasourceRequest.class, datasourceRequest) : ExcelUtils.getTableFields(datasourceRequest));
} catch (Exception e) {
DEException.throwException("Failed to create table " + toCreateTable + ", " + e.getMessage());
}
@ -539,13 +576,46 @@ public class DatasourceServer implements DatasourceApi {
}
}
List<CoreDatasource> dsList = datasourceMapper.selectList(queryWrapper);
for (CoreDatasource datasource : dsList) {
DatasourceDTO datasourceDTO = new DatasourceDTO();
BeanUtils.copyBean(datasourceDTO, datasource);
if (datasourceDTO.getType().contains(DatasourceConfiguration.DatasourceType.API.toString())) {
List<ApiDefinition> apiDefinitionList = JsonUtil.parseList(datasourceDTO.getConfiguration(), listTypeReference);
int success = 0;
for (ApiDefinition apiDefinition : apiDefinitionList) {
String status = null;
if (StringUtils.isNotEmpty(datasourceDTO.getStatus())) {
JsonNode jsonNode = null;
try {
list.add(convertCoreDatasource(datasource.getId(), false, datasource));
} catch (Exception e) {
e.printStackTrace();
jsonNode = objectMapper.readTree(datasourceDTO.getStatus());
for (JsonNode node : jsonNode) {
if (node.get("name").asText().equals(apiDefinition.getName())) {
status = node.get("status").asText();
}
}
apiDefinition.setStatus(status);
} catch (Exception ignore) {
}
}
if (StringUtils.isNotEmpty(status) && status.equalsIgnoreCase("Success")) {
success++;
}
}
if (success == apiDefinitionList.size()) {
datasourceDTO.setStatus("Success");
} else {
if (success > 0 && success < apiDefinitionList.size()) {
datasourceDTO.setStatus("Warning");
} else {
datasourceDTO.setStatus("Error");
}
}
}
list.add(datasourceDTO);
}
return list;
}
@ -558,7 +628,7 @@ public class DatasourceServer implements DatasourceApi {
@DeLog(id = "#p0", ot = LogOT.DELETE, st = LogST.DATASOURCE)
@Override
public void delete(Long datasourceId) throws DEException {
Objects.requireNonNull(CommonBeanFactory.getBean(DatasourceServer.class)).recursionDel(datasourceId);
Objects.requireNonNull(io.gisbi.utils.CommonBeanFactory.getBean(DatasourceServer.class)).recursionDel(datasourceId);
}
public void recursionDel(Long datasourceId) throws DEException {
@ -597,9 +667,8 @@ public class DatasourceServer implements DatasourceApi {
datasourceTaskServer.deleteByDSId(datasourceId);
}
datasourceMapper.deleteById(datasourceId);
if (!Arrays.asList("API", "Excel", "folder").contains(coreDatasource.getType())) {
if (notFullDs.stream().allMatch(e -> !coreDatasource.getType().contains(e))) {
calciteProvider.delete(coreDatasource);
}
@ -665,6 +734,32 @@ public class DatasourceServer implements DatasourceApi {
datasourceRequest.setDatasource(datasourceDTO);
if (coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.name())) {
List<DatasetTableDTO> datasetTableDTOS = (List<DatasetTableDTO>) invokeMethod(coreDatasource.getType(), "getApiTables", DatasourceRequest.class, datasourceRequest);
return datasetTableDTOS;
}
if (coreDatasource.getType().contains("Excel")) {
return ExcelUtils.getTables(datasourceRequest);
}
Provider provider = ProviderFactory.getProvider(datasourceDTO.getType());
return provider.getTables(datasourceRequest);
}
@Override
public List<DatasetTableDTO> getTableStatus(DatasetTableDTO datasetTableDTO) throws DEException {
CoreDatasource coreDatasource = dataSourceManage.getCoreDatasource(datasetTableDTO.getDatasourceId());
if (coreDatasource == null) {
DEException.throwException("无效数据源！");
}
List<DatasetTableDTO> datasetTableDTOS = new ArrayList<>();
DatasourceDTO datasourceDTO = new DatasourceDTO();
BeanUtils.copyBean(datasourceDTO, coreDatasource);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(datasourceDTO);
if (coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.name())) {
datasetTableDTOS = (List<DatasetTableDTO>) invokeMethod(coreDatasource.getType(), "getApiTables", DatasourceRequest.class, datasourceRequest);
}
if (coreDatasource.getType().equalsIgnoreCase(DatasourceConfiguration.DatasourceType.ExcelRemote.name())) {
datasetTableDTOS = ExcelUtils.getTables(datasourceRequest);
}
datasetTableDTOS.forEach(datasetTableDTO1 -> {
CoreDatasourceTaskLog log = datasourceTaskServer.lastSyncLogForTable(datasetTableDTO.getDatasourceId(), datasetTableDTO1.getTableName());
if (log != null) {
@ -674,12 +769,6 @@ public class DatasourceServer implements DatasourceApi {
});
return datasetTableDTOS;
}
if (coreDatasource.getType().equals("Excel")) {
return ExcelUtils.getTables(datasourceRequest);
}
Provider provider = ProviderFactory.getProvider(datasourceDTO.getType());
return provider.getTables(datasourceRequest);
}
@Override
public List<TableField> getTableField(Map<String, String> req) throws DEException {
@ -693,7 +782,7 @@ public class DatasourceServer implements DatasourceApi {
CoreDatasource coreDatasource = dataSourceManage.getCoreDatasource(Long.parseLong(datasourceId));
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(transDTO(coreDatasource));
if (coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.name()) || coreDatasource.getType().equals("Excel")) {
if (coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.name()) || coreDatasource.getType().contains("Excel")) {
datasourceRequest.setDatasource(transDTO(engineManage.getDeEngine()));
DatasourceSchemaDTO datasourceSchemaDTO = new DatasourceSchemaDTO();
BeanUtils.copyBean(datasourceSchemaDTO, engineManage.getDeEngine());
@ -704,7 +793,7 @@ public class DatasourceServer implements DatasourceApi {
Provider provider = ProviderFactory.getProvider(datasourceSchemaDTO.getType());
List<TableField> tableFields = (List<TableField>) provider.fetchTableField(datasourceRequest);
return tableFields.stream().filter(tableField -> {
return !tableField.getOriginName().equalsIgnoreCase("dataease_uuid");
return !tableField.getOriginName().equalsIgnoreCase("gisbi_uuid");
}).collect(Collectors.toList());
}
@ -757,13 +846,13 @@ public class DatasourceServer implements DatasourceApi {
private static final Integer replace = 0;
private static final Integer append = 1;
public ExcelFileData excelUpload(@RequestParam("file") MultipartFile file, @RequestParam("id") long datasourceId, @RequestParam("editType") Integer editType) throws DEException {
public ExcelFileData uploadFile(@RequestParam("file") MultipartFile file, @RequestParam("id") long datasourceId, @RequestParam("editType") Integer editType) throws DEException {
CoreDatasource coreDatasource = null;
if (ObjectUtils.isNotEmpty(datasourceId) && 0L != datasourceId) {
coreDatasource = dataSourceManage.getCoreDatasource(datasourceId);
}
ExcelUtils excelUtils = new ExcelUtils();
ExcelFileData excelFileData = excelUtils.excelSaveAndParse(file);
ExcelFileData excelFileData = excelUtils.excelSaveAndParse(file, String.valueOf(AuthUtils.getUser().getUserId()));
if (Objects.equals(editType, append)) { //按照excel sheet 名称匹配，替换：0；追加：1
if (coreDatasource != null) {
@ -814,6 +903,39 @@ public class DatasourceServer implements DatasourceApi {
return excelFileData;
}
public ExcelFileData loadRemoteFile(RemoteExcelRequest remoteExcelRequest) throws DEException, IOException {
remoteExcelRequest.setUserName(new String(Base64.getDecoder().decode(remoteExcelRequest.getUserName())));
remoteExcelRequest.setPasswd(new String(Base64.getDecoder().decode(remoteExcelRequest.getPasswd())));
ExcelFileData excelFileData = new ExcelUtils().parseRemoteExcel(remoteExcelRequest);
CoreDatasource coreDatasource = null;
if (ObjectUtils.isNotEmpty(remoteExcelRequest.getDatasourceId()) && 0L != remoteExcelRequest.getDatasourceId()) {
coreDatasource = dataSourceManage.getCoreDatasource(remoteExcelRequest.getDatasourceId());
}
if (coreDatasource != null) {
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(transDTO(coreDatasource));
List<DatasetTableDTO> datasetTableDTOS = ExcelUtils.getTables(datasourceRequest);
for (ExcelSheetData sheet : excelFileData.getSheets()) {
for (DatasetTableDTO datasetTableDTO : datasetTableDTOS) {
if (excelDataTableName(datasetTableDTO.getTableName()).equals(sheet.getTableName())) {
sheet.setDeTableName(datasetTableDTO.getTableName());
}
}
}
}
for (ExcelSheetData sheet : excelFileData.getSheets()) {
for (int i = 0; i < sheet.getFields().size() - 1; i++) {
for (int j = i + 1; j < sheet.getFields().size(); j++) {
if (sheet.getFields().get(i).getName().equalsIgnoreCase(sheet.getFields().get(j).getName())) {
DEException.throwException(sheet.getExcelLabel() + Translator.get("i18n_field_name_repeat") + sheet.getFields().get(i).getName());
}
}
}
}
return excelFileData;
}
private boolean isEqual(List<TableField> newTableFields, List<TableField> oldTableFields) {
if (CollectionUtils.isEmpty(newTableFields) || CollectionUtils.isEmpty(oldTableFields)) {
return false;
@ -868,6 +990,7 @@ public class DatasourceServer implements DatasourceApi {
public ApiDefinition checkApiDatasource(Map<String, String> request) throws DEException {
ApiDefinition apiDefinition = JsonUtil.parseObject(new String(java.util.Base64.getDecoder().decode(request.get("data"))), ApiDefinition.class);
apiDefinition.setType("table");
if (request.keySet().contains("type") && request.get("type").equals("apiStructure")) {
apiDefinition.setShowApiStructure(true);
}
@ -923,6 +1046,8 @@ public class DatasourceServer implements DatasourceApi {
String status = null;
if (coreDatasource.getType().startsWith("API")) {
status = (String) invokeMethod(coreDatasource.getType(), "checkAPIStatus", DatasourceRequest.class, datasourceRequest);
} else if (coreDatasource.getType().startsWith("Excel")) {
status = ExcelUtils.checkStatus(datasourceRequest);
} else {
Provider provider = ProviderFactory.getProvider(coreDatasource.getType());
status = provider.checkStatus(datasourceRequest);
@ -949,6 +1074,7 @@ public class DatasourceServer implements DatasourceApi {
PreviewSqlDTO previewSqlDTO = new PreviewSqlDTO();
previewSqlDTO.setSql(sql);
previewSqlDTO.setDatasourceId(id);
previewSqlDTO.setIsCross(false);
return datasetDataManage.previewSql(previewSqlDTO);
}
@ -980,7 +1106,12 @@ public class DatasourceServer implements DatasourceApi {
CoreDatasource coreDatasource = dataSourceManage.getCoreDatasource(dsId);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setDatasource(transDTO(coreDatasource));
List<DatasetTableDTO> datasetTableDTOS = (List<DatasetTableDTO>) invokeMethod(coreDatasource.getType(), "getApiTables", DatasourceRequest.class, datasourceRequest);
List<DatasetTableDTO> datasetTableDTOS = new ArrayList<>();
if (coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.toString())) {
datasetTableDTOS = (List<DatasetTableDTO>) invokeMethod(coreDatasource.getType(), "getApiTables", DatasourceRequest.class, datasourceRequest);
} else {
datasetTableDTOS = ExcelUtils.getTables(datasourceRequest);
}
for (int i = 0; i < pager.getRecords().size(); i++) {
for (int i1 = 0; i1 < datasetTableDTOS.size(); i1++) {
if (pager.getRecords().get(i).getTableName().equalsIgnoreCase(datasetTableDTOS.get(i1).getTableName())) {
@ -1186,6 +1317,17 @@ public class DatasourceServer implements DatasourceApi {
datasourceDTO.setStatus("Error");
}
}
} else {
if (hidePw) {
Provider provider = ProviderFactory.getProvider(datasourceDTO.getType());
provider.hidePW(datasourceDTO);
}
}
if (datasourceDTO.getType().contains(DatasourceConfiguration.DatasourceType.Excel.toString())) {
datasourceDTO.setFileName(ExcelUtils.getFileName(datasource));
datasourceDTO.setSize(ExcelUtils.getSize(datasource));
}
if (datasourceDTO.getType().equalsIgnoreCase(DatasourceConfiguration.DatasourceType.ExcelRemote.name()) || datasourceDTO.getType().contains(DatasourceConfiguration.DatasourceType.API.toString())) {
CoreDatasourceTask coreDatasourceTask = datasourceTaskServer.selectByDSId(datasourceDTO.getId());
TaskDTO taskDTO = new TaskDTO();
BeanUtils.copyBean(taskDTO, coreDatasourceTask);
@ -1194,15 +1336,6 @@ public class DatasourceServer implements DatasourceApi {
if (task != null) {
datasourceDTO.setLastSyncTime(task.getStartTime());
}
} else {
if (hidePw) {
Provider provider = ProviderFactory.getProvider(datasourceDTO.getType());
provider.hidePW(datasourceDTO);
}
}
if (datasourceDTO.getType().equalsIgnoreCase(DatasourceConfiguration.DatasourceType.Excel.toString())) {
datasourceDTO.setFileName(ExcelUtils.getFileName(datasource));
datasourceDTO.setSize(ExcelUtils.getSize(datasource));
}
datasourceDTO.setConfiguration(RsaUtils.symmetricEncrypt(datasourceDTO.getConfiguration()));
datasourceDTO.setCreator(coreUserManage.getUserName(Long.valueOf(datasourceDTO.getCreateBy())));
@ -1214,7 +1347,7 @@ public class DatasourceServer implements DatasourceApi {
BeanUtils.copyBean(datasourceDTO, coreDatasource);
try {
checkDatasourceStatus(datasourceDTO);
if (!Arrays.asList("API", "Excel", "folder").contains(coreDatasource.getType()) && !coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.name())) {
if (!Arrays.asList("API", "Excel", "folder").contains(coreDatasource.getType()) && !coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.name()) && !coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.ExcelRemote.name())) {
calciteProvider.updateDsPoolAfterCheckStatus(datasourceDTO);
}
} catch (DEException e) {
@ -1294,11 +1427,6 @@ public class DatasourceServer implements DatasourceApi {
Object resObj = null;
try {
Method method = getMethod(dsType, methodName, classes);
if (object instanceof DatasourceRequest && dsType.equals(DataSourceType.APILark.name())) {
Class<?> clazz = Class.forName(DatasourceRequest.class.getName());
Method setToken = clazz.getMethod("setToken", String.class);
setToken.invoke(object, dataSourceManage.getTenantAccessToken());
}
resObj = method.invoke(null, object);
} catch (Exception e) {
DEException.throwException(msg(e));
@ -1312,7 +1440,7 @@ public class DatasourceServer implements DatasourceApi {
if (exception.getCause() == null) {
return exception.getMessage();
}
if (exception instanceof DEException && !(exception.getCause() instanceof DEException)) {
if (exception instanceof DEException && (!(exception.getCause() instanceof DEException) && !(exception.getCause() instanceof InvocationTargetException))) {
return exception.getMessage();
}
exception = exception.getCause();

18
core/core-backend/src/main/java/io/gisbi/datasource/type/H2.java
View File

@ -1,11 +1,29 @@
package io.gisbi.datasource.type;
import io.gisbi.exception.DEException;
import io.gisbi.extensions.datasource.vo.DatasourceConfiguration;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
import java.util.Arrays;
import java.util.List;
@EqualsAndHashCode(callSuper = true)
@Data
@Component("h2")
public class H2 extends DatasourceConfiguration {
private String driver = "org.h2.Driver";
private List<String> illegalParameters = Arrays.asList("INIT", "RUNSCRIPT");
public String getJdbc() {
for (String illegalParameter : illegalParameters) {
if (jdbc.toUpperCase().replace("\\", "").contains(illegalParameter)) {
DEException.throwException("Has illegal parameter: " + jdbc);
}
}
return jdbc;
}
}

16
core/core-backend/src/main/java/io/gisbi/datasource/type/Mysql.java
View File

@ -27,22 +27,24 @@ public class Mysql extends DatasourceConfiguration {
}
return getJdbcUrl();
}
String jdbcUrl = "";
if (StringUtils.isEmpty(extraParams.trim())) {
return "jdbc:mysql://HOSTNAME:PORT/DATABASE"
jdbcUrl = "jdbc:mysql://HOSTNAME:PORT/DATABASE"
.replace("HOSTNAME", getLHost().trim())
.replace("PORT", getLPort().toString().trim())
.replace("DATABASE", getDataBase().trim());
} else {
for (String illegalParameter : illegalParameters) {
if (URLDecoder.decode(getExtraParams()).toLowerCase().contains(illegalParameter.toLowerCase()) || URLDecoder.decode(getExtraParams()).contains(illegalParameter.toLowerCase())) {
DEException.throwException("Illegal parameter: " + illegalParameter);
}
}
return "jdbc:mysql://HOSTNAME:PORT/DATABASE?EXTRA_PARAMS"
jdbcUrl = "jdbc:mysql://HOSTNAME:PORT/DATABASE?EXTRA_PARAMS"
.replace("HOSTNAME", getLHost().trim())
.replace("PORT", getLPort().toString().trim())
.replace("DATABASE", getDataBase().trim())
.replace("EXTRA_PARAMS", getExtraParams().trim());
}
for (String illegalParameter : illegalParameters) {
if (URLDecoder.decode(jdbcUrl).toLowerCase().contains(illegalParameter.toLowerCase()) || URLDecoder.decode(jdbcUrl).contains(illegalParameter.toLowerCase())) {
DEException.throwException("Illegal parameter: " + illegalParameter);
}
}
return jdbcUrl;
}
}

30
core/core-backend/src/main/java/io/gisbi/datasource/type/Pg.java
View File

@ -1,40 +1,58 @@
package io.gisbi.datasource.type;
import io.gisbi.exception.DEException;
import io.gisbi.extensions.datasource.vo.DatasourceConfiguration;
import lombok.Data;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
import java.net.URLDecoder;
import java.util.Arrays;
import java.util.List;
@Data
@Component("pg")
public class Pg extends DatasourceConfiguration {
private String driver = "org.postgresql.Driver";
private String extraParams = "";
private List<String> illegalParameters = Arrays.asList("socketFactory", "socketFactoryArg", "sslfactory", "sslhostnameverifier", "sslpasswordcallback", "authenticationPluginClassName");
public String getJdbc() {
if(StringUtils.isNoneEmpty(getUrlType()) && !getUrlType().equalsIgnoreCase("hostName")){
if (StringUtils.isNoneEmpty(getUrlType()) && !getUrlType().equalsIgnoreCase("hostName")) {
for (String illegalParameter : illegalParameters) {
if (URLDecoder.decode(getJdbcUrl()).contains(illegalParameter)) {
DEException.throwException("Illegal parameter: " + illegalParameter);
}
}
return getJdbcUrl();
}
if(StringUtils.isEmpty(extraParams.trim())){
String jdbcUrl = "";
if (StringUtils.isEmpty(extraParams.trim())) {
if (StringUtils.isEmpty(getSchema())) {
return "jdbc:postgresql://HOSTNAME:PORT/DATABASE"
jdbcUrl = "jdbc:postgresql://HOSTNAME:PORT/DATABASE"
.replace("HOSTNAME", getLHost().trim())
.replace("PORT", getLPort().toString().trim())
.replace("DATABASE", getDataBase().trim());
} else {
return "jdbc:postgresql://HOSTNAME:PORT/DATABASE?currentSchema=SCHEMA"
jdbcUrl = "jdbc:postgresql://HOSTNAME:PORT/DATABASE?currentSchema=SCHEMA"
.replace("HOSTNAME", getLHost().trim())
.replace("PORT", getLPort().toString().trim())
.replace("DATABASE", getDataBase().trim())
.replace("SCHEMA", getSchema().trim());
}
}else {
return "jdbc:postgresql://HOSTNAME:PORT/DATABASE?EXTRA_PARAMS"
} else {
jdbcUrl = "jdbc:postgresql://HOSTNAME:PORT/DATABASE?EXTRA_PARAMS"
.replace("HOSTNAME", getLHost().trim())
.replace("PORT", getLPort().toString().trim())
.replace("DATABASE", getDataBase().trim())
.replace("EXTRA_PARAMS", getExtraParams().trim());
}
for (String illegalParameter : illegalParameters) {
if (URLDecoder.decode(jdbcUrl).toLowerCase().contains(illegalParameter.toLowerCase()) || URLDecoder.decode(jdbcUrl).contains(illegalParameter.toLowerCase())) {
DEException.throwException("Illegal parameter: " + illegalParameter);
}
}
return jdbcUrl;
}
}

21
core/core-backend/src/main/java/io/gisbi/datasource/type/Redshift.java
View File

@ -1,23 +1,40 @@
package io.gisbi.datasource.type;
import io.gisbi.exception.DEException;
import io.gisbi.extensions.datasource.vo.DatasourceConfiguration;
import lombok.Data;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Component;
import java.net.URLDecoder;
import java.util.Arrays;
import java.util.List;
@Data
@Component("redshift")
public class Redshift extends DatasourceConfiguration {
private String driver = "com.amazon.redshift.jdbc42.Driver";
private String extraParams = "";
private List<String> illegalParameters = Arrays.asList("socketFactory", "socketFactoryArg", "sslfactory", "sslhostnameverifier", "sslpasswordcallback", "authenticationPluginClassName");
public String getJdbc() {
if(StringUtils.isNoneEmpty(getUrlType()) && !getUrlType().equalsIgnoreCase("hostName")){
if (StringUtils.isNoneEmpty(getUrlType()) && !getUrlType().equalsIgnoreCase("hostName")) {
for (String illegalParameter : illegalParameters) {
if (URLDecoder.decode(getJdbcUrl()).contains(illegalParameter)) {
DEException.throwException("Illegal parameter: " + illegalParameter);
}
}
return getJdbcUrl();
}
return "jdbc:redshift://HOSTNAME:PORT/DATABASE"
String jdbcUrl = "jdbc:redshift://HOSTNAME:PORT/DATABASE"
.replace("HOSTNAME", getLHost().trim())
.replace("PORT", getLPort().toString().trim())
.replace("DATABASE", getDataBase().trim());
for (String illegalParameter : illegalParameters) {
if (URLDecoder.decode(jdbcUrl).contains(illegalParameter)) {
DEException.throwException("Illegal parameter: " + illegalParameter);
}
}
return jdbcUrl;
}
}

2
core/core-backend/src/main/java/io/gisbi/engine/sql/SQLProvider.java
View File

@ -197,7 +197,7 @@ public class SQLProvider {
}
public static String sqlLimit(String sql, ChartViewDTO view) {
if (StringUtils.equalsIgnoreCase(view.getType(), "table-info")) {
if (StringUtils.equalsAnyIgnoreCase(view.getType(), "table-info", "table-normal")) {
return sql;
}
if (StringUtils.equalsIgnoreCase(view.getResultMode(), "custom")) {

23
core/core-backend/src/main/java/io/gisbi/engine/trans/CustomWhere2Str.java
View File

@ -9,6 +9,7 @@ import io.gisbi.extensions.datasource.dto.DatasetTableFieldDTO;
import io.gisbi.extensions.datasource.dto.DatasourceSchemaDTO;
import io.gisbi.extensions.datasource.model.SQLMeta;
import io.gisbi.extensions.datasource.model.SQLObj;
import io.gisbi.extensions.datasource.vo.DatasourceConfiguration;
import io.gisbi.extensions.view.filter.DynamicTimeSetting;
import io.gisbi.extensions.view.filter.FilterTreeItem;
import io.gisbi.extensions.view.filter.FilterTreeObj;
@ -101,10 +102,10 @@ public class CustomWhere2Str {
if (StringUtils.equalsIgnoreCase(dsType, "es")) {
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getOriginName());
} else {
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getDataeaseName());
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getGisbiName());
}
} else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 3) {
String groupFieldExp = Utils.transGroupFieldToSql(field, originFields);
String groupFieldExp = Utils.transGroupFieldToSql(field, originFields, isCross, dsMap, pluginManage);
// 给计算字段处加一个占位符，后续SQL方言转换后再替换
originName = String.format(SqlPlaceholderConstants.CALC_FIELD_PLACEHOLDER, field.getId());
fieldsDialect.put(originName, groupFieldExp);
@ -115,7 +116,7 @@ public class CustomWhere2Str {
if (StringUtils.equalsIgnoreCase(dsType, "es")) {
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getOriginName());
} else {
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getDataeaseName());
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getGisbiName());
}
}
if (field.getDeType() == 1) {
@ -167,7 +168,9 @@ public class CustomWhere2Str {
if (StringUtils.equalsIgnoreCase(item.getFilterType(), "enum")) {
if (ObjectUtils.isNotEmpty(item.getEnumValue())) {
if ((StringUtils.containsIgnoreCase(field.getType(), "NVARCHAR")
|| StringUtils.containsIgnoreCase(field.getType(), "NCHAR")) && !isCross) {
|| StringUtils.containsIgnoreCase(field.getType(), "NCHAR"))
&& !isCross
&& StringUtils.equalsIgnoreCase(dsType, DatasourceConfiguration.DatasourceType.sqlServer.getType())) {
res = "(" + whereName + " IN (" + item.getEnumValue().stream().map(str -> "'" + SQLConstants.MSSQL_N_PREFIX + str + "'").collect(Collectors.joining(",")) + "))";
} else {
res = "(" + whereName + " IN ('" + String.join("','", item.getEnumValue()) + "'))";
@ -193,14 +196,18 @@ public class CustomWhere2Str {
whereValue = "''";
} else if (StringUtils.containsIgnoreCase(item.getTerm(), "in") || StringUtils.containsIgnoreCase(item.getTerm(), "not in")) {
if ((StringUtils.containsIgnoreCase(field.getType(), "NVARCHAR")
|| StringUtils.containsIgnoreCase(field.getType(), "NCHAR")) && !isCross) {
|| StringUtils.containsIgnoreCase(field.getType(), "NCHAR"))
&& !isCross
&& StringUtils.equalsIgnoreCase(dsType, DatasourceConfiguration.DatasourceType.sqlServer.getType())) {
whereValue = "(" + Arrays.stream(value.split(",")).map(str -> "'" + SQLConstants.MSSQL_N_PREFIX + str + "'").collect(Collectors.joining(",")) + ")";
} else {
whereValue = "('" + String.join("','", value.split(",")) + "')";
}
} else if (StringUtils.containsIgnoreCase(item.getTerm(), "like")) {
if ((StringUtils.containsIgnoreCase(field.getType(), "NVARCHAR")
|| StringUtils.containsIgnoreCase(field.getType(), "NCHAR")) && !isCross) {
|| StringUtils.containsIgnoreCase(field.getType(), "NCHAR"))
&& !isCross
&& StringUtils.equalsIgnoreCase(dsType, DatasourceConfiguration.DatasourceType.sqlServer.getType())) {
whereValue = "'" + SQLConstants.MSSQL_N_PREFIX + "%" + value + "%'";
} else {
whereValue = "'%" + value + "%'";
@ -246,7 +253,9 @@ public class CustomWhere2Str {
}
} else {
if ((StringUtils.containsIgnoreCase(field.getType(), "NVARCHAR")
|| StringUtils.containsIgnoreCase(field.getType(), "NCHAR")) && !isCross) {
|| StringUtils.containsIgnoreCase(field.getType(), "NCHAR"))
&& !isCross
&& StringUtils.equalsIgnoreCase(dsType, DatasourceConfiguration.DatasourceType.sqlServer.getType())) {
whereValue = String.format(SQLConstants.WHERE_VALUE_VALUE_CH, value);
} else {
whereValue = String.format(SQLConstants.WHERE_VALUE_VALUE, value);

6
core/core-backend/src/main/java/io/gisbi/engine/trans/Dimension2SQLObj.java
View File

@ -57,10 +57,10 @@ public class Dimension2SQLObj {
if (StringUtils.equalsIgnoreCase(dsType, "es")) {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), x.getOriginName());
} else {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), x.getDataeaseName());
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), x.getGisbiName());
}
} else if (ObjectUtils.isNotEmpty(x.getExtField()) && Objects.equals(x.getExtField(), ExtFieldConstant.EXT_GROUP)) {
String groupFieldExp = Utils.transGroupFieldToSql(x, originFields);
String groupFieldExp = Utils.transGroupFieldToSql(x, originFields, isCross, dsMap, pluginManage);
// 给计算字段处加一个占位符，后续SQL方言转换后再替换
originField = String.format(SqlPlaceholderConstants.CALC_FIELD_PLACEHOLDER, x.getId());
fieldsDialect.put(originField, groupFieldExp);
@ -71,7 +71,7 @@ public class Dimension2SQLObj {
if (StringUtils.equalsIgnoreCase(dsType, "es")) {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), x.getOriginName());
} else {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), x.getDataeaseName());
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), x.getGisbiName());
}
}
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i);

28
core/core-backend/src/main/java/io/gisbi/engine/trans/ExtWhere2Str.java
View File

@ -9,6 +9,7 @@ import io.gisbi.extensions.datasource.dto.DatasetTableFieldDTO;
import io.gisbi.extensions.datasource.dto.DatasourceSchemaDTO;
import io.gisbi.extensions.datasource.model.SQLMeta;
import io.gisbi.extensions.datasource.model.SQLObj;
import io.gisbi.extensions.datasource.vo.DatasourceConfiguration;
import io.gisbi.extensions.view.dto.ChartExtFilterDTO;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
@ -71,10 +72,10 @@ public class ExtWhere2Str {
if (StringUtils.equalsIgnoreCase(dsType, "es")) {
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getOriginName());
} else {
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getDataeaseName());
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getGisbiName());
}
} else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 3) {
String groupFieldExp = Utils.transGroupFieldToSql(field, originFields);
String groupFieldExp = Utils.transGroupFieldToSql(field, originFields, isCross, dsMap, pluginManage);
// 给计算字段处加一个占位符，后续SQL方言转换后再替换
originName = String.format(SqlPlaceholderConstants.CALC_FIELD_PLACEHOLDER, field.getId());
fieldsDialect.put(originName, groupFieldExp);
@ -85,7 +86,7 @@ public class ExtWhere2Str {
if (StringUtils.equalsIgnoreCase(dsType, "es")) {
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getOriginName());
} else {
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getDataeaseName());
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getGisbiName());
}
}
@ -166,14 +167,16 @@ public class ExtWhere2Str {
break;
}
}
if (hasN && !isCross) {
if (hasN && !isCross && StringUtils.equalsIgnoreCase(dsType, DatasourceConfiguration.DatasourceType.sqlServer.getType())) {
whereValue = "(" + value.stream().map(str -> "'" + SQLConstants.MSSQL_N_PREFIX + str + "'").collect(Collectors.joining(",")) + ")";
} else {
whereValue = "('" + StringUtils.join(value, "','") + "')";
}
} else {
if ((StringUtils.containsIgnoreCase(request.getDatasetTableField().getType(), "NVARCHAR")
|| StringUtils.containsIgnoreCase(request.getDatasetTableField().getType(), "NCHAR")) && !isCross) {
|| StringUtils.containsIgnoreCase(request.getDatasetTableField().getType(), "NCHAR"))
&& !isCross
&& StringUtils.equalsIgnoreCase(dsType, DatasourceConfiguration.DatasourceType.sqlServer.getType())) {
whereValue = "(" + value.stream().map(str -> "'" + SQLConstants.MSSQL_N_PREFIX + str + "'").collect(Collectors.joining(",")) + ")";
} else {
if (request.getDatasetTableField().getDeType() == 2 || request.getDatasetTableField().getDeType() == 3) {
@ -196,14 +199,16 @@ public class ExtWhere2Str {
break;
}
}
if (hasN && !isCross) {
if (hasN && !isCross && StringUtils.equalsIgnoreCase(dsType, DatasourceConfiguration.DatasourceType.sqlServer.getType())) {
whereValue = "'" + SQLConstants.MSSQL_N_PREFIX + "%" + value.get(0) + "%'";
} else {
whereValue = "'%" + value.get(0) + "%'";
}
} else {
if ((StringUtils.containsIgnoreCase(request.getDatasetTableField().getType(), "NVARCHAR")
|| StringUtils.containsIgnoreCase(request.getDatasetTableField().getType(), "NCHAR")) && !isCross) {
|| StringUtils.containsIgnoreCase(request.getDatasetTableField().getType(), "NCHAR"))
&& !isCross
&& StringUtils.equalsIgnoreCase(dsType, DatasourceConfiguration.DatasourceType.sqlServer.getType())) {
whereValue = "'" + SQLConstants.MSSQL_N_PREFIX + "%" + value.get(0) + "%'";
} else {
whereValue = "'%" + value.get(0) + "%'";
@ -248,8 +253,9 @@ public class ExtWhere2Str {
List<DatasetTableFieldDTO> datasetTableFieldList = request.getDatasetTableFieldList();
boolean hasN = false;
for (DatasetTableFieldDTO dto : datasetTableFieldList) {
if (StringUtils.containsIgnoreCase(dto.getType(), "NVARCHAR")
|| StringUtils.containsIgnoreCase(dto.getType(), "NCHAR")) {
if ((StringUtils.containsIgnoreCase(dto.getType(), "NVARCHAR")
|| StringUtils.containsIgnoreCase(dto.getType(), "NCHAR"))
&& StringUtils.equalsIgnoreCase(dsType, DatasourceConfiguration.DatasourceType.sqlServer.getType())) {
hasN = true;
break;
}
@ -261,7 +267,9 @@ public class ExtWhere2Str {
}
} else {
if ((StringUtils.containsIgnoreCase(request.getDatasetTableField().getType(), "NVARCHAR")
|| StringUtils.containsIgnoreCase(request.getDatasetTableField().getType(), "NCHAR")) && !isCross) {
|| StringUtils.containsIgnoreCase(request.getDatasetTableField().getType(), "NCHAR"))
&& !isCross
&& StringUtils.equalsIgnoreCase(dsType, DatasourceConfiguration.DatasourceType.sqlServer.getType())) {
whereValue = String.format(SQLConstants.WHERE_VALUE_VALUE_CH, value.get(0));
} else {
whereValue = String.format(SQLConstants.WHERE_VALUE_VALUE, value.get(0));

6
core/core-backend/src/main/java/io/gisbi/engine/trans/Field2SQLObj.java
View File

@ -61,10 +61,10 @@ public class Field2SQLObj {
if (StringUtils.equalsIgnoreCase(dsType, "es")) {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), x.getOriginName());
} else {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), x.getDataeaseName());
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), x.getGisbiName());
}
} else if (ObjectUtils.isNotEmpty(x.getExtField()) && Objects.equals(x.getExtField(), ExtFieldConstant.EXT_GROUP)) {
String groupFieldExp = Utils.transGroupFieldToSql(x, originFields);
String groupFieldExp = Utils.transGroupFieldToSql(x, originFields, isCross, dsMap, pluginManage);
// 给计算字段处加一个占位符，后续SQL方言转换后再替换
originField = String.format(SqlPlaceholderConstants.CALC_FIELD_PLACEHOLDER, x.getId());
fieldsDialect.put(originField, groupFieldExp);
@ -75,7 +75,7 @@ public class Field2SQLObj {
if (StringUtils.equalsIgnoreCase(dsType, "es")) {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), x.getOriginName());
} else {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), x.getDataeaseName());
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), x.getGisbiName());
}
}
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i);

6
core/core-backend/src/main/java/io/gisbi/engine/trans/Order2SQLObj.java
View File

@ -63,10 +63,10 @@ public class Order2SQLObj {
if (StringUtils.equalsIgnoreCase(dsType, "es")) {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), f.getOriginName());
} else {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), f.getDataeaseName());
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), f.getGisbiName());
}
} else if (ObjectUtils.isNotEmpty(f.getExtField()) && Objects.equals(f.getExtField(), ExtFieldConstant.EXT_GROUP)) {
String groupFieldExp = Utils.transGroupFieldToSql(f, originFields);
String groupFieldExp = Utils.transGroupFieldToSql(f, originFields, isCross, dsMap, pluginManage);
// 给计算字段处加一个占位符，后续SQL方言转换后再替换
originField = String.format(SqlPlaceholderConstants.CALC_FIELD_PLACEHOLDER, f.getId());
fieldsDialect.put(originField, groupFieldExp);
@ -77,7 +77,7 @@ public class Order2SQLObj {
if (StringUtils.equalsIgnoreCase(dsType, "es")) {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), f.getOriginName());
} else {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), f.getDataeaseName());
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), f.getGisbiName());
}
}
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i);

6
core/core-backend/src/main/java/io/gisbi/engine/trans/Quota2SQLObj.java
View File

@ -57,10 +57,10 @@ public class Quota2SQLObj {
if (StringUtils.equalsIgnoreCase(dsType, "es")) {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), y.getOriginName());
} else {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), y.getDataeaseName());
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), y.getGisbiName());
}
} else if (ObjectUtils.isNotEmpty(y.getExtField()) && Objects.equals(y.getExtField(), ExtFieldConstant.EXT_GROUP)) {
String groupFieldExp = Utils.transGroupFieldToSql(y, originFields);
String groupFieldExp = Utils.transGroupFieldToSql(y, originFields, isCross, dsMap, pluginManage);
// 给计算字段处加一个占位符，后续SQL方言转换后再替换
originField = String.format(SqlPlaceholderConstants.CALC_FIELD_PLACEHOLDER, y.getId());
fieldsDialect.put(originField, groupFieldExp);
@ -71,7 +71,7 @@ public class Quota2SQLObj {
if (StringUtils.equalsIgnoreCase(dsType, "es")) {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), y.getOriginName());
} else {
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), y.getDataeaseName());
originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), y.getGisbiName());
}
}
String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i);

25
core/core-backend/src/main/java/io/gisbi/engine/trans/WhereTree2Str.java
View File

@ -1,8 +1,8 @@
package io.gisbi.engine.trans;
import io.gisbi.api.permissions.dataset.dto.DataSetRowPermissionsTreeDTO;
import io.gisbi.engine.constant.ExtFieldConstant;
import io.gisbi.constant.SQLConstants;
import io.gisbi.engine.constant.ExtFieldConstant;
import io.gisbi.engine.utils.Utils;
import io.gisbi.extensions.datasource.api.PluginManageApi;
import io.gisbi.extensions.datasource.constant.SqlPlaceholderConstants;
@ -11,6 +11,7 @@ import io.gisbi.extensions.datasource.dto.DatasetTableFieldDTO;
import io.gisbi.extensions.datasource.dto.DatasourceSchemaDTO;
import io.gisbi.extensions.datasource.model.SQLMeta;
import io.gisbi.extensions.datasource.model.SQLObj;
import io.gisbi.extensions.datasource.vo.DatasourceConfiguration;
import io.gisbi.extensions.view.dto.DatasetRowPermissionsTreeItem;
import io.gisbi.extensions.view.dto.DatasetRowPermissionsTreeObj;
import org.apache.commons.collections4.CollectionUtils;
@ -117,10 +118,10 @@ public class WhereTree2Str {
if (StringUtils.equalsIgnoreCase(dsType, "es")) {
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getOriginName());
} else {
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getDataeaseName());
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getGisbiName());
}
} else if (ObjectUtils.isNotEmpty(field.getExtField()) && Objects.equals(field.getExtField(), ExtFieldConstant.EXT_GROUP)) {
String groupFieldExp = Utils.transGroupFieldToSql(field, originFields);
String groupFieldExp = Utils.transGroupFieldToSql(field, originFields, isCross, dsMap, pluginManage);
// 给计算字段处加一个占位符，后续SQL方言转换后再替换
originName = String.format(SqlPlaceholderConstants.CALC_FIELD_PLACEHOLDER, field.getId());
fieldsDialect.put(originName, groupFieldExp);
@ -131,7 +132,7 @@ public class WhereTree2Str {
if (StringUtils.equalsIgnoreCase(dsType, "es")) {
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getOriginName());
} else {
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getDataeaseName());
originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getGisbiName());
}
}
if (field.getDeType() == 1) {
@ -180,7 +181,9 @@ public class WhereTree2Str {
if (StringUtils.equalsIgnoreCase(item.getFilterType(), "enum")) {
if (CollectionUtils.isNotEmpty(item.getEnumValue())) {
if ((StringUtils.containsIgnoreCase(field.getType(), "NVARCHAR")
|| StringUtils.containsIgnoreCase(field.getType(), "NCHAR")) && !isCross) {
|| StringUtils.containsIgnoreCase(field.getType(), "NCHAR"))
&& !isCross
&& StringUtils.equalsIgnoreCase(dsType, DatasourceConfiguration.DatasourceType.sqlServer.getType())) {
res = "(" + whereName + " IN (" + item.getEnumValue().stream().map(str -> "'" + SQLConstants.MSSQL_N_PREFIX + str + "'").collect(Collectors.joining(",")) + "))";
} else {
res = "(" + whereName + " IN ('" + String.join("','", item.getEnumValue()) + "'))";
@ -205,14 +208,18 @@ public class WhereTree2Str {
whereValue = "''";
} else if (StringUtils.containsIgnoreCase(item.getTerm(), "in") || StringUtils.containsIgnoreCase(item.getTerm(), "not in")) {
if ((StringUtils.containsIgnoreCase(field.getType(), "NVARCHAR")
|| StringUtils.containsIgnoreCase(field.getType(), "NCHAR")) && !isCross) {
|| StringUtils.containsIgnoreCase(field.getType(), "NCHAR"))
&& !isCross
&& StringUtils.equalsIgnoreCase(dsType, DatasourceConfiguration.DatasourceType.sqlServer.getType())) {
whereValue = "(" + Arrays.stream(value.split(",")).map(str -> "'" + SQLConstants.MSSQL_N_PREFIX + str + "'").collect(Collectors.joining(",")) + ")";
} else {
whereValue = "('" + String.join("','", value.split(",")) + "')";
}
} else if (StringUtils.containsIgnoreCase(item.getTerm(), "like")) {
if ((StringUtils.containsIgnoreCase(field.getType(), "NVARCHAR")
|| StringUtils.containsIgnoreCase(field.getType(), "NCHAR")) && !isCross) {
|| StringUtils.containsIgnoreCase(field.getType(), "NCHAR"))
&& !isCross
&& StringUtils.equalsIgnoreCase(dsType, DatasourceConfiguration.DatasourceType.sqlServer.getType())) {
whereValue = "'" + SQLConstants.MSSQL_N_PREFIX + "%" + value + "%'";
} else {
whereValue = "'%" + value + "%'";
@ -256,7 +263,9 @@ public class WhereTree2Str {
}
} else {
if ((StringUtils.containsIgnoreCase(field.getType(), "NVARCHAR")
|| StringUtils.containsIgnoreCase(field.getType(), "NCHAR")) && !isCross) {
|| StringUtils.containsIgnoreCase(field.getType(), "NCHAR"))
&& !isCross
&& StringUtils.equalsIgnoreCase(dsType, DatasourceConfiguration.DatasourceType.sqlServer.getType())) {
whereValue = String.format(SQLConstants.WHERE_VALUE_VALUE_CH, value);
} else {
whereValue = String.format(SQLConstants.WHERE_VALUE_VALUE, value);

44
core/core-backend/src/main/java/io/gisbi/engine/utils/Utils.java
View File

@ -1,7 +1,7 @@
package io.gisbi.engine.utils;
import io.gisbi.engine.constant.ExtFieldConstant;
import io.gisbi.constant.SQLConstants;
import io.gisbi.engine.constant.ExtFieldConstant;
import io.gisbi.exception.DEException;
import io.gisbi.extensions.datasource.api.PluginManageApi;
import io.gisbi.extensions.datasource.constant.SqlPlaceholderConstants;
@ -90,9 +90,9 @@ public class Utils {
// 计算字段允许二次引用，这里递归查询完整引用链
if (Objects.equals(ele.getExtField(), ExtFieldConstant.EXT_NORMAL)) {
if (isCross) {
originField = originField.replaceAll("\\[" + ele.getId() + "]", String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), ele.getDataeaseName()));
originField = originField.replaceAll("\\[" + ele.getId() + "]", String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), ele.getGisbiName()));
} else {
originField = originField.replaceAll("\\[" + ele.getId() + "]", datasourceType.getPrefix() + tableObj.getTableAlias() + datasourceType.getSuffix() + "." + datasourceType.getPrefix() + ele.getDataeaseName() + datasourceType.getSuffix());
originField = originField.replaceAll("\\[" + ele.getId() + "]", datasourceType.getPrefix() + tableObj.getTableAlias() + datasourceType.getSuffix() + "." + datasourceType.getPrefix() + ele.getGisbiName() + datasourceType.getSuffix());
}
} else {
originField = originField.replaceAll("\\[" + ele.getId() + "]", "(" + ele.getOriginName() + ")");
@ -161,6 +161,8 @@ public class Utils {
return "MM" + split + "dd";
case "H_m_s":
return "HH:mm:ss";
case "y_M_d_H":
return "yyyy" + split + "MM" + split + "dd" + " HH";
case "y_M_d_H_m":
return "yyyy" + split + "MM" + split + "dd" + " HH:mm";
case "y_M_d_H_m_s":
@ -496,7 +498,15 @@ public class Utils {
return null;
}
public static String transGroupFieldToSql(DatasetTableFieldDTO dto, List<DatasetTableFieldDTO> fields) {
public static String transGroupFieldToSql(DatasetTableFieldDTO dto, List<DatasetTableFieldDTO> fields, boolean isCross, Map<Long, DatasourceSchemaDTO> dsMap, PluginManageApi pluginManage) {
// 从fields里取最新的dto
for (DatasetTableFieldDTO fieldDTO : fields) {
if (Objects.equals(dto.getId(), fieldDTO.getId())) {
dto.setGroupList(fieldDTO.getGroupList());
dto.setOtherGroup(fieldDTO.getOtherGroup());
break;
}
}
// get origin field
DatasetTableFieldDTO originField = null;
for (DatasetTableFieldDTO ele : fields) {
@ -509,6 +519,22 @@ public class Utils {
DEException.throwException("Field not exists");
}
DsTypeDTO datasourceType = null;
if (dsMap != null && dsMap.entrySet().iterator().hasNext()) {
Map.Entry<Long, DatasourceSchemaDTO> next = dsMap.entrySet().iterator().next();
datasourceType = getDs(pluginManage, next.getValue().getType());
}
if (datasourceType == null) {
DEException.throwException("Datasource not exists");
}
String fieldName;
if (isCross) {
fieldName = originField.getGisbiName();
} else {
fieldName = datasourceType.getPrefix() + originField.getGisbiName() + datasourceType.getSuffix();
}
StringBuilder exp = new StringBuilder();
exp.append(" (CASE ");
if (originField.getDeType() == 0) {
@ -516,7 +542,7 @@ public class Utils {
exp.append(" WHEN ");
for (int i = 0; i < fieldGroupDTO.getText().size(); i++) {
String value = fieldGroupDTO.getText().get(i);
exp.append(originField.getDataeaseName()).append(" = ").append("'").append(transValue(value)).append("'");
exp.append(fieldName).append(" = ").append("'").append(transValue(value)).append("'");
if (i < fieldGroupDTO.getText().size() - 1) {
exp.append(" OR ");
}
@ -526,17 +552,17 @@ public class Utils {
} else if (originField.getDeType() == 1) {
for (FieldGroupDTO fieldGroupDTO : dto.getGroupList()) {
exp.append(" WHEN ");
exp.append(originField.getDataeaseName()).append(" >= ").append("'").append(fieldGroupDTO.getStartTime()).append("'");
exp.append(fieldName).append(" >= ").append("'").append(fieldGroupDTO.getStartTime()).append("'");
exp.append(" AND ");
exp.append(originField.getDataeaseName()).append(" <= ").append("'").append(fieldGroupDTO.getEndTime()).append("'");
exp.append(fieldName).append(" <= ").append("'").append(fieldGroupDTO.getEndTime()).append("'");
exp.append(" THEN '").append(transValue(fieldGroupDTO.getName())).append("'");
}
} else if (originField.getDeType() == 2 || originField.getDeType() == 3 || originField.getDeType() == 4) {
for (FieldGroupDTO fieldGroupDTO : dto.getGroupList()) {
exp.append(" WHEN ");
exp.append(originField.getDataeaseName()).append(StringUtils.equalsIgnoreCase(fieldGroupDTO.getMinTerm(), "le") ? " >= " : " > ").append(fieldGroupDTO.getMin());
exp.append(fieldName).append(StringUtils.equalsIgnoreCase(fieldGroupDTO.getMinTerm(), "le") ? " >= " : " > ").append(fieldGroupDTO.getMin());
exp.append(" AND ");
exp.append(originField.getDataeaseName()).append(StringUtils.equalsIgnoreCase(fieldGroupDTO.getMaxTerm(), "le") ? " <= " : " < ").append(fieldGroupDTO.getMax());
exp.append(fieldName).append(StringUtils.equalsIgnoreCase(fieldGroupDTO.getMaxTerm(), "le") ? " <= " : " < ").append(fieldGroupDTO.getMax());
exp.append(" THEN '").append(transValue(fieldGroupDTO.getName())).append("'");
}
}

6
core/core-backend/src/main/java/io/gisbi/exportCenter/dao/auto/entity/CoreExportTask.java
View File

@ -26,7 +26,7 @@ public class CoreExportTask implements Serializable {
private String fileSizeUnit;
private String exportFrom;
private Long exportFrom;
private String exportStatus;
@ -88,11 +88,11 @@ public class CoreExportTask implements Serializable {
this.fileSizeUnit = fileSizeUnit;
}
public String getExportFrom() {
public Long getExportFrom() {
return exportFrom;
}
public void setExportFrom(String exportFrom) {
public void setExportFrom(Long exportFrom) {
this.exportFrom = exportFrom;
}

4
core/core-backend/src/main/java/io/gisbi/exportCenter/manage/ExportCenterLimitManage.java
View File

@ -7,10 +7,10 @@ import org.springframework.stereotype.Component;
@Component("exportCenterLimitManage")
public class ExportCenterLimitManage {
@Value("${dataease.export.dataset.limit:100000}")
@Value("${gisbi.export.dataset.limit:100000}")
private Long datasetLimit;
@Value("${dataease.export.views.limit:100000}")
@Value("${gisbi.export.views.limit:100000}")
private Long viewLimit;
public Long getExportLimit(String type) {

680
core/core-backend/src/main/java/io/gisbi/exportCenter/manage/ExportCenterManage.java
View File

@ -1,81 +1,46 @@
package io.gisbi.exportCenter.manage;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.fasterxml.jackson.core.type.TypeReference;
import com.google.gson.Gson;
import io.gisbi.api.chart.dto.ViewDetailField;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import io.gisbi.api.chart.request.ChartExcelRequest;
import io.gisbi.api.chart.request.ChartExcelRequestInner;
import io.gisbi.api.dataset.dto.DataSetExportRequest;
import io.gisbi.api.dataset.union.DatasetGroupInfoDTO;
import io.gisbi.api.dataset.union.UnionDTO;
import io.gisbi.api.export.BaseExportApi;
import io.gisbi.api.xpack.dataFilling.DataFillingApi;
import io.gisbi.api.xpack.dataFilling.dto.DataFillFormTableDataRequest;
import io.gisbi.auth.bo.TokenUserBO;
import io.gisbi.chart.dao.auto.mapper.CoreChartViewMapper;
import io.gisbi.chart.server.ChartDataServer;
import io.gisbi.commons.utils.ExcelWatermarkUtils;
import io.gisbi.dataset.dao.auto.entity.CoreDatasetGroup;
import io.gisbi.dataset.dao.auto.mapper.CoreDatasetGroupMapper;
import io.gisbi.constant.LogOT;
import io.gisbi.constant.LogST;
import io.gisbi.dataset.manage.*;
import io.gisbi.datasource.utils.DatasourceUtils;
import io.gisbi.constant.DeTypeConstants;
import io.gisbi.engine.sql.SQLProvider;
import io.gisbi.engine.trans.Field2SQLObj;
import io.gisbi.engine.trans.Order2SQLObj;
import io.gisbi.engine.trans.Table2SQLObj;
import io.gisbi.engine.trans.WhereTree2Str;
import io.gisbi.engine.utils.Utils;
import io.gisbi.exception.DEException;
import io.gisbi.exportCenter.dao.auto.entity.CoreExportDownloadTask;
import io.gisbi.exportCenter.dao.auto.entity.CoreExportTask;
import io.gisbi.exportCenter.dao.auto.mapper.CoreExportDownloadTaskMapper;
import io.gisbi.exportCenter.dao.auto.mapper.CoreExportTaskMapper;
import io.gisbi.exportCenter.util.ExportCenterUtils;
import io.gisbi.extensions.datasource.api.PluginManageApi;
import io.gisbi.extensions.datasource.dto.DatasetTableFieldDTO;
import io.gisbi.extensions.datasource.dto.DatasourceRequest;
import io.gisbi.extensions.datasource.dto.DatasourceSchemaDTO;
import io.gisbi.extensions.datasource.factory.ProviderFactory;
import io.gisbi.extensions.datasource.model.SQLMeta;
import io.gisbi.extensions.datasource.provider.Provider;
import io.gisbi.extensions.view.dto.*;
import io.gisbi.i18n.Translator;
import io.gisbi.exportCenter.dao.ext.mapper.ExportTaskExtMapper;
import io.gisbi.log.DeLog;
import io.gisbi.model.ExportTaskDTO;
import io.gisbi.system.manage.CoreUserManage;
import io.gisbi.system.manage.SysParameterManage;
import io.gisbi.utils.*;
import io.gisbi.visualization.dao.auto.entity.VisualizationWatermark;
import io.gisbi.visualization.dao.auto.mapper.VisualizationWatermarkMapper;
import io.gisbi.visualization.dao.ext.mapper.ExtDataVisualizationMapper;
import io.gisbi.visualization.server.DataVisualizationServer;
import io.gisbi.websocket.WsMessage;
import io.gisbi.websocket.WsService;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.Resource;
import jakarta.servlet.http.HttpServletResponse;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import lombok.Data;
import org.apache.commons.lang3.StringUtils;
import org.apache.poi.ss.usermodel.*;
import org.apache.poi.xssf.streaming.SXSSFWorkbook;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Transactional;
import io.gisbi.visualization.dto.WatermarkContentDTO;
import io.gisbi.api.permissions.user.vo.UserFormVO;
import java.io.*;
import java.net.InetAddress;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.text.DecimalFormat;
import java.util.*;
import java.util.concurrent.Future;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@Component
@Transactional(rollbackFor = Exception.class)
@ -83,55 +48,30 @@ public class ExportCenterManage implements BaseExportApi {
@Resource
private CoreExportTaskMapper exportTaskMapper;
@Resource
private CoreExportDownloadTaskMapper coreExportDownloadTaskMapper;
@Resource
private ExportTaskExtMapper exportTaskExtMapper;
@Resource
private DatasetGroupManage datasetGroupManage;
@Resource
DataVisualizationServer dataVisualizationServer;
@Resource
private CoreChartViewMapper coreChartViewMapper;
@Autowired
private WsService wsService;
@Autowired(required = false)
private PluginManageApi pluginManage;
private ExportCenterDownLoadManage exportCenterDownLoadManage;
@Resource
private SysParameterManage sysParameterManage;
@Value("${dataease.export.core.size:10}")
@Value("${gisbi.export.core.size:10}")
private int core;
@Value("${dataease.export.max.size:10}")
@Value("${gisbi.export.max.size:10}")
private int max;
private final static String DATA_URL_TITLE = "data:image/jpeg;base64,";
@Value("${gisbi.path.exportData:/opt/gisbi/data/exportData/}")
@Value("${gisbi.path.exportData:/opt/gisbi2.0/data/exportData/}")
private String exportData_path;
@Resource
private VisualizationWatermarkMapper watermarkMapper;
@Resource
private ExtDataVisualizationMapper visualizationMapper;
public Integer getExtractPageSize() {
return extractPageSize;
}
@Value("${dataease.export.page.size:50000}")
private Integer extractPageSize;
static private List<String> STATUS = Arrays.asList("SUCCESS", "FAILED", "PENDING", "IN_PROGRESS", "ALL");
private ScheduledThreadPoolExecutor scheduledThreadPoolExecutor;
private int keepAliveSeconds = 600;
private Map<String, Future> Running_Task = new HashMap<>();
@Resource
private ChartDataServer chartDataServer;
@Resource
private CoreDatasetGroupMapper coreDatasetGroupMapper;
@Resource
private CoreUserManage coreUserManage;
@Resource
private DatasetSQLManage datasetSQLManage;
@Resource
private DatasetTableFieldManage datasetTableFieldManage;
@Resource
private DatasetDataManage datasetDataManage;
private final Long sheetLimit = 1000000L;
@Autowired(required = false)
private DataFillingApi dataFillingApi = null;
@ -140,56 +80,12 @@ public class ExportCenterManage implements BaseExportApi {
}
@PostConstruct
public void init() {
scheduledThreadPoolExecutor = new ScheduledThreadPoolExecutor(core);
scheduledThreadPoolExecutor.setKeepAliveTime(keepAliveSeconds, TimeUnit.SECONDS);
scheduledThreadPoolExecutor.setMaximumPoolSize(max);
}
@Scheduled(fixedRate = 5000)
public void checkRunningTask() {
Iterator<Map.Entry<String, Future>> iterator = Running_Task.entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry<String, Future> entry = iterator.next();
if (entry.getValue().isDone()) {
iterator.remove();
try {
CoreExportTask exportTask = exportTaskMapper.selectById(entry.getKey());
ExportTaskDTO exportTaskDTO = new ExportTaskDTO();
BeanUtils.copyBean(exportTaskDTO, exportTask);
setExportFromName(exportTaskDTO);
WsMessage message = new WsMessage(exportTask.getUserId(), "/task-export-topic", exportTaskDTO);
wsService.releaseMessage(message);
} catch (Exception e) {
}
}
}
}
public void download(String id, HttpServletResponse response) throws Exception {
if (coreExportDownloadTaskMapper.selectById(id) == null) {
DEException.throwException("任务不存在");
}
CoreExportTask exportTask = exportTaskMapper.selectById(id);
OutputStream outputStream = response.getOutputStream();
response.setContentType("application/vnd.ms-excel");
response.setHeader("Content-disposition", "attachment;filename=" + URLEncoder.encode(exportTask.getFileName(), StandardCharsets.UTF_8));
InputStream fileInputStream;
if (exportTask.getExportTime() < 1730277243491L) {
fileInputStream = new FileInputStream(exportData_path + id + "/" + exportTask.getFileName());
} else {
fileInputStream = new FileInputStream(exportData_path + id + "/" + id + ".xlsx");
}
byte[] buffer = new byte[4096];
int bytesRead;
while ((bytesRead = fileInputStream.read(buffer)) != -1) {
outputStream.write(buffer, 0, bytesRead);
}
outputStream.flush();
outputStream.close();
fileInputStream.close();
response.flushBuffer();
exportCenterDownLoadManage.download(exportTask, response);
}
public void delete(String id) {
@ -247,39 +143,69 @@ public class ExportCenterManage implements BaseExportApi {
FileUtils.deleteDirectoryRecursively(exportData_path + id);
if (exportTask.getExportFromType().equalsIgnoreCase("chart")) {
ChartExcelRequest request = JsonUtil.parseObject(exportTask.getParams(), ChartExcelRequest.class);
startViewTask(exportTask, request);
exportCenterDownLoadManage.startViewTask(exportTask, request);
}
if (exportTask.getExportFromType().equalsIgnoreCase("dataset")) {
DataSetExportRequest request = JsonUtil.parseObject(exportTask.getParams(), DataSetExportRequest.class);
startDatasetTask(exportTask, request);
exportCenterDownLoadManage.startDatasetTask(exportTask, request);
}
if (exportTask.getExportFromType().equalsIgnoreCase("data_filling")) {
HashMap request = JsonUtil.parseObject(exportTask.getParams(), HashMap.class);
startDataFillingTask(exportTask, request);
exportCenterDownLoadManage.startDataFillingTask(exportTask, request);
}
}
public List<ExportTaskDTO> exportTasks(String status) {
public IPage<ExportTaskDTO> pager(Page<ExportTaskDTO> page, String status) {
if (!STATUS.contains(status)) {
DEException.throwException("Invalid status: " + status);
}
QueryWrapper<CoreExportTask> queryWrapper = new QueryWrapper<>();
queryWrapper.eq("user_id", AuthUtils.getUser().getUserId());
if (!status.equalsIgnoreCase("ALL")) {
queryWrapper.eq("export_status", status);
}
queryWrapper.orderByDesc("export_time");
List<CoreExportTask> exportTasks = exportTaskMapper.selectList(queryWrapper);
List<ExportTaskDTO> result = new ArrayList<>();
exportTasks.forEach(exportTask -> {
ExportTaskDTO exportTaskDTO = new ExportTaskDTO();
BeanUtils.copyBean(exportTaskDTO, exportTask);
if (status.equalsIgnoreCase("ALL") || status.equalsIgnoreCase(exportTaskDTO.getExportStatus())) {
setExportFromAbsName(exportTaskDTO);
IPage<ExportTaskDTO> pager = exportTaskExtMapper.pager(page, queryWrapper);
List<ExportTaskDTO> records = pager.getRecords();
records.forEach(exportTask -> {
if (status.equalsIgnoreCase("ALL") || status.equalsIgnoreCase(exportTask.getExportStatus())) {
setExportFromAbsName(exportTask);
}
if (status.equalsIgnoreCase("ALL") || status.equalsIgnoreCase(exportTaskDTO.getExportStatus())) {
proxy().setOrg(exportTaskDTO);
if (status.equalsIgnoreCase("ALL") || status.equalsIgnoreCase(exportTask.getExportStatus())) {
proxy().setOrg(exportTask);
}
result.add(exportTaskDTO);
});
return pager;
}
public Map<String, Long> exportTasks() {
Map<String, Long> result = new HashMap<>();
QueryWrapper<CoreExportTask> queryWrapper = new QueryWrapper<>();
queryWrapper.eq("user_id", AuthUtils.getUser().getUserId());
queryWrapper.eq("export_status", "IN_PROGRESS");
result.put("IN_PROGRESS", exportTaskMapper.selectCount(queryWrapper));
queryWrapper.clear();
queryWrapper.eq("user_id", AuthUtils.getUser().getUserId());
queryWrapper.eq("export_status", "SUCCESS");
result.put("SUCCESS", exportTaskMapper.selectCount(queryWrapper));
queryWrapper.clear();
queryWrapper.eq("user_id", AuthUtils.getUser().getUserId());
queryWrapper.eq("export_status", "FAILED");
result.put("FAILED", exportTaskMapper.selectCount(queryWrapper));
queryWrapper.clear();
queryWrapper.eq("user_id", AuthUtils.getUser().getUserId());
queryWrapper.eq("export_status", "PENDING");
result.put("PENDING", exportTaskMapper.selectCount(queryWrapper));
queryWrapper.clear();
queryWrapper.eq("user_id", AuthUtils.getUser().getUserId());
result.put("ALL", exportTaskMapper.selectCount(queryWrapper));
return result;
}
@ -311,18 +237,6 @@ public class ExportCenterManage implements BaseExportApi {
}
}
private void setExportFromName(ExportTaskDTO exportTaskDTO) {
if (exportTaskDTO.getExportFromType().equalsIgnoreCase("chart")) {
exportTaskDTO.setExportFromName(coreChartViewMapper.selectById(exportTaskDTO.getExportFrom()).getTitle());
}
if (exportTaskDTO.getExportFromType().equalsIgnoreCase("dataset")) {
exportTaskDTO.setExportFromName(coreDatasetGroupMapper.selectById(exportTaskDTO.getExportFrom()).getName());
}
if (exportTaskDTO.getExportFromType().equalsIgnoreCase("data_filling")) {
exportTaskDTO.setExportFromName(getDataFillingApi().get(Long.parseLong(exportTaskDTO.getExportFrom())).getName());
}
}
private String hostName() {
String hostname = null;
try {
@ -334,11 +248,11 @@ public class ExportCenterManage implements BaseExportApi {
return hostname;
}
public void addTask(String exportFrom, String exportFromType, ChartExcelRequest request) {
public void addTask(String exportFrom, String exportFromType, ChartExcelRequest request, String busiFlag) {
CoreExportTask exportTask = new CoreExportTask();
exportTask.setId(UUID.randomUUID().toString());
exportTask.setId(IDUtils.snowID().toString());
exportTask.setUserId(AuthUtils.getUser().getUserId());
exportTask.setExportFrom(exportFrom);
exportTask.setExportFrom(Long.valueOf(exportFrom));
exportTask.setExportFromType(exportFromType);
exportTask.setExportStatus("PENDING");
exportTask.setFileName(request.getViewName() + ".xlsx");
@ -347,15 +261,20 @@ public class ExportCenterManage implements BaseExportApi {
exportTask.setParams(JsonUtil.toJSONString(request).toString());
exportTask.setExportMachineName(hostName());
exportTaskMapper.insert(exportTask);
startViewTask(exportTask, request);
if (busiFlag.equalsIgnoreCase("dashboard")) {
exportCenterDownLoadManage.startPanelViewTask(exportTask, request);
} else {
exportCenterDownLoadManage.startDataVViewTask(exportTask, request);
}
}
public void addTask(Long exportFrom, String exportFromType, DataSetExportRequest request) throws Exception {
datasetGroupManage.getDatasetGroupInfoDTO(exportFrom, null);
CoreExportTask exportTask = new CoreExportTask();
exportTask.setId(UUID.randomUUID().toString());
exportTask.setId(IDUtils.snowID().toString());
exportTask.setUserId(AuthUtils.getUser().getUserId());
exportTask.setExportFrom(String.valueOf(exportFrom));
exportTask.setExportFrom(exportFrom);
exportTask.setExportFromType(exportFromType);
exportTask.setExportStatus("PENDING");
exportTask.setFileName(request.getFilename() + ".xlsx");
@ -364,16 +283,16 @@ public class ExportCenterManage implements BaseExportApi {
exportTask.setParams(JsonUtil.toJSONString(request).toString());
exportTask.setExportMachineName(hostName());
exportTaskMapper.insert(exportTask);
startDatasetTask(exportTask, request);
exportCenterDownLoadManage.startDatasetTask(exportTask, request);
}
@Override
public void addTask(String exportFromId, String exportFromType, HashMap<String, Object> request, Long userId, Long org) {
CoreExportTask exportTask = new CoreExportTask();
request.put("org", org);
exportTask.setId(UUID.randomUUID().toString());
exportTask.setId(IDUtils.snowID().toString());
exportTask.setUserId(userId);
exportTask.setExportFrom(exportFromId);
exportTask.setExportFrom(Long.valueOf(exportFromId));
exportTask.setExportFromType(exportFromType);
exportTask.setExportStatus("PENDING");
exportTask.setFileName(request.get("name") + ".xlsx");
@ -383,399 +302,10 @@ public class ExportCenterManage implements BaseExportApi {
exportTask.setExportMachineName(hostName());
exportTaskMapper.insert(exportTask);
if (StringUtils.equals(exportFromType, "data_filling")) {
startDataFillingTask(exportTask, request);
exportCenterDownLoadManage.startDataFillingTask(exportTask, request);
}
}
private void startDataFillingTask(CoreExportTask exportTask, HashMap<String, Object> request) {
if (ObjectUtils.isEmpty(getDataFillingApi())) {
return;
}
String dataPath = exportData_path + exportTask.getId();
File directory = new File(dataPath);
boolean isCreated = directory.mkdir();
TokenUserBO tokenUserBO = AuthUtils.getUser();
Future future = scheduledThreadPoolExecutor.submit(() -> {
AuthUtils.setUser(tokenUserBO);
try {
exportTask.setExportStatus("IN_PROGRESS");
exportTaskMapper.updateById(exportTask);
getDataFillingApi().writeExcel(dataPath + "/" + exportTask.getId() + ".xlsx", new DataFillFormTableDataRequest().setId(Long.parseLong(exportTask.getExportFrom())).setWithoutLogs(true), exportTask.getUserId(), Long.parseLong(request.get("org").toString()));
exportTask.setExportProgress("100");
exportTask.setExportStatus("SUCCESS");
setFileSize(dataPath + "/" + exportTask.getId() + ".xlsx", exportTask);
} catch (Exception e) {
exportTask.setMsg(e.getMessage());
LogUtil.error("Failed to export data", e);
exportTask.setExportStatus("FAILED");
} finally {
exportTaskMapper.updateById(exportTask);
}
});
Running_Task.put(exportTask.getId(), future);
}
private void startDatasetTask(CoreExportTask exportTask, DataSetExportRequest request) {
String dataPath = exportData_path + exportTask.getId();
File directory = new File(dataPath);
boolean isCreated = directory.mkdir();
TokenUserBO tokenUserBO = AuthUtils.getUser();
Future future = scheduledThreadPoolExecutor.submit(() -> {
AuthUtils.setUser(tokenUserBO);
try {
exportTask.setExportStatus("IN_PROGRESS");
exportTaskMapper.updateById(exportTask);
CoreDatasetGroup coreDatasetGroup = coreDatasetGroupMapper.selectById(exportTask.getExportFrom());
if (coreDatasetGroup == null) {
throw new Exception("Not found dataset group: " + exportTask.getExportFrom());
}
DatasetGroupInfoDTO dto = new DatasetGroupInfoDTO();
BeanUtils.copyBean(dto, coreDatasetGroup);
dto.setUnionSql(null);
List<UnionDTO> unionDTOList = JsonUtil.parseList(coreDatasetGroup.getInfo(), new TypeReference<>() {
});
dto.setUnion(unionDTOList);
List<DatasetTableFieldDTO> dsFields = datasetTableFieldManage.selectByDatasetGroupId(Long.valueOf(exportTask.getExportFrom()));
List<DatasetTableFieldDTO> allFields = dsFields.stream().map(ele -> {
DatasetTableFieldDTO datasetTableFieldDTO = new DatasetTableFieldDTO();
BeanUtils.copyBean(datasetTableFieldDTO, ele);
datasetTableFieldDTO.setFieldShortName(ele.getDataeaseName());
return datasetTableFieldDTO;
}).collect(Collectors.toList());
Map<String, Object> sqlMap = datasetSQLManage.getUnionSQLForEdit(dto, null);
String sql = (String) sqlMap.get("sql");
if (ObjectUtils.isEmpty(allFields)) {
DEException.throwException(Translator.get("i18n_no_fields"));
}
Map<String, ColumnPermissionItem> desensitizationList = new HashMap<>();
// allFields = permissionManage.filterColumnPermissions(allFields, desensitizationList, dto.getId(), null);
if (ObjectUtils.isEmpty(allFields)) {
DEException.throwException(Translator.get("i18n_no_column_permission"));
}
dto.setAllFields(allFields);
datasetDataManage.buildFieldName(sqlMap, allFields);
Map<Long, DatasourceSchemaDTO> dsMap = (Map<Long, DatasourceSchemaDTO>) sqlMap.get("dsMap");
DatasourceUtils.checkDsStatus(dsMap);
List<String> dsList = new ArrayList<>();
for (Map.Entry<Long, DatasourceSchemaDTO> next : dsMap.entrySet()) {
dsList.add(next.getValue().getType());
}
boolean needOrder = Utils.isNeedOrder(dsList);
boolean crossDs = Utils.isCrossDs(dsMap);
if (!crossDs) {
if (datasetDataManage.notFullDs.contains(dsMap.entrySet().iterator().next().getValue().getType()) && (boolean) sqlMap.get("isFullJoin")) {
DEException.throwException(Translator.get("i18n_not_full"));
}
sql = Utils.replaceSchemaAlias(sql, dsMap);
}
// List<DataSetRowPermissionsTreeDTO> rowPermissionsTree = new ArrayList<>();
// TokenUserBO user = AuthUtils.getUser();
// if (user != null) {
// rowPermissionsTree = permissionManage.getRowPermissionsTree(dto.getId(), user.getUserId());
// }
// if (StringUtils.isNotEmpty(request.getExpressionTree())) {
// Gson gson = new Gson();
// DatasetRowPermissionsTreeObj datasetRowPermissionsTreeObj = JsonUtil.parseObject(request.getExpressionTree(), DatasetRowPermissionsTreeObj.class);
// permissionManage.getField(datasetRowPermissionsTreeObj);
// DataSetRowPermissionsTreeDTO dataSetRowPermissionsTreeDTO = new DataSetRowPermissionsTreeDTO();
// dataSetRowPermissionsTreeDTO.setTree(datasetRowPermissionsTreeObj);
// dataSetRowPermissionsTreeDTO.setExportData(true);
// rowPermissionsTree.add(dataSetRowPermissionsTreeDTO);
// }
Provider provider;
if (crossDs) {
provider = ProviderFactory.getDefaultProvider();
} else {
provider = ProviderFactory.getProvider(dsList.getFirst());
}
SQLMeta sqlMeta = new SQLMeta();
Table2SQLObj.table2sqlobj(sqlMeta, null, "(" + sql + ")", crossDs);
Field2SQLObj.field2sqlObj(sqlMeta, allFields, allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage);
// WhereTree2Str.transFilterTrees(sqlMeta, rowPermissionsTree, allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage);
Order2SQLObj.getOrders(sqlMeta, dto.getSortFields(), allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage);
String replaceSql = provider.rebuildSQL(SQLProvider.createQuerySQL(sqlMeta, false, false, false), sqlMeta, crossDs, dsMap);
Long totalCount = datasetDataManage.getDatasetTotal(dto, replaceSql, null);
Long curLimit = ExportCenterUtils.getExportLimit("dataset");
totalCount = totalCount > curLimit ? curLimit : totalCount;
Long sheetCount = (totalCount / sheetLimit) + (totalCount % sheetLimit > 0 ? 1 : 0);
Workbook wb = new SXSSFWorkbook();
CellStyle cellStyle = wb.createCellStyle();
Font font = wb.createFont();
font.setFontHeightInPoints((short) 12);
font.setBold(true);
cellStyle.setFont(font);
cellStyle.setFillForegroundColor(IndexedColors.GREY_25_PERCENT.getIndex());
cellStyle.setFillPattern(FillPatternType.SOLID_FOREGROUND);
for (Long s = 1L; s < sheetCount + 1; s++) {
Long sheetSize;
if (s.equals(sheetCount)) {
sheetSize = totalCount - (s - 1) * sheetLimit;
} else {
sheetSize = sheetLimit;
}
Long pageSize = (sheetSize / extractPageSize) + (sheetSize % extractPageSize > 0 ? 1 : 0);
Sheet detailsSheet = null;
List<List<String>> details = new ArrayList<>();
for (Long p = 0L; p < pageSize; p++) {
int beforeCount = (int) ((s - 1) * sheetLimit);
String querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, false, beforeCount + p.intValue() * extractPageSize, extractPageSize);
if (pageSize == 1) {
querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, false, 0, sheetSize.intValue());
}
querySQL = provider.rebuildSQL(querySQL, sqlMeta, crossDs, dsMap);
DatasourceRequest datasourceRequest = new DatasourceRequest();
datasourceRequest.setQuery(querySQL);
datasourceRequest.setDsList(dsMap);
Map<String, Object> previewData = datasetDataManage.buildPreviewData(provider.fetchResultField(datasourceRequest), allFields, desensitizationList);
List<Map<String, Object>> data = (List<Map<String, Object>>) previewData.get("data");
if (p.equals(0L)) {
detailsSheet = wb.createSheet("数据" + s);
List<String> header = new ArrayList<>();
for (DatasetTableFieldDTO field : allFields) {
header.add(field.getName());
}
details.add(header);
for (Map<String, Object> obj : data) {
List<String> row = new ArrayList<>();
for (DatasetTableFieldDTO field : allFields) {
String string = (String) obj.get(field.getDataeaseName());
row.add(string);
}
details.add(row);
}
if (CollectionUtils.isNotEmpty(details)) {
for (int i = 0; i < details.size(); i++) {
Row row = detailsSheet.createRow(i);
List<String> rowData = details.get(i);
if (rowData != null) {
for (int j = 0; j < rowData.size(); j++) {
Cell cell = row.createCell(j);
if (i == 0) {
cell.setCellValue(rowData.get(j));
cell.setCellStyle(cellStyle);
detailsSheet.setColumnWidth(j, 255 * 20);
} else {
if ((allFields.get(j).getDeType().equals(DeTypeConstants.DE_INT) || allFields.get(j).getDeType() == DeTypeConstants.DE_FLOAT) && StringUtils.isNotEmpty(rowData.get(j))) {
try {
cell.setCellValue(Double.valueOf(rowData.get(j)));
} catch (Exception e) {
cell.setCellValue(rowData.get(j));
}
} else {
cell.setCellValue(rowData.get(j));
}
}
}
}
}
}
} else {
details.clear();
for (Map<String, Object> obj : data) {
List<String> row = new ArrayList<>();
for (DatasetTableFieldDTO field : allFields) {
String string = (String) obj.get(field.getDataeaseName());
row.add(string);
}
details.add(row);
}
int lastNum = detailsSheet.getLastRowNum();
for (int i = 0; i < details.size(); i++) {
Row row = detailsSheet.createRow(i + lastNum + 1);
List<String> rowData = details.get(i);
if (rowData != null) {
for (int j = 0; j < rowData.size(); j++) {
Cell cell = row.createCell(j);
cell.setCellValue(rowData.get(j));
}
}
}
}
exportTask.setExportStatus("IN_PROGRESS");
double exportRogress2 = (double) ((double) s - 1) / ((double) sheetCount);
double exportRogress = (double) ((double) (p + 1) / (double) pageSize) * ((double) 1 / sheetCount);
DecimalFormat df = new DecimalFormat("#.##");
String formattedResult = df.format((exportRogress + exportRogress2) * 100);
exportTask.setExportProgress(formattedResult);
exportTaskMapper.updateById(exportTask);
}
}
this.addWatermarkTools(wb);
FileOutputStream fileOutputStream = new FileOutputStream(dataPath + "/" + exportTask.getId() + ".xlsx");
wb.write(fileOutputStream);
fileOutputStream.flush();
fileOutputStream.close();
wb.close();
exportTask.setExportProgress("100");
exportTask.setExportStatus("SUCCESS");
setFileSize(dataPath + "/" + exportTask.getId() + ".xlsx", exportTask);
} catch (Exception e) {
LogUtil.error("Failed to export data", e);
exportTask.setMsg(e.getMessage());
exportTask.setExportStatus("FAILED");
} finally {
exportTaskMapper.updateById(exportTask);
}
});
Running_Task.put(exportTask.getId(), future);
}
private void startViewTask(CoreExportTask exportTask, ChartExcelRequest request) {
String dataPath = exportData_path + exportTask.getId();
File directory = new File(dataPath);
boolean isCreated = directory.mkdir();
TokenUserBO tokenUserBO = AuthUtils.getUser();
Future future = scheduledThreadPoolExecutor.submit(() -> {
// LicenseUtil.validate();
AuthUtils.setUser(tokenUserBO);
try {
exportTask.setExportStatus("IN_PROGRESS");
exportTaskMapper.updateById(exportTask);
Workbook wb = new SXSSFWorkbook();
CellStyle cellStyle = wb.createCellStyle();
Font font = wb.createFont();
font.setFontHeightInPoints((short) 12);
font.setBold(true);
cellStyle.setFont(font);
cellStyle.setFillForegroundColor(IndexedColors.GREY_25_PERCENT.getIndex());
cellStyle.setFillPattern(FillPatternType.SOLID_FOREGROUND);
List<Object[]> details = new ArrayList<>();
Sheet detailsSheet;
Integer sheetIndex = 1;
if ("dataset".equals(request.getDownloadType()) || request.getViewInfo().getType().equalsIgnoreCase("table-info")) {
request.getViewInfo().getChartExtRequest().setPageSize(Long.valueOf(extractPageSize));
ChartViewDTO chartViewDTO = chartDataServer.findExcelData(request);
for (long i = 1; i < chartViewDTO.getTotalPage() + 1; i++) {
request.getViewInfo().getChartExtRequest().setGoPage(i);
chartDataServer.findExcelData(request);
details.addAll(request.getDetails());
if (((details.size() + extractPageSize) > sheetLimit) || i == chartViewDTO.getTotalPage()) {
detailsSheet = wb.createSheet("数据" + sheetIndex);
Integer[] excelTypes = request.getExcelTypes();
details.add(0, request.getHeader());
ViewDetailField[] detailFields = request.getDetailFields();
Object[] header = request.getHeader();
ChartDataServer.setExcelData(detailsSheet, cellStyle, header, details, detailFields, excelTypes, request.getViewInfo(), wb);
sheetIndex++;
details.clear();
exportTask.setExportStatus("IN_PROGRESS");
double exportRogress = (double) (i / (chartViewDTO.getTotalPage() + 1));
DecimalFormat df = new DecimalFormat("#.##");
String formattedResult = df.format((exportRogress) * 100);
exportTask.setExportProgress(formattedResult);
exportTaskMapper.updateById(exportTask);
}
}
} else {
downloadNotTableInfoData(request, wb);
}
this.addWatermarkTools(wb);
try (FileOutputStream outputStream = new FileOutputStream(dataPath + "/" + exportTask.getId() + ".xlsx")) {
wb.write(outputStream);
outputStream.flush();
}
wb.close();
exportTask.setExportProgress("100");
exportTask.setExportStatus("SUCCESS");
setFileSize(dataPath + "/" + exportTask.getId() + ".xlsx", exportTask);
} catch (Exception e) {
exportTask.setMsg(e.getMessage());
LogUtil.error("Failed to export data", e);
exportTask.setExportStatus("FAILED");
} finally {
exportTaskMapper.updateById(exportTask);
}
});
Running_Task.put(exportTask.getId(), future);
}
private void downloadNotTableInfoData(ChartExcelRequest request, Workbook wb) {
chartDataServer.findExcelData(request);
//给单元格设置样式
CellStyle cellStyle = wb.createCellStyle();
Font font = wb.createFont();
//设置字体大小
font.setFontHeightInPoints((short) 12);
//设置字体加粗
font.setBold(true);
//给字体设置样式
cellStyle.setFont(font);
//设置单元格背景颜色
cellStyle.setFillForegroundColor(IndexedColors.GREY_25_PERCENT.getIndex());
//设置单元格填充样式(使用纯色背景颜色填充)
cellStyle.setFillPattern(FillPatternType.SOLID_FOREGROUND);
if (CollectionUtils.isEmpty(request.getMultiInfo())) {
if (request.getViewInfo().getType().equalsIgnoreCase("chart-mix-dual-line")) {
} else {
List<Object[]> details = request.getDetails();
Integer[] excelTypes = request.getExcelTypes();
details.add(0, request.getHeader());
ViewDetailField[] detailFields = request.getDetailFields();
Object[] header = request.getHeader();
Sheet detailsSheet = wb.createSheet("数据");
if (request.getViewInfo().getType().equalsIgnoreCase("table-normal")) {
ChartDataServer.setExcelData(detailsSheet, cellStyle, header, details, detailFields, excelTypes, request.getViewInfo(), wb);
} else {
ChartDataServer.setExcelData(detailsSheet, cellStyle, header, details, detailFields, excelTypes, request.getViewInfo(), null);
}
}
} else {
//多个sheet
for (int i = 0; i < request.getMultiInfo().size(); i++) {
ChartExcelRequestInner requestInner = request.getMultiInfo().get(i);
List<Object[]> details = requestInner.getDetails();
Integer[] excelTypes = requestInner.getExcelTypes();
details.add(0, requestInner.getHeader());
ViewDetailField[] detailFields = requestInner.getDetailFields();
Object[] header = requestInner.getHeader();
//明细sheet
Sheet detailsSheet = wb.createSheet("数据 " + (i + 1));
ChartDataServer.setExcelData(detailsSheet, cellStyle, header, details, detailFields, excelTypes, request.getViewInfo(), null);
}
}
}
private void setFileSize(String filePath, CoreExportTask exportTask) {
File file = new File(filePath);
long length = file.length();
String unit = "Mb";
Double size = 0.0;
if ((double) length / 1024 / 1024 > 1) {
if ((double) length / 1024 / 1024 / 1024 > 1) {
unit = "Gb";
size = Double.valueOf(String.format("%.2f", (double) length / 1024 / 1024 / 1024));
} else {
size = Double.valueOf(String.format("%.2f", (double) length / 1024 / 1024));
}
} else {
unit = "Kb";
size = Double.valueOf(String.format("%.2f", (double) length / 1024));
}
exportTask.setFileSize(size);
exportTask.setFileSizeUnit(unit);
}
private static final String LOG_RETENTION = "30";
public void cleanLog() {
String key = "basic.exportFileLiveTime";
String val = sysParameterManage.singleVal(key);
@ -795,14 +325,46 @@ public class ExportCenterManage implements BaseExportApi {
public void addWatermarkTools(Workbook wb) {
VisualizationWatermark watermark = watermarkMapper.selectById("system_default");
WatermarkContentDTO watermarkContent = JsonUtil.parseObject(watermark.getSettingContent(), WatermarkContentDTO.class);
// if (watermarkContent.getEnable() && watermarkContent.getExcelEnable()) {
// UserFormVO userInfo = visualizationMapper.queryInnerUserInfo(AuthUtils.getUser().getUserId());
// // 在主逻辑中添加水印
// int watermarkPictureIdx = ExcelWatermarkUtils.addWatermarkImage(wb, watermarkContent, userInfo); // 生成水印图片并获取 ID
// for (Sheet sheet : wb) {
// ExcelWatermarkUtils.addWatermarkToSheet(sheet, watermarkPictureIdx); // 为每个 Sheet 添加水印
// }
// }
if (watermarkContent.getEnable() && watermarkContent.getExcelEnable()) {
UserFormVO userInfo = visualizationMapper.queryInnerUserInfo(AuthUtils.getUser().getUserId());
// 在主逻辑中添加水印
int watermarkPictureIdx = ExcelWatermarkUtils.addWatermarkImage(wb, watermarkContent, userInfo); // 生成水印图片并获取 ID
for (Sheet sheet : wb) {
ExcelWatermarkUtils.addWatermarkToSheet(sheet, watermarkPictureIdx); // 为每个 Sheet 添加水印
}
}
}
@DeLog(id = "#p0", ot = LogOT.DOWNLOAD, st = LogST.DATA)
public void generateDownloadUri(String id) {
CoreExportDownloadTask coreExportDownloadTask = coreExportDownloadTaskMapper.selectById(id);
if (coreExportDownloadTask != null) {
coreExportDownloadTask.setCreateTime(System.currentTimeMillis());
coreExportDownloadTaskMapper.updateById(coreExportDownloadTask);
} else {
coreExportDownloadTask = new CoreExportDownloadTask();
coreExportDownloadTask.setId(id);
coreExportDownloadTask.setCreateTime(System.currentTimeMillis());
coreExportDownloadTask.setValidTime(5L);
coreExportDownloadTaskMapper.insert(coreExportDownloadTask);
}
}
@Scheduled(fixedRate = 60 * 60 * 1000)
public void checkDownLoadInfos() {
coreExportDownloadTaskMapper.selectList(null).forEach(downLoadInfo -> {
if (System.currentTimeMillis() - downLoadInfo.getCreateTime() > downLoadInfo.getValidTime() * 60 * 1000) {
coreExportDownloadTaskMapper.deleteById(downLoadInfo.getId());
}
});
}
@Data
public class DownLoadInfo {
String id;
Long validTime; // 单位：minutes
Long createTime;
}
}

19
core/core-backend/src/main/java/io/gisbi/exportCenter/server/ExportCenterServer.java
View File

@ -1,5 +1,7 @@
package io.gisbi.exportCenter.server;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import io.gisbi.api.exportCenter.ExportCenterApi;
import io.gisbi.exportCenter.manage.ExportCenterManage;
import io.gisbi.exportCenter.util.ExportCenterUtils;
@ -11,6 +13,7 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import java.util.List;
import java.util.Map;
@RestController
@RequestMapping("/exportCenter")
@ -20,8 +23,14 @@ public class ExportCenterServer implements ExportCenterApi {
private ExportCenterManage exportCenterManage;
@Override
public List<ExportTaskDTO> exportTasks(String status) {
return exportCenterManage.exportTasks(status);
public Map<String, Long> exportTasks() {
return exportCenterManage.exportTasks();
}
@Override
public IPage<ExportTaskDTO> pager(int goPage, int pageSize, String status) {
Page<ExportTaskDTO> page = new Page<>(goPage, pageSize);
return exportCenterManage.pager(page, status);
}
@Override
@ -44,6 +53,12 @@ public class ExportCenterServer implements ExportCenterApi {
exportCenterManage.download(id, response);
}
@Override
public String generateDownloadUri(String id) throws Exception {
exportCenterManage.generateDownloadUri(id);
return "";
}
@Override
public void retry(String id) {
exportCenterManage.retry(id);

2
core/core-backend/src/main/java/io/gisbi/font/manage/FontManage.java
View File

@ -32,7 +32,7 @@ import java.util.UUID;
@Component
public class FontManage {
@Value("${gisbi.path.font:/opt/gisbi/data/font/}")
@Value("${gisbi.path.font:/opt/gisbi2.0/data/font/}")
private String path;
@Resource

2
core/core-backend/src/main/java/io/gisbi/license/manage/CoreLicManage.java
View File

@ -6,7 +6,7 @@ import org.springframework.stereotype.Component;
@Component
public class CoreLicManage {
@Value("1.0.0")
@Value("2.0.0")
private String version;
public String getVersion() {

2
core/core-backend/src/main/java/io/gisbi/license/server/LicenseServer.java
View File

@ -18,6 +18,6 @@ public class LicenseServer {
@GetMapping("/version")
public String version() {
return "1.0.0";
return "2.0.0";
}
}

12
core/core-backend/src/main/java/io/gisbi/listener/EhCacheStartListener.java
View File

@ -2,24 +2,16 @@ package io.gisbi.listener;
import io.gisbi.utils.ConfigUtils;
import org.springframework.beans.factory.config.YamlPropertiesFactoryBean;
import org.springframework.context.ApplicationContextInitializer;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.core.io.ClassPathResource;
import org.springframework.core.io.Resource;
import java.util.Objects;
public class EhCacheStartListener implements ApplicationContextInitializer<ConfigurableApplicationContext> {
@Override
public void initialize(ConfigurableApplicationContext applicationContext) {
Resource resource = new ClassPathResource("application.yml");
YamlPropertiesFactoryBean factory = new YamlPropertiesFactoryBean();
factory.setResources(resource);
String property = Objects.requireNonNull(factory.getObject()).getProperty("gisbi.login_timeout", "480");
String property = applicationContext.getEnvironment().getProperty("gisbi.login_timeout", String.class, "480");
System.setProperty("gisbi.login_timeout", property);
String ehcache = ConfigUtils.getConfig("gisbi.path.ehcache", "/opt/gisbi/cache");
String ehcache = ConfigUtils.getConfig("gisbi.path.ehcache", "/opt/gisbi2.0/cache");
System.setProperty("gisbi.path.ehcache", ehcache);
}
}

1
core/core-backend/src/main/java/io/gisbi/menu/server/MenuServer.java
View File

@ -19,7 +19,6 @@ public class MenuServer implements MenuApi {
@Resource
private MenuManage menuManage;
@I18n
@Override
public List<MenuVO> query() {
List<CoreMenu> coreMenus = menuManage.coreMenus();

11
core/core-backend/src/main/java/io/gisbi/resource/ResourceService.java
View File

@ -2,6 +2,7 @@ package io.gisbi.resource;
import io.gisbi.api.permissions.auth.dto.BusiPerCheckDTO;
import io.gisbi.constant.AuthEnum;
import io.gisbi.system.manage.CorePermissionManage;
import jakarta.annotation.Resource;
import org.springframework.stereotype.Component;
@ -10,11 +11,19 @@ import org.springframework.stereotype.Component;
*/
@Component
public class ResourceService {
@Resource
private CorePermissionManage corePermissionManage;
public boolean checkPermission(Long id) {
BusiPerCheckDTO dto = new BusiPerCheckDTO();
dto.setId(id);
dto.setAuthEnum(AuthEnum.READ);
return true;
boolean b;
try {
b = corePermissionManage.checkAuth(dto);
} catch (Exception e) {
b = false;
}
return b;
}
}

1
core/core-backend/src/main/java/io/gisbi/share/dao/ext/mapper/XpackShareExtMapper.java
View File

@ -18,6 +18,7 @@ public interface XpackShareExtMapper {
s.id as share_id,
v.id as resource_id,
v.mobile_layout as ext_flag,
v.status as ext_flag1,
v.type,
s.creator,
s.time,

2
core/core-backend/src/main/java/io/gisbi/share/dao/ext/po/XpackSharePO.java
View File

@ -30,4 +30,6 @@ public class XpackSharePO implements Serializable {
private Integer extFlag;
private Integer extFlag1;
}

11
core/core-backend/src/main/java/io/gisbi/share/interceptor/LinkInterceptor.java
View File

@ -19,8 +19,14 @@ import java.util.List;
@Component
public class LinkInterceptor implements HandlerInterceptor {
private final static String whiteListText = "/user/ipInfo, /apisix/check, /datasetData/enumValue, /datasetData/enumValueObj, /datasetData/getFieldTree, /dekey, /symmetricKey, /share/validate, /sysParameter/queryOnlineMap, /xpackComponent/viewPlugins, /v3/api-docs, /v3/api-docs/**, /swagger-ui, /swagger-ui/**, /swagger-ui.html, /swagger-resources";
private final static String whiteListText = "/user/ipInfo, /apisix/check, /datasetData/enumValue, /datasetData/enumValueObj, /datasetData/getFieldTree, /dekey, /symmetricKey, /share/validate, /sysParameter/queryOnlineMap, /xpackComponent/viewPlugins";
private final static String whiteStartListText = "/dataVisualization/findDvType/";
private boolean isWhiteStart(String url) {
List<String> whiteStartList = Arrays.stream(StringUtils.split(whiteStartListText, ",")).map(String::trim).toList();
return whiteStartList.stream().anyMatch(item -> StringUtils.startsWith(url, item));
}
@Override
public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws Exception {
@ -33,6 +39,7 @@ public class LinkInterceptor implements HandlerInterceptor {
if (deLinkPermit == null) {
List<String> whiteList = Arrays.stream(StringUtils.split(whiteListText, ",")).map(String::trim).toList();
String requestURI = ServletUtils.request().getRequestURI();
if (StringUtils.startsWith(requestURI, WhitelistUtils.getContextPath())) {
requestURI = requestURI.replaceFirst(WhitelistUtils.getContextPath(), "");
@ -40,7 +47,7 @@ public class LinkInterceptor implements HandlerInterceptor {
if (StringUtils.startsWith(requestURI, AuthConstant.DE_API_PREFIX)) {
requestURI = requestURI.replaceFirst(AuthConstant.DE_API_PREFIX, "");
}
boolean valid = whiteList.contains(requestURI) || WhitelistUtils.match(requestURI);
boolean valid = whiteList.contains(requestURI) || isWhiteStart(requestURI) || WhitelistUtils.match(requestURI);
if (!valid) {
DEException.throwException("分享链接Token不支持访问当前url[" + requestURI + "]");
}

2
core/core-backend/src/main/java/io/gisbi/share/manage/XpackShareManage.java
View File

@ -204,7 +204,7 @@ public class XpackShareManage {
return pos.stream().map(po ->
new XpackShareGridVO(
po.getShareId(), po.getResourceId(), po.getName(), po.getCreator().toString(),
po.getTime(), po.getExp(), 9, po.getExtFlag(), po.getType())).toList();
po.getTime(), po.getExp(), 9, po.getExtFlag(),po.getExtFlag1(), po.getType())).toList();
}
private XpackShareManage proxy() {

1
core/core-backend/src/main/java/io/gisbi/share/server/XpackShareServer.java
View File

@ -11,6 +11,7 @@ import io.gisbi.share.dao.auto.entity.XpackShare;
import io.gisbi.share.manage.XpackShareManage;
import jakarta.annotation.Resource;
import org.apache.commons.lang3.ObjectUtils;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

58
core/core-backend/src/main/java/io/gisbi/system/manage/SysParameterManage.java
View File

@ -13,12 +13,12 @@ import io.gisbi.utils.CommonBeanFactory;
import io.gisbi.utils.IDUtils;
import io.gisbi.utils.SystemSettingUtils;
import jakarta.annotation.Resource;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.CollectionUtils;
import java.util.*;
import java.util.stream.Collectors;
@ -26,10 +26,10 @@ import java.util.stream.Collectors;
@Component
public class SysParameterManage {
@Value("${dataease.show-demo-tips:false}")
@Value("${gisbi.show-demo-tips:false}")
private boolean showDemoTips;
@Value("${dataease.demo-tips-content:#{null}}")
@Value("${gisbi.demo-tips-content:#{null}}")
private String demoTipsContent;
private static final String MAP_KEY_PREFIX = "map.";
@ -52,30 +52,61 @@ public class SysParameterManage {
return null;
}
public OnlineMapEditor queryOnlineMap() {
public OnlineMapEditor queryOnlineMap(String mapType) {
if (StringUtils.isBlank(mapType)) {
List<CoreSysSetting> typeList = groupList(MAP_KEY_PREFIX + "mapType");
mapType = "gaode";
if (!CollectionUtils.isEmpty(typeList)) {
mapType = typeList.getFirst().getPval();
}
}
String prefix;
if (!StringUtils.equals(mapType, "gaode")) {
prefix = mapType + "." + MAP_KEY_PREFIX;
} else {
prefix = MAP_KEY_PREFIX;
}
var editor = new OnlineMapEditor();
List<String> fields = BeanUtils.getFieldNames(OnlineMapEditor.class);
Map<String, String> mapVal = groupVal(MAP_KEY_PREFIX);
Map<String, String> mapVal = groupVal(prefix);
fields.forEach(field -> {
String val = mapVal.get(MAP_KEY_PREFIX + field);
String val = mapVal.get(prefix + field);
if (StringUtils.isNotBlank(val)) {
BeanUtils.setFieldValueByName(editor, field, val, String.class);
}
});
editor.setMapType(mapType);
return editor;
}
public void saveOnlineMap(OnlineMapEditor editor) {
String mapType = editor.getMapType();
if (StringUtils.isBlank(mapType)) {
List<CoreSysSetting> typeList = groupList(MAP_KEY_PREFIX + "mapType");
mapType = "gaode";
if (!CollectionUtils.isEmpty(typeList)) {
mapType = typeList.getFirst().getPval();
}
}
List<String> fieldNames = BeanUtils.getFieldNames(OnlineMapEditor.class);
String finalMapType = mapType;
fieldNames.forEach(field -> {
String prefix = MAP_KEY_PREFIX;
if (!(StringUtils.equals(field, "mapType") || StringUtils.equals(finalMapType, "gaode"))) {
prefix = finalMapType + "." + MAP_KEY_PREFIX;
}
QueryWrapper<CoreSysSetting> queryWrapper = new QueryWrapper<>();
queryWrapper.eq("pkey", MAP_KEY_PREFIX + field);
queryWrapper.eq("pkey", prefix + field);
CoreSysSetting sysSetting = coreSysSettingMapper.selectOne(queryWrapper);
var val = (String) BeanUtils.getFieldValueByName(field, editor);
if (ObjectUtils.isEmpty(sysSetting)) {
sysSetting = new CoreSysSetting();
sysSetting.setId(IDUtils.snowID());
sysSetting.setPkey(MAP_KEY_PREFIX + field);
sysSetting.setPkey(prefix + field);
sysSetting.setPval(val == null ? "" : val);
sysSetting.setType("text");
sysSetting.setSort(1);
@ -135,15 +166,20 @@ public class SysParameterManage {
@Transactional
public void saveGroup(List<SettingItemVO> vos, String groupKey) {
QueryWrapper<CoreSysSetting> queryWrapper = new QueryWrapper<>();
queryWrapper.likeRight("pkey", groupKey);
coreSysSettingMapper.delete(queryWrapper);
List<CoreSysSetting> sysSettings = vos.stream().filter(vo -> !SystemSettingUtils.xpackSetting(vo.getPkey())).map(item -> {
CoreSysSetting sysSetting = BeanUtils.copyBean(new CoreSysSetting(), item);
sysSetting.setId(IDUtils.snowID());
return sysSetting;
}).collect(Collectors.toList());
if (CollectionUtils.isNotEmpty(sysSettings)) {
QueryWrapper<CoreSysSetting> queryWrapper = new QueryWrapper<>();
sysSettings.forEach(sysSetting -> {
queryWrapper.clear();
queryWrapper.eq("pkey", sysSetting.getPkey());
coreSysSettingMapper.delete(queryWrapper);
});
extCoreSysSettingMapper.saveBatch(sysSettings);
}
datasourceServer.addJob(sysSettings);
}

7
core/core-backend/src/main/java/io/gisbi/system/server/SysParameterServer.java
View File

@ -38,7 +38,12 @@ public class SysParameterServer implements SysParameterApi {
@Override
public OnlineMapEditor queryOnlineMap() {
return sysParameterManage.queryOnlineMap();
return sysParameterManage.queryOnlineMap(null);
}
@Override
public OnlineMapEditor queryOnlineMapByMapType(String type) {
return sysParameterManage.queryOnlineMap(type);
}
@Override

6
core/core-backend/src/main/java/io/gisbi/template/manage/TemplateCenterManage.java
View File

@ -7,12 +7,14 @@ import io.gisbi.api.template.dto.TemplateMarketPreviewInfoDTO;
import io.gisbi.api.template.response.*;
import io.gisbi.api.template.vo.MarketApplicationMetaDataVO;
import io.gisbi.api.template.vo.MarketApplicationSpecVO;
import io.gisbi.api.template.vo.MarketLatestReleaseVO;
import io.gisbi.api.template.vo.MarketMetaDataVO;
import io.gisbi.constant.CommonConstants;
import io.gisbi.exception.DEException;
import io.gisbi.i18n.Translator;
import io.gisbi.operation.manage.CoreOptRecentManage;
import io.gisbi.system.manage.SysParameterManage;
import io.gisbi.template.dao.auto.entity.VisualizationTemplateCategoryMap;
import io.gisbi.template.dao.auto.mapper.VisualizationTemplateCategoryMapMapper;
import io.gisbi.template.dao.ext.ExtVisualizationTemplateMapper;
import io.gisbi.utils.HttpClientConfig;
@ -67,7 +69,6 @@ public class TemplateCenterManage {
}
/**
* @param templateUrl template url
* @Description Get template file from template market
*/
public TemplateManageFileDTO getTemplateFromMarketV2(String templateName) {
@ -101,7 +102,8 @@ public class TemplateCenterManage {
config.setConnectTimeout(5000);
config.setSocketTimeout(10000);
config.setConnectionRequestTimeout(5000);
return HttpClientUtil.get(url, config);
return HttpClientUtil.
get(url, config);
}
private MarketTemplateV2BaseResponse templateQuery(Map<String, String> templateParams) {

2
core/core-backend/src/main/java/io/gisbi/template/manage/TemplateLocalParseManage.java
View File

@ -22,7 +22,7 @@ import java.time.LocalDateTime;
import java.time.temporal.ChronoUnit;
/**
* @Author bi-coder
* @author : WangJiaHao
* @date : 2024/5/7
*/
@Service

14
core/core-backend/src/main/java/io/gisbi/visualization/dao/auto/entity/DataVisualizationInfo.java
View File

@ -17,10 +17,7 @@ public class DataVisualizationInfo implements Serializable {
private static final long serialVersionUID = 1L;
private Long id;
/**
* 应用ID
*/
private String appId;
/**
* 名称
*/
@ -345,7 +342,6 @@ public class DataVisualizationInfo implements Serializable {
public String toString() {
return "DataVisualizationInfo{" +
"id = " + id +
", appId = " + appId +
", name = " + name +
", pid = " + pid +
", orgId = " + orgId +
@ -372,12 +368,4 @@ public class DataVisualizationInfo implements Serializable {
", checkVersion = " + checkVersion +
"}";
}
public String getAppId() {
return appId;
}
public void setAppId(String appId) {
this.appId = appId;
}
}

5
core/core-backend/src/main/java/io/gisbi/visualization/dao/auto/mapper/VisualizationWatermarkMapper.java
View File

@ -3,6 +3,7 @@ package io.gisbi.visualization.dao.auto.mapper;
import io.gisbi.visualization.dao.auto.entity.VisualizationWatermark;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Update;
/**
* <p>
@ -15,4 +16,8 @@ import org.apache.ibatis.annotations.Mapper;
@Mapper
public interface VisualizationWatermarkMapper extends BaseMapper<VisualizationWatermark> {
@Update("""
UPDATE `visualization_watermark` set `setting_content` = REPLACE(`setting_content`,'"enable":true','"enable":false')
""")
void disable();
}

3
core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/mapper/CoreStoreExtMapper.java
View File

@ -19,7 +19,8 @@ public interface CoreStoreExtMapper {
v.update_by as editor,
v.update_time as edit_time,
v.name,
v.mobile_layout as ext_flag
v.mobile_layout as ext_flag,
v.status as ext_flag1
from core_store s
inner join data_visualization_info v on s.resource_id = v.id
${ew.customSqlSegment}

2
core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/mapper/CoreVisualiationExtMapper.java
View File

@ -14,7 +14,7 @@ import java.util.Set;
public interface CoreVisualiationExtMapper {
@Select("""
select id, name, pid, node_type, mobile_layout as extraFlag from data_visualization_info
select id, name, pid, node_type, mobile_layout as extraFlag, status as extraFlag1 from data_visualization_info
${ew.customSqlSegment}
""")
List<VisualizationNodePO> queryNodes(@Param("ew") QueryWrapper<Object> queryWrapper);

56
core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/mapper/ExtDataVisualizationMapper.java
View File

@ -7,13 +7,16 @@ import io.gisbi.api.visualization.dto.VisualizationViewTableDTO;
import io.gisbi.api.visualization.vo.DataVisualizationBaseVO;
import io.gisbi.api.visualization.vo.DataVisualizationVO;
import io.gisbi.api.visualization.vo.VisualizationReportFilterVO;
import io.gisbi.api.visualization.vo.VisualizationResourceVO;
import io.gisbi.chart.dao.auto.entity.CoreChartView;
import io.gisbi.visualization.dao.ext.po.StorePO;
import io.gisbi.visualization.dao.ext.po.VisualizationResourcePO;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import org.apache.ibatis.annotations.Select;
import java.util.List;
import java.util.Map;
import java.util.Set;
@Mapper
@ -29,12 +32,12 @@ public interface ExtDataVisualizationMapper {
String findDvType(@Param("dvId") Long dvId);
void dvCopy(@Param("sourceDvId") Long sourceDvId,@Param("newDvId") Long newDvId,@Param("copyId") Long copyId);
void viewCopyWithDv(@Param("sourceDvId") Long sourceDvId,@Param("newDvId") Long newDvId,@Param("copyId") Long copyId);
void viewCopyWithDv(@Param("sourceDvId") Long sourceDvId,@Param("newDvId") Long newDvId,@Param("copyId") Long copyId,@Param("resourceTable") String resourceTable);
List<CoreChartView> findViewInfoByCopyId(@Param("copyId") Long copyId);
DataVisualizationVO findDvInfo(@Param("dvId") Long dvId,@Param("dvType") String dvType);
DataVisualizationVO findDvInfo(@Param("dvId") Long dvId,@Param("dvType") String dvType,@Param("resourceTable") String resourceTable);
IPage<VisualizationResourcePO> findRecent(IPage<VisualizationResourcePO> page, @Param("uid") Long uid, @Param("keyword") String keyword, @Param("ew") QueryWrapper<Object> ew);
IPage<VisualizationResourcePO> findRecent(IPage<VisualizationResourcePO> page, @Param("uid") Long uid, @Param("keyword") String keyword, @Param("ew") Map ew);
void copyLinkJump(@Param("copyId") Long copyId);
@ -50,9 +53,52 @@ public interface ExtDataVisualizationMapper {
List<VisualizationReportFilterVO> queryReportFilter(@Param("dvId") Long dvId,@Param("taskId") Long taskId);
void deleteDataVBatch(@Param("ids") Set<Long> ids);
void deleteDataVBatch(@Param("ids") Set<Long> ids,@Param("resourceTable") String resourceTable);
void deleteViewsBatch(@Param("ids") Set<Long> ids);
void deleteViewsBatch(@Param("ids") Set<Long> ids,@Param("resourceTable") String resourceTable);
UserFormVO queryInnerUserInfo(@Param("id") Long id);
void snapshotDataV(@Param("dvId") Long dvId);
void snapshotViews(@Param("dvId") Long dvId);
void snapshotLinkJumpTargetViewInfo(@Param("dvId") Long dvId);
void snapshotLinkJumpInfo(@Param("dvId") Long dvId);
void snapshotLinkJump(@Param("dvId") Long dvId);
void snapshotLinkageField(@Param("dvId") Long dvId);
void snapshotLinkage(@Param("dvId") Long dvId);
void snapshotOuterParamsTargetViewInfo(@Param("dvId") Long dvId);
void snapshotOuterParamsInfo(@Param("dvId") Long dvId);
void snapshotOuterParams(@Param("dvId") Long dvId);
void restoreDataV(@Param("dvId") Long dvId);
void restoreViews(@Param("dvId") Long dvId);
void restoreLinkJumpTargetViewInfo(@Param("dvId") Long dvId);
void restoreLinkJumpInfo(@Param("dvId") Long dvId);
void restoreLinkJump(@Param("dvId") Long dvId);
void restoreLinkageField(@Param("dvId") Long dvId);
void restoreLinkage(@Param("dvId") Long dvId);
void restoreOuterParamsTargetViewInfo(@Param("dvId") Long dvId);
void restoreOuterParamsInfo(@Param("dvId") Long dvId);
void restoreOuterParams(@Param("dvId") Long dvId);
@Select("select status from data_visualization_info where id = #{dvId}")
Integer findDvInfoStats(@Param("dvId") Long dvId);
}

16
core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/mapper/ExtVisualizationLinkJumpMapper.java
View File

@ -14,13 +14,15 @@ import java.util.List;
public interface ExtVisualizationLinkJumpMapper {
List<VisualizationLinkJumpDTO> queryWithDvId(@Param("dvId") Long dvId,@Param("uid") Long uid,@Param("isDesktop") Boolean isDesktop);
List<VisualizationLinkJumpDTO> queryWithDvIdSnapshot(@Param("dvId") Long dvId,@Param("uid") Long uid,@Param("isDesktop") Boolean isDesktop);
VisualizationLinkJumpDTO queryWithViewId(@Param("dvId") Long dvId,@Param("viewId") Long viewId,@Param("uid") Long uid,@Param("isDesktop") Boolean isDesktop);
void deleteJumpTargetViewInfo(@Param("dvId") Long dvId,@Param("viewId") Long viewId);
void deleteJumpTargetViewInfoSnapshot(@Param("dvId") Long dvId,@Param("viewId") Long viewId);
void deleteJumpInfo(@Param("dvId") Long dvId,@Param("viewId") Long viewId);
void deleteJumpInfoSnapshot(@Param("dvId") Long dvId,@Param("viewId") Long viewId);
void deleteJump(@Param("dvId") Long dvId,@Param("viewId") Long viewId);
void deleteJumpSnapshot(@Param("dvId") Long dvId,@Param("viewId") Long viewId);
void deleteJumpTargetViewInfoWithVisualization(@Param("dvId") Long dvId);
@ -28,8 +30,16 @@ public interface ExtVisualizationLinkJumpMapper {
void deleteJumpWithVisualization(@Param("dvId") Long dvId);
void deleteJumpTargetViewInfoWithVisualizationSnapshot(@Param("dvId") Long dvId);
void deleteJumpInfoWithVisualizationSnapshot(@Param("dvId") Long dvId);
void deleteJumpWithVisualizationSnapshot(@Param("dvId") Long dvId);
List<VisualizationLinkJumpDTO> getTargetVisualizationJumpInfo(@Param("request") VisualizationLinkJumpBaseRequest request);
List<VisualizationLinkJumpDTO> getTargetVisualizationJumpInfoSnapshot(@Param("request") VisualizationLinkJumpBaseRequest request);
void copyLinkJump(@Param("copyId")Long copyId);
void copyLinkJumpInfo(@Param("copyId")Long copyId);

8
core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/mapper/ExtVisualizationLinkageMapper.java
View File

@ -17,6 +17,10 @@ public interface ExtVisualizationLinkageMapper {
List<LinkageInfoDTO> getPanelAllLinkageInfo(@Param("dvId") Long dvId);
List<VisualizationLinkageDTO> getViewLinkageGatherSnapshot(@Param("dvId") Long dvId, @Param("sourceViewId") Long sourceViewId, @Param("targetViewIds") List<String> targetViewIds);
List<LinkageInfoDTO> getPanelAllLinkageInfoSnapshot(@Param("dvId") Long dvId);
List<DatasetTableFieldDTO> queryTableField(@Param("table_id") Long tableId);
List<DatasetTableFieldDTO> queryTableFieldWithViewId(@Param("viewId") Long viewId);
@ -25,6 +29,10 @@ public interface ExtVisualizationLinkageMapper {
void deleteViewLinkageField(@Param("dvId") Long dvId,@Param("sourceViewId") Long sourceViewId);
void deleteViewLinkageSnapshot(@Param("dvId") Long dvId,@Param("sourceViewId") Long sourceViewId);
void deleteViewLinkageFieldSnapshot(@Param("dvId") Long dvId,@Param("sourceViewId") Long sourceViewId);
void copyViewLinkage(@Param("copyId") Long copyId);
void copyViewLinkageField(@Param("copyId") Long copyId);

11
core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/mapper/ExtVisualizationOuterParamsMapper.java
View File

@ -4,6 +4,7 @@ package io.gisbi.visualization.dao.ext.mapper;
import io.gisbi.api.dataset.vo.CoreDatasetGroupVO;
import io.gisbi.api.visualization.dto.VisualizationOuterParamsDTO;
import io.gisbi.api.visualization.dto.VisualizationOuterParamsInfoDTO;
import io.gisbi.visualization.dao.auto.entity.SnapshotVisualizationOuterParamsInfo;
import io.gisbi.visualization.dao.auto.entity.VisualizationOuterParamsInfo;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
@ -13,7 +14,7 @@ import java.util.List;
@Mapper
public interface ExtVisualizationOuterParamsMapper {
VisualizationOuterParamsDTO queryWithVisualizationId(@Param("visualizationId") String visualizationId);
VisualizationOuterParamsDTO queryWithVisualizationIdSnapshot(@Param("visualizationId") String visualizationId);
void deleteOuterParamsTargetWithVisualizationId(@Param("visualizationId") String visualizationId);
@ -21,9 +22,15 @@ public interface ExtVisualizationOuterParamsMapper {
void deleteOuterParamsWithVisualizationId(@Param("visualizationId") String visualizationId);
void deleteOuterParamsTargetWithVisualizationIdSnapshot(@Param("visualizationId") String visualizationId);
void deleteOuterParamsInfoWithVisualizationIdSnapshot(@Param("visualizationId") String visualizationId);
void deleteOuterParamsWithVisualizationIdSnapshot(@Param("visualizationId") String visualizationId);
List<VisualizationOuterParamsInfoDTO> getVisualizationOuterParamsInfo(@Param("visualizationId") String visualizationId);
List<VisualizationOuterParamsInfo> getVisualizationOuterParamsInfoBase(@Param("visualizationId") String visualizationId);
List<SnapshotVisualizationOuterParamsInfo> getVisualizationOuterParamsInfoBase(@Param("visualizationId") String visualizationId);
List<CoreDatasetGroupVO> queryDsWithVisualizationId(@Param("visualizationId") String visualizationId);
}

2
core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/po/StorePO.java
View File

@ -26,4 +26,6 @@ public class StorePO implements Serializable {
private Integer extFlag;
private Integer extFlag1;
}

2
core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/po/VisualizationNodePO.java
View File

@ -15,5 +15,7 @@ public class VisualizationNodePO implements Serializable {
private String nodeType;
@Schema(description = "额外标识")
private int extraFlag;
@Schema(description = "额外标识1")
private int extraFlag1;
}

1
core/core-backend/src/main/java/io/gisbi/visualization/dto/VisualizationNodeBO.java
View File

@ -20,6 +20,7 @@ public class VisualizationNodeBO implements TreeBaseModel {
private Integer weight = 3;
private Long pid;
private Integer extraFlag;
private Integer extraFlag1;
}

46
core/core-backend/src/main/java/io/gisbi/visualization/manage/CoreVisualizationExportManage.java
View File

@ -6,8 +6,8 @@ import io.gisbi.chart.constant.ChartConstants;
import io.gisbi.chart.manage.ChartDataManage;
import io.gisbi.chart.manage.ChartViewManege;
import io.gisbi.constant.CommonConstants;
import io.gisbi.dataset.server.DatasetFieldServer;
import io.gisbi.constant.DeTypeConstants;
import io.gisbi.dataset.server.DatasetFieldServer;
import io.gisbi.exception.DEException;
import io.gisbi.exportCenter.util.ExportCenterUtils;
import io.gisbi.extensions.view.dto.ChartExtFilterDTO;
@ -52,15 +52,15 @@ public class CoreVisualizationExportManage {
private DatasetFieldServer datasetFieldServer;
public String getResourceName(Long dvId, String busiFlag) {
DataVisualizationVO visualization = extDataVisualizationMapper.findDvInfo(dvId, busiFlag);
DataVisualizationVO visualization = extDataVisualizationMapper.findDvInfo(dvId, busiFlag, "core");
if (ObjectUtils.isEmpty(visualization)) DEException.throwException("资源不存在或已经被删除...");
return visualization.getName();
}
public File exportExcel(Long dvId, String busiFlag, List<Long> viewIdList, boolean onlyDisplay) throws Exception {
DataVisualizationVO visualization = extDataVisualizationMapper.findDvInfo(dvId, busiFlag);
public File exportExcel(Long dvId, String busiFlag, List<Long> viewIdList, boolean onlyDisplay, String filterJson) throws Exception {
DataVisualizationVO visualization = extDataVisualizationMapper.findDvInfo(dvId, busiFlag, "core");
if (ObjectUtils.isEmpty(visualization)) DEException.throwException("资源不存在或已经被删除...");
List<ChartViewDTO> chartViewDTOS = chartViewManege.listBySceneId(dvId);
List<ChartViewDTO> chartViewDTOS = chartViewManege.listBySceneId(dvId, CommonConstants.RESOURCE_TABLE.CORE);
String componentsJson = visualization.getComponentData();
List<Map<String, Object>> components = JsonUtil.parseList(componentsJson, tokenType);
@ -70,13 +70,15 @@ public class CoreVisualizationExportManage {
chartViewDTOS = chartViewDTOS.stream().filter(item -> idList.contains(item.getId()) && viewIdList.contains(item.getId())).collect(Collectors.toList());
}
if (CollectionUtils.isEmpty(chartViewDTOS)) return null;
Map<String, ChartExtRequest> chartExtRequestMap = buildViewRequest(visualization, onlyDisplay);
Map<Long, ChartExtRequest> chartExtRequestMap = buildViewRequest(filterJson);
List<ExcelSheetModel> sheets = new ArrayList<>();
for (int i = 0; i < chartViewDTOS.size(); i++) {
ChartViewDTO view = chartViewDTOS.get(i);
ChartExtRequest extRequest = chartExtRequestMap.get(view.getId().toString());
ChartExtRequest extRequest = chartExtRequestMap.get(view.getId());
if (ObjectUtils.isNotEmpty(extRequest)) {
view.setChartExtRequest(extRequest);
} else {
view.setChartExtRequest(buildDefaultRequest());
}
view.getChartExtRequest().setUser(AuthUtils.getUser().getUserId());
view.setTitle((i + 1) + "-" + view.getTitle());
@ -96,11 +98,11 @@ public class CoreVisualizationExportManage {
if (CollectionUtils.isNotEmpty(fields)) {
fields.forEach(field -> {
Object name = field.getName();
Object dataeaseName = field.getDataeaseName();
Object gisbiName = field.getGisbiName();
Object deType = field.getDeType();
if (ObjectUtils.isNotEmpty(name) && ObjectUtils.isNotEmpty(dataeaseName)) {
if (ObjectUtils.isNotEmpty(name) && ObjectUtils.isNotEmpty(gisbiName)) {
heads.add(name.toString());
headKeys.add(dataeaseName.toString());
headKeys.add(gisbiName.toString());
if (deType == null) {
field.setDeType(DeTypeConstants.DE_STRING);
deType = DeTypeConstants.DE_STRING;
@ -187,6 +189,30 @@ public class CoreVisualizationExportManage {
private final TypeReference<List<Map<String, Object>>> tokenType = new TypeReference<List<Map<String, Object>>>() {
};
private Map<Long, ChartExtRequest> buildViewRequest(String filterJson) {
if (StringUtils.isBlank(filterJson)) {
return new HashMap<>();
}
Map<Long, ChartExtRequest> extRequestMap = JsonUtil.parseObject(filterJson, new TypeReference<Map<Long, ChartExtRequest>>() {
});
extRequestMap.forEach((key, chartExtRequest) -> {
chartExtRequest.setQueryFrom("panel");
chartExtRequest.setResultCount(Math.toIntExact(ExportCenterUtils.getExportLimit("view")));
chartExtRequest.setResultMode(ChartConstants.VIEW_RESULT_MODE.ALL);
chartExtRequest.setPageSize(ExportCenterUtils.getExportLimit("view"));
});
return extRequestMap;
}
private ChartExtRequest buildDefaultRequest() {
ChartExtRequest chartExtRequest = new ChartExtRequest();
chartExtRequest.setQueryFrom("panel");
chartExtRequest.setFilter(new ArrayList<>());
chartExtRequest.setResultCount(Math.toIntExact(ExportCenterUtils.getExportLimit("view")));
chartExtRequest.setResultMode(ChartConstants.VIEW_RESULT_MODE.ALL);
chartExtRequest.setPageSize(ExportCenterUtils.getExportLimit("view"));
return chartExtRequest;
}
private Map<String, ChartExtRequest> buildViewRequest(DataVisualizationVO panelDto, Boolean justView) {
String componentsJson = panelDto.getComponentData();

164
core/core-backend/src/main/java/io/gisbi/visualization/manage/CoreVisualizationManage.java
View File

@ -6,18 +6,23 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import io.gisbi.api.visualization.request.DataVisualizationBaseRequest;
import io.gisbi.api.visualization.request.VisualizationWorkbranchQueryRequest;
import io.gisbi.api.visualization.vo.VisualizationResourceVO;
import io.gisbi.chart.dao.ext.mapper.ExtChartViewMapper;
import io.gisbi.chart.manage.ChartViewManege;
import io.gisbi.commons.constants.DataVisualizationConstants;
import io.gisbi.commons.constants.OptConstants;
import io.gisbi.constant.BusiResourceEnum;
import io.gisbi.constant.CommonConstants;
import io.gisbi.exception.DEException;
import io.gisbi.license.config.XpackInteract;
import io.gisbi.model.BusiNodeRequest;
import io.gisbi.model.BusiNodeVO;
import io.gisbi.operation.manage.CoreOptRecentManage;
import io.gisbi.utils.*;
import io.gisbi.visualization.dao.auto.entity.DataVisualizationInfo;
import io.gisbi.visualization.dao.auto.entity.SnapshotDataVisualizationInfo;
import io.gisbi.visualization.dao.auto.mapper.DataVisualizationInfoMapper;
import io.gisbi.visualization.dao.ext.mapper.CoreVisualiationExtMapper;
import io.gisbi.visualization.dao.ext.mapper.ExtDataVisualizationMapper;
import io.gisbi.visualization.dao.auto.mapper.SnapshotDataVisualizationInfoMapper;
import io.gisbi.visualization.dao.ext.mapper.*;
import io.gisbi.visualization.dao.ext.po.VisualizationNodePO;
import io.gisbi.visualization.dao.ext.po.VisualizationResourcePO;
import io.gisbi.visualization.dto.VisualizationNodeBO;
@ -41,13 +46,31 @@ public class CoreVisualizationManage {
@Resource
private DataVisualizationInfoMapper mapper;
@Resource
private SnapshotDataVisualizationInfoMapper snapshotMapper;
@Resource
private ExtVisualizationLinkageMapper linkageMapper;
@Resource
private ExtVisualizationLinkJumpMapper linkJumpMapper;
@Resource
private ExtVisualizationOuterParamsMapper outerParamsMapper;
@Resource
private ExtDataVisualizationMapper extDataVisualizationMapper;
@Resource
private CoreOptRecentManage coreOptRecentManage;
@Resource
private ExtChartViewMapper extCoreChartMapper;
@Resource
private ChartViewManege chartViewManege;
@XpackInteract(value = "visualizationResourceTree", replace = true, invalid = true)
public List<BusiNodeVO> tree(BusiNodeRequest request) {
List<VisualizationNodeBO> nodes = new ArrayList<>();
if (ObjectUtils.isEmpty(request.getLeaf()) || !request.getLeaf()) {
@ -62,6 +85,10 @@ public class CoreVisualizationManage {
if (StringUtils.isNotBlank(info)) {
queryWrapper.notExists(String.format(info, "data_visualization_info.id"));
}
// 如果是编辑界面 只展示已发布的资源
if(CommonConstants.RESOURCE_TABLE.SNAPSHOT.equals(request.getResourceTable())){
queryWrapper.in("status", Arrays.asList(1,2));
}
queryWrapper.orderByDesc("create_time");
List<VisualizationNodePO> pos = extMapper.queryNodes(queryWrapper);
if (CollectionUtils.isNotEmpty(pos)) {
@ -70,7 +97,7 @@ public class CoreVisualizationManage {
return TreeUtils.mergeTree(nodes, BusiNodeVO.class, false);
}
@XpackInteract(value = "visualizationResourceTree", before = false)
public void delete(Long id) {
DataVisualizationInfo info = mapper.selectById(id);
if (ObjectUtils.isEmpty(info)) {
@ -93,14 +120,16 @@ public class CoreVisualizationManage {
}
}
// 删除可视化资源
extDataVisualizationMapper.deleteDataVBatch(delIds);
extDataVisualizationMapper.deleteDataVBatch(delIds,CommonConstants.RESOURCE_TABLE.CORE);
extDataVisualizationMapper.deleteDataVBatch(delIds,CommonConstants.RESOURCE_TABLE.SNAPSHOT);
// 删除图表信息
extDataVisualizationMapper.deleteViewsBatch(delIds);
extDataVisualizationMapper.deleteViewsBatch(delIds,CommonConstants.RESOURCE_TABLE.CORE);
extDataVisualizationMapper.deleteViewsBatch(delIds,CommonConstants.RESOURCE_TABLE.SNAPSHOT);
coreOptRecentManage.saveOpt(id, OptConstants.OPT_RESOURCE_TYPE.VISUALIZATION, OptConstants.OPT_TYPE.DELETE);
}
@XpackInteract(value = "visualizationResourceTree", before = false)
public void move(DataVisualizationBaseRequest request) {
if (!request.getMoveFromUpdate()) {
DataVisualizationInfo visualizationInfo = new DataVisualizationInfo();
@ -109,12 +138,15 @@ public class CoreVisualizationManage {
DEException.throwException("resource not exist");
}
visualizationInfo.setUpdateTime(System.currentTimeMillis());
SnapshotDataVisualizationInfo snapshotVisualizationInfo = new SnapshotDataVisualizationInfo();
BeanUtils.copyBean(snapshotVisualizationInfo, visualizationInfo);
coreOptRecentManage.saveOpt(visualizationInfo.getId(), OptConstants.OPT_RESOURCE_TYPE.VISUALIZATION, OptConstants.OPT_TYPE.UPDATE);
mapper.updateById(visualizationInfo);
snapshotMapper.updateById(snapshotVisualizationInfo);
}
}
@XpackInteract(value = "visualizationResourceTree", before = false)
public Long innerSave(DataVisualizationInfo visualizationInfo) {
visualizationInfo.setVersion(3);
return preInnerSave(visualizationInfo);
@ -126,22 +158,42 @@ public class CoreVisualizationManage {
visualizationInfo.setId(id);
}
visualizationInfo.setDeleteFlag(DataVisualizationConstants.DELETE_FLAG.AVAILABLE);
visualizationInfo.setStatus(visualizationInfo.getStatus());
visualizationInfo.setCreateBy(AuthUtils.getUser().getUserId().toString());
visualizationInfo.setUpdateBy(AuthUtils.getUser().getUserId().toString());
visualizationInfo.setCreateTime(System.currentTimeMillis());
visualizationInfo.setUpdateTime(System.currentTimeMillis());
visualizationInfo.setOrgId(AuthUtils.getUser().getDefaultOid());
mapper.insert(visualizationInfo);
// 镜像文件插入
SnapshotDataVisualizationInfo snapshotVisualizationInfo = new SnapshotDataVisualizationInfo();
BeanUtils.copyBean(snapshotVisualizationInfo,visualizationInfo);
snapshotMapper.insert(snapshotVisualizationInfo);
coreOptRecentManage.saveOpt(visualizationInfo.getId(), OptConstants.OPT_RESOURCE_TYPE.VISUALIZATION, OptConstants.OPT_TYPE.NEW);
return visualizationInfo.getId();
}
@XpackInteract(value = "visualizationResourceTree", before = false)
public void innerEdit(DataVisualizationInfo visualizationInfo) {
// 镜像和主表保持名称一致
visualizationInfo.setUpdateTime(System.currentTimeMillis());
visualizationInfo.setUpdateBy(AuthUtils.getUser().getUserId().toString());
visualizationInfo.setVersion(3);
mapper.updateById(visualizationInfo);
// 更新镜像
SnapshotDataVisualizationInfo snapshotVisualizationInfo = new SnapshotDataVisualizationInfo();
BeanUtils.copyBean(snapshotVisualizationInfo,visualizationInfo);
snapshotMapper.updateById(snapshotVisualizationInfo);
// 更新主表名称
DataVisualizationInfo coreVisualizationInfo = new DataVisualizationInfo();
coreVisualizationInfo.setId(visualizationInfo.getId());
coreVisualizationInfo.setStatus(visualizationInfo.getStatus());
coreVisualizationInfo.setPid(visualizationInfo.getPid());
coreVisualizationInfo.setContentId(visualizationInfo.getContentId());
coreVisualizationInfo.setName(visualizationInfo.getName());
coreVisualizationInfo.setUpdateTime(System.currentTimeMillis());
coreVisualizationInfo.setUpdateBy(AuthUtils.getUser().getUserId().toString());
coreVisualizationInfo.setVersion(3);
mapper.updateById(coreVisualizationInfo);
coreOptRecentManage.saveOpt(visualizationInfo.getId(), OptConstants.OPT_RESOURCE_TYPE.VISUALIZATION, OptConstants.OPT_TYPE.UPDATE);
}
@ -150,18 +202,18 @@ public class CoreVisualizationManage {
}
private VisualizationNodeBO rootNode() {
return new VisualizationNodeBO(0L, "root", false, 7, -1L, 0);
return new VisualizationNodeBO(0L, "root", false, 7, -1L, 0,1);
}
private VisualizationNodeBO convert(VisualizationNodePO po) {
return new VisualizationNodeBO(po.getId(), po.getName(), StringUtils.equals(po.getNodeType(), "leaf"), 9, po.getPid(), po.getExtraFlag());
return new VisualizationNodeBO(po.getId(), po.getName(), StringUtils.equals(po.getNodeType(), "leaf"), 9, po.getPid(), po.getExtraFlag(),po.getExtraFlag1());
}
public CoreVisualizationManage proxy() {
return CommonBeanFactory.getBean(this.getClass());
}
@XpackInteract(value = "perFilterManage", recursion = true, invalid = true)
public IPage<VisualizationResourceVO> query(int pageNum, int pageSize, VisualizationWorkbranchQueryRequest request) {
IPage<VisualizationResourcePO> visualizationResourcePOPageIPage = proxy().queryVisualizationPage(pageNum, pageSize, request);
if (ObjectUtils.isEmpty(visualizationResourcePOPageIPage)) {
@ -190,20 +242,98 @@ public class CoreVisualizationManage {
public IPage<VisualizationResourcePO> queryVisualizationPage(int goPage, int pageSize, VisualizationWorkbranchQueryRequest request) {
Long uid = AuthUtils.getUser().getUserId();
QueryWrapper<Object> queryWrapper = new QueryWrapper<>();
Map<String,Object> params = new HashMap<>();
if (StringUtils.isNotBlank(request.getType())) {
BusiResourceEnum busiResourceEnum = BusiResourceEnum.valueOf(request.getType().toUpperCase());
if (ObjectUtils.isEmpty(busiResourceEnum)) {
DEException.throwException("type is invalid");
}
queryWrapper.eq("dvResource.type", request.getType());
params.put("type",request.getType());
}
String info = CommunityUtils.getInfo();
if (StringUtils.isNotBlank(info)) {
queryWrapper.notExists(String.format(info, "core_opt_recent.resource_id"));
params.put("info",info);
}
queryWrapper.orderBy(true, request.isAsc(), "core_opt_recent.time");
params.put("isAsc",request.isAsc());
Page<VisualizationResourcePO> page = new Page<>(goPage, pageSize);
return extDataVisualizationMapper.findRecent(page, uid, request.getKeyword(), queryWrapper);
return extDataVisualizationMapper.findRecent(page, uid, request.getKeyword(), params);
}
@Transactional
public void removeSnapshot(Long dvId){
if(dvId != null){
// 清理历史数据
Set<Long> dvIds = new HashSet<>();
dvIds.add(dvId);
extDataVisualizationMapper.deleteDataVBatch(dvIds,CommonConstants.RESOURCE_TABLE.SNAPSHOT);
extCoreChartMapper.deleteViewsBySceneId(dvId,CommonConstants.RESOURCE_TABLE.SNAPSHOT);
linkageMapper.deleteViewLinkageFieldSnapshot(dvId,null);
linkageMapper.deleteViewLinkageSnapshot(dvId,null);
linkJumpMapper.deleteJumpTargetViewInfoWithVisualizationSnapshot(dvId);
linkJumpMapper.deleteJumpInfoWithVisualizationSnapshot(dvId);
linkJumpMapper.deleteJumpWithVisualizationSnapshot(dvId);
outerParamsMapper.deleteOuterParamsTargetWithVisualizationIdSnapshot(dvId.toString());
outerParamsMapper.deleteOuterParamsInfoWithVisualizationIdSnapshot(dvId.toString());
outerParamsMapper.deleteOuterParamsWithVisualizationIdSnapshot(dvId.toString());
//xpack 阈值告警
chartViewManege.removeThreshold(dvId,CommonConstants.RESOURCE_TABLE.SNAPSHOT);
}
}
@Transactional
public void removeDvCore(Long dvId){
if(dvId != null){
// 清理历史数据
Set<Long> dvIds = new HashSet<>();
dvIds.add(dvId);
extDataVisualizationMapper.deleteDataVBatch(dvIds,CommonConstants.RESOURCE_TABLE.CORE);
extCoreChartMapper.deleteViewsBySceneId(dvId,CommonConstants.RESOURCE_TABLE.CORE);
linkageMapper.deleteViewLinkageField(dvId,null);
linkageMapper.deleteViewLinkage(dvId,null);
linkJumpMapper.deleteJumpTargetViewInfoWithVisualization(dvId);
linkJumpMapper.deleteJumpInfoWithVisualization(dvId);
linkJumpMapper.deleteJumpWithVisualization(dvId);
outerParamsMapper.deleteOuterParamsTargetWithVisualizationId(dvId.toString());
outerParamsMapper.deleteOuterParamsInfoWithVisualizationId(dvId.toString());
outerParamsMapper.deleteOuterParamsWithVisualizationId(dvId.toString());
//xpack 阈值告警
chartViewManege.removeThreshold(dvId,CommonConstants.RESOURCE_TABLE.CORE);
}
}
@Transactional
public void dvSnapshotRecover(Long dvId){
// 清理历史数据
CoreVisualizationManage proxy = CommonBeanFactory.proxy(this.getClass());
assert proxy != null;
proxy.removeSnapshot(dvId);
// 导入新数据
extDataVisualizationMapper.snapshotDataV(dvId);
extDataVisualizationMapper.snapshotViews(dvId);
extDataVisualizationMapper.snapshotLinkJumpTargetViewInfo(dvId);
extDataVisualizationMapper.snapshotLinkJumpInfo(dvId);
extDataVisualizationMapper.snapshotLinkJump(dvId);
extDataVisualizationMapper.snapshotLinkageField(dvId);
extDataVisualizationMapper.snapshotLinkage(dvId);
extDataVisualizationMapper.snapshotOuterParamsTargetViewInfo(dvId);
extDataVisualizationMapper.snapshotOuterParamsInfo(dvId);
extDataVisualizationMapper.snapshotOuterParams(dvId);
//xpack 阈值告警
chartViewManege.restoreThreshold(dvId,CommonConstants.RESOURCE_TABLE.SNAPSHOT);
}
@Transactional
public void dvRestore(Long dvId){
extDataVisualizationMapper.restoreDataV(dvId);
extDataVisualizationMapper.restoreViews(dvId);
extDataVisualizationMapper.restoreLinkJumpTargetViewInfo(dvId);
extDataVisualizationMapper.restoreLinkJumpInfo(dvId);
extDataVisualizationMapper.restoreLinkJump(dvId);
extDataVisualizationMapper.restoreLinkageField(dvId);
extDataVisualizationMapper.restoreLinkage(dvId);
extDataVisualizationMapper.restoreOuterParamsTargetViewInfo(dvId);
extDataVisualizationMapper.restoreOuterParamsInfo(dvId);
extDataVisualizationMapper.restoreOuterParams(dvId);
//xpack 阈值告警
chartViewManege.restoreThreshold(dvId,CommonConstants.RESOURCE_TABLE.CORE);
}
}

4
core/core-backend/src/main/java/io/gisbi/visualization/manage/VisualizationStoreManage.java
View File

@ -8,6 +8,7 @@ import io.gisbi.api.visualization.request.VisualizationWorkbranchQueryRequest;
import io.gisbi.api.visualization.vo.VisualizationStoreVO;
import io.gisbi.constant.BusiResourceEnum;
import io.gisbi.exception.DEException;
import io.gisbi.license.config.XpackInteract;
import io.gisbi.utils.AuthUtils;
import io.gisbi.utils.CommonBeanFactory;
import io.gisbi.utils.CommunityUtils;
@ -65,6 +66,7 @@ public class VisualizationStoreManage {
return coreStoreMapper.exists(queryWrapper);
}
@XpackInteract(value = "perFilterManage", recursion = true, invalid = true)
public IPage<VisualizationStoreVO> query(int pageNum, int pageSize, VisualizationWorkbranchQueryRequest request) {
IPage<StorePO> storePOIPage = proxy().queryStorePage(pageNum, pageSize, request);
if (ObjectUtils.isEmpty(storePOIPage)) return null;
@ -88,7 +90,7 @@ public class VisualizationStoreManage {
new VisualizationStoreVO(
po.getStoreId(), po.getResourceId(), po.getName(),
po.getType(), String.valueOf(po.getCreator()), ObjectUtils.isEmpty(po.getEditor()) ? null : String.valueOf(po.getEditor()),
po.getEditTime(), 9, po.getExtFlag())).toList();
po.getEditTime(), 9, po.getExtFlag(), po.getExtFlag1())).toList();
}
public IPage<StorePO> queryStorePage(int goPage, int pageSize, VisualizationWorkbranchQueryRequest request) {

2
core/core-backend/src/main/java/io/gisbi/visualization/manage/VisualizationTemplateExtendDataManage.java
View File

@ -15,7 +15,7 @@ import java.util.List;
/**
* @Author bi-coder
* @author : WangJiaHao
* @date : 2023/11/13 13:25
*/
@Service

194
core/core-backend/src/main/java/io/gisbi/visualization/server/DataVisualizationServer.java
View File

@ -2,7 +2,10 @@ package io.gisbi.visualization.server;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.fasterxml.jackson.core.type.TypeReference;
import io.gisbi.api.dataset.union.DatasetGroupInfoDTO;
import io.gisbi.api.dataset.union.DatasetTableInfoDTO;
import io.gisbi.api.dataset.union.UnionDTO;
import io.gisbi.api.template.dto.TemplateManageFileDTO;
import io.gisbi.api.template.dto.VisualizationTemplateExtendDataDTO;
import io.gisbi.api.visualization.DataVisualizationApi;
@ -14,6 +17,7 @@ import io.gisbi.api.visualization.vo.*;
import io.gisbi.auth.DeLinkPermit;
import io.gisbi.chart.dao.auto.entity.CoreChartView;
import io.gisbi.chart.dao.auto.mapper.CoreChartViewMapper;
import io.gisbi.chart.dao.ext.mapper.ExtChartViewMapper;
import io.gisbi.chart.manage.ChartDataManage;
import io.gisbi.chart.manage.ChartViewManege;
import io.gisbi.commons.constants.DataVisualizationConstants;
@ -28,11 +32,14 @@ import io.gisbi.dataset.dao.auto.mapper.CoreDatasetTableFieldMapper;
import io.gisbi.dataset.dao.auto.mapper.CoreDatasetTableMapper;
import io.gisbi.dataset.manage.DatasetDataManage;
import io.gisbi.dataset.manage.DatasetGroupManage;
import io.gisbi.dataset.manage.DatasetSQLManage;
import io.gisbi.dataset.utils.DatasetUtils;
import io.gisbi.datasource.dao.auto.entity.CoreDatasource;
import io.gisbi.datasource.dao.auto.mapper.CoreDatasourceMapper;
import io.gisbi.datasource.provider.ExcelUtils;
import io.gisbi.datasource.server.DatasourceServer;
import io.gisbi.exception.DEException;
import io.gisbi.extensions.datasource.dto.DatasetTableDTO;
import io.gisbi.extensions.datasource.vo.DatasourceConfiguration;
import io.gisbi.extensions.view.dto.ChartViewDTO;
import io.gisbi.i18n.Translator;
@ -50,19 +57,23 @@ import io.gisbi.template.dao.ext.ExtVisualizationTemplateMapper;
import io.gisbi.template.manage.TemplateCenterManage;
import io.gisbi.utils.*;
import io.gisbi.visualization.dao.auto.entity.DataVisualizationInfo;
import io.gisbi.visualization.dao.auto.entity.SnapshotDataVisualizationInfo;
import io.gisbi.visualization.dao.auto.entity.VisualizationWatermark;
import io.gisbi.visualization.dao.auto.mapper.DataVisualizationInfoMapper;
import io.gisbi.visualization.dao.auto.mapper.SnapshotCoreChartViewMapper;
import io.gisbi.visualization.dao.auto.mapper.SnapshotDataVisualizationInfoMapper;
import io.gisbi.visualization.dao.auto.mapper.VisualizationWatermarkMapper;
import io.gisbi.visualization.dao.ext.mapper.ExtDataVisualizationMapper;
import io.gisbi.visualization.manage.CoreBusiManage;
import io.gisbi.visualization.manage.CoreVisualizationManage;
import io.gisbi.visualization.utils.VisualizationUtils;
import jakarta.annotation.Resource;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.collections4.MapUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.util.CollectionUtils;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
@ -129,7 +140,8 @@ public class DataVisualizationServer implements DataVisualizationApi {
@Resource
private CoreDatasetTableFieldMapper coreDatasetTableFieldMapper;
@Autowired
@Resource
private CoreDatasourceMapper coreDatasourceMapper;
@Resource
@ -143,9 +155,16 @@ public class DataVisualizationServer implements DataVisualizationApi {
@Resource
private DatasourceServer datasourceServer;
@Resource
private SnapshotDataVisualizationInfoMapper snapshotMapper;
@Resource
private ExtChartViewMapper extChartViewMapper;
@Resource
private DatasetSQLManage datasetSQLManage;
@Override
public DataVisualizationVO findCopyResource(Long dvId, String busiFlag) {
DataVisualizationVO result = Objects.requireNonNull(CommonBeanFactory.proxy(this.getClass())).findById(new DataVisualizationBaseRequest(dvId, busiFlag));
DataVisualizationVO result = Objects.requireNonNull(CommonBeanFactory.proxy(this.getClass())).findById(new DataVisualizationBaseRequest(dvId, busiFlag, CommonConstants.RESOURCE_TABLE.SNAPSHOT,DataVisualizationConstants.QUERY_SOURCE.MAIN_EDIT));
if (result != null && result.getPid() == -1) {
return result;
} else {
@ -159,7 +178,17 @@ public class DataVisualizationServer implements DataVisualizationApi {
public DataVisualizationVO findById(DataVisualizationBaseRequest request) {
Long dvId = request.getId();
String busiFlag = request.getBusiFlag();
DataVisualizationVO result = extDataVisualizationMapper.findDvInfo(dvId, busiFlag);
String resourceTable = request.getResourceTable();
// 如果是编辑查询 则进行镜像检查
if (DataVisualizationConstants.QUERY_SOURCE.MAIN_EDIT.equals(request.getSource())) {
QueryWrapper<SnapshotDataVisualizationInfo> queryWrapper = new QueryWrapper<>();
queryWrapper.eq("id", dvId);
queryWrapper.in("status", Arrays.asList(CommonConstants.DV_STATUS.UNPUBLISHED, CommonConstants.DV_STATUS.SAVED_UNPUBLISHED)); // 状态为0 未发布 和 2 已保存未发布的 不需要重置镜像
if (!snapshotMapper.exists(queryWrapper)) {
coreVisualizationManage.dvSnapshotRecover(dvId);
}
}
DataVisualizationVO result = extDataVisualizationMapper.findDvInfo(dvId, busiFlag, resourceTable);
if (result != null) {
// get creator
String userName = coreUserManage.getUserName(Long.valueOf(result.getCreateBy()));
@ -167,9 +196,10 @@ public class DataVisualizationServer implements DataVisualizationApi {
result.setCreatorName(userName);
}
//获取图表信息
List<ChartViewDTO> chartViewDTOS = chartViewManege.listBySceneId(dvId);
List<ChartViewDTO> chartViewDTOS = chartViewManege.listBySceneId(dvId, resourceTable);
if (!CollectionUtils.isEmpty(chartViewDTOS)) {
Map<Long, ChartViewDTO> viewInfo = chartViewDTOS.stream().collect(Collectors.toMap(ChartViewDTO::getId, chartView -> chartView));
// 增加过滤当前使用的图表信息
Map<Long, ChartViewDTO> viewInfo = chartViewDTOS.stream().filter(item -> result.getComponentData().indexOf("\"id\":\"" + item.getId()) > 0).collect(Collectors.toMap(ChartViewDTO::getId, chartView -> chartView));
result.setCanvasViewInfo(viewInfo);
}
VisualizationWatermark watermark = watermarkMapper.selectById("system_default");
@ -208,6 +238,14 @@ public class DataVisualizationServer implements DataVisualizationApi {
@Override
@Transactional
public String saveCanvas(DataVisualizationBaseRequest request) throws Exception {
/*
* 发布兼容逻辑
* saveCanvas 为初次保存 包括 模板 应用 普通创建 所有变更操作都走snapshot表
* 1.如果是文件夹直接保存在主表中，如果是仪表板（数据大屏），主表和镜像表各保存一份 主表仅作为权限和预览控制此时主表状态为‘未发布’
* 2.编辑检查：如果存在未发布的仪表板snapshot，则默认加载snapshot进行编辑所有操作均为snapshot操作
* 3.发布（重新发布）：将snapshot表中的所有数据复制到主表中，同时变更主表状态为‘已发布’
* 4.如果对已发布的仪表板编辑并存在已保存的镜像，此时仪表板状态为‘已保存未发布’
*/
boolean isAppSave = false;
Long time = System.currentTimeMillis();
// 如果是应用 则新进行应用校验 数据集名称和 数据源名称校验
@ -220,18 +258,21 @@ public class DataVisualizationServer implements DataVisualizationApi {
Map<Long, Long> datasourceIdMap = new HashMap<>();
Map<Long, Map<String, String>> dsTableNamesMap = new HashMap<>();
List<Long> newDatasourceId = new ArrayList<>();
List<Long> excelDatasourceId = new ArrayList<>();
Map<String, String> excelTableNamesMap = new HashMap<>();
if (appData != null) {
isAppSave = true;
try {
List<AppCoreDatasourceVO> appCoreDatasourceVO = appData.getDatasourceInfo();
// app 数据源 excel 表名映射
appCoreDatasourceVO.forEach(datasourceOld -> {
newDatasourceId.add(datasourceOld.getSystemDatasourceId());
// Excel 数据表明映射
if (StringUtils.isNotEmpty(datasourceOld.getConfiguration())) {
if (datasourceOld.getType().equals(DatasourceConfiguration.DatasourceType.Excel.name())) {
dsTableNamesMap.put(datasourceOld.getId(), ExcelUtils.getTableNamesMap(datasourceOld.getConfiguration()));
if (datasourceOld.getType().equals(DatasourceConfiguration.DatasourceType.API.name())) {
DEException.throwException(Translator.get("i18n_app_error_no_api"));
} else if (datasourceOld.getType().equals(DatasourceConfiguration.DatasourceType.Excel.name())) {
dsTableNamesMap.put(datasourceOld.getId(), ExcelUtils.getTableNamesMap(datasourceOld.getType(), datasourceOld.getConfiguration()));
} else if (datasourceOld.getType().contains(DatasourceConfiguration.DatasourceType.API.name())) {
dsTableNamesMap.put(datasourceOld.getId(), (Map<String, String>) datasourceServer.invokeMethod(datasourceOld.getType(), "getTableNamesMap", String.class, datasourceOld.getConfiguration()));
}
@ -243,7 +284,8 @@ public class DataVisualizationServer implements DataVisualizationApi {
// Excel 数据表明映射
if (StringUtils.isNotEmpty(datasourceNew.getConfiguration())) {
if (datasourceNew.getType().equals(DatasourceConfiguration.DatasourceType.Excel.name())) {
dsTableNamesMap.put(datasourceNew.getId(), ExcelUtils.getTableNamesMap(datasourceNew.getConfiguration()));
dsTableNamesMap.put(datasourceNew.getId(), ExcelUtils.getTableNamesMap(datasourceNew.getType(), datasourceNew.getConfiguration()));
excelDatasourceId.add(datasourceNew.getId());
} else if (datasourceNew.getType().contains(DatasourceConfiguration.DatasourceType.API.name())) {
dsTableNamesMap.put(datasourceNew.getId(), (Map<String, String>) datasourceServer.invokeMethod(datasourceNew.getType(), "getTableNamesMap", String.class, datasourceNew.getConfiguration()));
}
@ -257,7 +299,7 @@ public class DataVisualizationServer implements DataVisualizationApi {
datasetFolderNewRequest.setName(datasetFolderName);
datasetFolderNewRequest.setNodeType("folder");
datasetFolderNewRequest.setPid(datasetFolderPid);
DatasetGroupInfoDTO datasetFolderNew = datasetGroupManage.save(datasetFolderNewRequest, false);
DatasetGroupInfoDTO datasetFolderNew = datasetGroupManage.save(datasetFolderNewRequest, false, false);
Long datasetFolderNewId = datasetFolderNew.getId();
//新建数据集
appData.getDatasetGroupsInfo().forEach(appDatasetGroup -> {
@ -332,10 +374,13 @@ public class DataVisualizationServer implements DataVisualizationApi {
//表名映射更新
Map<String, String> appDsTableNamesMap = dsTableNamesMap.get(key);
Map<String, String> systemDsTableNamesMap = dsTableNamesMap.get(value);
if (!CollectionUtils.isEmpty(appDsTableNamesMap) && !CollectionUtils.isEmpty(systemDsTableNamesMap)) {
if (MapUtils.isNotEmpty(appDsTableNamesMap)) {
appDsTableNamesMap.forEach((keyName, valueName) -> {
if (StringUtils.isNotEmpty(systemDsTableNamesMap.get(keyName))) {
if (MapUtils.isNotEmpty(systemDsTableNamesMap) && StringUtils.isNotEmpty(systemDsTableNamesMap.get(keyName))) {
dsGroup.setInfo(dsGroup.getInfo().replaceAll(valueName, systemDsTableNamesMap.get(keyName)));
excelTableNamesMap.put(valueName, systemDsTableNamesMap.get(keyName));
} else {
dsGroup.setInfo(dsGroup.getInfo().replaceAll(valueName, "excel_can_not_find"));
}
});
}
@ -345,6 +390,14 @@ public class DataVisualizationServer implements DataVisualizationApi {
dsGroup.setName(dsGroup.getName() + "-" + UUID.randomUUID().toString());
}
dsGroupNameSave.add(dsGroup.getName());
if(dsGroup.getIsCross() == null){
if(dsGroup.getUnion() == null){
dsGroup.setUnion(JsonUtil.parseList(dsGroup.getInfo(), new TypeReference<>() {
}));
}
datasetSQLManage.mergeDatasetCrossDefault(dsGroup);
}
excelAdaptor(dsGroup,excelTableNamesMap,excelDatasourceId);
datasetGroupManage.innerSave(dsGroup);
});
@ -370,7 +423,7 @@ public class DataVisualizationServer implements DataVisualizationApi {
//表名映射更新
Map<String, String> appDsTableNamesMap = dsTableNamesMap.get(key);
Map<String, String> systemDsTableNamesMap = dsTableNamesMap.get(value);
if (!CollectionUtils.isEmpty(appDsTableNamesMap) && !CollectionUtils.isEmpty(systemDsTableNamesMap)) {
if (MapUtils.isNotEmpty(appDsTableNamesMap) && MapUtils.isNotEmpty(systemDsTableNamesMap)) {
appDsTableNamesMap.forEach((keyName, valueName) -> {
if (StringUtils.isNotEmpty(systemDsTableNamesMap.get(keyName))) {
componentDataStr.set(componentDataStr.get().replaceAll(key.toString(), value.toString()));
@ -392,8 +445,12 @@ public class DataVisualizationServer implements DataVisualizationApi {
if (DataVisualizationConstants.RESOURCE_OPT_TYPE.COPY.equals(request.getOptType())) {
// 复制更新 新建权限插入
visualizationInfoMapper.deleteById(request.getId());
snapshotMapper.deleteById(request.getId());
visualizationInfo.setNodeType(DataVisualizationConstants.NODE_TYPE.LEAF);
}
// 文件夹走默认发布 非文件夹默认未发布
visualizationInfo.setStatus(DataVisualizationConstants.NODE_TYPE.FOLDER.equals(visualizationInfo.getNodeType())
? CommonConstants.DV_STATUS.PUBLISHED : CommonConstants.DV_STATUS.UNPUBLISHED);
Long newDvId = coreVisualizationManage.innerSave(visualizationInfo);
request.setId(newDvId);
// 还原ID信息
@ -429,6 +486,26 @@ public class DataVisualizationServer implements DataVisualizationApi {
return newDvId.toString();
}
private void excelAdaptor(DatasetGroupInfoDTO dsInfo,Map<String, String> excelTableNamesMap ,List<Long> excelDsId) {
List<UnionDTO> unionDTOList = JsonUtil.parseList(dsInfo.getInfo(), new TypeReference<>() {
});
if(CollectionUtils.isNotEmpty(excelDsId) && MapUtils.isNotEmpty(excelTableNamesMap)){
for(UnionDTO unionDTO : unionDTOList) {
DatasetTableDTO tableDTO =unionDTO.getCurrentDs();
if(excelDsId.contains(tableDTO.getDatasourceId())){
DatasetTableInfoDTO infoDTO = JsonUtil.parseObject(tableDTO.getInfo(), DatasetTableInfoDTO.class);
String s = new String(Base64.getDecoder().decode(infoDTO.getSql()));
excelTableNamesMap.forEach((key, value) -> {
infoDTO.setSql(Base64.getEncoder().encodeToString(s.replaceAll(key, value).getBytes()));
});
tableDTO.setInfo((String)JsonUtil.toJSONString(infoDTO));
}
}
}
dsInfo.setInfo((String)JsonUtil.toJSONString(unionDTOList));
}
@Override
public String appCanvasNameCheck(DataVisualizationBaseRequest request) throws Exception {
Long datasetFolderPid = request.getDatasetFolderPid();
@ -436,6 +513,7 @@ public class DataVisualizationServer implements DataVisualizationApi {
QueryWrapper<CoreDatasetGroup> queryWrapper = new QueryWrapper<>();
queryWrapper.eq("name", datasetFolderName);
queryWrapper.eq("pid", datasetFolderPid);
queryWrapper.eq("node_type", DataVisualizationConstants.NODE_TYPE.FOLDER);
if (coreDatasetGroupMapper.exists(queryWrapper)) {
return "repeat";
} else {
@ -462,7 +540,10 @@ public class DataVisualizationServer implements DataVisualizationApi {
@DeLog(id = "#p0.id", ot = LogOT.MODIFY, stExp = "#p0.type")
@Override
@Transactional
public void updateCanvas(DataVisualizationBaseRequest request) {
public DataVisualizationVO updateCanvas(DataVisualizationBaseRequest request) {
for (Map.Entry<Long, ChartViewDTO> ele : request.getCanvasViewInfo().entrySet()) {
DatasetUtils.viewDecode(ele.getValue());
}
Long dvId = request.getId();
if (dvId == null) {
DEException.throwException("ID can not be null");
@ -485,10 +566,48 @@ public class DataVisualizationServer implements DataVisualizationApi {
coreVisualizationManage.move(request);
}
}
// 状态修改统一为后端操作：历史状态检查 如果 状态为 0（未发布） 或者 2（已发布未保存）则状态不变
// 如果当前状态为 1 则状态修改为 2（已发布未保存）
Integer curStatus = extDataVisualizationMapper.findDvInfoStats(dvId);
visualizationInfo.setStatus(curStatus == 1 ? CommonConstants.DV_STATUS.SAVED_UNPUBLISHED : curStatus);
coreVisualizationManage.innerEdit(visualizationInfo);
//保存图表信息
chartDataManage.saveChartViewFromVisualization(request.getComponentData(), dvId, request.getCanvasViewInfo());
return new DataVisualizationVO(visualizationInfo.getStatus());
}
@Override
@Transactional
public void updatePublishStatus(DataVisualizationBaseRequest request) {
/**
* 如果当前传入状态是1（已发布），则原始状态0（未发布）-》1（已发布）；2（已保存未发布）-》1（已发布）
* 统一处理为1.删除主表数据，2.将镜像表数据统一copy到主表 不删除镜像数据（发布状态后镜像数据和主表数据是保持一致的）
* 其他状态仅更新主表和镜像表状态
* */
Long dvId = request.getId();
DataVisualizationInfo visualizationInfo = new DataVisualizationInfo();
visualizationInfo.setMobileLayout(request.getMobileLayout());
visualizationInfo.setId(dvId);
visualizationInfo.setName(request.getName());
visualizationInfo.setStatus(request.getStatus());
coreVisualizationManage.innerEdit(visualizationInfo);
if (CommonConstants.DV_STATUS.PUBLISHED == request.getStatus()) {
coreVisualizationManage.removeDvCore(dvId);
coreVisualizationManage.dvRestore(dvId);
// chartViewManege.publishThreshold(dvId, request.getActiveViewIds());
} else if (CommonConstants.DV_STATUS.UNPUBLISHED == request.getStatus()) {
// chartViewManege.publishThreshold(dvId, request.getActiveViewIds());
}
}
@Override
public void recoverToPublished(DataVisualizationBaseRequest request) {
coreVisualizationManage.dvSnapshotRecover(request.getId());
DataVisualizationInfo visualizationInfo = new DataVisualizationInfo();
visualizationInfo.setId(request.getId());
visualizationInfo.setName(request.getName());
visualizationInfo.setStatus(CommonConstants.DV_STATUS.PUBLISHED);
coreVisualizationManage.innerEdit(visualizationInfo);
}
/**
@ -527,6 +646,9 @@ public class DataVisualizationServer implements DataVisualizationApi {
@Override
public List<BusiNodeVO> tree(BusiNodeRequest request) {
if (StringUtils.isEmpty(request.getResourceTable())) {
request.setResourceTable(CommonConstants.RESOURCE_TABLE.SNAPSHOT);
}
String busiFlag = request.getBusiFlag();
if (busiFlag.equals("dashboard-dataV")) {
BusiNodeRequest requestDv = new BusiNodeRequest();
@ -609,7 +731,8 @@ public class DataVisualizationServer implements DataVisualizationApi {
newDv.setPid(request.getPid());
newDv.setCreateTime(System.currentTimeMillis());
// 复制图表 chart_view
extDataVisualizationMapper.viewCopyWithDv(sourceDvId, newDvId, copyId);
extDataVisualizationMapper.viewCopyWithDv(sourceDvId, newDvId, copyId, CommonConstants.RESOURCE_TABLE.CORE);
extDataVisualizationMapper.viewCopyWithDv(sourceDvId, newDvId, copyId, CommonConstants.RESOURCE_TABLE.SNAPSHOT);
List<CoreChartView> viewList = extDataVisualizationMapper.findViewInfoByCopyId(copyId);
if (!CollectionUtils.isEmpty(viewList)) {
String componentData = newDv.getComponentData();
@ -660,7 +783,6 @@ public class DataVisualizationServer implements DataVisualizationApi {
String name = null;
String dvType = null;
Integer version = null;
//内部模板新建
if (DataVisualizationConstants.NEW_PANEL_FROM.NEW_INNER_TEMPLATE.equals(newFrom)) {
VisualizationTemplate visualizationTemplate = templateMapper.selectById(request.getTemplateId());
@ -776,10 +898,10 @@ public class DataVisualizationServer implements DataVisualizationApi {
@Override
public List<VisualizationViewTableDTO> detailList(Long dvId) {
List<VisualizationViewTableDTO> result = extDataVisualizationMapper.getVisualizationViewDetails(dvId);
DataVisualizationInfo dvInfo = visualizationInfoMapper.selectById(dvId);
SnapshotDataVisualizationInfo dvInfo = snapshotMapper.selectById(dvId);
if (dvInfo != null && !CollectionUtils.isEmpty(result)) {
String componentData = dvInfo.getComponentData();
return result.stream().filter(item -> componentData.indexOf(String.valueOf(item.getId())) > 0).toList();
return result.stream().filter(item -> componentData.indexOf("\"id\":\"" + item.getId()) > 0).toList();
} else {
return result;
}
@ -810,6 +932,9 @@ public class DataVisualizationServer implements DataVisualizationApi {
if (CollectionUtils.isEmpty(datasourceVOInfo)) {
DEException.throwException("当前不存在数据源无法导出");
} else if (datasourceVOInfo.stream()
.anyMatch(datasource -> DatasourceConfiguration.DatasourceType.API.name().equals(datasource.getType()))) {
DEException.throwException(Translator.get("i18n_app_error_no_api"));
}
List<VisualizationLinkageVO> linkageVOInfo = appTemplateMapper.findAppLinkageInfo(dvId);
@ -843,27 +968,28 @@ public class DataVisualizationServer implements DataVisualizationApi {
if (AuthUtils.getUser().getDefaultOid() != null) {
wrapper.eq("org_id", AuthUtils.getUser().getDefaultOid());
}
if (visualizationInfoMapper.exists(wrapper)) {
List<DataVisualizationInfo> existList = visualizationInfoMapper.selectList(wrapper);
if (CollectionUtils.isNotEmpty(existList) && existList.stream().anyMatch(item -> item.getName().equals(request.getName().trim()))) {
DEException.throwException("当前名称已经存在");
}
}
public String getAbsPath(String id) {
CoreChartView coreChartView = coreChartViewMapper.selectById(id);
if (coreChartView == null) {
public String getAbsPath(Long id) {
ChartViewDTO viewDTO = chartViewManege.findChartViewAround(String.valueOf(id));
if (viewDTO == null) {
return null;
}
if (coreChartView.getSceneId() == null) {
return coreChartView.getTitle();
if (viewDTO.getPid() == null) {
return viewDTO.getTitle();
}
List<DataVisualizationInfo> parents = getParents(coreChartView.getSceneId());
List<DataVisualizationInfo> parents = getParents(viewDTO.getPid());
StringBuilder stringBuilder = new StringBuilder();
parents.forEach(ele -> {
if (ObjectUtils.isNotEmpty(ele)) {
stringBuilder.append(ele.getName()).append("/");
}
});
stringBuilder.append(coreChartView.getTitle());
stringBuilder.append(viewDTO.getTitle());
return stringBuilder.toString();
}
@ -887,4 +1013,16 @@ public class DataVisualizationServer implements DataVisualizationApi {
}
}
public List<Long> getEnabledViewIds(Long dvId, String resourceTable) {
List<Long> result = new ArrayList<>();
DataVisualizationVO dvInfo = extDataVisualizationMapper.findDvInfo(dvId, null, resourceTable);
List<CoreChartView> views = extChartViewMapper.selectListCustom(dvId, resourceTable);
if (CollectionUtils.isNotEmpty(views) && dvInfo != null) {
String componentData = dvInfo.getComponentData();
result = views.stream().filter(item -> componentData.indexOf("\"id\":\"" + item.getId()) > 0).map(CoreChartView::getId)
.collect(Collectors.toList());
}
return result;
}
}

7
core/core-backend/src/main/java/io/gisbi/visualization/server/StaticResourceServer.java
View File

@ -34,7 +34,7 @@ import java.util.Map;
@RequestMapping("/staticResource")
public class StaticResourceServer implements StaticResourceApi {
@Value("${gisbi.path.static-resource:/opt/gisbi/data/static-resource/}")
@Value("${gisbi.path.static-resource:/opt/gisbi2.0/data/static-resource/}")
private String staticDir;
@Override
@ -113,11 +113,6 @@ public class StaticResourceServer implements StaticResourceApi {
return result;
}
@Override
public Map<String, String> urlTest() {
return null;
}
private static boolean isValidSVG(MultipartFile file){
if (file == null || file.isEmpty()) {
return false;

72
core/core-backend/src/main/java/io/gisbi/visualization/server/VisualizationLinkJumpService.java
View File

@ -11,19 +11,14 @@ import io.gisbi.api.visualization.vo.VisualizationViewTableVO;
import io.gisbi.auth.DeLinkPermit;
import io.gisbi.chart.dao.auto.entity.CoreChartView;
import io.gisbi.chart.dao.auto.mapper.CoreChartViewMapper;
import io.gisbi.constant.CommonConstants;
import io.gisbi.extensions.datasource.dto.DatasetTableFieldDTO;
import io.gisbi.utils.AuthUtils;
import io.gisbi.utils.BeanUtils;
import io.gisbi.utils.IDUtils;
import io.gisbi.utils.ModelUtils;
import io.gisbi.visualization.dao.auto.entity.DataVisualizationInfo;
import io.gisbi.visualization.dao.auto.entity.VisualizationLinkJump;
import io.gisbi.visualization.dao.auto.entity.VisualizationLinkJumpInfo;
import io.gisbi.visualization.dao.auto.entity.VisualizationLinkJumpTargetViewInfo;
import io.gisbi.visualization.dao.auto.mapper.DataVisualizationInfoMapper;
import io.gisbi.visualization.dao.auto.mapper.VisualizationLinkJumpInfoMapper;
import io.gisbi.visualization.dao.auto.mapper.VisualizationLinkJumpMapper;
import io.gisbi.visualization.dao.auto.mapper.VisualizationLinkJumpTargetViewInfoMapper;
import io.gisbi.visualization.dao.auto.entity.*;
import io.gisbi.visualization.dao.auto.mapper.*;
import io.gisbi.visualization.dao.ext.mapper.ExtVisualizationLinkJumpMapper;
import io.gisbi.visualization.dao.ext.mapper.ExtVisualizationLinkageMapper;
import jakarta.annotation.Resource;
@ -59,9 +54,22 @@ public class VisualizationLinkJumpService implements VisualizationLinkJumpApi {
@Resource
private VisualizationLinkJumpTargetViewInfoMapper visualizationLinkJumpTargetViewInfoMapper;
@Resource
private SnapshotVisualizationLinkJumpMapper snapshotVisualizationLinkJumpMapper;
@Resource
private SnapshotVisualizationLinkJumpInfoMapper snapshotVisualizationLinkJumpInfoMapper;
@Resource
private SnapshotVisualizationLinkJumpTargetViewInfoMapper snapshotVisualizationLinkJumpTargetViewInfoMapper;
@Resource
private CoreChartViewMapper coreChartViewMapper;
@Resource
private SnapshotCoreChartViewMapper snapshotCoreChartViewMapper;
@Resource
private DataVisualizationInfoMapper dataVisualizationInfoMapper;
@ -73,9 +81,14 @@ public class VisualizationLinkJumpService implements VisualizationLinkJumpApi {
@DeLinkPermit
//获取仪表板的跳转信息
@Override
public VisualizationLinkJumpBaseResponse queryVisualizationJumpInfo(Long dvId) {
public VisualizationLinkJumpBaseResponse queryVisualizationJumpInfo(Long dvId, String resourceTable) {
Map<String, VisualizationLinkJumpInfoDTO> resultBase = new HashMap<>();
List<VisualizationLinkJumpDTO> resultLinkJumpList = extVisualizationLinkJumpMapper.queryWithDvId(dvId, AuthUtils.getUser().getUserId(), ModelUtils.isDesktop());
List<VisualizationLinkJumpDTO> resultLinkJumpList = null;
if (CommonConstants.RESOURCE_TABLE.SNAPSHOT.equals(resourceTable)) {
resultLinkJumpList = extVisualizationLinkJumpMapper.queryWithDvIdSnapshot(dvId, AuthUtils.getUser().getUserId(), ModelUtils.isDesktop());
} else {
resultLinkJumpList = extVisualizationLinkJumpMapper.queryWithDvId(dvId, AuthUtils.getUser().getUserId(), ModelUtils.isDesktop());
}
Optional.ofNullable(resultLinkJumpList).orElse(new ArrayList<>()).forEach(resultLinkJump -> {
if (resultLinkJump.getChecked()) {
Long sourceViewId = resultLinkJump.getSourceViewId();
@ -111,30 +124,30 @@ public class VisualizationLinkJumpService implements VisualizationLinkJumpApi {
Assert.notNull(dvId, "dvId cannot be null");
Assert.notNull(viewId, "viewId cannot be null");
//清理原有数据
extVisualizationLinkJumpMapper.deleteJumpTargetViewInfo(dvId, viewId);
extVisualizationLinkJumpMapper.deleteJumpInfo(dvId, viewId);
extVisualizationLinkJumpMapper.deleteJump(dvId, viewId);
extVisualizationLinkJumpMapper.deleteJumpTargetViewInfoSnapshot(dvId, viewId);
extVisualizationLinkJumpMapper.deleteJumpInfoSnapshot(dvId, viewId);
extVisualizationLinkJumpMapper.deleteJumpSnapshot(dvId, viewId);
// 插入新的数据
Long linkJumpId = IDUtils.snowID();
jumpDTO.setId(linkJumpId);
VisualizationLinkJump insertParam = new VisualizationLinkJump();
SnapshotVisualizationLinkJump insertParam = new SnapshotVisualizationLinkJump();
BeanUtils.copyBean(insertParam, jumpDTO);
visualizationLinkJumpMapper.insert(insertParam);
snapshotVisualizationLinkJumpMapper.insert(insertParam);
Optional.ofNullable(jumpDTO.getLinkJumpInfoArray()).orElse(new ArrayList<>()).forEach(linkJumpInfo -> {
Long linkJumpInfoId = IDUtils.snowID();
linkJumpInfo.setId(linkJumpInfoId);
linkJumpInfo.setLinkJumpId(linkJumpId);
VisualizationLinkJumpInfo insertJumpInfoParam = new VisualizationLinkJumpInfo();
SnapshotVisualizationLinkJumpInfo insertJumpInfoParam = new SnapshotVisualizationLinkJumpInfo();
BeanUtils.copyBean(insertJumpInfoParam, linkJumpInfo);
visualizationLinkJumpInfoMapper.insert(insertJumpInfoParam);
snapshotVisualizationLinkJumpInfoMapper.insert(insertJumpInfoParam);
Optional.ofNullable(linkJumpInfo.getTargetViewInfoList()).orElse(new ArrayList<>()).forEach(targetViewInfo -> {
Long targetViewInfoId = IDUtils.snowID();
targetViewInfo.setTargetId(targetViewInfoId);
targetViewInfo.setLinkJumpInfoId(linkJumpInfoId);
VisualizationLinkJumpTargetViewInfo insertTargetViewInfoParam = new VisualizationLinkJumpTargetViewInfo();
SnapshotVisualizationLinkJumpTargetViewInfo insertTargetViewInfoParam = new SnapshotVisualizationLinkJumpTargetViewInfo();
BeanUtils.copyBean(insertTargetViewInfoParam, targetViewInfo);
visualizationLinkJumpTargetViewInfoMapper.insert(insertTargetViewInfoParam);
snapshotVisualizationLinkJumpTargetViewInfoMapper.insert(insertTargetViewInfoParam);
});
});
}
@ -142,7 +155,12 @@ public class VisualizationLinkJumpService implements VisualizationLinkJumpApi {
@DeLinkPermit("#p0.targetDvId")
@Override
public VisualizationLinkJumpBaseResponse queryTargetVisualizationJumpInfo(VisualizationLinkJumpBaseRequest request) {
List<VisualizationLinkJumpDTO> result = extVisualizationLinkJumpMapper.getTargetVisualizationJumpInfo(request);
List<VisualizationLinkJumpDTO> result = null;
if (CommonConstants.RESOURCE_TABLE.SNAPSHOT.equals(request.getResourceTable())) {
result = extVisualizationLinkJumpMapper.getTargetVisualizationJumpInfoSnapshot(request);
} else {
result = extVisualizationLinkJumpMapper.getTargetVisualizationJumpInfo(request);
}
return new VisualizationLinkJumpBaseResponse(null, Optional.ofNullable(result).orElse(new ArrayList<>()).stream().filter(item -> StringUtils.isNotEmpty(item.getSourceInfo())).collect(Collectors.toMap(VisualizationLinkJumpDTO::getSourceInfo, VisualizationLinkJumpDTO::getTargetInfoList)));
}
@ -161,25 +179,25 @@ public class VisualizationLinkJumpService implements VisualizationLinkJumpApi {
outParamsJumpInfo = new ArrayList<>();
componentData = "[]";
}
return new VisualizationComponentDTO(componentData,result,outParamsJumpInfo);
return new VisualizationComponentDTO(componentData, result, outParamsJumpInfo);
}
@Override
public VisualizationLinkJumpBaseResponse updateJumpSetActive(VisualizationLinkJumpBaseRequest request) {
CoreChartView coreChartView = new CoreChartView();
SnapshotCoreChartView coreChartView = new SnapshotCoreChartView();
coreChartView.setId(Long.valueOf(request.getSourceViewId()));
coreChartView.setJumpActive(request.getActiveStatus());
coreChartViewMapper.updateById(coreChartView);
return queryVisualizationJumpInfo(request.getSourceDvId());
snapshotCoreChartViewMapper.updateById(coreChartView);
return queryVisualizationJumpInfo(request.getSourceDvId(), CommonConstants.RESOURCE_TABLE.SNAPSHOT);
}
@Override
public void removeJumpSet(VisualizationLinkJumpDTO jumpDTO) {
//清理原有数据
extVisualizationLinkJumpMapper.deleteJumpTargetViewInfo(jumpDTO.getSourceDvId(), jumpDTO.getSourceViewId());
extVisualizationLinkJumpMapper.deleteJumpInfo(jumpDTO.getSourceDvId(), jumpDTO.getSourceViewId());
extVisualizationLinkJumpMapper.deleteJump(jumpDTO.getSourceDvId(), jumpDTO.getSourceViewId());
extVisualizationLinkJumpMapper.deleteJumpTargetViewInfoSnapshot(jumpDTO.getSourceDvId(), jumpDTO.getSourceViewId());
extVisualizationLinkJumpMapper.deleteJumpInfoSnapshot(jumpDTO.getSourceDvId(), jumpDTO.getSourceViewId());
extVisualizationLinkJumpMapper.deleteJumpSnapshot(jumpDTO.getSourceDvId(), jumpDTO.getSourceViewId());
}
}

Some files were not shown because too many files have changed in this diff Show More