From adabf1e604b38f5175e68c7751cd3caab1cecc54 Mon Sep 17 00:00:00 2001 From: root <13910913995@163.com> Date: Mon, 23 Jun 2025 10:34:29 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=B9=B6=E6=8F=90=E4=BA=A4?= =?UTF-8?q?=E4=BA=86=E5=90=8E=E7=AB=AF=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/core-backend/pom.xml | 41 +- .../java/io/gisbi/MybatisPlusGenerator.java | 2 +- .../charts/impl/DefaultChartHandler.java | 142 ++- .../chart/charts/impl/YoyChartHandler.java | 4 +- .../chart/charts/impl/bar/BarHandler.java | 32 +- .../chart/charts/impl/line/LineHandler.java | 31 +- .../charts/impl/line/StackAreaHandler.java | 31 +- .../charts/impl/map/SymbolicMapHandler.java | 15 +- .../chart/charts/impl/mix/MixHandler.java | 67 +- .../impl/numeric/NumericalChartHandler.java | 11 +- .../chart/charts/impl/pie/PieHandler.java | 80 +- .../impl/table/TableHeatmapHandler.java | 2 + .../charts/impl/table/TableInfoHandler.java | 37 +- .../charts/impl/table/TableNormalHandler.java | 196 +++- .../charts/impl/table/TablePivotHandler.java | 64 +- .../dao/ext/mapper/ExtChartViewMapper.java | 40 +- .../gisbi/chart/manage/ChartDataManage.java | 91 +- .../gisbi/chart/manage/ChartViewManege.java | 71 +- .../manage/ChartViewThresholdManage.java | 36 +- .../gisbi/chart/server/ChartDataServer.java | 246 ++++- .../gisbi/chart/server/ChartViewServer.java | 21 +- .../io/gisbi/chart/utils/ChartDataBuild.java | 39 +- .../constants/DataVisualizationConstants.java | 3 + .../io/gisbi/commons/utils/EncryptUtils.java | 1 + .../commons/utils/ExcelWatermarkUtils.java | 41 +- .../gisbi/commons/utils/SqlparserUtils.java | 61 +- .../java/io/gisbi/config/DeMvcConfig.java | 8 +- .../dao/auto/entity/CoreDatasetGroup.java | 28 +- .../auto/entity/CoreDatasetTableField.java | 16 +- .../auto/mapper/CoreDatasetTableMapper.java | 4 +- .../dao/ext/mapper/CoreDataSetExtMapper.java | 2 +- .../dataset/manage/DatasetDataManage.java | 315 +++--- .../dataset/manage/DatasetGroupManage.java | 82 +- .../dataset/manage/DatasetSQLManage.java | 125 ++- .../manage/DatasetTableFieldManage.java | 77 +- .../dataset/manage/PermissionManage.java | 6 +- .../dataset/server/DatasetDataServer.java | 73 +- .../dataset/server/DatasetFieldServer.java | 5 +- .../dataset/server/DatasetTreeServer.java | 25 +- .../io/gisbi/dataset/utils/DatasetUtils.java | 99 +- .../datasource/manage/DataSourceManage.java | 28 +- .../manage/DatasourceSyncManage.java | 46 +- .../gisbi/datasource/manage/EngineManage.java | 21 +- .../gisbi/datasource/provider/ApiUtils.java | 85 +- .../datasource/provider/CalciteProvider.java | 412 ++++++-- .../gisbi/datasource/provider/ExcelUtils.java | 358 +++++-- .../datasource/provider/H2EngineProvider.java | 9 +- .../provider/MysqlEngineProvider.java | 6 +- .../server/DatasourceDriverServer.java | 2 +- .../datasource/server/DatasourceServer.java | 244 +++-- .../java/io/gisbi/datasource/type/H2.java | 18 + .../java/io/gisbi/datasource/type/Mysql.java | 16 +- .../java/io/gisbi/datasource/type/Pg.java | 30 +- .../io/gisbi/datasource/type/Redshift.java | 21 +- .../java/io/gisbi/engine/sql/SQLProvider.java | 2 +- .../gisbi/engine/trans/CustomWhere2Str.java | 23 +- .../gisbi/engine/trans/Dimension2SQLObj.java | 6 +- .../io/gisbi/engine/trans/ExtWhere2Str.java | 28 +- .../io/gisbi/engine/trans/Field2SQLObj.java | 6 +- .../io/gisbi/engine/trans/Order2SQLObj.java | 6 +- .../io/gisbi/engine/trans/Quota2SQLObj.java | 6 +- .../io/gisbi/engine/trans/WhereTree2Str.java | 25 +- .../java/io/gisbi/engine/utils/Utils.java | 44 +- .../dao/auto/entity/CoreExportTask.java | 6 +- .../manage/ExportCenterLimitManage.java | 4 +- .../manage/ExportCenterManage.java | 680 +++---------- .../server/ExportCenterServer.java | 19 +- .../java/io/gisbi/font/manage/FontManage.java | 2 +- .../gisbi/license/manage/CoreLicManage.java | 2 +- .../gisbi/license/server/LicenseServer.java | 2 +- .../gisbi/listener/EhCacheStartListener.java | 12 +- .../java/io/gisbi/menu/server/MenuServer.java | 1 - .../io/gisbi/resource/ResourceService.java | 11 +- .../dao/ext/mapper/XpackShareExtMapper.java | 1 + .../gisbi/share/dao/ext/po/XpackSharePO.java | 2 + .../share/interceptor/LinkInterceptor.java | 11 +- .../gisbi/share/manage/XpackShareManage.java | 2 +- .../gisbi/share/server/XpackShareServer.java | 1 + .../system/manage/SysParameterManage.java | 60 +- .../system/server/SysParameterServer.java | 7 +- .../template/manage/TemplateCenterManage.java | 6 +- .../manage/TemplateLocalParseManage.java | 2 +- .../auto/entity/DataVisualizationInfo.java | 14 +- .../mapper/VisualizationWatermarkMapper.java | 5 + .../dao/ext/mapper/CoreStoreExtMapper.java | 3 +- .../ext/mapper/CoreVisualiationExtMapper.java | 2 +- .../mapper/ExtDataVisualizationMapper.java | 56 +- .../ExtVisualizationLinkJumpMapper.java | 16 +- .../mapper/ExtVisualizationLinkageMapper.java | 8 + .../ExtVisualizationOuterParamsMapper.java | 11 +- .../visualization/dao/ext/po/StorePO.java | 2 + .../dao/ext/po/VisualizationNodePO.java | 2 + .../dto/VisualizationNodeBO.java | 1 + .../manage/CoreVisualizationExportManage.java | 46 +- .../manage/CoreVisualizationManage.java | 164 ++- .../manage/VisualizationStoreManage.java | 4 +- ...VisualizationTemplateExtendDataManage.java | 2 +- .../server/DataVisualizationServer.java | 194 +++- .../server/StaticResourceServer.java | 7 +- .../server/VisualizationLinkJumpService.java | 72 +- .../server/VisualizationLinkageService.java | 70 +- .../VisualizationOuterParamsService.java | 57 +- .../server/VisualizationStoreServer.java | 1 + .../utils/VisualizationExcelUtils.java | 4 +- .../src/main/resources/application.yml | 2 +- .../main/resources/i18n/core_en_US.properties | 7 + .../main/resources/i18n/core_zh_CN.properties | 8 + .../main/resources/i18n/core_zh_TW.properties | 204 ---- .../mybatis/ExtDataVisualizationMapper.xml | 932 ++++++++++++++++-- .../ExtVisualizationLinkJumpMapper.xml | 242 +++-- .../mybatis/ExtVisualizationLinkageMapper.xml | 120 ++- .../ExtVisualizationOuterParamsMapper.xml | 97 +- core/pom.xml | 25 +- pom.xml | 1 + .../java/io/gisbi/model/BusiNodeRequest.java | 2 + sdk/sdk-bundle/pom.xml | 8 +- 116 files changed, 4759 insertions(+), 2173 deletions(-) delete mode 100644 core/core-backend/src/main/resources/i18n/core_zh_TW.properties diff --git a/core/core-backend/pom.xml b/core/core-backend/pom.xml index f738ba5..2587cad 100644 --- a/core/core-backend/pom.xml +++ b/core/core-backend/pom.xml @@ -3,33 +3,40 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - gisbi + core io.gisbi - 1.0.0 + 2.0.0 + 4.0.0 jar core-backend + + + commons-net + commons-net + 3.8.0 + com.google.guava guava - ${guava.version} + 33.0.0-jre io.gisbi api-base - 1.0.0 + 2.0.0 io.gisbi common - 1.0.0 + 2.0.0 io.gisbi api-sync - 1.0.0 + 2.0.0 com.mysql @@ -38,18 +45,18 @@ org.apache.commons commons-dbcp2 - ${commons-dbcp2.version} + 2.6.0 org.antlr antlr - ${antlr.version} + 3.5.2 junit junit - ${junit.version} + 4.13.2 test @@ -73,7 +80,6 @@ hutool-all 5.8.8 - org.springframework.boot spring-boot-starter-websocket @@ -93,15 +99,10 @@ org.apache.commons commons-lang3 - - cn.hutool - hutool-all - 5.8.38 - io.gisbi api-permissions - 1.0.0 + 2.0.0 @@ -125,12 +126,12 @@ org.seleniumhq.selenium selenium-java - ${selenium-java.version} + 4.19.1 org.eclipse.angus angus-mail - ${angus-mail.version} + 2.0.3 org.eclipse.angus @@ -141,13 +142,13 @@ com.itextpdf itext7-core - ${itextpdf.version} + 8.0.4 pom com.vladsch.flexmark flexmark-all - ${flexmark.version} + 0.62.2 diff --git a/core/core-backend/src/main/java/io/gisbi/MybatisPlusGenerator.java b/core/core-backend/src/main/java/io/gisbi/MybatisPlusGenerator.java index c5db994..a621a48 100644 --- a/core/core-backend/src/main/java/io/gisbi/MybatisPlusGenerator.java +++ b/core/core-backend/src/main/java/io/gisbi/MybatisPlusGenerator.java @@ -14,7 +14,7 @@ public class MybatisPlusGenerator { * 第一 我嫌麻烦 * 第二 后面配置会放到nacos读起来更麻烦了 */ - private static final String url = "jdbc:mysql://localhost:3306/dataease?autoReconnect=false&useUnicode=true&characterEncoding=UTF-8&characterSetResults=UTF-8&zeroDateTimeBehavior=convertToNull&useSSL=false"; + private static final String url = "jdbc:mysql://localhost:3306/gisbi?autoReconnect=false&useUnicode=true&characterEncoding=UTF-8&characterSetResults=UTF-8&zeroDateTimeBehavior=convertToNull&useSSL=false"; private static final String username = "root"; private static final String password = "123456"; diff --git a/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/DefaultChartHandler.java b/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/DefaultChartHandler.java index 7f00f9c..1fd82ad 100644 --- a/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/DefaultChartHandler.java +++ b/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/DefaultChartHandler.java @@ -1,12 +1,14 @@ package io.gisbi.chart.charts.impl; +import com.beust.jcommander.Strings; +import io.gisbi.api.dataset.union.DatasetGroupInfoDTO; import io.gisbi.chart.charts.ChartHandlerManager; import io.gisbi.chart.constant.ChartConstants; import io.gisbi.chart.manage.ChartDataManage; import io.gisbi.chart.manage.ChartViewManege; import io.gisbi.chart.utils.ChartDataBuild; -import io.gisbi.dataset.manage.DatasetTableFieldManage; import io.gisbi.constant.SQLConstants; +import io.gisbi.dataset.manage.DatasetTableFieldManage; import io.gisbi.engine.sql.SQLProvider; import io.gisbi.engine.trans.Dimension2SQLObj; import io.gisbi.engine.trans.Quota2SQLObj; @@ -82,7 +84,7 @@ public class DefaultChartHandler extends AbstractChartPlugin { var desensitizationList = (Map) formatResult.getContext().get("desensitizationList"); if (MapUtils.isNotEmpty(desensitizationList)) { formatResult.getAxisMap().forEach((axis, fields) -> { - fields.removeIf(f -> desensitizationList.containsKey(f.getDataeaseName())); + fields.removeIf(f -> desensitizationList.containsKey(f.getGisbiName())); }); } return (T) new CustomFilterResult(filterList, formatResult.getContext()); @@ -107,8 +109,9 @@ public class DefaultChartHandler extends AbstractChartPlugin { dsList.add(next.getValue().getType()); } boolean needOrder = Utils.isNeedOrder(dsList); - boolean crossDs = Utils.isCrossDs(dsMap); + boolean crossDs = ((DatasetGroupInfoDTO) formatResult.getContext().get("dataset")).getIsCross(); DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setIsCross(crossDs); datasourceRequest.setDsList(dsMap); var xAxis = formatResult.getAxisMap().get(ChartAxis.xAxis); var yAxis = formatResult.getAxisMap().get(ChartAxis.yAxis); @@ -168,8 +171,8 @@ public class DefaultChartHandler extends AbstractChartPlugin { dataMap.putAll(calcResult.getData()); dataMap.putAll(mapTableNormal); dataMap.put("sourceFields", allFields); - mergeAssistField(calcResult.getDynamicAssistFields(), calcResult.getAssistData()); - dataMap.put("dynamicAssistLines", calcResult.getDynamicAssistFields()); + List chartSeniorAssistDTOS = mergeAssistField(calcResult.getDynamicAssistFields(), calcResult.getAssistData(), calcResult.getDynamicAssistFieldsOriginList(), calcResult.getAssistDataOriginList()); + dataMap.put("dynamicAssistLines", chartSeniorAssistDTOS); view.setData(dataMap); view.setSql(Base64.getEncoder().encodeToString(calcResult.getQuerySql().getBytes())); view.setDrill(isDrill); @@ -177,18 +180,30 @@ public class DefaultChartHandler extends AbstractChartPlugin { return view; } - - protected void mergeAssistField(List dynamicAssistFields, List assistData) { - if (ObjectUtils.isEmpty(assistData)) { - return; - } - String[] strings = assistData.get(0); - for (int i = 0; i < dynamicAssistFields.size(); i++) { - if (i < strings.length) { - ChartSeniorAssistDTO chartSeniorAssistDTO = dynamicAssistFields.get(i); - chartSeniorAssistDTO.setValue(strings[i]); + protected List mergeAssistField(List dynamicAssistFields, List assistData, List dynamicAssistFieldsOriginList, List assistDataOriginList) { + List list = new ArrayList<>(); + if (ObjectUtils.isNotEmpty(assistData)) { + String[] strings = assistData.getFirst(); + for (int i = 0; i < dynamicAssistFields.size(); i++) { + if (i < strings.length) { + ChartSeniorAssistDTO chartSeniorAssistDTO = dynamicAssistFields.get(i); + chartSeniorAssistDTO.setValue(strings[i]); + list.add(chartSeniorAssistDTO); + } } } + + if (ObjectUtils.isNotEmpty(assistDataOriginList)) { + String[] stringsOriginList = assistDataOriginList.getLast();// 取最后一项，如果有其他运算逻辑需要取明细数据可增加逻辑 + for (int i = 0; i < dynamicAssistFieldsOriginList.size(); i++) { + if (i < stringsOriginList.length) { + ChartSeniorAssistDTO chartSeniorAssistDTO = dynamicAssistFieldsOriginList.get(i); + chartSeniorAssistDTO.setValue(stringsOriginList[i]); + list.add(chartSeniorAssistDTO); + } + } + } + return list; } protected List getDynamicAssistFields(ChartViewDTO view) { @@ -370,7 +385,7 @@ public class DefaultChartHandler extends AbstractChartPlugin { return conditionField; } - protected String assistSQL(String sql, List assistFields, Map dsMap) { + protected String assistSQL(String sql, List assistFields, Map dsMap, boolean crossDs) { // get datasource prefix and suffix String dsType = dsMap.entrySet().iterator().next().getValue().getType(); String prefix = ""; @@ -391,25 +406,54 @@ public class DefaultChartHandler extends AbstractChartPlugin { } } - boolean crossDs = Utils.isCrossDs(dsMap); - StringBuilder stringBuilder = new StringBuilder(); + List fieldList = new ArrayList<>(); for (int i = 0; i < assistFields.size(); i++) { ChartViewFieldDTO dto = assistFields.get(i); + if (StringUtils.equalsIgnoreCase(dto.getSummary(), "last_item")) { + continue; + } if (crossDs) { - if (i == (assistFields.size() - 1)) { - stringBuilder.append(dto.getSummary() + "(" + dto.getOriginName() + ")"); - } else { - stringBuilder.append(dto.getSummary() + "(" + dto.getOriginName() + "),"); - } + fieldList.add(dto.getSummary() + "(" + dto.getOriginName() + ")"); } else { - if (i == (assistFields.size() - 1)) { - stringBuilder.append(dto.getSummary() + "(" + prefix + dto.getOriginName() + suffix + ")"); + fieldList.add(dto.getSummary() + "(" + prefix + dto.getOriginName() + suffix + ")"); + } + } + return "SELECT " + Strings.join(",", fieldList) + " FROM (" + sql + ") tmp"; + } + + protected String assistSQLOriginList(String sql, List assistFields, Map dsMap, boolean crossDs) { + // get datasource prefix and suffix + String dsType = dsMap.entrySet().iterator().next().getValue().getType(); + String prefix = ""; + String suffix = ""; + if (Arrays.stream(DatasourceConfiguration.DatasourceType.values()).map(DatasourceConfiguration.DatasourceType::getType).toList().contains(dsType)) { + DatasourceConfiguration.DatasourceType datasourceType = DatasourceConfiguration.DatasourceType.valueOf(dsType); + prefix = datasourceType.getPrefix(); + suffix = datasourceType.getSuffix(); + } else { + List xpackPluginsDatasourceVOS = pluginManage.queryPluginDs(); + List list = xpackPluginsDatasourceVOS.stream().filter(ele -> StringUtils.equals(ele.getType(), dsType)).toList(); + if (ObjectUtils.isNotEmpty(list)) { + XpackPluginsDatasourceVO first = list.getFirst(); + prefix = first.getPrefix(); + suffix = first.getSuffix(); + } else { + DEException.throwException("当前数据源插件不存在"); + } + } + + List fieldList = new ArrayList<>(); + for (int i = 0; i < assistFields.size(); i++) { + ChartViewFieldDTO dto = assistFields.get(i); + if (StringUtils.equalsIgnoreCase(dto.getSummary(), "last_item")) { + if (crossDs) { + fieldList.add(dto.getOriginName()); } else { - stringBuilder.append(dto.getSummary() + "(" + prefix + dto.getOriginName() + suffix + "),"); + fieldList.add(prefix + dto.getOriginName() + suffix); } } } - return "SELECT " + stringBuilder + " FROM (" + sql + ") tmp"; + return "SELECT " + Strings.join(",", fieldList) + " FROM (" + sql + ") tmp"; } protected List mergeIds(List xAxisExt, List extStack) { @@ -522,26 +566,29 @@ public class DefaultChartHandler extends AbstractChartPlugin { if (CollectionUtils.isEmpty(data)) { break; } - if (StringUtils.containsAny(chartType, "group", "stack")) { + if (Objects.isNull(extStack)) { + extStack = Arrays.asList(); + } + if (Objects.isNull(xAxisExt)) { + xAxisExt = Arrays.asList(); + } + boolean isStack = StringUtils.containsIgnoreCase(chartType, "stack") && CollectionUtils.isNotEmpty(extStack); + boolean isGroup = StringUtils.containsIgnoreCase(chartType, "group") + || (CollectionUtils.isNotEmpty(xAxisExt)); + if (isStack || isGroup) { if (CollectionUtils.isEmpty(xAxis)) { break; } - if (StringUtils.containsIgnoreCase(chartType, "stack") && extStack.isEmpty()) { - break; - } - if (StringUtils.containsIgnoreCase(chartType, "group") && xAxisExt.isEmpty()) { - break; - } final Map mainIndexMap = new HashMap<>(); final List> mainMatrix = new ArrayList<>(); // 排除group和stack的字段 List groupStackAxisIds = mergeIds(xAxisExt, extStack); - List finalXAxisBase = xAxis.stream().filter(ele->!groupStackAxisIds.contains(String.valueOf(ele.getId()))).toList(); - if (CollectionUtils.isEmpty(finalXAxisBase) && CollectionUtils.isNotEmpty(xAxis)) { - finalXAxisBase.add(xAxis.get(0)); + List xAxisBase = xAxis.stream().filter(ele -> !groupStackAxisIds.contains(String.valueOf(ele.getId()))).collect(Collectors.toList()); + if (CollectionUtils.isEmpty(xAxisBase) && CollectionUtils.isNotEmpty(xAxis)) { + xAxisBase.add(xAxis.get(0)); } data.forEach(item -> { - String[] mainAxisArr = Arrays.copyOfRange(item, 0, finalXAxisBase.size()); + String[] mainAxisArr = Arrays.copyOfRange(item, 0, xAxisBase.size()); String mainAxis = StringUtils.join(mainAxisArr, '-'); Integer index = mainIndexMap.get(mainAxis); if (index == null) { @@ -555,21 +602,24 @@ public class DefaultChartHandler extends AbstractChartPlugin { } }); int finalDataIndex = dataIndex; - int subEndIndex = finalXAxisBase.size(); - if (StringUtils.containsIgnoreCase(chartType, "group")) { + int subEndIndex = xAxisBase.size(); + if (CollectionUtils.isNotEmpty(xAxisExt) + || StringUtils.containsIgnoreCase(chartType, "group") + || StringUtils.containsIgnoreCase(chartType, "-mix")) { subEndIndex += xAxisExt.size(); } if (StringUtils.containsIgnoreCase(chartType, "stack")) { subEndIndex += extStack.size(); } int finalSubEndIndex = subEndIndex; + // 存储上次的值 + Map preDataMap = new HashMap<>(); //滑动累加 for (int k = 1; k < mainMatrix.size(); k++) { List preDataItems = mainMatrix.get(k - 1); List curDataItems = mainMatrix.get(k); - Map preDataMap = new HashMap<>(); preDataItems.forEach(preDataItem -> { - String[] groupStackAxisArr = Arrays.copyOfRange(preDataItem, finalXAxisBase.size(), finalSubEndIndex); + String[] groupStackAxisArr = Arrays.copyOfRange(preDataItem, xAxisBase.size(), finalSubEndIndex); String groupStackAxis = StringUtils.join(groupStackAxisArr, '-'); String preVal = preDataItem[finalDataIndex]; if (StringUtils.isBlank(preVal)) { @@ -578,13 +628,19 @@ public class DefaultChartHandler extends AbstractChartPlugin { preDataMap.put(groupStackAxis, new BigDecimal(preVal)); }); curDataItems.forEach(curDataItem -> { - String[] groupStackAxisArr = Arrays.copyOfRange(curDataItem, finalXAxisBase.size(), finalSubEndIndex); + String[] groupStackAxisArr = Arrays.copyOfRange(curDataItem, xAxisBase.size(), finalSubEndIndex); String groupStackAxis = StringUtils.join(groupStackAxisArr, '-'); BigDecimal preValue = preDataMap.get(groupStackAxis); if (preValue != null) { curDataItem[finalDataIndex] = new BigDecimal(curDataItem[finalDataIndex]) .add(preValue) .toString(); + } else { + if (preDataMap.containsKey(groupStackAxis)) { + curDataItem[finalDataIndex] = new BigDecimal(curDataItem[finalDataIndex]) + .add(preDataMap.get(groupStackAxis)) + .toString(); + } } }); } diff --git a/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/YoyChartHandler.java b/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/YoyChartHandler.java index d4af6b9..438d0af 100644 --- a/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/YoyChartHandler.java +++ b/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/YoyChartHandler.java @@ -2,6 +2,7 @@ package io.gisbi.chart.charts.impl; import com.fasterxml.jackson.core.type.TypeReference; +import io.gisbi.api.dataset.union.DatasetGroupInfoDTO; import io.gisbi.engine.sql.SQLProvider; import io.gisbi.engine.trans.ExtWhere2Str; import io.gisbi.engine.utils.Utils; @@ -74,7 +75,7 @@ public class YoyChartHandler extends DefaultChartHandler { dsList.add(next.getValue().getType()); } boolean needOrder = Utils.isNeedOrder(dsList); - boolean crossDs = Utils.isCrossDs(dsMap); + boolean crossDs = ((DatasetGroupInfoDTO) formatResult.getContext().get("dataset")).getIsCross(); // 这里拿到的可能有一年前的数据 var expandedResult = (T) super.calcChartResult(view, formatResult, filterResult, sqlMap, sqlMeta, provider); // 检查同环比过滤，拿到实际数据 @@ -86,6 +87,7 @@ public class YoyChartHandler extends DefaultChartHandler { var originSql = SQLProvider.createQuerySQL(sqlMeta, true, needOrder, view); originSql = provider.rebuildSQL(originSql, sqlMeta, crossDs, dsMap); var request = new DatasourceRequest(); + request.setIsCross(crossDs); request.setDsList(dsMap); request.setQuery(originSql); logger.debug("calcite yoy sql: " + originSql); diff --git a/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/bar/BarHandler.java b/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/bar/BarHandler.java index 7e83893..8510cb0 100644 --- a/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/bar/BarHandler.java +++ b/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/bar/BarHandler.java @@ -1,5 +1,6 @@ package io.gisbi.chart.charts.impl.bar; +import io.gisbi.api.dataset.union.DatasetGroupInfoDTO; import io.gisbi.chart.charts.impl.YoyChartHandler; import io.gisbi.engine.utils.Utils; import io.gisbi.extensions.datasource.dto.DatasourceRequest; @@ -8,6 +9,8 @@ import io.gisbi.extensions.datasource.model.SQLMeta; import io.gisbi.extensions.datasource.provider.Provider; import io.gisbi.extensions.view.dto.*; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; import java.util.ArrayList; @@ -42,7 +45,7 @@ public class BarHandler extends YoyChartHandler { dsList.add(next.getValue().getType()); } boolean needOrder = Utils.isNeedOrder(dsList); - boolean crossDs = Utils.isCrossDs(dsMap); + boolean crossDs = ((DatasetGroupInfoDTO) formatResult.getContext().get("dataset")).getIsCross(); var result = (T) super.calcChartResult(view, formatResult, filterResult, sqlMap, sqlMeta, provider); try { //如果有同环比过滤,应该用原始sql @@ -52,13 +55,28 @@ public class BarHandler extends YoyChartHandler { var assistFields = getAssistFields(dynamicAssistFields, yAxis); if (CollectionUtils.isNotEmpty(assistFields)) { var req = new DatasourceRequest(); + req.setIsCross(crossDs); req.setDsList(dsMap); - var assistSql = assistSQL(originSql, assistFields, dsMap); - req.setQuery(assistSql); - logger.debug("calcite assistSql sql: " + assistSql); - var assistData = (List) provider.fetchResultField(req).get("data"); - result.setAssistData(assistData); - result.setDynamicAssistFields(dynamicAssistFields); + + List assists = dynamicAssistFields.stream().filter(ele -> !StringUtils.equalsIgnoreCase(ele.getSummary(), "last_item")).toList(); + if (ObjectUtils.isNotEmpty(assists)) { + var assistSql = assistSQL(originSql, assistFields, dsMap, crossDs); + req.setQuery(assistSql); + logger.debug("calcite assistSql sql: " + assistSql); + var assistData = (List) provider.fetchResultField(req).get("data"); + result.setAssistData(assistData); + result.setDynamicAssistFields(assists); + } + + List assistsOriginList = dynamicAssistFields.stream().filter(ele -> StringUtils.equalsIgnoreCase(ele.getSummary(), "last_item")).toList(); + if (ObjectUtils.isNotEmpty(assistsOriginList)) { + var assistSqlOriginList = assistSQLOriginList(originSql, assistFields, dsMap, crossDs); + req.setQuery(assistSqlOriginList); + logger.debug("calcite assistSql sql origin list: " + assistSqlOriginList); + var assistDataOriginList = (List) provider.fetchResultField(req).get("data"); + result.setAssistDataOriginList(assistDataOriginList); + result.setDynamicAssistFieldsOriginList(assistsOriginList); + } } } catch (Exception e) { e.printStackTrace(); diff --git a/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/line/LineHandler.java b/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/line/LineHandler.java index 5f90baa..1265f41 100644 --- a/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/line/LineHandler.java +++ b/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/line/LineHandler.java @@ -1,5 +1,6 @@ package io.gisbi.chart.charts.impl.line; +import io.gisbi.api.dataset.union.DatasetGroupInfoDTO; import io.gisbi.chart.charts.impl.YoyChartHandler; import io.gisbi.chart.utils.ChartDataBuild; import io.gisbi.extensions.datasource.dto.DatasourceRequest; @@ -9,6 +10,8 @@ import io.gisbi.extensions.datasource.provider.Provider; import io.gisbi.extensions.view.dto.*; import lombok.Getter; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; import java.util.ArrayList; @@ -55,6 +58,7 @@ public class LineHandler extends YoyChartHandler { for (Map.Entry next : dsMap.entrySet()) { dsList.add(next.getValue().getType()); } + boolean crossDs = ((DatasetGroupInfoDTO) formatResult.getContext().get("dataset")).getIsCross(); var result = (T) super.calcChartResult(view, formatResult, filterResult, sqlMap, sqlMeta, provider); try { //如果有同环比过滤,应该用原始sql @@ -64,13 +68,28 @@ public class LineHandler extends YoyChartHandler { var assistFields = getAssistFields(dynamicAssistFields, yAxis); if (CollectionUtils.isNotEmpty(assistFields)) { var req = new DatasourceRequest(); + req.setIsCross(crossDs); req.setDsList(dsMap); - var assistSql = assistSQL(originSql, assistFields, dsMap); - req.setQuery(assistSql); - logger.debug("calcite assistSql sql: " + assistSql); - var assistData = (List) provider.fetchResultField(req).get("data"); - result.setAssistData(assistData); - result.setDynamicAssistFields(dynamicAssistFields); + + List assists = dynamicAssistFields.stream().filter(ele -> !StringUtils.equalsIgnoreCase(ele.getSummary(), "last_item")).toList(); + if (ObjectUtils.isNotEmpty(assists)) { + var assistSql = assistSQL(originSql, assistFields, dsMap, crossDs); + req.setQuery(assistSql); + logger.debug("calcite assistSql sql: " + assistSql); + var assistData = (List) provider.fetchResultField(req).get("data"); + result.setAssistData(assistData); + result.setDynamicAssistFields(assists); + } + + List assistsOriginList = dynamicAssistFields.stream().filter(ele -> StringUtils.equalsIgnoreCase(ele.getSummary(), "last_item")).toList(); + if (ObjectUtils.isNotEmpty(assistsOriginList)) { + var assistSqlOriginList = assistSQLOriginList(originSql, assistFields, dsMap, crossDs); + req.setQuery(assistSqlOriginList); + logger.debug("calcite assistSql sql origin list: " + assistSqlOriginList); + var assistDataOriginList = (List) provider.fetchResultField(req).get("data"); + result.setAssistDataOriginList(assistDataOriginList); + result.setDynamicAssistFieldsOriginList(assistsOriginList); + } } } catch (Exception e) { e.printStackTrace(); diff --git a/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/line/StackAreaHandler.java b/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/line/StackAreaHandler.java index 70b9a4f..fe67e90 100644 --- a/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/line/StackAreaHandler.java +++ b/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/line/StackAreaHandler.java @@ -1,5 +1,6 @@ package io.gisbi.chart.charts.impl.line; +import io.gisbi.api.dataset.union.DatasetGroupInfoDTO; import io.gisbi.chart.charts.impl.YoyChartHandler; import io.gisbi.chart.utils.ChartDataBuild; import io.gisbi.engine.utils.Utils; @@ -11,6 +12,7 @@ import io.gisbi.extensions.view.dto.*; import lombok.Getter; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; import java.util.ArrayList; @@ -89,7 +91,7 @@ public class StackAreaHandler extends YoyChartHandler { dsList.add(next.getValue().getType()); } boolean needOrder = Utils.isNeedOrder(dsList); - boolean crossDs = Utils.isCrossDs(dsMap); + boolean crossDs = ((DatasetGroupInfoDTO) formatResult.getContext().get("dataset")).getIsCross(); var result = (T) super.calcChartResult(view, formatResult, filterResult, sqlMap, sqlMeta, provider); try { //如果有同环比过滤,应该用原始sql @@ -99,13 +101,28 @@ public class StackAreaHandler extends YoyChartHandler { var assistFields = getAssistFields(dynamicAssistFields, yAxis); if (CollectionUtils.isNotEmpty(assistFields)) { var req = new DatasourceRequest(); + req.setIsCross(crossDs); req.setDsList(dsMap); - var assistSql = assistSQL(originSql, assistFields, dsMap); - req.setQuery(assistSql); - logger.debug("calcite assist sql: " + assistSql); - var assistData = (List) provider.fetchResultField(req).get("data"); - result.setAssistData(assistData); - result.setDynamicAssistFields(dynamicAssistFields); + + List assists = dynamicAssistFields.stream().filter(ele -> !StringUtils.equalsIgnoreCase(ele.getSummary(), "last_item")).toList(); + if (ObjectUtils.isNotEmpty(assists)) { + var assistSql = assistSQL(originSql, assistFields, dsMap, crossDs); + req.setQuery(assistSql); + logger.debug("calcite assistSql sql: " + assistSql); + var assistData = (List) provider.fetchResultField(req).get("data"); + result.setAssistData(assistData); + result.setDynamicAssistFields(assists); + } + + List assistsOriginList = dynamicAssistFields.stream().filter(ele -> StringUtils.equalsIgnoreCase(ele.getSummary(), "last_item")).toList(); + if (ObjectUtils.isNotEmpty(assistsOriginList)) { + var assistSqlOriginList = assistSQLOriginList(originSql, assistFields, dsMap, crossDs); + req.setQuery(assistSqlOriginList); + logger.debug("calcite assistSql sql origin list: " + assistSqlOriginList); + var assistDataOriginList = (List) provider.fetchResultField(req).get("data"); + result.setAssistDataOriginList(assistDataOriginList); + result.setDynamicAssistFieldsOriginList(assistsOriginList); + } } } catch (Exception e) { e.printStackTrace(); diff --git a/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/map/SymbolicMapHandler.java b/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/map/SymbolicMapHandler.java index b19f87a..30ed621 100644 --- a/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/map/SymbolicMapHandler.java +++ b/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/map/SymbolicMapHandler.java @@ -1,5 +1,6 @@ package io.gisbi.chart.charts.impl.map; +import io.gisbi.api.dataset.union.DatasetGroupInfoDTO; import io.gisbi.chart.charts.impl.GroupChartHandler; import io.gisbi.chart.utils.ChartDataBuild; import io.gisbi.dataset.manage.DatasetDataManage; @@ -60,22 +61,23 @@ public class SymbolicMapHandler extends GroupChartHandler { boolean needOrder = Utils.isNeedOrder(dsList); boolean crossDs = Utils.isCrossDs(dsMap); DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setIsCross(((DatasetGroupInfoDTO) formatResult.getContext().get("dataset")).getIsCross()); datasourceRequest.setDsList(dsMap); var xAxis = formatResult.getAxisMap().get(ChartAxis.xAxis); var extBubble = formatResult.getAxisMap().get(ChartAxis.extBubble); var allFields = (List) filterResult.getContext().get("allFields"); - List countField =chartViewManege.transFieldDTO(Collections.singletonList(chartViewManege.createCountField(view.getTableId()))); + List countField = chartViewManege.transFieldDTO(Collections.singletonList(chartViewManege.createCountField(view.getTableId()))); List datasetTableFieldDTOList = FieldUtil.transFields(allFields); SQLMeta sqlMeta1 = new SQLMeta(); BeanUtils.copyBean(sqlMeta1, sqlMeta); Dimension2SQLObj.dimension2sqlObj(sqlMeta, xAxis, datasetTableFieldDTOList, crossDs, dsMap, Utils.getParams(datasetTableFieldDTOList), view.getCalParams(), pluginManage); List yAxis = new ArrayList<>(); - if(!extBubble.isEmpty() && !"*".equals(extBubble.get(0).getDataeaseName())){ + if (!extBubble.isEmpty() && !"*".equals(extBubble.get(0).getGisbiName())) { yAxis.addAll(extBubble); } yAxis.addAll(countField); datasetTableFieldDTOList.addAll(FieldUtil.transFields(countField)); - formatResult.getAxisMap().put(ChartAxis.yAxis,countField); + formatResult.getAxisMap().put(ChartAxis.yAxis, countField); Quota2SQLObj.quota2sqlObj(sqlMeta, yAxis, datasetTableFieldDTOList, crossDs, dsMap, Utils.getParams(datasetTableFieldDTOList), view.getCalParams(), pluginManage); String querySql = SQLProvider.createQuerySQL(sqlMeta, true, needOrder, view); querySql = provider.rebuildSQL(querySql, sqlMeta, crossDs, dsMap); @@ -102,6 +104,7 @@ public class SymbolicMapHandler extends GroupChartHandler { logger.debug("calcite data preview sql: " + querySQL); // 调用数据源的calcite获得data DatasourceRequest datasourceRequest1 = new DatasourceRequest(); + datasourceRequest1.setIsCross(((DatasetGroupInfoDTO) formatResult.getContext().get("dataset")).getIsCross()); datasourceRequest1.setQuery(querySQL); datasourceRequest1.setDsList(dsMap); detailData = (List) provider.fetchResultField(datasourceRequest1).get("data"); @@ -115,7 +118,7 @@ public class SymbolicMapHandler extends GroupChartHandler { calcResult.setContext(filterResult.getContext()); calcResult.setQuerySql(querySql); calcResult.setOriginData(data); - formatResult.getAxisMap().put(ChartAxis.yAxis,new ArrayList<>()); + formatResult.getAxisMap().put(ChartAxis.yAxis, new ArrayList<>()); return calcResult; } @@ -142,8 +145,8 @@ public class SymbolicMapHandler extends GroupChartHandler { dataMap.putAll(calcResult.getData()); dataMap.putAll(mapTableNormal); dataMap.put("sourceFields", allFields); - mergeAssistField(calcResult.getDynamicAssistFields(), calcResult.getAssistData()); - dataMap.put("dynamicAssistLines", calcResult.getDynamicAssistFields()); + List chartSeniorAssistDTOS = mergeAssistField(calcResult.getDynamicAssistFields(), calcResult.getAssistData(), calcResult.getDynamicAssistFieldsOriginList(), calcResult.getAssistDataOriginList()); + dataMap.put("dynamicAssistLines", chartSeniorAssistDTOS); view.setData(dataMap); view.setSql(Base64.getEncoder().encodeToString(calcResult.getQuerySql().getBytes())); view.setDrill(isDrill); diff --git a/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/mix/MixHandler.java b/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/mix/MixHandler.java index 9977cee..bd538c0 100644 --- a/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/mix/MixHandler.java +++ b/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/mix/MixHandler.java @@ -1,5 +1,6 @@ package io.gisbi.chart.charts.impl.mix; +import io.gisbi.api.dataset.union.DatasetGroupInfoDTO; import io.gisbi.chart.charts.impl.YoyChartHandler; import io.gisbi.chart.utils.ChartDataBuild; import io.gisbi.engine.utils.Utils; @@ -10,6 +11,7 @@ import io.gisbi.extensions.datasource.provider.Provider; import io.gisbi.extensions.view.dto.*; import lombok.Getter; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; @@ -80,7 +82,7 @@ public class MixHandler extends YoyChartHandler { dsList.add(next.getValue().getType()); } boolean needOrder = Utils.isNeedOrder(dsList); - boolean crossDs = Utils.isCrossDs(dsMap); + boolean crossDs = ((DatasetGroupInfoDTO) formatResult.getContext().get("dataset")).getIsCross(); var leftResult = (T) super.calcChartResult(view, formatResult, filterResult, sqlMap, sqlMeta, provider); var dynamicAssistFields = getDynamicAssistFields(view); try { @@ -91,13 +93,28 @@ public class MixHandler extends YoyChartHandler { var assistFields = getAssistFields(leftAssistFields, yAxis); if (CollectionUtils.isNotEmpty(assistFields)) { var req = new DatasourceRequest(); + req.setIsCross(crossDs); req.setDsList(dsMap); - var assistSql = assistSQL(originSql, assistFields, dsMap); - req.setQuery(assistSql); - logger.debug("calcite assistSql sql: " + assistSql); - var assistData = (List) provider.fetchResultField(req).get("data"); - leftResult.setAssistData(assistData); - leftResult.setDynamicAssistFields(leftAssistFields); + + List assists = leftAssistFields.stream().filter(ele -> !StringUtils.equalsIgnoreCase(ele.getSummary(), "last_item")).toList(); + if (ObjectUtils.isNotEmpty(assists)) { + var assistSql = assistSQL(originSql, assistFields, dsMap, crossDs); + req.setQuery(assistSql); + logger.debug("calcite assistSql sql: " + assistSql); + var assistData = (List) provider.fetchResultField(req).get("data"); + leftResult.setAssistData(assistData); + leftResult.setDynamicAssistFields(assists); + } + + List assistsOriginList = leftAssistFields.stream().filter(ele -> StringUtils.equalsIgnoreCase(ele.getSummary(), "last_item")).toList(); + if (ObjectUtils.isNotEmpty(assistsOriginList)) { + var assistSqlOriginList = assistSQLOriginList(originSql, assistFields, dsMap, crossDs); + req.setQuery(assistSqlOriginList); + logger.debug("calcite assistSql sql origin list: " + assistSqlOriginList); + var assistDataOriginList = (List) provider.fetchResultField(req).get("data"); + leftResult.setAssistDataOriginList(assistDataOriginList); + leftResult.setDynamicAssistFieldsOriginList(assistsOriginList); + } } } catch (Exception e) { e.printStackTrace(); @@ -143,12 +160,28 @@ public class MixHandler extends YoyChartHandler { var assistFields = getAssistFields(rightAssistFields, yAxis); if (CollectionUtils.isNotEmpty(assistFields)) { var req = new DatasourceRequest(); + req.setIsCross(crossDs); req.setDsList(dsMap); - var assistSql = assistSQL(originSql, assistFields, dsMap); - req.setQuery(assistSql); - var assistData = (List) provider.fetchResultField(req).get("data"); - rightResult.setAssistData(assistData); - rightResult.setDynamicAssistFields(rightAssistFields); + + List assists = rightAssistFields.stream().filter(ele -> !StringUtils.equalsIgnoreCase(ele.getSummary(), "last_item")).toList(); + if (ObjectUtils.isNotEmpty(assists)) { + var assistSql = assistSQL(originSql, assistFields, dsMap, crossDs); + req.setQuery(assistSql); + logger.debug("calcite assistSql sql: " + assistSql); + var assistData = (List) provider.fetchResultField(req).get("data"); + rightResult.setAssistData(assistData); + rightResult.setDynamicAssistFields(assists); + } + + List assistsOriginList = rightAssistFields.stream().filter(ele -> StringUtils.equalsIgnoreCase(ele.getSummary(), "last_item")).toList(); + if (ObjectUtils.isNotEmpty(assistsOriginList)) { + var assistSqlOriginList = assistSQLOriginList(originSql, assistFields, dsMap, crossDs); + req.setQuery(assistSqlOriginList); + logger.debug("calcite assistSql sql origin list: " + assistSqlOriginList); + var assistDataOriginList = (List) provider.fetchResultField(req).get("data"); + rightResult.setAssistDataOriginList(assistDataOriginList); + rightResult.setDynamicAssistFieldsOriginList(assistsOriginList); + } } } catch (Exception e) { e.printStackTrace(); @@ -169,12 +202,12 @@ public class MixHandler extends YoyChartHandler { var leftFields = new ArrayList(); leftFields.addAll(formatResult.getAxisMap().get(ChartAxis.xAxis)); leftFields.addAll(formatResult.getAxisMap().get(ChartAxis.yAxis)); - mergeAssistField(leftCalcResult.getDynamicAssistFields(), leftCalcResult.getAssistData()); + List chartSeniorAssistDTOSLeft = mergeAssistField(leftCalcResult.getDynamicAssistFields(), leftCalcResult.getAssistData(), leftCalcResult.getDynamicAssistFieldsOriginList(), leftCalcResult.getAssistDataOriginList()); var leftOriginData = leftCalcResult.getOriginData(); var leftTable = ChartDataBuild.transTableNormal(leftFields, view, leftOriginData, desensitizationList); var leftData = new HashMap(leftTable); leftData.putAll(leftCalcResult.getData()); - leftData.put("dynamicAssistLines", leftCalcResult.getDynamicAssistFields()); + leftData.put("dynamicAssistLines", chartSeniorAssistDTOSLeft); var rightCalcResult = (ChartCalcDataResult) calcResult.getData().get("right"); var rightFields = new ArrayList(); @@ -183,12 +216,12 @@ public class MixHandler extends YoyChartHandler { rightFields.addAll(subAxisMap.get(ChartAxis.xAxis)); rightFields.addAll(subAxisMap.get(ChartAxis.yAxis)); - mergeAssistField(rightCalcResult.getDynamicAssistFields(), rightCalcResult.getAssistData()); + List chartSeniorAssistDTOSRight = mergeAssistField(rightCalcResult.getDynamicAssistFields(), rightCalcResult.getAssistData(), rightCalcResult.getDynamicAssistFieldsOriginList(), rightCalcResult.getAssistDataOriginList()); var rightOriginData = rightCalcResult.getOriginData(); var rightTable = ChartDataBuild.transTableNormal(rightFields, view, rightOriginData, desensitizationList); var rightData = new HashMap(rightTable); rightData.putAll(rightCalcResult.getData()); - rightData.put("dynamicAssistLines", rightCalcResult.getDynamicAssistFields()); + rightData.put("dynamicAssistLines", chartSeniorAssistDTOSRight); // 构建结果 Map chartData = new TreeMap<>(); @@ -205,7 +238,7 @@ public class MixHandler extends YoyChartHandler { var isDrill = CollectionUtils.isNotEmpty(drillFilters); view.setDrillFilters(drillFilters); view.setDrill(isDrill); - view.setSql(leftCalcResult.getQuerySql()); + view.setSql(Base64.getEncoder().encodeToString(leftCalcResult.getQuerySql().getBytes())); view.setData(chartData); return view; } diff --git a/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/numeric/NumericalChartHandler.java b/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/numeric/NumericalChartHandler.java index 1b91c0c..2c6041a 100644 --- a/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/numeric/NumericalChartHandler.java +++ b/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/numeric/NumericalChartHandler.java @@ -1,5 +1,6 @@ package io.gisbi.chart.charts.impl.numeric; +import io.gisbi.api.dataset.union.DatasetGroupInfoDTO; import io.gisbi.chart.charts.impl.DefaultChartHandler; import io.gisbi.chart.utils.ChartDataBuild; import io.gisbi.engine.sql.SQLProvider; @@ -33,6 +34,7 @@ public class NumericalChartHandler extends DefaultChartHandler { boolean needOrder = Utils.isNeedOrder(dsList); boolean crossDs = Utils.isCrossDs(dsMap); DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setIsCross(((DatasetGroupInfoDTO) formatResult.getContext().get("dataset")).getIsCross()); datasourceRequest.setDsList(dsMap); var xAxis = formatResult.getAxisMap().get(ChartAxis.xAxis); var yAxis = formatResult.getAxisMap().get(ChartAxis.yAxis); @@ -67,14 +69,11 @@ public class NumericalChartHandler extends DefaultChartHandler { String summary = (String) maxField.get("summary"); DatasetTableFieldDTO datasetTableField = datasetTableFieldManage.selectById(id); if (ObjectUtils.isNotEmpty(datasetTableField)) { - if (datasetTableField.getDeType() == 0 || datasetTableField.getDeType() == 1 || datasetTableField.getDeType() == 5) { - if (!StringUtils.containsIgnoreCase(summary, "count")) { - DEException.throwException(Translator.get("i18n_gauge_field_change")); - } - } ChartViewFieldDTO dto = new ChartViewFieldDTO(); BeanUtils.copyBean(dto, datasetTableField); - dto.setSummary(summary); + if (StringUtils.isEmpty(dto.getSummary())) { + dto.setSummary(summary); + } return dto; } else { DEException.throwException(Translator.get("i18n_gauge_field_delete")); diff --git a/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/pie/PieHandler.java b/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/pie/PieHandler.java index a05cc7b..d4c50c6 100644 --- a/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/pie/PieHandler.java +++ b/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/pie/PieHandler.java @@ -1,11 +1,15 @@ package io.gisbi.chart.charts.impl.pie; import io.gisbi.chart.charts.impl.YoyChartHandler; -import io.gisbi.extensions.view.dto.AxisFormatResult; -import io.gisbi.extensions.view.dto.ChartAxis; -import io.gisbi.extensions.view.dto.ChartViewDTO; +import io.gisbi.extensions.view.dto.*; import org.springframework.stereotype.Component; +import java.math.BigDecimal; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + @Component public class PieHandler extends YoyChartHandler { @Override @@ -26,4 +30,74 @@ public class PieHandler extends YoyChartHandler { result.getAxisMap().put(ChartAxis.extTooltip, view.getExtTooltip()); return result; } + + @Override + public ChartViewDTO buildChart(ChartViewDTO view, ChartCalcDataResult calcResult, AxisFormatResult formatResult, CustomFilterResult filterResult) { + ChartViewDTO result = super.buildChart(view, calcResult, formatResult, filterResult); + filterPositiveData(result, "data", AxisChartDataAntVDTO.class); + filterPositiveData(result, "tableRow", Map.class, view.getYAxis().get(0).getGisbiName()); + return result; + } + + /** + * 过滤正数数据根据data + * @param result + * @param key + * @param clazz + * @param + */ + private void filterPositiveData(ChartViewDTO result, String key, Class clazz) { + if (result.getData().containsKey(key)) { + List list = ((List) result.getData().get(key)) + .stream() + .filter(item -> { + if (clazz == AxisChartDataAntVDTO.class) { + if (Objects.isNull(((AxisChartDataAntVDTO) item).getValue())) return false; + return ((AxisChartDataAntVDTO) item).getValue().compareTo(BigDecimal.ZERO) >= 0; + } else if (clazz == Map.class) { + return isPositive(((Map) item).get("value")); + } + return false; + }) + .collect(Collectors.toList()); + result.getData().put(key, list); + } + } + + /** + * 过滤正数数据根据tableRow + * @param result + * @param key + * @param clazz + * @param yAxisName + * @param + */ + private void filterPositiveData(ChartViewDTO result, String key, Class clazz, String yAxisName) { + if (result.getData().containsKey(key)) { + List list = ((List) result.getData().get(key)) + .stream() + .filter(item -> { + if (clazz == Map.class) { + Object value = ((Map) item).get(yAxisName); + return isPositive(value); + } + return false; + }) + .collect(Collectors.toList()); + result.getData().put(key, list); + } + } + + private boolean isPositive(Object value) { + if (value instanceof String) { + try { + return new BigDecimal((String) value).compareTo(BigDecimal.ZERO) >= 0; + } catch (NumberFormatException e) { + return false; + } + } else if (value instanceof BigDecimal) { + return ((BigDecimal) value).compareTo(BigDecimal.ZERO) >= 0; + } + return false; + } } diff --git a/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/table/TableHeatmapHandler.java b/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/table/TableHeatmapHandler.java index dae3acf..1cfe918 100644 --- a/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/table/TableHeatmapHandler.java +++ b/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/table/TableHeatmapHandler.java @@ -1,5 +1,6 @@ package io.gisbi.chart.charts.impl.table; +import io.gisbi.api.dataset.union.DatasetGroupInfoDTO; import io.gisbi.chart.charts.impl.DefaultChartHandler; import io.gisbi.chart.utils.ChartDataBuild; import io.gisbi.engine.sql.SQLProvider; @@ -55,6 +56,7 @@ public class TableHeatmapHandler extends DefaultChartHandler { boolean needOrder = Utils.isNeedOrder(dsList); boolean crossDs = Utils.isCrossDs(dsMap); DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setIsCross(((DatasetGroupInfoDTO) formatResult.getContext().get("dataset")).getIsCross()); datasourceRequest.setDsList(dsMap); var xAxis = formatResult.getAxisMap().get(ChartAxis.xAxis); var yAxis = formatResult.getAxisMap().get(ChartAxis.yAxis); diff --git a/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/table/TableInfoHandler.java b/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/table/TableInfoHandler.java index 2d08831..3b3ad7e 100644 --- a/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/table/TableInfoHandler.java +++ b/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/table/TableInfoHandler.java @@ -1,6 +1,7 @@ package io.gisbi.chart.charts.impl.table; import io.gisbi.api.chart.dto.PageInfo; +import io.gisbi.api.dataset.union.DatasetGroupInfoDTO; import io.gisbi.chart.charts.impl.DefaultChartHandler; import io.gisbi.engine.sql.SQLProvider; import io.gisbi.engine.trans.Dimension2SQLObj; @@ -65,6 +66,11 @@ public class TableInfoHandler extends DefaultChartHandler { return (T) new CustomFilterResult(filterList, formatResult.getContext()); } + @Override + public Map buildResult(ChartViewDTO view, AxisFormatResult formatResult, CustomFilterResult filterResult, List data) { + return new HashMap<>(); + } + @Override public T calcChartResult(ChartViewDTO view, AxisFormatResult formatResult, CustomFilterResult filterResult, Map sqlMap, SQLMeta sqlMeta, Provider provider) { var chartExtRequest = view.getChartExtRequest(); @@ -73,8 +79,9 @@ public class TableInfoHandler extends DefaultChartHandler { for (Map.Entry next : dsMap.entrySet()) { dsList.add(next.getValue().getType()); } - boolean crossDs = Utils.isCrossDs(dsMap); + boolean crossDs = ((DatasetGroupInfoDTO) formatResult.getContext().get("dataset")).getIsCross(); DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setIsCross(crossDs); datasourceRequest.setDsList(dsMap); var xAxis = formatResult.getAxisMap().get(ChartAxis.xAxis); var allFields = (List) filterResult.getContext().get("allFields"); @@ -150,12 +157,28 @@ public class TableInfoHandler extends DefaultChartHandler { if (CollectionUtils.isNotEmpty(assistFields)) { var req = new DatasourceRequest(); req.setDsList(dsMap); - var assistSql = assistSQL(querySql, assistFields, dsMap); - req.setQuery(assistSql); - logger.debug("calcite assistSql sql: " + assistSql); - var assistData = (List) provider.fetchResultField(req).get("data"); - calcResult.setAssistData(assistData); - calcResult.setDynamicAssistFields(dynamicAssistFields); + + List assists = dynamicAssistFields.stream().filter(ele -> !StringUtils.equalsIgnoreCase(ele.getSummary(), "last_item")).toList(); + if (ObjectUtils.isNotEmpty(assists)) { + var assistSql = assistSQL(originSql, assistFields, dsMap, crossDs); + var tmpSql = provider.rebuildSQL(assistSql, sqlMeta, crossDs, dsMap); + req.setQuery(tmpSql); + logger.debug("calcite assistSql sql: " + tmpSql); + var assistData = (List) provider.fetchResultField(req).get("data"); + calcResult.setAssistData(assistData); + calcResult.setDynamicAssistFields(assists); + } + + List assistsOriginList = dynamicAssistFields.stream().filter(ele -> StringUtils.equalsIgnoreCase(ele.getSummary(), "last_item")).toList(); + if (ObjectUtils.isNotEmpty(assistsOriginList)) { + var assistSqlOriginList = assistSQLOriginList(originSql, assistFields, dsMap, crossDs); + var tmpSql = provider.rebuildSQL(assistSqlOriginList, sqlMeta, crossDs, dsMap); + req.setQuery(tmpSql); + logger.debug("calcite assistSql sql origin list: " + tmpSql); + var assistDataOriginList = (List) provider.fetchResultField(req).get("data"); + calcResult.setAssistDataOriginList(assistDataOriginList); + calcResult.setDynamicAssistFieldsOriginList(assistsOriginList); + } } } catch (Exception e) { e.printStackTrace(); diff --git a/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/table/TableNormalHandler.java b/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/table/TableNormalHandler.java index 7fb8135..d6b0f12 100644 --- a/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/table/TableNormalHandler.java +++ b/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/table/TableNormalHandler.java @@ -1,49 +1,213 @@ package io.gisbi.chart.charts.impl.table; -import io.gisbi.chart.charts.impl.YoyChartHandler; +import com.fasterxml.jackson.core.type.TypeReference; +import io.gisbi.api.chart.dto.PageInfo; +import io.gisbi.api.dataset.union.DatasetGroupInfoDTO; +import io.gisbi.chart.charts.impl.DefaultChartHandler; +import io.gisbi.engine.sql.SQLProvider; +import io.gisbi.engine.trans.Dimension2SQLObj; +import io.gisbi.engine.trans.ExtWhere2Str; +import io.gisbi.engine.trans.Quota2SQLObj; +import io.gisbi.engine.utils.Utils; import io.gisbi.extensions.datasource.dto.DatasourceRequest; import io.gisbi.extensions.datasource.dto.DatasourceSchemaDTO; import io.gisbi.extensions.datasource.model.SQLMeta; import io.gisbi.extensions.datasource.provider.Provider; import io.gisbi.extensions.view.dto.*; +import io.gisbi.extensions.view.util.ChartDataUtil; +import io.gisbi.extensions.view.util.FieldUtil; +import io.gisbi.utils.JsonUtil; import lombok.Getter; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; -import java.util.List; -import java.util.Map; +import java.util.*; /** * @author jianneng * @date 2024/9/11 11:37 **/ @Component -public class TableNormalHandler extends YoyChartHandler { +public class TableNormalHandler extends DefaultChartHandler { @Getter - private String type = "table-normal"; + private final String type = "table-normal"; + + @Override + public T customFilter(ChartViewDTO view, List filterList, AxisFormatResult formatResult) { + var chartExtRequest = view.getChartExtRequest(); + Map mapAttr = view.getCustomAttr(); + Map basicStyle = (Map) mapAttr.get("basicStyle"); + var tablePageMode = (String) basicStyle.get("tablePageMode"); + formatResult.getContext().put("tablePageMode", tablePageMode); + if (StringUtils.equalsIgnoreCase(tablePageMode, "page")) { + if (chartExtRequest.getGoPage() == null) { + chartExtRequest.setGoPage(1L); + } + if (chartExtRequest.getPageSize() == null) { + int pageSize = (int) basicStyle.get("tablePageSize"); + if (StringUtils.equalsIgnoreCase(view.getResultMode(), "custom")) { + chartExtRequest.setPageSize(Math.min(pageSize, view.getResultCount().longValue())); + } else { + chartExtRequest.setPageSize((long) pageSize); + } + } + } else { + if (StringUtils.equalsIgnoreCase(view.getResultMode(), "custom")) { + chartExtRequest.setGoPage(1L); + chartExtRequest.setPageSize(view.getResultCount().longValue()); + } else if (!view.getIsExcelExport()) { + chartExtRequest.setGoPage(null); + chartExtRequest.setPageSize(null); + } + } + var yAxis = formatResult.getAxisMap().get(ChartAxis.yAxis); + String originFilterJson = (String) JsonUtil.toJSONString(filterList); + List originFilter = JsonUtil.parseList(originFilterJson, new TypeReference<>() { + }); + boolean yoyFiltered = checkYoyFilter(originFilter, yAxis); + if (yoyFiltered) { + formatResult.getContext().put("expandedFilter", originFilter); + formatResult.getContext().put("yoyFiltered", true); + } + return (T) new CustomFilterResult(filterList, formatResult.getContext()); + } + + @Override + public Map buildResult(ChartViewDTO view, AxisFormatResult formatResult, CustomFilterResult filterResult, List data) { + return new HashMap<>(); + } @Override public T calcChartResult(ChartViewDTO view, AxisFormatResult formatResult, CustomFilterResult filterResult, Map sqlMap, SQLMeta sqlMeta, Provider provider) { + var chartExtRequest = view.getChartExtRequest(); var dsMap = (Map) sqlMap.get("dsMap"); - var result = (T) super.calcChartResult(view, formatResult, filterResult, sqlMap, sqlMeta, provider); + List dsList = new ArrayList<>(); + for (Map.Entry next : dsMap.entrySet()) { + dsList.add(next.getValue().getType()); + } + boolean crossDs = ((DatasetGroupInfoDTO) formatResult.getContext().get("dataset")).getIsCross(); + DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setIsCross(crossDs); + datasourceRequest.setDsList(dsMap); + var xAxis = formatResult.getAxisMap().get(ChartAxis.xAxis); + var yAxis = formatResult.getAxisMap().get(ChartAxis.yAxis); + + var allFields = (List) filterResult.getContext().get("allFields"); + PageInfo pageInfo = new PageInfo(); + pageInfo.setGoPage(chartExtRequest.getGoPage()); + if (StringUtils.equalsIgnoreCase(view.getResultMode(), "custom")) { + pageInfo.setPageSize(Math.min(view.getResultCount() - (chartExtRequest.getGoPage() - 1) * chartExtRequest.getPageSize(), chartExtRequest.getPageSize())); + } else { + pageInfo.setPageSize(chartExtRequest.getPageSize()); + } + Dimension2SQLObj.dimension2sqlObj(sqlMeta, xAxis, FieldUtil.transFields(allFields), crossDs, dsMap, Utils.getParams(FieldUtil.transFields(allFields)), view.getCalParams(), pluginManage); + Quota2SQLObj.quota2sqlObj(sqlMeta, yAxis, FieldUtil.transFields(allFields), crossDs, dsMap, Utils.getParams(FieldUtil.transFields(allFields)), view.getCalParams(), pluginManage); + String originSql = SQLProvider.createQuerySQL(sqlMeta, true, !StringUtils.equalsIgnoreCase(dsMap.values().iterator().next().getType(), "es"), view);// 分页强制加排序 + String limit = ((pageInfo.getGoPage() != null && pageInfo.getPageSize() != null) ? " LIMIT " + pageInfo.getPageSize() + " OFFSET " + (pageInfo.getGoPage() - 1) * chartExtRequest.getPageSize() : ""); + var querySql = originSql + limit; + + var tablePageMode = (String) filterResult.getContext().get("tablePageMode"); + var totalPageSql = "SELECT COUNT(*) FROM (" + SQLProvider.createQuerySQLNoSort(sqlMeta, true, view) + ") COUNT_TEMP"; + if (StringUtils.isNotEmpty(totalPageSql) && StringUtils.equalsIgnoreCase(tablePageMode, "page")) { + totalPageSql = provider.rebuildSQL(totalPageSql, sqlMeta, crossDs, dsMap); + datasourceRequest.setQuery(totalPageSql); + datasourceRequest.setTotalPageFlag(true); + logger.debug("calcite total sql: " + totalPageSql); + List tmpData = (List) provider.fetchResultField(datasourceRequest).get("data"); + var totalItems = ObjectUtils.isEmpty(tmpData) ? 0 : Long.valueOf(tmpData.get(0)[0]); + if (StringUtils.equalsIgnoreCase(view.getResultMode(), "custom")) { + totalItems = totalItems <= view.getResultCount() ? totalItems : view.getResultCount(); + } + var totalPage = (totalItems / pageInfo.getPageSize()) + (totalItems % pageInfo.getPageSize() > 0 ? 1 : 0); + view.setTotalItems(totalItems); + view.setTotalPage(totalPage); + } + + querySql = provider.rebuildSQL(querySql, sqlMeta, crossDs, dsMap); + datasourceRequest.setQuery(querySql); + logger.debug("calcite chart sql: " + querySql); + List data = (List) provider.fetchResultField(datasourceRequest).get("data"); + //自定义排序 + data = ChartDataUtil.resultCustomSort(xAxis, yAxis, view.getSortPriority(), data); + + var yoyFiltered = filterResult.getContext().get("yoyFiltered") != null; + if (yoyFiltered) { + // 这里没加分页，因为加了分页参数可能会把原始数据挤出去 + var expandedFilter = (List) filterResult.getContext().get("expandedFilter"); + ExtWhere2Str.extWhere2sqlOjb(sqlMeta, expandedFilter, FieldUtil.transFields(allFields), crossDs, dsMap, Utils.getParams(FieldUtil.transFields(allFields)), view.getCalParams(), pluginManage); + var expandedSql = SQLProvider.createQuerySQL(sqlMeta, true, !StringUtils.equalsIgnoreCase(dsMap.values().iterator().next().getType(), "es"), view); + expandedSql = provider.rebuildSQL(expandedSql, sqlMeta, crossDs, dsMap); + var expandedReq = new DatasourceRequest(); + expandedReq.setIsCross(crossDs); + expandedReq.setDsList(dsMap); + expandedReq.setQuery(expandedSql); + logger.debug("expanded sql: " + expandedSql); + var expandedData = (List) provider.fetchResultField(expandedReq).get("data"); + logger.debug("expanded data: " + expandedData); + quickCalc(xAxis, yAxis, Collections.emptyList(), Collections.emptyList(), view.getType(), expandedData); + var resultData = new ArrayList(); + for (String[] originDataLine : data) { + var originDim = new StringBuilder(); + for (int i = 0; i < xAxis.size(); i++) { + originDim.append(originDataLine[i]); + } + for (String[] expandedDataLine : expandedData) { + var expandedDim = new StringBuilder(); + for (int i = 0; i < xAxis.size(); i++) { + expandedDim.append(expandedDataLine[i]); + } + if (StringUtils.equals(originDim, expandedDim)) { + resultData.add(expandedDataLine); + break; + } + } + } + data = resultData; + } else { + quickCalc(xAxis, yAxis, Collections.emptyList(), Collections.emptyList(), view.getType(), data); + } + //数据重组逻辑可重载 + var result = this.buildResult(view, formatResult, filterResult, data); + T calcResult = (T) new ChartCalcDataResult(); + calcResult.setData(result); + calcResult.setContext(filterResult.getContext()); + calcResult.setQuerySql(querySql); + calcResult.setOriginData(data); try { - var originSql = result.getQuerySql(); var dynamicAssistFields = getDynamicThresholdFields(view); - var yAxis = formatResult.getAxisMap().get(ChartAxis.yAxis); - var assistFields = getAssistFields(dynamicAssistFields, yAxis); + var assistFields = getAssistFields(dynamicAssistFields, yAxis, xAxis); if (CollectionUtils.isNotEmpty(assistFields)) { var req = new DatasourceRequest(); + req.setIsCross(crossDs); req.setDsList(dsMap); - var assistSql = assistSQL(originSql, assistFields, dsMap); - req.setQuery(assistSql); - logger.debug("calcite assistSql sql: " + assistSql); - var assistData = (List) provider.fetchResultField(req).get("data"); - result.setAssistData(assistData); - result.setDynamicAssistFields(dynamicAssistFields); + + List assists = dynamicAssistFields.stream().filter(ele -> !StringUtils.equalsIgnoreCase(ele.getSummary(), "last_item")).toList(); + if (ObjectUtils.isNotEmpty(assists)) { + var assistSql = assistSQL(originSql, assistFields, dsMap, crossDs); + var tmpSql = provider.rebuildSQL(assistSql, sqlMeta, crossDs, dsMap); + req.setQuery(tmpSql); + logger.debug("calcite assistSql sql: " + tmpSql); + var assistData = (List) provider.fetchResultField(req).get("data"); + calcResult.setAssistData(assistData); + calcResult.setDynamicAssistFields(assists); + } + + List assistsOriginList = dynamicAssistFields.stream().filter(ele -> StringUtils.equalsIgnoreCase(ele.getSummary(), "last_item")).toList(); + if (ObjectUtils.isNotEmpty(assistsOriginList)) { + var assistSqlOriginList = assistSQLOriginList(originSql, assistFields, dsMap, crossDs); + var tmpSql = provider.rebuildSQL(assistSqlOriginList, sqlMeta, crossDs, dsMap); + req.setQuery(tmpSql); + logger.debug("calcite assistSql sql origin list: " + tmpSql); + var assistDataOriginList = (List) provider.fetchResultField(req).get("data"); + calcResult.setAssistDataOriginList(assistDataOriginList); + calcResult.setDynamicAssistFieldsOriginList(assistsOriginList); + } } } catch (Exception e) { e.printStackTrace(); } - return result; + return calcResult; } } diff --git a/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/table/TablePivotHandler.java b/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/table/TablePivotHandler.java index b7bcd76..13f3b48 100644 --- a/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/table/TablePivotHandler.java +++ b/core/core-backend/src/main/java/io/gisbi/chart/charts/impl/table/TablePivotHandler.java @@ -1,5 +1,6 @@ package io.gisbi.chart.charts.impl.table; +import io.gisbi.api.dataset.union.DatasetGroupInfoDTO; import io.gisbi.chart.charts.impl.GroupChartHandler; import io.gisbi.constant.DeTypeConstants; import io.gisbi.engine.constant.ExtFieldConstant; @@ -18,6 +19,7 @@ import io.gisbi.utils.IDUtils; import io.gisbi.utils.JsonUtil; import lombok.Getter; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; import reactor.util.function.Tuple2; @@ -34,7 +36,8 @@ public class TablePivotHandler extends GroupChartHandler { @Override public T calcChartResult(ChartViewDTO view, AxisFormatResult formatResult, CustomFilterResult filterResult, Map sqlMap, SQLMeta sqlMeta, Provider provider) { T result = super.calcChartResult(view, formatResult, filterResult, sqlMap, sqlMeta, provider); - Map customCalc = calcCustomExpr(view, filterResult, sqlMap, sqlMeta, provider); + Map customCalc = calcCustomExpr(view, formatResult, filterResult, sqlMap, sqlMeta, provider); + boolean crossDs = ((DatasetGroupInfoDTO) formatResult.getContext().get("dataset")).getIsCross(); result.getData().put("customCalc", customCalc); try { var dsMap = (Map) sqlMap.get("dsMap"); @@ -44,13 +47,28 @@ public class TablePivotHandler extends GroupChartHandler { var assistFields = getAssistFields(dynamicAssistFields, yAxis); if (CollectionUtils.isNotEmpty(assistFields)) { var req = new DatasourceRequest(); + req.setIsCross(crossDs); req.setDsList(dsMap); - var assistSql = assistSQL(originSql, assistFields, dsMap); - req.setQuery(assistSql); - logger.debug("calcite assistSql sql: " + assistSql); - var assistData = (List) provider.fetchResultField(req).get("data"); - result.setAssistData(assistData); - result.setDynamicAssistFields(dynamicAssistFields); + + List assists = dynamicAssistFields.stream().filter(ele -> !StringUtils.equalsIgnoreCase(ele.getSummary(), "last_item")).toList(); + if (ObjectUtils.isNotEmpty(assists)) { + var assistSql = assistSQL(originSql, assistFields, dsMap, crossDs); + req.setQuery(assistSql); + logger.debug("calcite assistSql sql: " + assistSql); + var assistData = (List) provider.fetchResultField(req).get("data"); + result.setAssistData(assistData); + result.setDynamicAssistFields(assists); + } + + List assistsOriginList = dynamicAssistFields.stream().filter(ele -> StringUtils.equalsIgnoreCase(ele.getSummary(), "last_item")).toList(); + if (ObjectUtils.isNotEmpty(assistsOriginList)) { + var assistSqlOriginList = assistSQLOriginList(originSql, assistFields, dsMap, crossDs); + req.setQuery(assistSqlOriginList); + logger.debug("calcite assistSql sql origin list: " + assistSqlOriginList); + var assistDataOriginList = (List) provider.fetchResultField(req).get("data"); + result.setAssistDataOriginList(assistDataOriginList); + result.setDynamicAssistFieldsOriginList(assistsOriginList); + } } } catch (Exception e) { e.printStackTrace(); @@ -58,7 +76,18 @@ public class TablePivotHandler extends GroupChartHandler { return result; } - private Map calcCustomExpr(ChartViewDTO view, CustomFilterResult filterResult, Map sqlMap, SQLMeta sqlMeta, Provider provider) { + @Override + public Map buildResult(ChartViewDTO view, AxisFormatResult formatResult, CustomFilterResult filterResult, List data) { + var result = new HashMap(); + var yoyFiltered = filterResult.getContext().get("yoyFiltered") != null; + // 带过滤同环比直接返回原始数据,再由视图重新组装 + if (yoyFiltered) { + result.put("data", data); + } + return result; + } + + private Map calcCustomExpr(ChartViewDTO view, AxisFormatResult formatResult, CustomFilterResult filterResult, Map sqlMap, SQLMeta sqlMeta, Provider provider) { Object totalStr = JsonUtil.toJSONString(view.getCustomAttr().get("tableTotal")); TableTotal tableTotal = JsonUtil.parseObject((String) totalStr, TableTotal.class); var dsMap = (Map) sqlMap.get("dsMap"); @@ -67,7 +96,7 @@ public class TablePivotHandler extends GroupChartHandler { dsList.add(next.getValue().getType()); } boolean needOrder = Utils.isNeedOrder(dsList); - boolean crossDs = Utils.isCrossDs(dsMap); + boolean crossDs = ((DatasetGroupInfoDTO) formatResult.getContext().get("dataset")).getIsCross(); DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDsList(dsMap); var allFields = (List) filterResult.getContext().get("allFields"); @@ -178,7 +207,7 @@ public class TablePivotHandler extends GroupChartHandler { var tmpData = new HashMap(); for (int i = 0; i < yAxis.size(); i++) { var a = yAxis.get(i); - tmpData.put(a.getDataeaseName(), data.getFirst()[i]); + tmpData.put(a.getGisbiName(), data.getFirst()[i]); } tmp.put("data", tmpData); tmp.put("sql", Base64.getEncoder().encodeToString(querySql.getBytes())); @@ -208,8 +237,8 @@ public class TablePivotHandler extends GroupChartHandler { } } // 列总计里面的行小计 - if (col.isShowGrandTotals() && row.isShowGrandTotals() && rowAxis.size() >= 2) { - var yAxis = getCustomFields(view, row.getCalcTotals().getCfg()); + if (col.isShowGrandTotals() && row.isShowSubTotals() && rowAxis.size() >= 2) { + var yAxis = getCustomFields(view, row.getCalcSubTotals().getCfg()); if (!yAxis.isEmpty()) { var tmpData = new ArrayList>(); dataMap.put("rowSubInColTotal", tmpData); @@ -243,11 +272,11 @@ public class TablePivotHandler extends GroupChartHandler { var tmpList = new ArrayList>(); tmpData.add(tmpList); var subRow = rowAxis.subList(0, i + 1); - var xAxis = new ArrayList<>(subRow); for (int j = 0; j < colAxis.size(); j++) { if (j == colAxis.size() - 1) { break; } + var xAxis = new ArrayList<>(subRow); var subCol = colAxis.subList(0, j + 1); xAxis.addAll(subCol); var tmpAllList = new ArrayList<>(allFields); @@ -272,7 +301,7 @@ public class TablePivotHandler extends GroupChartHandler { var rowData = data.getFirst(); for (int i = 0; i < rowData.length; i++) { var qAxis = quotaAxis.get(i); - rootResult.put(qAxis.getDataeaseName(), rowData[i]); + rootResult.put(qAxis.getGisbiName(), rowData[i]); } return rootResult; } @@ -291,7 +320,7 @@ public class TablePivotHandler extends GroupChartHandler { if (j == dimAxis.size() - 1) { for (int k = 0; k < quotaAxis.size(); k++) { var qAxis = quotaAxis.get(k); - curSubMap.put(qAxis.getDataeaseName(), rowData[j + k + 1]); + curSubMap.put(qAxis.getGisbiName(), rowData[j + k + 1]); } } } @@ -303,6 +332,7 @@ public class TablePivotHandler extends GroupChartHandler { List allFields, boolean crossDs, Map dsMap, ChartViewDTO view, Provider provider, boolean needOrder) { DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setIsCross(crossDs); datasourceRequest.setDsList(dsMap); Dimension2SQLObj.dimension2sqlObj(sqlMeta, xAxis, FieldUtil.transFields(allFields), crossDs, dsMap, Utils.getParams(FieldUtil.transFields(allFields)), view.getCalParams(), pluginManage); Quota2SQLObj.quota2sqlObj(sqlMeta, yAxis, FieldUtil.transFields(allFields), crossDs, dsMap, Utils.getParams(FieldUtil.transFields(allFields)), view.getCalParams(), pluginManage); @@ -326,10 +356,10 @@ public class TablePivotHandler extends GroupChartHandler { } private List getCustomFields(ChartViewDTO view, List cfgList) { - var quotaIds = view.getYAxis().stream().map(ChartViewFieldDTO::getDataeaseName).collect(Collectors.toSet()); + var quotaIds = view.getYAxis().stream().map(ChartViewFieldDTO::getGisbiName).collect(Collectors.toSet()); var customFields = new ArrayList(); for (TableCalcTotalCfg totalCfg : cfgList) { - if (!quotaIds.contains(totalCfg.getDataeaseName())) { + if (!quotaIds.contains(totalCfg.getGisbiName())) { continue; } if (StringUtils.equalsIgnoreCase(totalCfg.getAggregation(), "CUSTOM")) { diff --git a/core/core-backend/src/main/java/io/gisbi/chart/dao/ext/mapper/ExtChartViewMapper.java b/core/core-backend/src/main/java/io/gisbi/chart/dao/ext/mapper/ExtChartViewMapper.java index c517bf8..34f1ade 100644 --- a/core/core-backend/src/main/java/io/gisbi/chart/dao/ext/mapper/ExtChartViewMapper.java +++ b/core/core-backend/src/main/java/io/gisbi/chart/dao/ext/mapper/ExtChartViewMapper.java @@ -1,7 +1,9 @@ package io.gisbi.chart.dao.ext.mapper; import io.gisbi.api.chart.vo.ViewSelectorVO; +import io.gisbi.chart.dao.auto.entity.CoreChartView; import io.gisbi.chart.dao.ext.entity.ChartBasePO; +import io.gisbi.extensions.view.dto.ChartViewDTO; import org.apache.ibatis.annotations.Mapper; import org.apache.ibatis.annotations.Param; import org.apache.ibatis.annotations.Select; @@ -16,29 +18,21 @@ public interface ExtChartViewMapper { """) List queryViewOption(@Param("resourceId") Long resourceId); + ChartBasePO queryChart(@Param("id") Long id, @Param("resourceTable")String resourceTable); + + List selectListCustom(@Param("sceneId") Long sceneId, @Param("resourceTable") String resourceTable); + + void deleteViewsBySceneId(@Param("sceneId") Long sceneId, @Param("resourceTable") String resourceTable); + @Select(""" - select - ccv.id as chart_id, - ccv.title as chart_name, - ccv.type as chart_type, - ccv.table_id, - dvi.id as resource_id, - dvi.name as resource_name, - dvi.type as resource_type, - ccv.x_axis, - ccv.x_axis_ext, - ccv.y_axis, - ccv.y_axis_ext, - ccv.ext_stack, - ccv.ext_bubble, - ccv.ext_label, - ccv.ext_tooltip, - ccv.flow_map_start_name, - ccv.flow_map_end_name, - ccv.ext_color - from core_chart_view ccv - left join data_visualization_info dvi on dvi.id = ccv.scene_id - where ccv.id = #{id} + SELECT id, scene_id as pid, title, type FROM ( + SELECT id, scene_id, title, type FROM core_chart_view + WHERE id = #{viewId} + UNION ALL + SELECT id, scene_id, title, type FROM snapshot_core_chart_view + WHERE id = #{viewId} + ) combined_views + LIMIT 1 """) - ChartBasePO queryChart(@Param("id") Long id); + ChartViewDTO findChartViewAround(@Param("viewId") String viewId); } diff --git a/core/core-backend/src/main/java/io/gisbi/chart/manage/ChartDataManage.java b/core/core-backend/src/main/java/io/gisbi/chart/manage/ChartDataManage.java index 17d9a99..b01d3f1 100644 --- a/core/core-backend/src/main/java/io/gisbi/chart/manage/ChartDataManage.java +++ b/core/core-backend/src/main/java/io/gisbi/chart/manage/ChartDataManage.java @@ -1,14 +1,16 @@ package io.gisbi.chart.manage; import io.gisbi.api.dataset.union.DatasetGroupInfoDTO; - +import io.gisbi.api.permissions.auth.dto.BusiPerCheckDTO; import io.gisbi.api.permissions.dataset.dto.DataSetRowPermissionsTreeDTO; import io.gisbi.chart.charts.ChartHandlerManager; import io.gisbi.chart.constant.ChartConstants; +import io.gisbi.constant.AuthEnum; import io.gisbi.dataset.manage.DatasetGroupManage; import io.gisbi.dataset.manage.DatasetSQLManage; import io.gisbi.dataset.manage.DatasetTableFieldManage; import io.gisbi.dataset.manage.PermissionManage; +import io.gisbi.dataset.utils.DatasetUtils; import io.gisbi.engine.sql.SQLProvider; import io.gisbi.engine.trans.*; import io.gisbi.engine.utils.SQLUtils; @@ -28,6 +30,7 @@ import io.gisbi.extensions.view.plugin.AbstractChartPlugin; import io.gisbi.extensions.view.util.FieldUtil; import io.gisbi.i18n.Translator; import io.gisbi.result.ResultCode; +import io.gisbi.system.manage.CorePermissionManage; import io.gisbi.utils.AuthUtils; import io.gisbi.utils.BeanUtils; import jakarta.annotation.Resource; @@ -58,12 +61,14 @@ public class ChartDataManage { @Resource private ChartViewManege chartViewManege; @Resource + private PermissionManage permissionManage; + @Resource private ChartFilterTreeService chartFilterTreeService; @Resource private ChartHandlerManager chartHandlerManager; - @Resource - private PermissionManage permissionManage; + @Resource + private CorePermissionManage corePermissionManage; @Autowired(required = false) private PluginManageApi pluginManage; @@ -132,17 +137,18 @@ public class ChartDataManage { // row permission List rowPermissionsTree = permissionManage.getRowPermissionsTree(table.getId(), chartExtRequest.getUser()); //将没有权限的列删掉 - List dataeaseNames = columnPermissionFields.stream().map(DatasetTableFieldDTO::getDataeaseName).collect(Collectors.toList()); + List gisbiNames = columnPermissionFields.stream().map(DatasetTableFieldDTO::getGisbiName).collect(Collectors.toList()); //计数字段 - dataeaseNames.add("*"); + gisbiNames.add("*"); AxisFormatResult formatResult = chartHandler.formatAxis(view); + formatResult.getContext().put("dataset", table); formatResult.getContext().put("desensitizationList", desensitizationList); var xAxis = formatResult.getAxisMap().get(ChartAxis.xAxis); var yAxis = formatResult.getAxisMap().get(ChartAxis.yAxis); formatResult.getContext().put("allFields", allFields); var axisMap = formatResult.getAxisMap(); axisMap.forEach((axis, fields) -> { - fields.removeIf(fieldDTO -> !dataeaseNames.contains(fieldDTO.getDataeaseName())); + fields.removeIf(fieldDTO -> !gisbiNames.contains(fieldDTO.getGisbiName())); }); // 过滤来自仪表板的条件 @@ -225,11 +231,17 @@ public class ChartDataManage { } List filters = new ArrayList<>(); + FilterTreeObj customLinkageFilter = null; // 联动条件 if (ObjectUtils.isNotEmpty(chartExtRequest.getLinkageFilters())) { - filters.addAll(chartExtRequest.getLinkageFilters()); + for (ChartExtFilterDTO linkageFilter : chartExtRequest.getLinkageFilters()) { + if (3 == linkageFilter.getFilterType()) { + customLinkageFilter = linkageFilter.getCustomFilter(); + } else { + filters.add(linkageFilter); + } + } } - // 外部参数条件 if (ObjectUtils.isNotEmpty(chartExtRequest.getOuterParamsFilters())) { filters.addAll(chartExtRequest.getOuterParamsFilters()); @@ -357,13 +369,17 @@ public class ChartDataManage { } // 字段过滤器 FilterTreeObj fieldCustomFilter = view.getCustomFilter(); + // 指标表联动时 使用的CustomFilter + if (customLinkageFilter != null) { + fieldCustomFilter = customLinkageFilter; + } chartFilterTreeService.searchFieldAndSet(fieldCustomFilter); fieldCustomFilter = chartFilterTreeService.charReplace(fieldCustomFilter); // 获取dsMap,union sql Map sqlMap = datasetSQLManage.getUnionSQLForEdit(table, chartExtRequest); String sql = (String) sqlMap.get("sql"); Map dsMap = (Map) sqlMap.get("dsMap"); - boolean crossDs = Utils.isCrossDs(dsMap); + boolean crossDs = table.getIsCross(); if (!crossDs) { sql = Utils.replaceSchemaAlias(sql, dsMap); } @@ -592,11 +608,11 @@ public class ChartDataManage { int getIndex = 0; for (int i = 0; i < fieldList.size(); i++) { ChartViewFieldDTO item = fieldList.get(i); - if (StringUtils.equalsIgnoreCase(item.getSort(), "custom_sort")) { + if (StringUtils.equalsIgnoreCase(item.getSort(), "custom_sort")) {// 此处与已有的自定义字段对比 chartViewFieldDTO = item; index = i; } - if (Objects.equals(item.getId(), field.getId())) { + if (Objects.equals(item.getId(), field.getId())) {// 获得当前自定义的字段 getIndex = i; } } @@ -648,7 +664,7 @@ public class ChartDataManage { // 获取图表所有字段配置 List allFields = getAllChartFields(view); - // 处理堆叠/分组图表的特殊x轴配置 + // 针对分组切换堆叠时会遇到的问题 if (StringUtils.equalsIgnoreCase(view.getType(), "bar-stack") || StringUtils.equalsIgnoreCase(view.getType(), "chart-mix-stack")) { view.setXAxisExt(new ArrayList<>()); } @@ -668,7 +684,6 @@ public class ChartDataManage { xAxis.addAll(xAxisExt); } List yAxis = new ArrayList<>(view.getYAxis()); - // 合并扩展y轴配置（针对混合图表类型） if (StringUtils.containsIgnoreCase(view.getType(), "chart-mix")) { List yAxisExt = new ArrayList<>(view.getYAxisExt()); yAxis.addAll(yAxisExt); @@ -678,6 +693,10 @@ public class ChartDataManage { List sizeField = getSizeField(view); yAxis.addAll(sizeField); } + List extStack = new ArrayList<>(view.getExtStack()); + List extBubble = new ArrayList<>(view.getExtBubble()); + FilterTreeObj fieldCustomFilter = view.getCustomFilter(); + List drill = new ArrayList<>(view.getDrillFields()); // 获取数据集权限信息 DatasetGroupInfoDTO table = datasetGroupManage.getDatasetGroupInfoDTO(view.getTableId(), null); @@ -715,6 +734,7 @@ public class ChartDataManage { } break; case "table-normal": + break; case "bar-group": case "bar-group-stack": case "flow-map": @@ -739,6 +759,7 @@ public class ChartDataManage { // 初始化数据源请求对象 DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDsList(dsMap); + datasourceRequest.setIsCross(crossDs); // 根据数据源类型选择查询提供器 Provider provider; @@ -750,8 +771,10 @@ public class ChartDataManage { // 执行数据查询 List data = new ArrayList<>(); + String querySql = null; - if (table.getMode() == 0) { // 直连模式处理 + //如果不是插件图表 走原生逻辑 + if (table.getMode() == 0) {// 直连 if (ObjectUtils.isEmpty(dsMap)) { DEException.throwException(Translator.get("i18n_datasource_delete")); } @@ -768,21 +791,20 @@ public class ChartDataManage { Table2SQLObj.table2sqlobj(sqlMeta, null, "(" + sql + ")", crossDs); WhereTree2Str.transFilterTrees(sqlMeta, rowPermissionsTree, transFields(allFields), crossDs, dsMap, Utils.getParams(transFields(allFields)), view.getCalParams(), pluginManage); - // 根据图表类型生成不同SQL逻辑 if (StringUtils.equalsAnyIgnoreCase(view.getType(), "indicator", "gauge", "liquid")) { Quota2SQLObj.quota2sqlObj(sqlMeta, yAxis, transFields(allFields), crossDs, dsMap, Utils.getParams(transFields(allFields)), view.getCalParams(), pluginManage); querySql = SQLProvider.createQuerySQL(sqlMeta, true, needOrder, view); } else if (StringUtils.containsIgnoreCase(view.getType(), "stack")) { List xFields = new ArrayList<>(); xFields.addAll(xAxis); - xFields.addAll(view.getExtStack()); + xFields.addAll(extStack); Dimension2SQLObj.dimension2sqlObj(sqlMeta, xFields, transFields(allFields), crossDs, dsMap, Utils.getParams(transFields(allFields)), view.getCalParams(), pluginManage); Quota2SQLObj.quota2sqlObj(sqlMeta, yAxis, transFields(allFields), crossDs, dsMap, Utils.getParams(transFields(allFields)), view.getCalParams(), pluginManage); querySql = SQLProvider.createQuerySQL(sqlMeta, true, needOrder, view); } else if (StringUtils.containsIgnoreCase(view.getType(), "scatter")) { List yFields = new ArrayList<>(); yFields.addAll(yAxis); - yFields.addAll(view.getExtBubble()); + yFields.addAll(extBubble); Dimension2SQLObj.dimension2sqlObj(sqlMeta, xAxis, transFields(allFields), crossDs, dsMap, Utils.getParams(transFields(allFields)), view.getCalParams(), pluginManage); Quota2SQLObj.quota2sqlObj(sqlMeta, yFields, transFields(allFields), crossDs, dsMap, Utils.getParams(transFields(allFields)), view.getCalParams(), pluginManage); querySql = SQLProvider.createQuerySQL(sqlMeta, true, needOrder, view); @@ -804,7 +826,6 @@ public class ChartDataManage { return data; } - private List getAllChartFields(ChartViewDTO view) { // get all fields Map> stringListMap = chartViewManege.listByDQ(view.getTableId(), view.getId(), view); @@ -837,14 +858,13 @@ public class ChartDataManage { DEException.throwException(e); } } else { - // 如果当前图表ID未存在于检查数据中，则记录为废弃ID disuseChartIdList.add(chartViewDTO.getId()); } }); - // 如果存在需要废弃的图表ID列表，则批量标记为废弃 - if (CollectionUtils.isNotEmpty(disuseChartIdList)) { - chartViewManege.disuse(disuseChartIdList); - } + // 阈值告警处理 统一在发布时处理 +// if (CollectionUtils.isNotEmpty(disuseChartIdList)) { +// chartViewManege.disuse(disuseChartIdList); +// } } } @@ -879,19 +899,18 @@ public class ChartDataManage { // 执行SQL查询获取原始数据 List sqlData = sqlData(view, view.getChartExtRequest(), fieldId); - - // 根据自定义排序规则对数据进行排序 - List result = customSort( - Optional.ofNullable(targetField.getCustomSort()).orElse(new ArrayList<>()), - sqlData, - 0 - ); - - // 提取第一列数据并去重生成最终结果列表 - return result.stream() - .map(i -> i[0]) - .distinct() - .collect(Collectors.toList()); + List result = customSort(Optional.ofNullable(targetField.getCustomSort()).orElse(new ArrayList<>()), sqlData, 0); + return result.stream().map(i -> i[0]).distinct().collect(Collectors.toList()); } + public void encodeData(ChartViewDTO chartViewDTO) { + if (chartViewDTO.getData() != null) { + if (chartViewDTO.getType().startsWith("chart-mix")) { + DatasetUtils.listEncode((List) ((Map) chartViewDTO.getData().get("left")).get("sourceFields")); + DatasetUtils.listEncode((List) ((Map) chartViewDTO.getData().get("right")).get("sourceFields")); + } else { + DatasetUtils.listEncode((List) chartViewDTO.getData().get("sourceFields")); + } + } + } } diff --git a/core/core-backend/src/main/java/io/gisbi/chart/manage/ChartViewManege.java b/core/core-backend/src/main/java/io/gisbi/chart/manage/ChartViewManege.java index 2604295..e459b51 100644 --- a/core/core-backend/src/main/java/io/gisbi/chart/manage/ChartViewManege.java +++ b/core/core-backend/src/main/java/io/gisbi/chart/manage/ChartViewManege.java @@ -10,6 +10,7 @@ import io.gisbi.chart.dao.auto.entity.CoreChartView; import io.gisbi.chart.dao.auto.mapper.CoreChartViewMapper; import io.gisbi.chart.dao.ext.entity.ChartBasePO; import io.gisbi.chart.dao.ext.mapper.ExtChartViewMapper; +import io.gisbi.constant.CommonConstants; import io.gisbi.dataset.dao.auto.entity.CoreDatasetTableField; import io.gisbi.dataset.dao.auto.mapper.CoreDatasetTableFieldMapper; import io.gisbi.dataset.manage.DatasetTableFieldManage; @@ -25,13 +26,16 @@ import io.gisbi.extensions.datasource.dto.FieldGroupDTO; import io.gisbi.extensions.datasource.model.SQLObj; import io.gisbi.extensions.view.dto.*; import io.gisbi.extensions.view.filter.FilterTreeObj; +import io.gisbi.i18n.Lang; import io.gisbi.i18n.Translator; import io.gisbi.utils.BeanUtils; import io.gisbi.utils.IDUtils; import io.gisbi.utils.JsonUtil; import io.gisbi.utils.LogUtil; import io.gisbi.visualization.dao.auto.entity.DataVisualizationInfo; +import io.gisbi.visualization.dao.auto.entity.SnapshotCoreChartView; import io.gisbi.visualization.dao.auto.mapper.DataVisualizationInfoMapper; +import io.gisbi.visualization.dao.auto.mapper.SnapshotCoreChartViewMapper; import jakarta.annotation.Resource; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -55,6 +59,8 @@ public class ChartViewManege { @Resource private CoreChartViewMapper coreChartViewMapper; @Resource + private SnapshotCoreChartViewMapper snapshotCoreChartViewMapper; + @Resource private ChartDataManage chartDataManage; @Resource private CoreDatasetTableFieldMapper coreDatasetTableFieldMapper; @@ -82,18 +88,19 @@ public class ChartViewManege { if (id == null) { DEException.throwException(Translator.get("i18n_no_id")); } - CoreChartView coreChartView = coreChartViewMapper.selectById(id); - CoreChartView record = transDTO2Record(chartViewDTO); + SnapshotCoreChartView coreChartView = snapshotCoreChartViewMapper.selectById(id); + SnapshotCoreChartView record = transDTO2Record(chartViewDTO); if (ObjectUtils.isEmpty(coreChartView)) { - coreChartViewMapper.insert(record); + snapshotCoreChartViewMapper.deleteById(record.getId()); + snapshotCoreChartViewMapper.insert(record); } else { - UpdateWrapper updateWrapper = new UpdateWrapper<>(); + UpdateWrapper updateWrapper = new UpdateWrapper<>(); updateWrapper.eq("id", record.getId()); //富文本允许设置空的tableId 这里额外更新一下 if (record.getTableId() == null) { updateWrapper.set("table_id", null); } - coreChartViewMapper.update(record, updateWrapper); + snapshotCoreChartViewMapper.update(record, updateWrapper); } return chartViewDTO; } @@ -113,10 +120,20 @@ public class ChartViewManege { coreChartViewMapper.delete(wrapper); } - public ChartViewDTO getDetails(Long id) { - CoreChartView coreChartView = coreChartViewMapper.selectById(id); - if (ObjectUtils.isEmpty(coreChartView)) { - return null; + public ChartViewDTO getDetails(Long id, String resourceTable) { + CoreChartView coreChartView = null; + if (CommonConstants.RESOURCE_TABLE.SNAPSHOT.equals(resourceTable)) { + SnapshotCoreChartView snapshotCoreChartView = snapshotCoreChartViewMapper.selectById(id); + if (ObjectUtils.isEmpty(snapshotCoreChartView)) { + return null; + } + coreChartView = new CoreChartView(); + BeanUtils.copyBean(coreChartView, snapshotCoreChartView); + } else { + coreChartView = coreChartViewMapper.selectById(id); + if (ObjectUtils.isEmpty(coreChartView)) { + return null; + } } ChartViewDTO dto = transRecord2DTO(coreChartView); return dto; @@ -125,10 +142,10 @@ public class ChartViewManege { /** * sceneId 为仪表板或者数据大屏id */ - public List listBySceneId(Long sceneId) { + public List listBySceneId(Long sceneId, String resourceTable) { QueryWrapper wrapper = new QueryWrapper<>(); wrapper.eq("scene_id", sceneId); - List chartViewDTOS = transChart(coreChartViewMapper.selectList(wrapper)); + List chartViewDTOS = transChart(extChartViewMapper.selectListCustom(sceneId, resourceTable)); if (!CollectionUtils.isEmpty(chartViewDTOS)) { List tableIds = chartViewDTOS.stream() .map(ChartViewDTO::getTableId) @@ -192,8 +209,8 @@ public class ChartViewManege { }).collect(Collectors.toList()); } - public ChartViewDTO getChart(Long id) throws Exception { - ChartViewDTO details = getDetails(id); + public ChartViewDTO getChart(Long id, String resourceTable) throws Exception { + ChartViewDTO details = getDetails(id, resourceTable); if (details == null) { return null; } @@ -220,9 +237,8 @@ public class ChartViewManege { }).collect(Collectors.toList()); // filter column disable field Map desensitizationList = new HashMap<>(); -// List datasetTableFieldDTOS = permissionManage.filterColumnPermissions(collect, desensitizationList, id, null); List datasetTableFieldDTOS = collect; - datasetTableFieldDTOS.forEach(ele -> ele.setDesensitized(desensitizationList.containsKey(ele.getDataeaseName()))); + datasetTableFieldDTOS.forEach(ele -> ele.setDesensitized(desensitizationList.containsKey(ele.getGisbiName()))); datasetTableFieldDTOS.add(createCountField(id)); List list = transFieldDTO(datasetTableFieldDTOS); @@ -282,8 +298,8 @@ public class ChartViewManege { coreDatasetTableField.setExtField(2); coreDatasetTableField.setOriginName("[" + id + "]"); coreDatasetTableField.setId(IDUtils.snowID()); - coreDatasetTableField.setDataeaseName(TableUtils.fieldNameShort(coreDatasetTableField.getId() + "_" + coreDatasetTableField.getOriginName())); - coreDatasetTableField.setFieldShortName(coreDatasetTableField.getDataeaseName()); + coreDatasetTableField.setGisbiName(TableUtils.fieldNameShort(coreDatasetTableField.getId() + "_" + coreDatasetTableField.getOriginName())); + coreDatasetTableField.setFieldShortName(coreDatasetTableField.getGisbiName()); coreDatasetTableFieldMapper.insert(coreDatasetTableField); } @@ -306,8 +322,8 @@ public class ChartViewManege { coreDatasetTableFieldMapper.delete(queryWrapper); } - public ChartBaseVO chartBaseInfo(Long id) { - ChartBasePO po = extChartViewMapper.queryChart(id); + public ChartBaseVO chartBaseInfo(Long id, String resourceTable) { + ChartBasePO po = extChartViewMapper.queryChart(id, resourceTable); if (ObjectUtils.isEmpty(po)) return null; ChartBaseVO vo = BeanUtils.copyBean(new ChartBaseVO(), po); TypeReference> tokenType = new TypeReference<>() { @@ -336,7 +352,7 @@ public class ChartViewManege { dto.setDatasetGroupId(id); dto.setOriginName("*"); dto.setName("记录数*"); - dto.setDataeaseName("*"); + dto.setGisbiName("*"); dto.setType("INT"); dto.setChecked(true); dto.setColumnIndex(999); @@ -353,6 +369,7 @@ public class ChartViewManege { BeanUtils.copyBean(dto, ele); dto.setDateStyle("y_M_d"); dto.setDatePattern("date_sub"); + dto.setDateShowFormat("y_M_d"); dto.setChartType("bar"); if (dto.getId() == -1L || dto.getDeType() == 0 || dto.getDeType() == 1 || dto.getDeType() == 7) { @@ -365,7 +382,7 @@ public class ChartViewManege { chartFieldCompareDTO.setType("none"); dto.setCompareCalc(chartFieldCompareDTO); - dto.setFormatterCfg(new FormatterCfgDTO()); + dto.setFormatterCfg(new FormatterCfgDTO().setUnitLanguage(Lang.isChinese() ? "ch" : "en")); dto.setSort("none"); dto.setFilter(Collections.emptyList()); @@ -373,8 +390,8 @@ public class ChartViewManege { }).collect(Collectors.toList()); } - public CoreChartView transDTO2Record(ChartViewDTO dto) throws Exception { - CoreChartView record = new CoreChartView(); + public SnapshotCoreChartView transDTO2Record(ChartViewDTO dto) throws Exception { + SnapshotCoreChartView record = new SnapshotCoreChartView(); BeanUtils.copyBean(record, dto); record.setxAxis(objectMapper.writeValueAsString(dto.getXAxis())); @@ -400,7 +417,7 @@ public class ChartViewManege { record.setFlowMapStartName(objectMapper.writeValueAsString(dto.getFlowMapStartName())); record.setFlowMapEndName(objectMapper.writeValueAsString(dto.getFlowMapEndName())); record.setExtColor(objectMapper.writeValueAsString(dto.getExtColor())); - + record.setSortPriority(objectMapper.writeValueAsString(dto.getSortPriority())); return record; } @@ -434,6 +451,8 @@ public class ChartViewManege { dto.setFlowMapStartName(JsonUtil.parseList(record.getFlowMapStartName(), tokenType)); dto.setFlowMapEndName(JsonUtil.parseList(record.getFlowMapEndName(), tokenType)); dto.setExtColor(JsonUtil.parseList(record.getExtColor(), tokenType)); + dto.setSortPriority(JsonUtil.parseList(record.getSortPriority(), new TypeReference>() { + })); return dto; @@ -462,4 +481,8 @@ public class ChartViewManege { return result; } } + + public ChartViewDTO findChartViewAround(String viewId) { + return extChartViewMapper.findChartViewAround(viewId); + } } diff --git a/core/core-backend/src/main/java/io/gisbi/chart/manage/ChartViewThresholdManage.java b/core/core-backend/src/main/java/io/gisbi/chart/manage/ChartViewThresholdManage.java index a77a680..fdc27e3 100644 --- a/core/core-backend/src/main/java/io/gisbi/chart/manage/ChartViewThresholdManage.java +++ b/core/core-backend/src/main/java/io/gisbi/chart/manage/ChartViewThresholdManage.java @@ -20,6 +20,7 @@ import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; +import java.math.BigDecimal; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.*; @@ -34,8 +35,8 @@ public class ChartViewThresholdManage { @Resource private ChartViewManege chartViewManege; - public String convertThresholdRules(Long chartId, String thresholdRules) { - ChartViewDTO details = chartViewManege.getDetails(chartId); + public String convertThresholdRules(Long chartId, String thresholdRules, String resourceTable) { + ChartViewDTO details = chartViewManege.getDetails(chartId, resourceTable); return convertThresholdRules(details, thresholdRules); } @@ -251,7 +252,7 @@ public class ChartViewThresholdManage { String thresholdRules = request.getThresholdRules(); Long chartId = request.getChartId(); try { - ChartViewDTO chart = chartViewManege.getChart(chartId); + ChartViewDTO chart = chartViewManege.getChart(chartId, request.getResourceTable()); Map data = null; if (ObjectUtils.isEmpty(chart) || MapUtils.isEmpty(data = chart.getData())) { return new ThresholdCheckVO(false, null, "查询图表异常！", null); @@ -280,9 +281,16 @@ public class ChartViewThresholdManage { DatasetTableFieldDTO fieldDTO = fieldMap.get(id); if (ObjectUtils.isEmpty(fieldDTO)) continue; String fieldDTOName = fieldDTO.getName(); - String dataeaseName = fieldDTO.getDataeaseName(); - List valueList = rows.stream().map(row -> ObjectUtils.isEmpty(row.get(dataeaseName)) ? null : row.get(dataeaseName).toString()).collect(Collectors.toList()); - String replacement = fieldDTOName + ": " + JsonUtil.toJSONString(valueList); + String gisbiName = fieldDTO.getGisbiName(); + String replacement = null; + if (fieldDTO.getDeType().equals(DeTypeConstants.DE_FLOAT) || fieldDTO.getDeType().equals(DeTypeConstants.DE_INT)) { + List valueList = rows.stream().map(row -> ObjectUtils.isEmpty(row.get(gisbiName)) ? null : stripTrailingZeros2String(row.get(gisbiName))).collect(Collectors.toList()); + replacement = fieldDTOName + ": " + JsonUtil.toJSONString(valueList); + } else { + List valueList = rows.stream().map(row -> ObjectUtils.isEmpty(row.get(gisbiName)) ? null : row.get(gisbiName).toString()).collect(Collectors.toList()); + replacement = fieldDTOName + ": " + JsonUtil.toJSONString(valueList); + } + // 替换文本 matcher.appendReplacement(sb, replacement); } @@ -297,6 +305,14 @@ public class ChartViewThresholdManage { } } + private String stripTrailingZeros2String(Object value) { + if (ObjectUtils.isEmpty(value)) { + return null; + } + if (!(value instanceof BigDecimal)) return value.toString(); + return ((BigDecimal) value).stripTrailingZeros().toPlainString(); + } + private void chartDynamicMap(List> rows, FilterTreeObj conditionTree, Map fieldMap) { List items = conditionTree.getItems(); items.forEach(item -> { @@ -404,13 +420,13 @@ public class ChartViewThresholdManage { private String formatValue(List> rows, FilterTreeItem item) { DatasetTableFieldDTO field = item.getField(); - String dataeaseName = field.getDataeaseName(); + String gisbiName = field.getGisbiName(); String value = item.getValue(); Float tempFVal = StringUtils.equalsAny(value, "min", "max") ? null : 0f; int validLen = 0; for (Map row : rows) { - Object o = row.get(dataeaseName); + Object o = row.get(gisbiName); if (ObjectUtils.isEmpty(o)) continue; float fvalue = Float.parseFloat(o.toString()); if (StringUtils.equals("min", value)) { @@ -487,10 +503,10 @@ public class ChartViewThresholdManage { } private boolean rowMatch(Map row, FilterTreeItem item, DatasetTableFieldDTO fieldDTO) { - String dataeaseName = fieldDTO.getDataeaseName(); + String gisbiName = fieldDTO.getGisbiName(); String filterType = item.getFilterType(); Integer deType = fieldDTO.getDeType(); - Object valueObj = row.get(dataeaseName); + Object valueObj = row.get(gisbiName); if (StringUtils.equals(filterType, "enum")) { List enumValue = item.getEnumValue(); return ObjectUtils.isNotEmpty(valueObj) && enumValue.contains(valueObj); diff --git a/core/core-backend/src/main/java/io/gisbi/chart/server/ChartDataServer.java b/core/core-backend/src/main/java/io/gisbi/chart/server/ChartDataServer.java index 2675930..c90f161 100644 --- a/core/core-backend/src/main/java/io/gisbi/chart/server/ChartDataServer.java +++ b/core/core-backend/src/main/java/io/gisbi/chart/server/ChartDataServer.java @@ -5,7 +5,6 @@ import io.gisbi.api.chart.ChartDataApi; import io.gisbi.api.chart.dto.ViewDetailField; import io.gisbi.api.chart.request.ChartExcelRequest; import io.gisbi.api.chart.request.ChartExcelRequestInner; -import io.gisbi.auth.DeLinkPermit; import io.gisbi.chart.constant.ChartConstants; import io.gisbi.chart.manage.ChartDataManage; import io.gisbi.constant.AuthConstant; @@ -14,11 +13,13 @@ import io.gisbi.constant.CommonConstants; import io.gisbi.dataset.server.DatasetFieldServer; import io.gisbi.dataset.manage.PermissionManage; import io.gisbi.constant.DeTypeConstants; +import io.gisbi.dataset.utils.DatasetUtils; import io.gisbi.exception.DEException; import io.gisbi.exportCenter.manage.ExportCenterManage; import io.gisbi.exportCenter.util.ExportCenterUtils; import io.gisbi.extensions.datasource.dto.DatasetTableFieldDTO; import io.gisbi.extensions.view.dto.*; +import io.gisbi.i18n.Lang; import io.gisbi.result.ResultCode; import io.gisbi.utils.JsonUtil; import io.gisbi.utils.LogUtil; @@ -69,12 +70,12 @@ public class ChartDataServer implements ChartDataApi { @Resource private DatasetFieldServer datasetFieldServer; - @Value("${dataease.export.page.size:50000}") + @Value("${gisbi.export.page.size:50000}") private Integer extractPageSize; private final Long sheetLimit = 1000000L; - @DeLinkPermit("#p0.sceneId") + @Override public ChartViewDTO getData(ChartViewDTO chartViewDTO) throws Exception { try { @@ -82,7 +83,11 @@ public class ChartDataServer implements ChartDataApi { if (CommonConstants.VIEW_DATA_FROM.TEMPLATE.equalsIgnoreCase(chartViewDTO.getDataFrom())) { return extendDataManage.getChartDataInfo(chartViewDTO.getId(), chartViewDTO); } else { - return chartDataManage.calcData(chartViewDTO); + DatasetUtils.viewDecode(chartViewDTO); + ChartViewDTO dto = chartDataManage.calcData(chartViewDTO); + DatasetUtils.viewEncode(dto); + chartDataManage.encodeData(dto); + return dto; } } catch (Exception e) { DEException.throwException(ResultCode.DATA_IS_WRONG.code(), e.getMessage() + "\n\n" + ExceptionUtils.getStackTrace(e)); @@ -103,8 +108,8 @@ public class ChartDataServer implements ChartDataApi { viewDTO.setType("table-info"); viewDTO.setRender("antv"); List sourceFields = datasetFieldServer.listByDatasetGroup(viewDTO.getTableId()); - List fileNames = permissionManage.filterColumnPermissions(sourceFields, new HashMap<>(), viewDTO.getTableId(), null).stream().map(DatasetTableFieldDTO::getDataeaseName).collect(Collectors.toList()); - sourceFields = sourceFields.stream().filter(datasetTableFieldDTO -> fileNames.contains(datasetTableFieldDTO.getDataeaseName())).collect(Collectors.toList()); + List fileNames = permissionManage.filterColumnPermissions(sourceFields, new HashMap<>(), viewDTO.getTableId(), null).stream().map(DatasetTableFieldDTO::getGisbiName).collect(Collectors.toList()); + sourceFields = sourceFields.stream().filter(datasetTableFieldDTO -> fileNames.contains(datasetTableFieldDTO.getGisbiName())).collect(Collectors.toList()); dsHeader = sourceFields.stream().map(DatasetTableFieldDTO::getName).toArray(String[]::new); dsTypes = sourceFields.stream().map(DatasetTableFieldDTO::getDeType).toArray(Integer[]::new); TypeReference> listTypeReference = new TypeReference>() { @@ -120,7 +125,11 @@ public class ChartDataServer implements ChartDataApi { } else { viewDTO.setResultCount(viewLimit); } - chartViewInfo = getData(viewDTO); + if (CommonConstants.VIEW_DATA_FROM.TEMPLATE.equalsIgnoreCase(viewDTO.getDataFrom())) { + chartViewInfo = extendDataManage.getChartDataInfo(viewDTO.getId(), viewDTO); + } else { + chartViewInfo = chartDataManage.calcData(viewDTO); + } List tableRow = (List) chartViewInfo.getData().get("sourceData"); if ("dataset".equals(request.getDownloadType())) { request.setHeader(dsHeader); @@ -214,13 +223,12 @@ public class ChartDataServer implements ChartDataApi { } - @DeLinkPermit("#p0.dvId") @Override public void innerExportDetails(ChartExcelRequest request, HttpServletResponse response) throws Exception { HttpServletRequest httpServletRequest = ((ServletRequestAttributes) RequestContextHolder.getRequestAttributes()).getRequest(); String linkToken = httpServletRequest.getHeader(AuthConstant.LINK_TOKEN_KEY); - LogUtil.info(request.getViewInfo().getId() + " " + StringUtils.isNotEmpty(linkToken) + " " + request.isDataEaseBi()); - if ((StringUtils.isNotEmpty(linkToken) && !request.isDataEaseBi()) || (request.isDataEaseBi() && StringUtils.isEmpty(linkToken))) { + LogUtil.info(request.getViewInfo().getId() + " " + StringUtils.isNotEmpty(linkToken) + " " + request.isGisbiBi()); + if ((StringUtils.isNotEmpty(linkToken) && !request.isGisbiBi()) || (request.isGisbiBi() && StringUtils.isEmpty(linkToken))) { OutputStream outputStream = response.getOutputStream(); try { Workbook wb = new SXSSFWorkbook(); @@ -238,7 +246,7 @@ public class ChartDataServer implements ChartDataApi { //设置单元格填充样式(使用纯色背景颜色填充) cellStyle.setFillPattern(FillPatternType.SOLID_FOREGROUND); - if ("dataset".equals(request.getDownloadType()) || request.getViewInfo().getType().equalsIgnoreCase("table-info")) { + if ("dataset".equals(request.getDownloadType()) || request.getViewInfo().getType().equalsIgnoreCase("table-info") || request.getViewInfo().getType().equalsIgnoreCase("table-normal")) { List details = new ArrayList<>(); Sheet detailsSheet; Integer sheetIndex = 1; @@ -251,9 +259,21 @@ public class ChartDataServer implements ChartDataApi { if ((details.size() + extractPageSize) > sheetLimit || i == chartViewDTO.getTotalPage()) { detailsSheet = wb.createSheet("数据" + sheetIndex); Integer[] excelTypes = request.getExcelTypes(); - details.add(0, request.getHeader()); - ViewDetailField[] detailFields = request.getDetailFields(); Object[] header = request.getHeader(); + request.getViewInfo().setXAxis(request.getViewInfo().getXAxis().stream().filter(ele -> !ele.isHide()).collect(Collectors.toList())); + request.getViewInfo().setYAxis(request.getViewInfo().getYAxis().stream().filter(ele -> !ele.isHide()).collect(Collectors.toList())); + request.getViewInfo().setXAxisExt(request.getViewInfo().getXAxisExt().stream().filter(ele -> !ele.isHide()).collect(Collectors.toList())); + request.getViewInfo().setYAxisExt(request.getViewInfo().getYAxisExt().stream().filter(ele -> !ele.isHide()).collect(Collectors.toList())); + request.getViewInfo().setExtStack(request.getViewInfo().getExtStack().stream().filter(ele -> !ele.isHide()).collect(Collectors.toList())); + List xAxis = new ArrayList<>(); + xAxis.addAll(request.getViewInfo().getXAxis()); + xAxis.addAll(request.getViewInfo().getYAxis()); + xAxis.addAll(request.getViewInfo().getXAxisExt()); + xAxis.addAll(request.getViewInfo().getYAxisExt()); + xAxis.addAll(request.getViewInfo().getExtStack()); + header = Arrays.stream(request.getHeader()).filter(item -> xAxis.stream().map(DatasetTableFieldDTO::getName).collect(Collectors.toList()).contains(item)).collect(Collectors.toList()).toArray(); + details.add(0, header); + ViewDetailField[] detailFields = request.getDetailFields(); ChartDataServer.setExcelData(detailsSheet, cellStyle, header, details, detailFields, excelTypes, request.getViewInfo(), wb); sheetIndex++; details.clear(); @@ -268,7 +288,11 @@ public class ChartDataServer implements ChartDataApi { ViewDetailField[] detailFields = request.getDetailFields(); Object[] header = request.getHeader(); Sheet detailsSheet = wb.createSheet("数据"); - setExcelData(detailsSheet, cellStyle, header, details, detailFields, excelTypes, request.getViewInfo(), null); + if (request.getViewInfo().getType().equalsIgnoreCase("table-normal")) { + setExcelData(detailsSheet, cellStyle, header, details, detailFields, excelTypes, request.getViewInfo(), wb); + } else { + setExcelData(detailsSheet, cellStyle, header, details, detailFields, excelTypes, request.getViewInfo(), null); + } } else { for (int i = 0; i < request.getMultiInfo().size(); i++) { ChartExcelRequestInner requestInner = request.getMultiInfo().get(i); @@ -293,12 +317,10 @@ public class ChartDataServer implements ChartDataApi { DEException.throwException(e); } } else { - exportCenterManage.addTask(request.getViewId(), "chart", request); - return; + exportCenterManage.addTask(request.getViewId(), "chart", request, request.getBusiFlag()); } } - @DeLinkPermit("#p0.dvId") @Override public void innerExportDataSetDetails(ChartExcelRequest request, HttpServletResponse response) throws Exception { this.innerExportDetails(request, response); @@ -318,7 +340,9 @@ public class ChartDataServer implements ChartDataApi { xAxis.addAll(viewInfo.getXAxisExt()); xAxis.addAll(viewInfo.getYAxisExt()); xAxis.addAll(viewInfo.getExtStack()); - + xAxis.addAll(viewInfo.getDrillFields()); + TableHeader tableHeader = null; + Integer totalDepth = 0; if (viewInfo.getType().equalsIgnoreCase("table-normal") || viewInfo.getType().equalsIgnoreCase("table-info")) { for (ChartViewFieldDTO xAxi : xAxis) { if (xAxi.getDeType().equals(DeTypeConstants.DE_INT) || xAxi.getDeType().equals(DeTypeConstants.DE_FLOAT)) { @@ -328,6 +352,22 @@ public class ChartDataServer implements ChartDataApi { styles.add(null); } } + + Map customAttr = viewInfo.getCustomAttr(); + Map tableHeaderMap = (Map) customAttr.get("tableHeader"); + if (tableHeaderMap.get("headerGroup") != null && Boolean.parseBoolean(tableHeaderMap.get("headerGroup").toString())) { + var tmpHeader = JsonUtil.parseObject((String) JsonUtil.toJSONString(customAttr.get("tableHeader")), TableHeader.class); + // 校验字段数量和顺序 + if (validateHeaderGroup(tmpHeader, viewInfo.getXAxis())) { + tableHeader = tmpHeader; + for (TableHeader.ColumnInfo column : tableHeader.getHeaderGroupConfig().getColumns()) { + totalDepth = Math.max(totalDepth, getDepth(column, 1)); + } + for (TableHeader.ColumnInfo column : tableHeader.getHeaderGroupConfig().getColumns()) { + setWidth(column, 1); + } + } + } } boolean mergeHead = false; @@ -374,9 +414,31 @@ public class ChartDataServer implements ChartDataApi { mergeHead = true; } if (CollectionUtils.isNotEmpty(details) && (!mergeHead || details.size() > 2)) { - int realDetailRowIndex = 2; + int realDetailRowIndex = tableHeader == null ? 2 : totalDepth; + if (tableHeader != null) { + cellStyle.setBorderTop(BorderStyle.THIN); + cellStyle.setBorderRight(BorderStyle.THIN); + cellStyle.setBorderBottom(BorderStyle.THIN); + cellStyle.setBorderLeft(BorderStyle.THIN); + Map rowMap = new HashMap<>(); + for (Integer i = 0; i < totalDepth; i++) { + rowMap.put("row" + i, detailsSheet.createRow(i)); + } + int width = 0; + Integer depth = 0; + for (TableHeader.ColumnInfo column : tableHeader.getHeaderGroupConfig().getColumns()) { + createCell(tableHeader, column, width, depth, detailsSheet, cellStyle, totalDepth, rowMap, xAxis); + width = width + column.getWidth(); + } + } for (int i = (mergeHead ? 2 : 0); i < details.size(); i++) { - Row row = detailsSheet.createRow(realDetailRowIndex > 2 ? realDetailRowIndex : i); + int rowIndex = i; + if (tableHeader != null) { + rowIndex = realDetailRowIndex - 1 + i; + } else { + rowIndex = realDetailRowIndex > 2 ? realDetailRowIndex : i; + } + Row row = detailsSheet.createRow(rowIndex); Object[] rowData = details.get(i); if (rowData != null) { for (int j = 0; j < rowData.length; j++) { @@ -417,7 +479,7 @@ public class ChartDataServer implements ChartDataApi { try { if ((viewInfo.getType().equalsIgnoreCase("table-normal") || viewInfo.getType().equalsIgnoreCase("table-info")) && (xAxis.get(j).getDeType().equals(DeTypeConstants.DE_INT) || xAxis.get(j).getDeType().equals(DeTypeConstants.DE_FLOAT))) { try { - FormatterCfgDTO formatterCfgDTO = xAxis.get(j).getFormatterCfg() == null ? new FormatterCfgDTO() : xAxis.get(j).getFormatterCfg(); + FormatterCfgDTO formatterCfgDTO = xAxis.get(j).getFormatterCfg() == null ? new FormatterCfgDTO().setUnitLanguage(Lang.isChinese() ? "ch" : "en") : xAxis.get(j).getFormatterCfg(); row.getCell(j).setCellStyle(styles.get(j)); row.getCell(j).setCellValue(Double.valueOf(cellValue(formatterCfgDTO, new BigDecimal(cellValObj.toString())))); } catch (Exception e) { @@ -436,10 +498,11 @@ public class ChartDataServer implements ChartDataApi { } else { if (!viewInfo.getType().equalsIgnoreCase("circle-packing")) { Map senior = viewInfo.getSenior(); + viewInfo.getCustomAttr().get(""); ChartSeniorFunctionCfgDTO functionCfgDTO = JsonUtil.parseObject((String) JsonUtil.toJSONString(senior.get("functionCfg")), ChartSeniorFunctionCfgDTO.class); if (functionCfgDTO != null && StringUtils.isNotEmpty(functionCfgDTO.getEmptyDataStrategy()) && functionCfgDTO.getEmptyDataStrategy().equalsIgnoreCase("setZero")) { if ((viewInfo.getType().equalsIgnoreCase("table-normal") || viewInfo.getType().equalsIgnoreCase("table-info"))) { - if (functionCfgDTO.getEmptyDataFieldCtrl().contains(xAxis.get(j).getDataeaseName())) { + if (functionCfgDTO.getEmptyDataFieldCtrl().contains(xAxis.get(j).getGisbiName())) { cell.setCellValue(0); } } else { @@ -454,6 +517,123 @@ public class ChartDataServer implements ChartDataApi { } } + private static boolean validateHeaderGroup(TableHeader header, List fields) { + if (header == null) { + return false; + } + var columns = header.getHeaderGroupConfig().getColumns(); + if (CollectionUtils.isEmpty(columns)) { + return false; + } + var leafColumn = getHeaderLeafColumn(columns); + if (CollectionUtils.isEmpty(leafColumn) || leafColumn.size() != fields.size()) { + return false; + } + for (int i = 0; i < leafColumn.size(); i++) { + var a = leafColumn.get(i); + var b = fields.get(i).getGisbiName(); + if (!StringUtils.equals(a, b)) { + return false; + } + } + return true; + } + + private static List getHeaderLeafColumn(List columns) { + var result = new ArrayList(); + for (TableHeader.ColumnInfo column : columns) { + if (CollectionUtils.isEmpty(column.getChildren())) { + result.add(column.getKey()); + } else { + result.addAll(getHeaderLeafColumn(column.getChildren())); + } + } + return result; + } + + private static Integer getDepth(TableHeader.ColumnInfo column, Integer parentDepth) { + if (org.springframework.util.CollectionUtils.isEmpty(column.getChildren())) { + return parentDepth; + } else { + Integer depth = 0; + for (TableHeader.ColumnInfo child : column.getChildren()) { + depth = Math.max(depth, getDepth(child, parentDepth + 1)); + } + return depth; + } + } + + private static void createCell(TableHeader tableHeader, TableHeader.ColumnInfo column, Integer width, Integer depth, Sheet sheet, CellStyle cellStyle, Integer totaalDepth, Map rowMap, List xAxis) { + if (org.springframework.util.CollectionUtils.isEmpty(column.getChildren())) { + Integer toDepth = totaalDepth - 1 > depth ? totaalDepth - 1 : depth; + if (depth.equals(toDepth)) { + Cell cell = rowMap.get("row" + depth).createCell(width); + cell.setCellStyle(cellStyle); + cell.setCellValue(getDeFieldName(xAxis, column.getKey())); + } else { + for (int i = depth; i <= toDepth; i++) { + Cell cell1 = rowMap.get("row" + i).createCell(width); + cell1.setCellValue(getDeFieldName(xAxis, column.getKey())); + cell1.setCellStyle(cellStyle); + } + CellRangeAddress region = new CellRangeAddress(depth, toDepth, width, width); + sheet.addMergedRegion(region); + + Cell mergedCell = rowMap.get("row" + depth).getCell(width); + mergedCell.setCellStyle(cellStyle); + + } + } else { + Cell cell1 = rowMap.get("row" + depth).createCell(width); + cell1.setCellValue(getGroupName(tableHeader, column.getKey())); + cell1.setCellStyle(cellStyle); + Cell cell2 = rowMap.get("row" + depth).createCell(width + column.getWidth() - 1); + cell2.setCellValue(getGroupName(tableHeader, column.getKey())); + cell2.setCellStyle(cellStyle); + CellRangeAddress region = new CellRangeAddress(depth, depth, width, width + column.getWidth() - 1); + sheet.addMergedRegion(region); + Cell mergedCell = rowMap.get("row" + depth).getCell(width); + mergedCell.setCellStyle(cellStyle); + int subWith = width; + for (TableHeader.ColumnInfo child : column.getChildren()) { + createCell(tableHeader, child, subWith, depth + 1, sheet, cellStyle, totaalDepth, rowMap, xAxis); + subWith = subWith + child.getWidth(); + } + } + } + + private static String getGroupName(TableHeader tableHeader, String key) { + for (TableHeader.MetaInfo metaInfo : tableHeader.getHeaderGroupConfig().getMeta()) { + if (metaInfo.getField().equals(key)) { + return metaInfo.getName(); + } + } + return ""; + } + + private static String getDeFieldName(List xAxis, String key) { + for (ChartViewFieldDTO xAxi : xAxis) { + if (xAxi.getGisbiName().equals(key)) { + return StringUtils.isNotBlank(xAxi.getChartShowName()) ? xAxi.getChartShowName() : xAxi.getName(); + } + } + return ""; + } + + private static Integer setWidth(TableHeader.ColumnInfo column, Integer parentWidth) { + if (org.springframework.util.CollectionUtils.isEmpty(column.getChildren())) { + column.setWidth(parentWidth); + return parentWidth; + } else { + Integer depth = 0; + for (TableHeader.ColumnInfo child : column.getChildren()) { + depth = depth + setWidth(child, 1); + } + column.setWidth(depth); + return depth; + } + } + private static String cellValue(FormatterCfgDTO formatterCfgDTO, BigDecimal value) { if (formatterCfgDTO.getType().equalsIgnoreCase("percent")) { return value.toString(); @@ -474,22 +654,22 @@ public class ChartDataServer implements ChartDataApi { if (formatter.getType().equals("auto")) { String[] valueSplit = String.valueOf(value).split("."); if (StringUtils.isEmpty(value) || !value.contains(".")) { - formatStr = "0"; + formatStr = "General"; } else { formatStr = "0." + new String(new char[valueSplit.length]).replace('\0', '0'); } switch (formatter.getUnit()) { case 1000: - formatStr = formatStr + "千"; + formatStr = formatStr + "\"千\""; break; case 10000: - formatStr = formatStr + "万"; + formatStr = formatStr + "\"万\""; break; case 1000000: - formatStr = formatStr + "百万"; + formatStr = formatStr + "\"百万\""; break; case 100000000: - formatStr = formatStr + "'亿'"; + formatStr = formatStr + "\"亿\""; break; default: break; @@ -501,7 +681,7 @@ public class ChartDataServer implements ChartDataApi { if (formatter.getSuffix().equals("%")) { formatStr = formatStr + "\"%\""; } else { - formatStr = formatStr + formatter.getSuffix(); + formatStr = formatStr + "\"" + formatter.getSuffix() + "\""; } } } @@ -513,16 +693,16 @@ public class ChartDataServer implements ChartDataApi { } switch (formatter.getUnit()) { case 1000: - formatStr = formatStr + "千"; + formatStr = formatStr + "\"千\""; break; case 10000: - formatStr = formatStr + "万"; + formatStr = formatStr + "\"万\""; break; case 1000000: - formatStr = formatStr + "百万"; + formatStr = formatStr + "\"百万\""; break; case 100000000: - formatStr = formatStr + "'亿'"; + formatStr = formatStr + "\"亿\""; break; default: break; @@ -534,7 +714,7 @@ public class ChartDataServer implements ChartDataApi { if (formatter.getSuffix().equals("%")) { formatStr = formatStr + "\"%\""; } else { - formatStr = formatStr + formatter.getSuffix(); + formatStr = formatStr + "\"" + formatter.getSuffix() + "\""; } } } else if (formatter.getType().equals("percent")) { diff --git a/core/core-backend/src/main/java/io/gisbi/chart/server/ChartViewServer.java b/core/core-backend/src/main/java/io/gisbi/chart/server/ChartViewServer.java index 076d986..858bbb8 100644 --- a/core/core-backend/src/main/java/io/gisbi/chart/server/ChartViewServer.java +++ b/core/core-backend/src/main/java/io/gisbi/chart/server/ChartViewServer.java @@ -2,11 +2,13 @@ package io.gisbi.chart.server; import io.gisbi.api.chart.ChartViewApi; import io.gisbi.api.chart.vo.ChartBaseVO; -import io.gisbi.extensions.view.dto.ChartViewDTO; -import io.gisbi.extensions.view.dto.ChartViewFieldDTO; import io.gisbi.api.chart.vo.ViewSelectorVO; import io.gisbi.chart.manage.ChartViewManege; +import io.gisbi.constant.CommonConstants; +import io.gisbi.dataset.utils.DatasetUtils; import io.gisbi.exception.DEException; +import io.gisbi.extensions.view.dto.ChartViewDTO; +import io.gisbi.extensions.view.dto.ChartViewFieldDTO; import io.gisbi.result.ResultCode; import jakarta.annotation.Resource; import org.springframework.web.bind.annotation.RequestMapping; @@ -27,7 +29,7 @@ public class ChartViewServer implements ChartViewApi { @Override public ChartViewDTO getData(Long id) throws Exception { try { - return chartViewManege.getChart(id); + return chartViewManege.getChart(id, CommonConstants.RESOURCE_TABLE.CORE); } catch (Exception e) { DEException.throwException(ResultCode.DATA_IS_WRONG.code(), e.getMessage()); } @@ -36,7 +38,10 @@ public class ChartViewServer implements ChartViewApi { @Override public Map> listByDQ(Long id, Long chartId, ChartViewDTO dto) { - return chartViewManege.listByDQ(id, chartId, dto); + Map> stringListMap = chartViewManege.listByDQ(id, chartId, dto); + DatasetUtils.listEncode(stringListMap.get("dimensionList")); + DatasetUtils.listEncode(stringListMap.get("quotaList")); + return stringListMap; } @Override @@ -50,8 +55,8 @@ public class ChartViewServer implements ChartViewApi { } @Override - public ChartViewDTO getDetail(Long id) { - return chartViewManege.getDetails(id); + public ChartViewDTO getDetail(Long id, String resourceTable) { + return chartViewManege.getDetails(id, resourceTable); } @Override @@ -75,7 +80,7 @@ public class ChartViewServer implements ChartViewApi { } @Override - public ChartBaseVO chartBaseInfo(Long id) { - return chartViewManege.chartBaseInfo(id); + public ChartBaseVO chartBaseInfo(Long id, String resourceTable) { + return chartViewManege.chartBaseInfo(id, resourceTable); } } diff --git a/core/core-backend/src/main/java/io/gisbi/chart/utils/ChartDataBuild.java b/core/core-backend/src/main/java/io/gisbi/chart/utils/ChartDataBuild.java index 4d46e38..fcf405c 100644 --- a/core/core-backend/src/main/java/io/gisbi/chart/utils/ChartDataBuild.java +++ b/core/core-backend/src/main/java/io/gisbi/chart/utils/ChartDataBuild.java @@ -1180,13 +1180,13 @@ public class ChartDataBuild { Map> groupDataList = detailData.stream().collect(Collectors.groupingBy(item -> "(" + StringUtils.join(ArrayUtils.subarray(item, 0, xEndIndex), ")-de-(") + ")")); tableRow.forEach(row -> { - String key = xAxis.stream().map(x -> String.format(format, row.get(x.getDataeaseName()).toString())).collect(Collectors.joining("-de-")); + String key = xAxis.stream().map(x -> String.format(format, row.get(x.getGisbiName()).toString())).collect(Collectors.joining("-de-")); List detailFieldValueList = groupDataList.get(key); List> detailValueMapList = detailFieldValueList.stream().map((detailArr -> { Map temp = new HashMap<>(); for (int i = 0; i < realDetailFields.size(); i++) { ChartViewFieldDTO realDetailField = realDetailFields.get(i); - temp.put(realDetailField.getDataeaseName(), detailArr[detailIndex + i]); + temp.put(realDetailField.getGisbiName(), detailArr[detailIndex + i]); } return temp; })).collect(Collectors.toList()); @@ -1196,7 +1196,7 @@ public class ChartDataBuild { ChartViewFieldDTO detailFieldDTO = new ChartViewFieldDTO(); detailFieldDTO.setId(IDUtils.snowID()); detailFieldDTO.setName("detail"); - detailFieldDTO.setDataeaseName("detail"); + detailFieldDTO.setGisbiName("detail"); fields.add(detailFieldDTO); map.put("fields", fields); map.put("detailFields", realDetailFields); @@ -1299,21 +1299,21 @@ public class ChartDataBuild { data.forEach(ele -> { Map d = new HashMap<>(); for (int i = 0; i < fields.size(); i++) { - if (ObjectUtils.isNotEmpty(desensitizationList.keySet()) && desensitizationList.containsKey(fields.get(i).getDataeaseName())) { - String desensitizationValue = desensitizationValue(desensitizationList.get(fields.get(i).getDataeaseName()), String.valueOf(ele[i])); + if (ObjectUtils.isNotEmpty(desensitizationList.keySet()) && desensitizationList.containsKey(fields.get(i).getGisbiName())) { + String desensitizationValue = desensitizationValue(desensitizationList.get(fields.get(i).getGisbiName()), String.valueOf(ele[i])); ele[i] = desensitizationValue; - d.put(fields.get(i).getDataeaseName(), desensitizationValue); + d.put(fields.get(i).getGisbiName(), desensitizationValue); continue; } if (i == ele.length) break; ChartViewFieldDTO chartViewFieldDTO = fields.get(i); if (chartViewFieldDTO.getDeType() == 0 || chartViewFieldDTO.getDeType() == 1 || chartViewFieldDTO.getDeType() == 5 || chartViewFieldDTO.getDeType() == 7) { - d.put(fields.get(i).getDataeaseName(), StringUtils.isEmpty(ele[i]) ? "" : ele[i]); + d.put(fields.get(i).getGisbiName(), StringUtils.isEmpty(ele[i]) ? "" : ele[i]); } else if (chartViewFieldDTO.getDeType() == 2 || chartViewFieldDTO.getDeType() == 3 || chartViewFieldDTO.getDeType() == 4) { if (view.getIsExcelExport()) { - d.put(fields.get(i).getDataeaseName(), StringUtils.isEmpty(ele[i]) ? "" : ele[i]); + d.put(fields.get(i).getGisbiName(), StringUtils.isEmpty(ele[i]) ? "" : ele[i]); } else { - d.put(fields.get(i).getDataeaseName(), StringUtils.isEmpty(ele[i]) ? null : new BigDecimal(ele[i]).setScale(8, RoundingMode.HALF_UP)); + d.put(fields.get(i).getGisbiName(), StringUtils.isEmpty(ele[i]) ? null : new BigDecimal(ele[i]).setScale(8, RoundingMode.HALF_UP)); } } } @@ -1383,10 +1383,7 @@ public class ChartDataBuild { chartDimensionDTO.setValue(row[j]); dimensionList.add(chartDimensionDTO); } - ChartDimensionDTO chartDimensionDTO = new ChartDimensionDTO(); - chartDimensionDTO.setId(extStack.get(0).getId()); - chartDimensionDTO.setValue(row[xAxis.size()]); - dimensionList.add(chartDimensionDTO); + axisChartDataDTO.setDimensionList(dimensionList); if (ObjectUtils.isNotEmpty(yAxis)) { @@ -1785,19 +1782,19 @@ public class ChartDataBuild { List> tableRow = (List>) map.get("tableRow"); final int xEndIndex = detailIndex; Map> groupDataList = detailData.stream().collect(Collectors.groupingBy(item -> "(" + StringUtils.join(ArrayUtils.subarray(item, 0, xEndIndex), ")-de-(") + ")")); - String extBubbleDataeaseName = ObjectUtils.isNotEmpty(extBubble) ? extBubble.get(0).getDataeaseName() : ""; + String extBubblegisbiName = ObjectUtils.isNotEmpty(extBubble) ? extBubble.get(0).getGisbiName() : ""; tableRow.forEach(row -> { - BigDecimal rowValue = row.get(extBubbleDataeaseName) == null ? BigDecimal.ZERO : new BigDecimal(row.get(extBubbleDataeaseName).toString()); - String key = xAxis.stream().map(x -> String.format(format, row.get(x.getDataeaseName()).toString())).collect(Collectors.joining("-de-")); + BigDecimal rowValue = row.get(extBubblegisbiName) == null ? BigDecimal.ZERO : new BigDecimal(row.get(extBubblegisbiName).toString()); + String key = xAxis.stream().map(x -> String.format(format, row.get(x.getGisbiName()).toString())).collect(Collectors.joining("-de-")); List detailFieldValueList = groupDataList.get(key); List> detailValueMapList = Optional.ofNullable(detailFieldValueList).orElse(new ArrayList<>()).stream().map((detailArr -> { Map temp = new HashMap<>(); for (int i = 0; i < realDetailFields.size(); i++) { ChartViewFieldDTO realDetailField = realDetailFields.get(i); - if (StringUtils.equalsIgnoreCase(extBubbleDataeaseName, realDetailField.getDataeaseName())) { - temp.put(realDetailField.getDataeaseName(), rowValue); + if (StringUtils.equalsIgnoreCase(extBubblegisbiName, realDetailField.getGisbiName())) { + temp.put(realDetailField.getGisbiName(), rowValue); } else { - temp.put(realDetailField.getDataeaseName(), detailArr[detailIndex + i]); + temp.put(realDetailField.getGisbiName(), detailArr[detailIndex + i]); } } return temp; @@ -1807,10 +1804,10 @@ public class ChartDataBuild { }); // 先过滤掉所有记录数字段 List filterCountAxis = fields.stream() - .filter(item -> !StringUtils.equalsIgnoreCase(item.getDataeaseName(), "*")) + .filter(item -> !StringUtils.equalsIgnoreCase(item.getGisbiName(), "*")) .collect(Collectors.toList()); // 如果气泡大小是记录数，添加到字段列表中 - if (ObjectUtils.isNotEmpty(extBubble) && "*".equals(extBubble.get(0).getDataeaseName())) { + if (ObjectUtils.isNotEmpty(extBubble) && "*".equals(extBubble.get(0).getGisbiName())) { filterCountAxis.addAll(yAxis); } map.put("fields", filterCountAxis); diff --git a/core/core-backend/src/main/java/io/gisbi/commons/constants/DataVisualizationConstants.java b/core/core-backend/src/main/java/io/gisbi/commons/constants/DataVisualizationConstants.java index d5e47b7..1c56219 100644 --- a/core/core-backend/src/main/java/io/gisbi/commons/constants/DataVisualizationConstants.java +++ b/core/core-backend/src/main/java/io/gisbi/commons/constants/DataVisualizationConstants.java @@ -10,6 +10,9 @@ public class DataVisualizationConstants { // 主工程 public static final String MAIN = "main"; + + // 主工程编辑区 + public static final String MAIN_EDIT = "main-edit"; } //新建仪表板来源 diff --git a/core/core-backend/src/main/java/io/gisbi/commons/utils/EncryptUtils.java b/core/core-backend/src/main/java/io/gisbi/commons/utils/EncryptUtils.java index b5dcc58..bf1ab08 100644 --- a/core/core-backend/src/main/java/io/gisbi/commons/utils/EncryptUtils.java +++ b/core/core-backend/src/main/java/io/gisbi/commons/utils/EncryptUtils.java @@ -10,6 +10,7 @@ public class EncryptUtils extends CodingUtil { private static final String secretKey = "www.fit2cloud.co"; private static final String iv = "1234567890123456"; + public static Object aesEncrypt(Object o) { if (o == null) { return null; diff --git a/core/core-backend/src/main/java/io/gisbi/commons/utils/ExcelWatermarkUtils.java b/core/core-backend/src/main/java/io/gisbi/commons/utils/ExcelWatermarkUtils.java index 56529fc..86f5b09 100644 --- a/core/core-backend/src/main/java/io/gisbi/commons/utils/ExcelWatermarkUtils.java +++ b/core/core-backend/src/main/java/io/gisbi/commons/utils/ExcelWatermarkUtils.java @@ -1,5 +1,6 @@ package io.gisbi.commons.utils; +import io.gisbi.api.permissions.user.vo.UserFormVO; import io.gisbi.i18n.Translator; import io.gisbi.utils.IPUtils; import io.gisbi.visualization.dto.WatermarkContentDTO; @@ -20,31 +21,31 @@ public class ExcelWatermarkUtils { private static SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); -// public static String transContent(WatermarkContentDTO watermarkContent, UserFormVO userInfo) { -// String content = ""; -// switch (watermarkContent.getType()) { -// case "custom" -> content = watermarkContent.getContent(); -// case "nickName" -> content = "${nickName}"; -// case "ip" -> content = "${ip}"; -// case "time" -> content = "${time}"; -// default -> content = "${username}"; -// } -// String nickName = userInfo.getName().contains("i18n_") ?Translator.get(userInfo.getName()):userInfo.getName(); -// content = content.replaceAll("\\$\\{ip}", IPUtils.get() == null ? "127.0.0.1" : IPUtils.get()); -// content = content.replaceAll("\\$\\{username}", userInfo.getAccount()); -// content = content.replaceAll("\\$\\{nickName}", nickName); -// content = content.replaceAll("\\$\\{time}", sdf.format(new Date())); -// return content; -// } + public static String transContent(WatermarkContentDTO watermarkContent, UserFormVO userInfo) { + String content = ""; + switch (watermarkContent.getType()) { + case "custom" -> content = watermarkContent.getContent(); + case "nickName" -> content = "${nickName}"; + case "ip" -> content = "${ip}"; + case "time" -> content = "${time}"; + default -> content = "${username}"; + } + String nickName = userInfo.getName().contains("i18n_") ?Translator.get(userInfo.getName()):userInfo.getName(); + content = content.replaceAll("\\$\\{ip}", IPUtils.get() == null ? "127.0.0.1" : IPUtils.get()); + content = content.replaceAll("\\$\\{username}", userInfo.getAccount()); + content = content.replaceAll("\\$\\{nickName}", nickName); + content = content.replaceAll("\\$\\{time}", sdf.format(new Date())); + return content; + } /** * 添加水印图片到工作簿并返回图片 ID */ -// public static int addWatermarkImage(Workbook wb, WatermarkContentDTO watermarkContent, UserFormVO userInfo) { -// byte[] imageBytes = createTextImage(transContent(watermarkContent, userInfo), watermarkContent); // 生成文字水印图片 -// return wb.addPicture(imageBytes, Workbook.PICTURE_TYPE_PNG); // 添加到工作簿并返回 ID -// } + public static int addWatermarkImage(Workbook wb, WatermarkContentDTO watermarkContent, UserFormVO userInfo) { + byte[] imageBytes = createTextImage(transContent(watermarkContent, userInfo), watermarkContent); // 生成文字水印图片 + return wb.addPicture(imageBytes, Workbook.PICTURE_TYPE_PNG); // 添加到工作簿并返回 ID + } public static void addWatermarkToSheet(Sheet sheet, Integer pictureIdx) { Drawing drawing = sheet.createDrawingPatriarch(); diff --git a/core/core-backend/src/main/java/io/gisbi/commons/utils/SqlparserUtils.java b/core/core-backend/src/main/java/io/gisbi/commons/utils/SqlparserUtils.java index e6591d3..27eb09c 100644 --- a/core/core-backend/src/main/java/io/gisbi/commons/utils/SqlparserUtils.java +++ b/core/core-backend/src/main/java/io/gisbi/commons/utils/SqlparserUtils.java @@ -1,6 +1,9 @@ package io.gisbi.commons.utils; import com.fasterxml.jackson.core.type.TypeReference; +import io.gisbi.api.permissions.user.vo.UserFormVO; +import io.gisbi.api.permissions.variable.dto.SysVariableValueDto; +import io.gisbi.api.permissions.variable.dto.SysVariableValueItem; import io.gisbi.exception.DEException; import io.gisbi.extensions.datasource.api.PluginManageApi; import io.gisbi.extensions.datasource.dto.DatasourceSchemaDTO; @@ -18,7 +21,6 @@ import net.sf.jsqlparser.parser.CCJSqlParserUtil; import net.sf.jsqlparser.statement.Statement; import net.sf.jsqlparser.statement.select.*; import net.sf.jsqlparser.util.deparser.ExpressionDeParser; -import net.sf.jsqlparser.util.deparser.SelectDeParser; import org.apache.calcite.sql.*; import org.apache.calcite.sql.parser.SqlParser; import org.apache.calcite.sql.util.SqlShuttle; @@ -37,24 +39,29 @@ import static org.apache.calcite.sql.SqlKind.*; public class SqlparserUtils { public static final String regex = "\\$\\{(.*?)\\}"; - public static final String regex2 = "\\[(.*?)\\]"; - private static final String SubstitutedParams = "DATAEASE_PATAMS_BI"; + public static final String regex2 = "\\$f2cde\\[(.*?)\\]"; + private static final String SubstitutedParams = "gisbi_PATAMS_BI"; private static final String SysParamsSubstitutedParams = "DeSysParams_"; private static final String SubstitutedSql = " 'DE-BI' = 'DE-BI' "; private boolean removeSysParams; + boolean hasVariables = false; + private UserFormVO userEntity; private final List> sysParams = new ArrayList<>(); + private static final String deVariablePattern = "\\$DE_PARAM\\{(.*?)\\}"; + + public String handleVariableDefaultValue(String sql, String sqlVariableDetails, boolean isEdit, boolean isFromDataSet, List parameters, boolean isCross, Map dsMap, PluginManageApi pluginManage, UserFormVO userEntity) { + Pattern r = Pattern.compile(deVariablePattern); + Matcher m = r.matcher(sql); + if (m.find()) { + return new DeSqlparserUtils().handleVariableDefaultValue(sql, sqlVariableDetails, isEdit, isFromDataSet, parameters, isCross, dsMap, pluginManage, userEntity); + } - public String handleVariableDefaultValue(String sql, String sqlVariableDetails, boolean isEdit, boolean isFromDataSet, List parameters, boolean isCross, Map dsMap, PluginManageApi pluginManage) { DatasourceSchemaDTO ds = dsMap.entrySet().iterator().next().getValue(); if (StringUtils.isEmpty(sql)) { DEException.throwException(Translator.get("i18n_sql_not_empty")); } - try { - this.removeSysParams = true; - removeVariables(sql, ds.getType()); - } catch (Exception e) { - DEException.throwException(e); - } + this.userEntity = userEntity; + hasVariables = false; sql = sql.trim(); if (sql.endsWith(";")) { sql = sql.substring(0, sql.length() - 1); @@ -139,40 +146,62 @@ public class SqlparserUtils { return sql; } + private static boolean isParams(String paramId) { + if (Arrays.asList("sysParams.userId", "sysParams.userEmail", "sysParams.userName").contains(paramId)) { + return true; + } + boolean isLong = false; + try { + Long.valueOf(paramId); + isLong = true; + } catch (Exception e) { + isLong = false; + } + if (paramId.length() >= 18 && isLong) { + return true; + } + return false; + } private String removeVariables(final String sql, String dsType) throws Exception { String tmpSql = sql.replaceAll("(?m)^\\s*$[\n\r]{0,}", ""); Pattern pattern = Pattern.compile(regex); Matcher matcher = pattern.matcher(tmpSql); - boolean hasVariables = false; while (matcher.find()) { hasVariables = true; tmpSql = tmpSql.replace(matcher.group(), SubstitutedParams); } if (removeSysParams) { for (Map sysParam : sysParams) { - tmpSql = tmpSql.replaceAll(sysParam.get("replace"), sysParam.get("origin")); + tmpSql = tmpSql.replace(sysParam.get("replace"), sysParam.get("origin")); } pattern = Pattern.compile(regex2); matcher = pattern.matcher(tmpSql); while (matcher.find()) { + String paramId = matcher.group().substring(7, matcher.group().length() - 1); + if (!isParams(paramId)) { + continue; + } hasVariables = true; tmpSql = tmpSql.replace(matcher.group(), SubstitutedParams); } } else { pattern = Pattern.compile(regex2); matcher = pattern.matcher(tmpSql); - sysParams.clear(); while (matcher.find()) { + String paramId = matcher.group().substring(7, matcher.group().length() - 1); + if (!isParams(paramId)) { + continue; + } hasVariables = true; - tmpSql = tmpSql.replace(matcher.group(), SysParamsSubstitutedParams + matcher.group().substring(1, matcher.group().length() - 1)); + tmpSql = tmpSql.replace(matcher.group(), SysParamsSubstitutedParams + matcher.group().substring(7, matcher.group().length() - 1)); Map sysParam = new HashMap<>(); sysParam.put("origin", matcher.group()); - sysParam.put("replace", SysParamsSubstitutedParams + matcher.group().substring(1, matcher.group().length() - 1)); + sysParam.put("replace", SysParamsSubstitutedParams + matcher.group().substring(7, matcher.group().length() - 1)); sysParams.add(sysParam); } } - if(!hasVariables && !sql.contains(SubstitutedParams)){ + if (!hasVariables && !sql.contains(SubstitutedParams)) { return sql; } Statement statement = CCJSqlParserUtil.parse(tmpSql); diff --git a/core/core-backend/src/main/java/io/gisbi/config/DeMvcConfig.java b/core/core-backend/src/main/java/io/gisbi/config/DeMvcConfig.java index 7c25dc9..199a348 100644 --- a/core/core-backend/src/main/java/io/gisbi/config/DeMvcConfig.java +++ b/core/core-backend/src/main/java/io/gisbi/config/DeMvcConfig.java @@ -25,20 +25,20 @@ public class DeMvcConfig implements WebMvcConfigurer { @Override public void addResourceHandlers(ResourceHandlerRegistry registry) { String workDir = FILE_PROTOCOL + ensureSuffix(WORK_DIR, FILE_SEPARATOR); - String uploadUrlPattern = ensureBoth(URL_SEPARATOR + UPLOAD_URL_PREFIX, URL_SEPARATOR) + "**"; + String uploadUrlPattern = ensureBoth(URL_SEPARATOR + UPLOAD_URL_PREFIX, AuthConstant.DE_API_PREFIX, URL_SEPARATOR) + "**"; registry.addResourceHandler(uploadUrlPattern).addResourceLocations(workDir); // map String mapDir = FILE_PROTOCOL + ensureSuffix(MAP_DIR, FILE_SEPARATOR); - String mapUrlPattern = ensureBoth(MAP_URL, URL_SEPARATOR) + "**"; + String mapUrlPattern = ensureBoth(MAP_URL, AuthConstant.DE_API_PREFIX, URL_SEPARATOR) + "**"; registry.addResourceHandler(mapUrlPattern).addResourceLocations(mapDir); String geoDir = FILE_PROTOCOL + ensureSuffix(CUSTOM_MAP_DIR, FILE_SEPARATOR); - String geoUrlPattern = ensureBoth(GEO_URL, URL_SEPARATOR) + "**"; + String geoUrlPattern = ensureBoth(GEO_URL, AuthConstant.DE_API_PREFIX, URL_SEPARATOR) + "**"; registry.addResourceHandler(geoUrlPattern).addResourceLocations(geoDir); String i18nDir = FILE_PROTOCOL + ensureSuffix(I18N_DIR, FILE_SEPARATOR); - String i18nUrlPattern = ensureBoth(I18N_URL, URL_SEPARATOR) + "**"; + String i18nUrlPattern = ensureBoth(I18N_URL, AuthConstant.DE_API_PREFIX, URL_SEPARATOR) + "**"; registry.addResourceHandler(i18nUrlPattern).addResourceLocations(i18nDir); } diff --git a/core/core-backend/src/main/java/io/gisbi/dataset/dao/auto/entity/CoreDatasetGroup.java b/core/core-backend/src/main/java/io/gisbi/dataset/dao/auto/entity/CoreDatasetGroup.java index cccc84f..6116400 100644 --- a/core/core-backend/src/main/java/io/gisbi/dataset/dao/auto/entity/CoreDatasetGroup.java +++ b/core/core-backend/src/main/java/io/gisbi/dataset/dao/auto/entity/CoreDatasetGroup.java @@ -5,7 +5,7 @@ import java.io.Serializable; /** *

- * + * 数据集分组表 *

* * @Author bi-coder @@ -71,6 +71,9 @@ public class CoreDatasetGroup implements Serializable { */ private Long createTime; + /** + * Quartz 实例 ID + */ private String qrtzInstance; /** @@ -93,6 +96,11 @@ public class CoreDatasetGroup implements Serializable { */ private String unionSql; + /** + * 是否跨源 + */ + private Boolean isCross; + public Long getId() { return id; } @@ -213,11 +221,18 @@ public class CoreDatasetGroup implements Serializable { this.unionSql = unionSql; } + public Boolean getIsCross() { + return isCross; + } + + public void setIsCross(Boolean isCross) { + this.isCross = isCross; + } + @Override public String toString() { return "CoreDatasetGroup{" + "id = " + id + - ", appId = " + appId + ", name = " + name + ", pid = " + pid + ", level = " + level + @@ -232,14 +247,7 @@ public class CoreDatasetGroup implements Serializable { ", updateBy = " + updateBy + ", lastUpdateTime = " + lastUpdateTime + ", unionSql = " + unionSql + + ", isCross = " + isCross + "}"; } - - public String getAppId() { - return appId; - } - - public void setAppId(String appId) { - this.appId = appId; - } } diff --git a/core/core-backend/src/main/java/io/gisbi/dataset/dao/auto/entity/CoreDatasetTableField.java b/core/core-backend/src/main/java/io/gisbi/dataset/dao/auto/entity/CoreDatasetTableField.java index 140030b..6fd060a 100644 --- a/core/core-backend/src/main/java/io/gisbi/dataset/dao/auto/entity/CoreDatasetTableField.java +++ b/core/core-backend/src/main/java/io/gisbi/dataset/dao/auto/entity/CoreDatasetTableField.java @@ -57,9 +57,9 @@ public class CoreDatasetTableField implements Serializable { private String description; /** - * de字段名用作唯一标识 + * 字段名用作唯一标识 */ - private String dataeaseName; + private String gisbiName; /** * de字段别名 @@ -92,7 +92,7 @@ public class CoreDatasetTableField implements Serializable { private Integer size; /** - * dataease字段类型：0-文本，1-时间，2-整型数值，3-浮点数值，4-布尔，5-地理位置，6-二进制 + * gisbi字段类型：0-文本，1-时间，2-整型数值，3-浮点数值，4-布尔，5-地理位置，6-二进制 */ private Integer deType; @@ -205,12 +205,12 @@ public class CoreDatasetTableField implements Serializable { this.description = description; } - public String getDataeaseName() { - return dataeaseName; + public String getGisbiName() { + return gisbiName; } - public void setDataeaseName(String dataeaseName) { - this.dataeaseName = dataeaseName; + public void setGisbiName(String gisbiName) { + this.gisbiName = gisbiName; } public String getFieldShortName() { @@ -352,7 +352,7 @@ public class CoreDatasetTableField implements Serializable { ", originName = " + originName + ", name = " + name + ", description = " + description + - ", dataeaseName = " + dataeaseName + + ", gisbiName = " + gisbiName + ", fieldShortName = " + fieldShortName + ", groupList = " + groupList + ", otherGroup = " + otherGroup + diff --git a/core/core-backend/src/main/java/io/gisbi/dataset/dao/auto/mapper/CoreDatasetTableMapper.java b/core/core-backend/src/main/java/io/gisbi/dataset/dao/auto/mapper/CoreDatasetTableMapper.java index 72109de..27bf299 100644 --- a/core/core-backend/src/main/java/io/gisbi/dataset/dao/auto/mapper/CoreDatasetTableMapper.java +++ b/core/core-backend/src/main/java/io/gisbi/dataset/dao/auto/mapper/CoreDatasetTableMapper.java @@ -1,10 +1,8 @@ package io.gisbi.dataset.dao.auto.mapper; -import io.gisbi.application.system.domain.Role; -import io.gisbi.dataset.dao.auto.entity.CoreDatasetTable; import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import io.gisbi.dataset.dao.auto.entity.CoreDatasetTable; import org.apache.ibatis.annotations.Mapper; -import org.apache.ibatis.annotations.Param; import org.apache.ibatis.annotations.Select; import java.util.List; diff --git a/core/core-backend/src/main/java/io/gisbi/dataset/dao/ext/mapper/CoreDataSetExtMapper.java b/core/core-backend/src/main/java/io/gisbi/dataset/dao/ext/mapper/CoreDataSetExtMapper.java index 0b92735..16e9d0e 100644 --- a/core/core-backend/src/main/java/io/gisbi/dataset/dao/ext/mapper/CoreDataSetExtMapper.java +++ b/core/core-backend/src/main/java/io/gisbi/dataset/dao/ext/mapper/CoreDataSetExtMapper.java @@ -19,6 +19,6 @@ public interface CoreDataSetExtMapper { """) List query(@Param("ew") QueryWrapper queryWrapper); - @Select("select id, name, node_type, create_by, create_time, update_by, last_update_time from core_dataset_group where id = #{id}") + @Select("select id, name, node_type, create_by, create_time, update_by, last_update_time, is_cross from core_dataset_group where id = #{id}") DataSetBarVO queryBarInfo(@Param("id") Long id); } diff --git a/core/core-backend/src/main/java/io/gisbi/dataset/manage/DatasetDataManage.java b/core/core-backend/src/main/java/io/gisbi/dataset/manage/DatasetDataManage.java index 8881206..73c9aa1 100644 --- a/core/core-backend/src/main/java/io/gisbi/dataset/manage/DatasetDataManage.java +++ b/core/core-backend/src/main/java/io/gisbi/dataset/manage/DatasetDataManage.java @@ -1,20 +1,23 @@ package io.gisbi.dataset.manage; -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import io.gisbi.api.chart.dto.DeSortField; import io.gisbi.api.dataset.dto.*; import io.gisbi.api.dataset.union.DatasetGroupInfoDTO; import io.gisbi.api.dataset.union.DatasetTableInfoDTO; +import io.gisbi.api.permissions.auth.dto.BusiPerCheckDTO; +import io.gisbi.api.permissions.dataset.api.RowPermissionsApi; +import io.gisbi.api.permissions.user.vo.UserFormVO; import io.gisbi.chart.utils.ChartDataBuild; import io.gisbi.commons.utils.SqlparserUtils; +import io.gisbi.constant.AuthEnum; +import io.gisbi.constant.SQLConstants; import io.gisbi.dataset.constant.DatasetTableType; -import io.gisbi.dataset.dao.auto.entity.CoreDatasetTable; -import io.gisbi.dataset.dao.auto.entity.CoreDatasetTableField; import io.gisbi.dataset.dao.auto.mapper.CoreDatasetTableFieldMapper; import io.gisbi.dataset.dao.auto.mapper.CoreDatasetTableMapper; import io.gisbi.dataset.utils.DatasetUtils; import io.gisbi.dataset.utils.FieldUtils; +import io.gisbi.dataset.utils.SqlUtils; import io.gisbi.dataset.utils.TableUtils; import io.gisbi.datasource.dao.auto.entity.CoreDatasource; import io.gisbi.datasource.dao.auto.mapper.CoreDatasourceMapper; @@ -22,7 +25,6 @@ import io.gisbi.datasource.manage.DataSourceManage; import io.gisbi.datasource.manage.EngineManage; import io.gisbi.datasource.utils.DatasourceUtils; import io.gisbi.engine.constant.ExtFieldConstant; -import io.gisbi.constant.SQLConstants; import io.gisbi.engine.sql.SQLProvider; import io.gisbi.engine.trans.*; import io.gisbi.engine.utils.SQLUtils; @@ -40,10 +42,8 @@ import io.gisbi.extensions.view.dto.ChartExtRequest; import io.gisbi.extensions.view.dto.ColumnPermissionItem; import io.gisbi.extensions.view.dto.SqlVariableDetails; import io.gisbi.i18n.Translator; -import io.gisbi.utils.BeanUtils; -import io.gisbi.utils.IDUtils; -import io.gisbi.utils.JsonUtil; -import io.gisbi.utils.TreeUtils; +import io.gisbi.system.manage.CorePermissionManage; +import io.gisbi.utils.*; import jakarta.annotation.Resource; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -58,6 +58,7 @@ import java.util.*; import java.util.stream.Collectors; import static io.gisbi.chart.manage.ChartDataManage.START_END_SEPARATOR; +import static io.gisbi.dataset.utils.TableUtils.format; /** * @Author bi-coder @@ -79,6 +80,10 @@ public class DatasetDataManage { @Autowired(required = false) private PluginManageApi pluginManage; @Resource + private CorePermissionManage corePermissionManage; + @Autowired(required = false) + private RowPermissionsApi rowPermissionsApi; + @Resource private CoreDatasetTableFieldMapper coreDatasetTableFieldMapper; @Resource private CoreDatasetTableMapper coreDatasetTableMapper; @@ -87,7 +92,11 @@ public class DatasetDataManage { private static Logger logger = LoggerFactory.getLogger(DatasetDataManage.class); - public static final List notFullDs = List.of("mysql", "mariadb", "Excel", "API"); + private RowPermissionsApi getRowPermissionsApi() { + return rowPermissionsApi; + } + + public static final List notFullDs = List.of("mysql", "mariadb", "Excel", "API", "H2", "h2"); public List getTableFields(DatasetTableDTO datasetTableDTO) throws Exception { List list = null; @@ -97,7 +106,7 @@ public class DatasetDataManage { if (StringUtils.equalsIgnoreCase(type, DatasetTableType.DB) || StringUtils.equalsIgnoreCase(type, DatasetTableType.SQL)) { CoreDatasource coreDatasource = dataSourceManage.getCoreDatasource(datasetTableDTO.getDatasourceId()); DatasourceSchemaDTO datasourceSchemaDTO = new DatasourceSchemaDTO(); - if (StringUtils.equalsIgnoreCase("excel", coreDatasource.getType()) || coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.name())) { + if (coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.Excel.name()) || coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.name())) { coreDatasource = engineManage.getDeEngine(); } if (StringUtils.isNotEmpty(coreDatasource.getStatus()) && "Error".equalsIgnoreCase(coreDatasource.getStatus())) { @@ -108,6 +117,7 @@ public class DatasetDataManage { Provider provider = ProviderFactory.getProvider(coreDatasource.getType()); DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setIsCross(datasetTableDTO.getIsCross()); datasourceRequest.setDsList(Map.of(datasourceSchemaDTO.getId(), datasourceSchemaDTO)); String sql; if (StringUtils.equalsIgnoreCase(type, DatasetTableType.DB)) { @@ -115,23 +125,34 @@ public class DatasetDataManage { sql = TableUtils.tableName2Sql(datasourceSchemaDTO, tableInfoDTO.getTable()) + " LIMIT 0 OFFSET 0"; // replace schema alias, trans dialect Map map = JsonUtil.parseObject(datasourceSchemaDTO.getConfiguration(), Map.class); - if (ObjectUtils.isNotEmpty(map.get("schema"))) { - sql = sql.replaceAll(SqlPlaceholderConstants.KEYWORD_PREFIX_REGEX + datasourceSchemaDTO.getSchemaAlias() + SqlPlaceholderConstants.KEYWORD_SUFFIX_REGEX, map.get("schema").toString()); + if (!datasourceRequest.getIsCross()) { + if (ObjectUtils.isNotEmpty(map.get("schema"))) { + sql = sql.replaceAll(SqlPlaceholderConstants.KEYWORD_PREFIX_REGEX + datasourceSchemaDTO.getSchemaAlias() + SqlPlaceholderConstants.KEYWORD_SUFFIX_REGEX, String.format(format, map.get("schema").toString()) ); + } else { + sql = sql.replaceAll(SqlPlaceholderConstants.KEYWORD_PREFIX_REGEX + datasourceSchemaDTO.getSchemaAlias() + SqlPlaceholderConstants.KEYWORD_SUFFIX_REGEX + "\\.", ""); + } + sql = provider.transSqlDialect(sql, datasourceRequest.getDsList()); } else { sql = sql.replaceAll(SqlPlaceholderConstants.KEYWORD_PREFIX_REGEX + datasourceSchemaDTO.getSchemaAlias() + SqlPlaceholderConstants.KEYWORD_SUFFIX_REGEX + "\\.", ""); + String tableSchema = datasetSQLManage.putObj2Map(datasourceRequest.getDsList(), datasetTableDTO, datasourceRequest.getIsCross()); + sql = SqlUtils.addSchema(sql, tableSchema); } - sql = provider.transSqlDialect(sql, datasourceRequest.getDsList()); } else { // parser sql params and replace default value String s = new String(Base64.getDecoder().decode(tableInfoDTO.getSql())); - String originSql = new SqlparserUtils().handleVariableDefaultValue(s, datasetTableDTO.getSqlVariableDetails(), false, false, null, false, datasourceRequest.getDsList(), pluginManage); + String originSql = new SqlparserUtils().handleVariableDefaultValue(s, datasetTableDTO.getSqlVariableDetails(), true, false, null, datasourceRequest.getIsCross(), datasourceRequest.getDsList(), pluginManage, getUserEntity()); originSql = provider.replaceComment(originSql); // add sql table schema - sql = SQLUtils.buildOriginPreviewSql(SqlPlaceholderConstants.TABLE_PLACEHOLDER, 0, 0); - sql = provider.transSqlDialect(sql, datasourceRequest.getDsList()); - // replace placeholder - sql = provider.replaceTablePlaceHolder(sql, originSql); + if (!datasourceRequest.getIsCross()) { + sql = SQLUtils.buildOriginPreviewSql(SqlPlaceholderConstants.TABLE_PLACEHOLDER, 0, 0); + sql = provider.transSqlDialect(sql, datasourceRequest.getDsList()); + // replace placeholder + sql = provider.replaceTablePlaceHolder(sql, originSql); + } else { + String tableSchema = datasetSQLManage.putObj2Map(datasourceRequest.getDsList(), datasetTableDTO, datasourceRequest.getIsCross()); + sql = SqlUtils.addSchema(originSql, tableSchema); + } } datasourceRequest.setQuery(sql.replaceAll("\r\n", " ") .replaceAll("\n", " ")); @@ -189,7 +210,11 @@ public class DatasetDataManage { }).collect(Collectors.toList()); } - public Map previewDataWithLimit(DatasetGroupInfoDTO datasetGroupInfoDTO, Integer start, Integer count, boolean checkPermission) throws Exception { + public Map previewDataWithLimit(DatasetGroupInfoDTO datasetGroupInfoDTO, Integer start, Integer count, boolean checkPermission, boolean encode) throws Exception { + if (encode) { + DatasetUtils.dsDecode(datasetGroupInfoDTO); + } + Map sqlMap = datasetSQLManage.getUnionSQLForEdit(datasetGroupInfoDTO, null); String sql = (String) sqlMap.get("sql"); @@ -214,7 +239,7 @@ public class DatasetDataManage { dsList.add(next.getValue().getType()); } boolean needOrder = Utils.isNeedOrder(dsList); - boolean crossDs = Utils.isCrossDs(dsMap); + boolean crossDs = datasetGroupInfoDTO.getIsCross(); if (!crossDs) { if (notFullDs.contains(dsMap.entrySet().iterator().next().getValue().getType()) && (boolean) sqlMap.get("isFullJoin")) { DEException.throwException(Translator.get("i18n_not_full")); @@ -253,16 +278,20 @@ public class DatasetDataManage { DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setQuery(querySQL); datasourceRequest.setDsList(dsMap); + datasourceRequest.setIsCross(crossDs); Map data = provider.fetchResultField(datasourceRequest); Map map = new LinkedHashMap<>(); // 重新构造data - Map previewData = buildPreviewData(data, fields, desensitizationList); + Map previewData = buildPreviewData(data, fields, desensitizationList, encode); map.put("data", previewData); if (ObjectUtils.isEmpty(datasetGroupInfoDTO.getId())) { map.put("allFields", fields); } else { List fieldList = datasetTableFieldManage.selectByDatasetGroupId(datasetGroupInfoDTO.getId()); + if (encode) { + DatasetUtils.listEncode(fieldList); + } map.put("allFields", fieldList); } map.put("sql", Base64.getEncoder().encodeToString(querySQL.getBytes())); @@ -278,32 +307,33 @@ public class DatasetDataManage { return 0L; } - public Long getDatasetCountWithWhere(Long datasetGroupId) throws Exception { - DatasetGroupInfoDTO datasetGroupInfoDTO = datasetGroupManage.getForCount(datasetGroupId); - Map sqlMap = datasetSQLManage.getUnionSQLForEdit(datasetGroupInfoDTO, null); - String sql = (String) sqlMap.get("sql"); + public Long getDatasetCountWithWhere(Long datasetGroupId) { + try { + DatasetGroupInfoDTO datasetGroupInfoDTO = datasetGroupManage.getForCount(datasetGroupId); + Map sqlMap = datasetSQLManage.getUnionSQLForEdit(datasetGroupInfoDTO, null); + String sql = (String) sqlMap.get("sql"); - // 获取allFields - List fields = datasetGroupInfoDTO.getAllFields(); - if (ObjectUtils.isEmpty(fields)) { - DEException.throwException(Translator.get("i18n_no_fields")); - } - - buildFieldName(sqlMap, fields); - - Map dsMap = (Map) sqlMap.get("dsMap"); - DatasourceUtils.checkDsStatus(dsMap); - List dsList = new ArrayList<>(); - for (Map.Entry next : dsMap.entrySet()) { - dsList.add(next.getValue().getType()); - } - boolean crossDs = Utils.isCrossDs(dsMap); - if (!crossDs) { - if (notFullDs.contains(dsMap.entrySet().iterator().next().getValue().getType()) && (boolean) sqlMap.get("isFullJoin")) { - DEException.throwException(Translator.get("i18n_not_full")); + // 获取allFields + List fields = datasetGroupInfoDTO.getAllFields(); + if (ObjectUtils.isEmpty(fields)) { + DEException.throwException(Translator.get("i18n_no_fields")); + } + + buildFieldName(sqlMap, fields); + + Map dsMap = (Map) sqlMap.get("dsMap"); + DatasourceUtils.checkDsStatus(dsMap); + List dsList = new ArrayList<>(); + for (Map.Entry next : dsMap.entrySet()) { + dsList.add(next.getValue().getType()); + } + boolean crossDs = datasetGroupInfoDTO.getIsCross(); + if (!crossDs) { + if (notFullDs.contains(dsMap.entrySet().iterator().next().getValue().getType()) && (boolean) sqlMap.get("isFullJoin")) { + DEException.throwException(Translator.get("i18n_not_full")); + } + sql = Utils.replaceSchemaAlias(sql, dsMap); } - sql = Utils.replaceSchemaAlias(sql, dsMap); - } // List rowPermissionsTree = new ArrayList<>(); // TokenUserBO user = AuthUtils.getUser(); @@ -311,27 +341,30 @@ public class DatasetDataManage { // rowPermissionsTree = permissionManage.getRowPermissionsTree(datasetGroupInfoDTO.getId(), user.getUserId()); // } - Provider provider; - if (crossDs) { - provider = ProviderFactory.getDefaultProvider(); - } else { - provider = ProviderFactory.getProvider(dsList.getFirst()); - } + Provider provider; + if (crossDs) { + provider = ProviderFactory.getDefaultProvider(); + } else { + provider = ProviderFactory.getProvider(dsList.getFirst()); + } - // build query sql - SQLMeta sqlMeta = new SQLMeta(); - Table2SQLObj.table2sqlobj(sqlMeta, null, "(" + sql + ")", crossDs); - Field2SQLObj.field2sqlObj(sqlMeta, fields, fields, crossDs, dsMap, Utils.getParams(fields), null, pluginManage); -// WhereTree2Str.transFilterTrees(sqlMeta, rowPermissionsTree, fields, crossDs, dsMap, Utils.getParams(fields), null, pluginManage); - Order2SQLObj.getOrders(sqlMeta, datasetGroupInfoDTO.getSortFields(), fields, crossDs, dsMap, Utils.getParams(fields), null, pluginManage); - String replaceSql = provider.rebuildSQL(SQLProvider.createQuerySQL(sqlMeta, false, false, false), sqlMeta, crossDs, dsMap); - return getDatasetTotal(datasetGroupInfoDTO, replaceSql, null); + // build query sql + SQLMeta sqlMeta = new SQLMeta(); + Table2SQLObj.table2sqlobj(sqlMeta, null, "(" + sql + ")", crossDs); + Field2SQLObj.field2sqlObj(sqlMeta, fields, fields, crossDs, dsMap, Utils.getParams(fields), null, pluginManage); +// WhereTree2Str.transFilterTrees(sqlMeta, rowPermissionsTree, fields, crossDs, dsMap, Utils.getParams(fields), null, pluginManage); + Order2SQLObj.getOrders(sqlMeta, datasetGroupInfoDTO.getSortFields(), fields, crossDs, dsMap, Utils.getParams(fields), null, pluginManage); + String replaceSql = provider.rebuildSQL(SQLProvider.createQuerySQL(sqlMeta, false, false, false), sqlMeta, crossDs, dsMap); + return getDatasetTotal(datasetGroupInfoDTO, replaceSql, null); + } catch (Exception e) { + return null; + } } public Long getDatasetTotal(DatasetGroupInfoDTO datasetGroupInfoDTO, String s, ChartExtRequest request) throws Exception { Map sqlMap = datasetSQLManage.getUnionSQLForEdit(datasetGroupInfoDTO, request); Map dsMap = (Map) sqlMap.get("dsMap"); - boolean crossDs = Utils.isCrossDs(dsMap); + boolean crossDs = datasetGroupInfoDTO.getIsCross(); String sql; if (StringUtils.isEmpty(s)) { sql = (String) sqlMap.get("sql"); @@ -349,6 +382,7 @@ public class DatasetDataManage { DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setQuery(querySQL); datasourceRequest.setDsList(dsMap); + datasourceRequest.setIsCross(crossDs); Provider provider; if (crossDs) { @@ -390,17 +424,17 @@ public class DatasetDataManage { return map; } -// private UserFormVO getUserEntity() { -// if (getRowPermissionsApi() == null) { -// return null; -// } -// return getRowPermissionsApi().getUserById(AuthUtils.getUser().getUserId()); -// } + private UserFormVO getUserEntity() { + if (getRowPermissionsApi() == null) { + return null; + } + return getRowPermissionsApi().getUserById(AuthUtils.getUser().getUserId()); + } public Map previewSql(PreviewSqlDTO dto) throws DEException { CoreDatasource coreDatasource = dataSourceManage.getCoreDatasource(dto.getDatasourceId()); DatasourceSchemaDTO datasourceSchemaDTO = new DatasourceSchemaDTO(); - if (coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.name()) || coreDatasource.getType().equalsIgnoreCase("Excel")) { + if (coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.name()) || coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.Excel.name())) { BeanUtils.copyBean(datasourceSchemaDTO, engineManage.getDeEngine()); } else { BeanUtils.copyBean(datasourceSchemaDTO, coreDatasource); @@ -417,38 +451,60 @@ public class DatasetDataManage { dsMap.put(datasourceSchemaDTO.getId(), datasourceSchemaDTO); DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDsList(dsMap); + datasourceRequest.setIsCross(dto.getIsCross()); Provider provider = ProviderFactory.getProvider(datasourceSchemaDTO.getType()); // parser sql params and replace default value String s = new String(Base64.getDecoder().decode(dto.getSql())); - String originSql = new SqlparserUtils().handleVariableDefaultValue(datasetSQLManage.subPrefixSuffixChar(s), dto.getSqlVariableDetails(), true, true, null, false, dsMap, pluginManage); + String originSql = new SqlparserUtils().handleVariableDefaultValue(datasetSQLManage.subPrefixSuffixChar(s), dto.getSqlVariableDetails(), true, true, null, dto.getIsCross(), dsMap, pluginManage, getUserEntity()); originSql = provider.replaceComment(originSql); // sql 作为临时表，外层加上limit String sql; + if (dto.getIsCross()) { + DatasetTableDTO currentDs = new DatasetTableDTO(); + BeanUtils.copyBean(currentDs, dto); + currentDs.setType("sql"); + String tableSchema = datasetSQLManage.putObj2Map(dsMap, currentDs, dto.getIsCross()); + sql = SqlUtils.addSchema(originSql, tableSchema); + if (Utils.isNeedOrder(List.of(datasourceSchemaDTO.getType()))) { + // 先根据sql获取表字段 + String sqlField = SQLUtils.buildOriginPreviewSql(sql, 0, 0); + datasourceRequest.setQuery(sqlField); - if (Utils.isNeedOrder(List.of(datasourceSchemaDTO.getType()))) { - // 先根据sql获取表字段 - String sqlField = SQLUtils.buildOriginPreviewSql(SqlPlaceholderConstants.TABLE_PLACEHOLDER, 0, 0); - - sqlField = provider.transSqlDialect(sqlField, datasourceRequest.getDsList()); - // replace placeholder - sqlField = provider.replaceTablePlaceHolder(sqlField, originSql); - datasourceRequest.setQuery(sqlField); - - // 获取数据源表的原始字段 - List list = provider.fetchTableField(datasourceRequest); - if (ObjectUtils.isEmpty(list)) { - return null; + // 获取数据源表的原始字段 + List list = provider.fetchTableField(datasourceRequest); + if (ObjectUtils.isEmpty(list)) { + return null; + } + sql = SQLUtils.buildOriginPreviewSqlWithOrderBy(sql, 100, 0, String.format(SQLConstants.FIELD_DOT_FIX, list.get(0).getOriginName()) + " ASC "); + } else { + sql = SQLUtils.buildOriginPreviewSql(sql, 100, 0); } - sql = SQLUtils.buildOriginPreviewSqlWithOrderBy(SqlPlaceholderConstants.TABLE_PLACEHOLDER, 100, 0, String.format(SQLConstants.FIELD_DOT_FIX, list.get(0).getOriginName()) + " ASC "); } else { - sql = SQLUtils.buildOriginPreviewSql(SqlPlaceholderConstants.TABLE_PLACEHOLDER, 100, 0); + if (Utils.isNeedOrder(List.of(datasourceSchemaDTO.getType()))) { + // 先根据sql获取表字段 + String sqlField = SQLUtils.buildOriginPreviewSql(SqlPlaceholderConstants.TABLE_PLACEHOLDER, 0, 0); + + sqlField = provider.transSqlDialect(sqlField, datasourceRequest.getDsList()); + // replace placeholder + sqlField = provider.replaceTablePlaceHolder(sqlField, originSql); + datasourceRequest.setQuery(sqlField); + + // 获取数据源表的原始字段 + List list = provider.fetchTableField(datasourceRequest); + if (ObjectUtils.isEmpty(list)) { + return null; + } + sql = SQLUtils.buildOriginPreviewSqlWithOrderBy(SqlPlaceholderConstants.TABLE_PLACEHOLDER, 100, 0, String.format(SQLConstants.FIELD_DOT_FIX, list.get(0).getOriginName()) + " ASC "); + } else { + sql = SQLUtils.buildOriginPreviewSql(SqlPlaceholderConstants.TABLE_PLACEHOLDER, 100, 0); + } + sql = provider.transSqlDialect(sql, datasourceRequest.getDsList()); + // replace placeholder + sql = provider.replaceTablePlaceHolder(sql, originSql); } - sql = provider.transSqlDialect(sql, datasourceRequest.getDsList()); - // replace placeholder - sql = provider.replaceTablePlaceHolder(sql, originSql); logger.debug("calcite data preview sql: " + sql); datasourceRequest.setQuery(sql); @@ -456,14 +512,14 @@ public class DatasetDataManage { // 重新构造data List fList = (List) data.get("fields"); List fields = transFields(fList, false); - Map previewData = buildPreviewData(data, fields, new HashMap<>()); + Map previewData = buildPreviewData(data, fields, new HashMap<>(), false); Map map = new LinkedHashMap<>(); map.put("data", previewData); map.put("sql", Base64.getEncoder().encodeToString(sql.getBytes())); return map; } - public Map buildPreviewData(Map data, List fields, Map desensitizationList) { + public Map buildPreviewData(Map data, List fields, Map desensitizationList, boolean isEncode) { Map map = new LinkedHashMap<>(); List dataList = (List) data.get("data"); List> dataObjectList = new ArrayList<>(); @@ -479,11 +535,11 @@ public class DatasetDataManage { BigDecimal bigDecimal = new BigDecimal(res); res = String.format("%.8f", bigDecimal); } - if (desensitizationList.keySet().contains(fields.get(j).getDataeaseName())) { - obj.put(fields.get(j).getDataeaseName(), ChartDataBuild.desensitizationValue(desensitizationList.get(fields.get(j).getDataeaseName()), String.valueOf(res))); + if (desensitizationList.keySet().contains(fields.get(j).getGisbiName())) { + obj.put(fields.get(j).getGisbiName(), ChartDataBuild.desensitizationValue(desensitizationList.get(fields.get(j).getGisbiName()), String.valueOf(res))); } else { - obj.put(ObjectUtils.isNotEmpty(fields.get(j).getDataeaseName()) ? - fields.get(j).getDataeaseName() : fields.get(j).getOriginName(), res); + obj.put(ObjectUtils.isNotEmpty(fields.get(j).getGisbiName()) ? + fields.get(j).getGisbiName() : fields.get(j).getOriginName(), res); } } } @@ -491,6 +547,10 @@ public class DatasetDataManage { } } + if (isEncode) { + DatasetUtils.listEncode(fields); + } + map.put("fields", fields); map.put("data", dataObjectList); return map; @@ -506,27 +566,27 @@ public class DatasetDataManage { for (DatasetTableFieldDTO fieldDTO : unionFields) { if (Objects.equals(datasetTableFieldDTO.getDatasetTableId(), fieldDTO.getDatasetTableId()) && Objects.equals(datasetTableFieldDTO.getOriginName(), fieldDTO.getOriginName())) { - datasetTableFieldDTO.setDataeaseName(fieldDTO.getDataeaseName()); + datasetTableFieldDTO.setGisbiName(fieldDTO.getGisbiName()); datasetTableFieldDTO.setFieldShortName(fieldDTO.getFieldShortName()); } } } if (Objects.equals(datasetTableFieldDTO.getExtField(), ExtFieldConstant.EXT_CALC)) { - String dataeaseName = TableUtils.fieldNameShort(datasetTableFieldDTO.getId() + "_" + datasetTableFieldDTO.getOriginName()); - datasetTableFieldDTO.setDataeaseName(dataeaseName); - datasetTableFieldDTO.setFieldShortName(dataeaseName); + String gisbiName = TableUtils.fieldNameShort(datasetTableFieldDTO.getId() + "_" + datasetTableFieldDTO.getOriginName()); + datasetTableFieldDTO.setGisbiName(gisbiName); + datasetTableFieldDTO.setFieldShortName(gisbiName); datasetTableFieldDTO.setDeExtractType(datasetTableFieldDTO.getDeType()); } if (Objects.equals(datasetTableFieldDTO.getExtField(), ExtFieldConstant.EXT_GROUP)) { - String dataeaseName = TableUtils.fieldNameShort(datasetTableFieldDTO.getId() + "_" + datasetTableFieldDTO.getOriginName()); - datasetTableFieldDTO.setDataeaseName(dataeaseName); - datasetTableFieldDTO.setFieldShortName(dataeaseName); + String gisbiName = TableUtils.fieldNameShort(datasetTableFieldDTO.getId() + "_" + datasetTableFieldDTO.getOriginName()); + datasetTableFieldDTO.setGisbiName(gisbiName); + datasetTableFieldDTO.setFieldShortName(gisbiName); datasetTableFieldDTO.setDeExtractType(0); datasetTableFieldDTO.setDeType(0); datasetTableFieldDTO.setGroupType("d"); } } else { - datasetTableFieldDTO.setDataeaseName(dto.getDataeaseName()); + datasetTableFieldDTO.setGisbiName(dto.getGisbiName()); datasetTableFieldDTO.setFieldShortName(dto.getFieldShortName()); } } @@ -546,7 +606,7 @@ public class DatasetDataManage { allFields.addAll(datasetGroupInfoDTO.getAllFields()); Map dsMap = (Map) sqlMap.get("dsMap"); - boolean crossDs = Utils.isCrossDs(dsMap); + boolean crossDs = datasetGroupInfoDTO.getIsCross(); if (!crossDs) { sql = Utils.replaceSchemaAlias(sql, dsMap); } @@ -579,7 +639,7 @@ public class DatasetDataManage { } Field2SQLObj.field2sqlObj(sqlMeta, fields, allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage); -// WhereTree2Str.transFilterTrees(sqlMeta, null, allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage); + WhereTree2Str.transFilterTrees(sqlMeta, null, allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage); Order2SQLObj.getOrders(sqlMeta, datasetGroupInfoDTO.getSortFields(), allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage); String querySQL; querySQL = SQLProvider.createQuerySQL(sqlMeta, false, needOrder, !StringUtils.equalsIgnoreCase(dsType, "es")); @@ -591,6 +651,7 @@ public class DatasetDataManage { DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setQuery(querySQL); datasourceRequest.setDsList(dsMap); + datasourceRequest.setIsCross(crossDs); Map data = provider.fetchResultField(datasourceRequest); List dataList = (List) data.get("data"); @@ -635,13 +696,13 @@ public class DatasetDataManage { Long datasetGroupId = field.getDatasetGroupId(); // check permission -// BusiPerCheckDTO dto = new BusiPerCheckDTO(); -// dto.setId(datasetGroupId); -// dto.setAuthEnum(AuthEnum.READ); -// boolean checked = corePermissionManage.checkAuth(dto); -// if (!checked) { -// DEException.throwException(Translator.get("i18n_no_dataset_permission")); -// } + BusiPerCheckDTO dto = new BusiPerCheckDTO(); + dto.setId(datasetGroupId); + dto.setAuthEnum(AuthEnum.READ); + boolean checked = corePermissionManage.checkAuth(dto); + if (!checked) { + DEException.throwException(Translator.get("i18n_no_dataset_permission")); + } if (field.getChartId() != null) { allFields.addAll(datasetTableFieldManage.getChartCalcFields(field.getChartId())); } @@ -653,7 +714,7 @@ public class DatasetDataManage { allFields.addAll(datasetGroupInfoDTO.getAllFields()); Map dsMap = (Map) sqlMap.get("dsMap"); - boolean crossDs = Utils.isCrossDs(dsMap); + boolean crossDs = datasetGroupInfoDTO.getIsCross(); if (!crossDs) { sql = Utils.replaceSchemaAlias(sql, dsMap); } @@ -713,6 +774,7 @@ public class DatasetDataManage { DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setQuery(querySQL); datasourceRequest.setDsList(dsMap); + datasourceRequest.setIsCross(crossDs); Map data = provider.fetchResultField(datasourceRequest); List dataList = (List) data.get("data"); @@ -738,9 +800,9 @@ public class DatasetDataManage { tmpData.set(i, val); } } - if (desensitizationList.keySet().contains(field.getDataeaseName())) { + if (desensitizationList.keySet().contains(field.getGisbiName())) { for (int i = 0; i < tmpData.size(); i++) { - previewData.add(ChartDataBuild.desensitizationValue(desensitizationList.get(field.getDataeaseName()), tmpData.get(i))); + previewData.add(ChartDataBuild.desensitizationValue(desensitizationList.get(field.getGisbiName()), tmpData.get(i))); } } else { previewData = tmpData; @@ -819,7 +881,7 @@ public class DatasetDataManage { allFields.addAll(datasetGroupInfoDTO.getAllFields()); dsMap = (Map) sqlMap.get("dsMap"); - crossDs = Utils.isCrossDs(dsMap); + crossDs = datasetGroupInfoDTO.getIsCross(); if (!crossDs) { sql = Utils.replaceSchemaAlias(sql, dsMap); } @@ -975,6 +1037,7 @@ public class DatasetDataManage { DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setQuery(querySQL); datasourceRequest.setDsList(dsMap); + datasourceRequest.setIsCross(crossDs); Map data = provider.fetchResultField(datasourceRequest); List dataList = (List) data.get("data"); @@ -1012,8 +1075,8 @@ public class DatasetDataManage { BigDecimal bigDecimal = new BigDecimal(val); val = String.format("%.8f", bigDecimal); } - if (desensitizationList.containsKey(field.getDataeaseName())) { - String str = ChartDataBuild.desensitizationValue(desensitizationList.get(field.getDataeaseName()), val); + if (desensitizationList.containsKey(field.getGisbiName())) { + String str = ChartDataBuild.desensitizationValue(desensitizationList.get(field.getGisbiName()), val); map.put(field.getId() + "", str); } else { map.put(field.getId() + "", val); @@ -1049,7 +1112,7 @@ public class DatasetDataManage { allFields.addAll(datasetGroupInfoDTO.getAllFields()); Map dsMap = (Map) sqlMap.get("dsMap"); - boolean crossDs = Utils.isCrossDs(dsMap); + boolean crossDs = datasetGroupInfoDTO.getIsCross(); if (!crossDs) { sql = Utils.replaceSchemaAlias(sql, dsMap); } @@ -1110,6 +1173,7 @@ public class DatasetDataManage { DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setQuery(querySQL); datasourceRequest.setDsList(dsMap); + datasourceRequest.setIsCross(crossDs); Map data = provider.fetchResultField(datasourceRequest); List rows = (List) data.get("data"); @@ -1118,13 +1182,16 @@ public class DatasetDataManage { Set pkSet = new HashSet<>(); rows = rows.stream().filter(row -> { boolean hasEmpty = false; + int emptyCount = 0; for (String s : row) { if (StringUtils.isBlank(s)) { - hasEmpty = true; - break; + emptyCount++; + hasEmpty = true; // 标记已遇到第一个null + } else if (hasEmpty) { + return false; // 在null后出现非null元素，不符合要求 } } - return !hasEmpty; + return emptyCount != row.length; }).toList(); List treeNodes = rows.stream().map(row -> buildTreeNode(row, pkSet)).flatMap(Collection::stream).collect(Collectors.toList()); List tree = DatasetUtils.mergeDuplicateTree(treeNodes, "root"); @@ -1136,7 +1203,9 @@ public class DatasetDataManage { List parentPkList = new ArrayList<>(); for (int i = 0; i < row.length; i++) { String text = row[i]; - + if (StringUtils.isEmpty(text)) { + continue; + } parentPkList.add(text); String val = String.join(TreeUtils.SEPARATOR, parentPkList); String parentVal = i == 0 ? TreeUtils.DEFAULT_ROOT : row[i - 1]; @@ -1358,7 +1427,7 @@ private String buildPrimaryKeyQuerySQL(String dbType, String tablename) { logger.debug("执行插入数据的SQL: {}", sql); // 执行插入操作 - int result= provider.executeUpdate(datasourceRequest); + int result= provider.executeUpdate(datasourceRequest,null).getCount(); if (result==1) { return true; // process result set @@ -1501,7 +1570,7 @@ private String buildPrimaryKeyQuerySQL(String dbType, String tablename) { logger.debug("执行更新数据的SQL: {}", sql); // 执行更新操作 - int result= provider.executeUpdate(datasourceRequest); + int result= provider.executeUpdate(datasourceRequest,null).getCount(); if (result==1) { return true; // process result set @@ -1540,7 +1609,7 @@ private String buildPrimaryKeyQuerySQL(String dbType, String tablename) { logger.debug("执行删除数据的SQL: {}", sql); // 执行删除操作 - int result= provider.executeUpdate(datasourceRequest); + int result= provider.executeUpdate(datasourceRequest,null).getCount(); if (result==1) { return true; // process result set diff --git a/core/core-backend/src/main/java/io/gisbi/dataset/manage/DatasetGroupManage.java b/core/core-backend/src/main/java/io/gisbi/dataset/manage/DatasetGroupManage.java index 4d83c9a..6479e89 100644 --- a/core/core-backend/src/main/java/io/gisbi/dataset/manage/DatasetGroupManage.java +++ b/core/core-backend/src/main/java/io/gisbi/dataset/manage/DatasetGroupManage.java @@ -33,10 +33,9 @@ import jakarta.annotation.Resource; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; - +import io.gisbi.dataset.utils.DatasetUtils; import java.util.*; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -80,8 +79,7 @@ public class DatasetGroupManage { @Transactional - public DatasetGroupInfoDTO save(DatasetGroupInfoDTO datasetGroupInfoDTO, boolean rename) throws Exception { - lock.lock(); + public DatasetGroupInfoDTO save(DatasetGroupInfoDTO datasetGroupInfoDTO, boolean rename, boolean encode) throws Exception { try { boolean isCreate; // 用于重命名获取pid @@ -123,6 +121,9 @@ public class DatasetGroupManage { } // node_type=dataset需要创建dataset_table和field if (StringUtils.equalsIgnoreCase(datasetGroupInfoDTO.getNodeType(), "dataset")) { + if (encode) { + DatasetUtils.dsDecode(datasetGroupInfoDTO); + } List tableIds = new ArrayList<>(); List fieldIds = new ArrayList<>(); // 解析tree，保存 @@ -131,12 +132,16 @@ public class DatasetGroupManage { // 删除不要的table和field datasetTableManage.deleteByDatasetGroupUpdate(datasetGroupInfoDTO.getId(), tableIds); datasetTableFieldManage.deleteByDatasetGroupUpdate(datasetGroupInfoDTO.getId(), fieldIds); + if (encode) { + DatasetUtils.dsEncode(datasetGroupInfoDTO); + } + } + if (StringUtils.isNotEmpty(datasetGroupInfoDTO.getUnionSql())) { + datasetGroupInfoDTO.setUnionSql(DatasetUtils.getEncode(datasetGroupInfoDTO.getUnionSql())); } return datasetGroupInfoDTO; } catch (Exception e) { DEException.throwException(e.getMessage()); - } finally { - lock.unlock(); } return null; } @@ -214,6 +219,7 @@ public class DatasetGroupManage { public List tree(BusiNodeRequest request) { + QueryWrapper queryWrapper = new QueryWrapper<>(); if (ObjectUtils.isNotEmpty(request.getLeaf())) { queryWrapper.eq("node_type", request.getLeaf() ? "dataset" : "folder"); @@ -329,7 +335,7 @@ public class DatasetGroupManage { if (ObjectUtils.isEmpty(datasetGroupInfoDTO.getUnion())) { return; } - datasetDataManage.previewDataWithLimit(datasetGroupInfoDTO, 0, 1, false); + datasetDataManage.previewDataWithLimit(datasetGroupInfoDTO, 0, 1, false, false); // table和field均由前端生成id（如果没有id） Long datasetGroupId = datasetGroupInfoDTO.getId(); List allFields = datasetGroupInfoDTO.getAllFields(); @@ -345,28 +351,28 @@ public class DatasetGroupManage { for (DatasetTableFieldDTO fieldDTO : unionFields) { if (Objects.equals(datasetTableFieldDTO.getDatasetTableId(), fieldDTO.getDatasetTableId()) && Objects.equals(datasetTableFieldDTO.getOriginName(), fieldDTO.getOriginName())) { - datasetTableFieldDTO.setDataeaseName(fieldDTO.getDataeaseName()); + datasetTableFieldDTO.setGisbiName(fieldDTO.getGisbiName()); datasetTableFieldDTO.setFieldShortName(fieldDTO.getFieldShortName()); } } } if (Objects.equals(datasetTableFieldDTO.getExtField(), ExtFieldConstant.EXT_CALC)) { - String dataeaseName = TableUtils.fieldNameShort(datasetTableFieldDTO.getId() + "_" + datasetTableFieldDTO.getOriginName()); - datasetTableFieldDTO.setDataeaseName(dataeaseName); - datasetTableFieldDTO.setFieldShortName(dataeaseName); + String gisbiName = TableUtils.fieldNameShort(datasetTableFieldDTO.getId() + "_" + datasetTableFieldDTO.getOriginName()); + datasetTableFieldDTO.setGisbiName(gisbiName); + datasetTableFieldDTO.setFieldShortName(gisbiName); datasetTableFieldDTO.setDeExtractType(datasetTableFieldDTO.getDeType()); } if (Objects.equals(datasetTableFieldDTO.getExtField(), ExtFieldConstant.EXT_GROUP)) { - String dataeaseName = TableUtils.fieldNameShort(datasetTableFieldDTO.getId() + "_" + datasetTableFieldDTO.getOriginName()); - datasetTableFieldDTO.setDataeaseName(dataeaseName); - datasetTableFieldDTO.setFieldShortName(dataeaseName); + String gisbiName = TableUtils.fieldNameShort(datasetTableFieldDTO.getId() + "_" + datasetTableFieldDTO.getOriginName()); + datasetTableFieldDTO.setGisbiName(gisbiName); + datasetTableFieldDTO.setFieldShortName(gisbiName); datasetTableFieldDTO.setDeExtractType(0); datasetTableFieldDTO.setDeType(0); datasetTableFieldDTO.setGroupType("d"); } datasetTableFieldDTO.setDatasetGroupId(datasetGroupId); } else { - datasetTableFieldDTO.setDataeaseName(dto.getDataeaseName()); + datasetTableFieldDTO.setGisbiName(dto.getGisbiName()); datasetTableFieldDTO.setFieldShortName(dto.getFieldShortName()); } datasetTableFieldDTO = datasetTableFieldManage.save(datasetTableFieldDTO); @@ -390,7 +396,7 @@ public class DatasetGroupManage { List allFields = dsFields.stream().map(ele -> { DatasetTableFieldDTO datasetTableFieldDTO = new DatasetTableFieldDTO(); BeanUtils.copyBean(datasetTableFieldDTO, ele); - datasetTableFieldDTO.setFieldShortName(ele.getDataeaseName()); + datasetTableFieldDTO.setFieldShortName(ele.getGisbiName()); return datasetTableFieldDTO; }).collect(Collectors.toList()); @@ -426,10 +432,12 @@ public class DatasetGroupManage { List allFields = dsFields.stream().map(ele -> { DatasetTableFieldDTO datasetTableFieldDTO = new DatasetTableFieldDTO(); BeanUtils.copyBean(datasetTableFieldDTO, ele); - datasetTableFieldDTO.setFieldShortName(ele.getDataeaseName()); + datasetTableFieldDTO.setFieldShortName(ele.getGisbiName()); return datasetTableFieldDTO; }).collect(Collectors.toList()); + DatasetUtils.listEncode(allFields); + dto.setAllFields(allFields); } return dto; @@ -462,7 +470,7 @@ public class DatasetGroupManage { List allFields = dsFields.stream().map(ele -> { DatasetTableFieldDTO datasetTableFieldDTO = new DatasetTableFieldDTO(); BeanUtils.copyBean(datasetTableFieldDTO, ele); - datasetTableFieldDTO.setFieldShortName(ele.getDataeaseName()); + datasetTableFieldDTO.setFieldShortName(ele.getGisbiName()); return datasetTableFieldDTO; }).collect(Collectors.toList()); @@ -470,7 +478,7 @@ public class DatasetGroupManage { if ("preview".equalsIgnoreCase(type)) { // 请求数据 - Map map = datasetDataManage.previewDataWithLimit(dto, 0, 100, true); + Map map = datasetDataManage.previewDataWithLimit(dto, 0, 100, true, false); // 获取data,sql Map data = (Map) map.get("data"); String sql = (String) map.get("sql"); @@ -583,6 +591,8 @@ public class DatasetGroupManage { List dimensionList = fields.stream().filter(ele -> StringUtils.equalsIgnoreCase(ele.getGroupType(), "d")).toList(); List quotaList = fields.stream().filter(ele -> StringUtils.equalsIgnoreCase(ele.getGroupType(), "q")).toList(); Map> map = new LinkedHashMap<>(); + DatasetUtils.listEncode(dimensionList); + DatasetUtils.listEncode(quotaList); map.put("dimensionList", dimensionList); map.put("quotaList", quotaList); dto.setFields(map); @@ -592,4 +602,38 @@ public class DatasetGroupManage { } return result; } + + public List getAllList() { + List coreDatasetGroupList = coreDatasetGroupMapper.selectList(new QueryWrapper<>()); + if (CollectionUtils.isEmpty(coreDatasetGroupList)) { + return new ArrayList<>(); + } + List list = new ArrayList<>(); + for (CoreDatasetGroup coreDatasetGroup : coreDatasetGroupList) { + DatasetGroupInfoDTO dto = new DatasetGroupInfoDTO(); + BeanUtils.copyBean(dto, coreDatasetGroup); + dto.setUnionSql(null); + if (StringUtils.equalsIgnoreCase(dto.getNodeType(), "dataset")) { + List unionDTOList = JsonUtil.parseList(coreDatasetGroup.getInfo(), new TypeReference<>() { + }); + dto.setUnion(unionDTOList); + + // 获取field + List dsFields = datasetTableFieldManage.selectByDatasetGroupId(coreDatasetGroup.getId()); + List allFields = dsFields.stream().map(ele -> { + DatasetTableFieldDTO datasetTableFieldDTO = new DatasetTableFieldDTO(); + BeanUtils.copyBean(datasetTableFieldDTO, ele); + datasetTableFieldDTO.setFieldShortName(ele.getGisbiName()); + return datasetTableFieldDTO; + }).collect(Collectors.toList()); + + DatasetUtils.listEncode(allFields); + + dto.setAllFields(allFields); + + list.add(dto); + } + } + return list; + } } diff --git a/core/core-backend/src/main/java/io/gisbi/dataset/manage/DatasetSQLManage.java b/core/core-backend/src/main/java/io/gisbi/dataset/manage/DatasetSQLManage.java index d0b55cc..b244a41 100644 --- a/core/core-backend/src/main/java/io/gisbi/dataset/manage/DatasetSQLManage.java +++ b/core/core-backend/src/main/java/io/gisbi/dataset/manage/DatasetSQLManage.java @@ -1,8 +1,14 @@ package io.gisbi.dataset.manage; import io.gisbi.api.dataset.union.*; +import io.gisbi.api.permissions.auth.dto.BusiPerCheckDTO; +import io.gisbi.api.permissions.dataset.api.RowPermissionsApi; +import io.gisbi.api.permissions.user.vo.UserFormVO; import io.gisbi.commons.utils.SqlparserUtils; +import io.gisbi.constant.AuthEnum; import io.gisbi.dataset.constant.DatasetTableType; +import io.gisbi.dataset.dao.auto.entity.CoreDatasetGroup; +import io.gisbi.dataset.dao.auto.mapper.CoreDatasetGroupMapper; import io.gisbi.dataset.utils.DatasetTableTypeConstants; import io.gisbi.dataset.utils.SqlUtils; import io.gisbi.dataset.utils.TableUtils; @@ -27,6 +33,9 @@ import io.gisbi.extensions.view.dto.ChartExtFilterDTO; import io.gisbi.extensions.view.dto.ChartExtRequest; import io.gisbi.extensions.view.dto.SqlVariableDetails; import io.gisbi.i18n.Translator; +import io.gisbi.license.utils.LicenseUtil; +import io.gisbi.system.manage.CorePermissionManage; +import io.gisbi.utils.AuthUtils; import io.gisbi.utils.BeanUtils; import io.gisbi.utils.JsonUtil; import jakarta.annotation.Resource; @@ -53,10 +62,23 @@ public class DatasetSQLManage { @Resource private EngineManage engineManage; + @Resource + private CorePermissionManage corePermissionManage; + @Autowired(required = false) private PluginManageApi pluginManage; + @Autowired(required = false) + private RowPermissionsApi rowPermissionsApi; @Resource private DataSourceManage dataSourceManage; + @Resource + private DatasetGroupManage datasetGroupManage; + @Resource + private CoreDatasetGroupMapper coreDatasetGroupMapper; + + private RowPermissionsApi getRowPermissionsApi() { + return rowPermissionsApi; + } private static Logger logger = LoggerFactory.getLogger(DatasetSQLManage.class); @@ -109,22 +131,12 @@ public class DatasetSQLManage { List unionList = new ArrayList<>(); List checkedFields = new ArrayList<>(); String sql = ""; - if (ObjectUtils.isEmpty(union)) { return null; } - Set allDs = getAllDs(union); - boolean isCross = allDs.size() > 1; - + boolean isCross = dataTableInfoDTO.getIsCross(); DatasetTableDTO currentDs = union.get(0).getCurrentDs(); - - // get datasource and schema,put map - String tableSchema = putObj2Map(dsMap, currentDs, isCross); - // get table - DatasetTableInfoDTO infoDTO = JsonUtil.parseObject(currentDs.getInfo(), DatasetTableInfoDTO.class); - - SQLObj tableName = getUnionTable(currentDs, infoDTO, tableSchema, 0, filterParameters(chartExtRequest, currentDs.getId()), chartExtRequest == null, isCross, dsMap); - + SQLObj tableName = null; for (int i = 0; i < union.size(); i++) { UnionDTO unionDTO = union.get(i); DatasetTableDTO datasetTable = unionDTO.getCurrentDs(); @@ -137,7 +149,9 @@ public class DatasetSQLManage { schema = putObj2Map(dsMap, datasetTable, isCross); } SQLObj table = getUnionTable(datasetTable, tableInfo, schema, i, filterParameters(chartExtRequest, currentDs.getId()), chartExtRequest == null, isCross, dsMap); - + if (i == 0) { + tableName = table; + } // 获取前端传过来选中的字段 List fields = unionDTO.getCurrentDsFields(); fields = fields.stream().filter(DatasetTableFieldDTO::getChecked).collect(Collectors.toList()); @@ -145,14 +159,13 @@ public class DatasetSQLManage { String[] array = fields.stream() .map(f -> { String alias; - if (StringUtils.isEmpty(f.getDataeaseName())) { + if (StringUtils.isEmpty(f.getGisbiName())) { alias = TableUtils.fieldNameShort(table.getTableAlias() + "_" + f.getOriginName()); } else { - alias = f.getDataeaseName(); + alias = f.getGisbiName(); } - f.setFieldShortName(alias); - f.setDataeaseName(f.getFieldShortName()); + f.setgisbiName(f.getFieldShortName()); f.setDatasetTableId(datasetTable.getId()); String prefix = ""; String suffix = ""; @@ -320,14 +333,14 @@ public class DatasetSQLManage { String[] array = fields.stream() .map(f -> { String alias; - if (StringUtils.isEmpty(f.getDataeaseName())) { + if (StringUtils.isEmpty(f.getGisbiName())) { alias = TableUtils.fieldNameShort(table.getTableAlias() + "_" + f.getOriginName()); } else { - alias = f.getDataeaseName(); + alias = f.getGisbiName(); } f.setFieldShortName(alias); - f.setDataeaseName(f.getFieldShortName()); + f.setgisbiName(f.getFieldShortName()); f.setDatasetTableId(datasetTable.getId()); String prefix = ""; String suffix = ""; @@ -398,19 +411,21 @@ public class DatasetSQLManage { BeanUtils.copyBean(dto, datasourceType); return dto; } else { - List xpackPluginsDatasourceVOS = pluginManage.queryPluginDs(); - List list = xpackPluginsDatasourceVOS.stream().filter(ele -> StringUtils.equals(ele.getType(), type)).toList(); - if (ObjectUtils.isNotEmpty(list)) { - XpackPluginsDatasourceVO first = list.getFirst(); - DsTypeDTO dto = new DsTypeDTO(); - dto.setName(first.getName()); - dto.setCatalog(first.getCategory()); - dto.setType(first.getType()); - dto.setPrefix(first.getPrefix()); - dto.setSuffix(first.getSuffix()); - return dto; - } else { - DEException.throwException(Translator.get("i18n_dataset_plugin_error")); + if (LicenseUtil.licenseValid()) { + List xpackPluginsDatasourceVOS = pluginManage.queryPluginDs(); + List list = xpackPluginsDatasourceVOS.stream().filter(ele -> StringUtils.equals(ele.getType(), type)).toList(); + if (ObjectUtils.isNotEmpty(list)) { + XpackPluginsDatasourceVO first = list.getFirst(); + DsTypeDTO dto = new DsTypeDTO(); + dto.setName(first.getName()); + dto.setCatalog(first.getCategory()); + dto.setType(first.getType()); + dto.setPrefix(first.getPrefix()); + dto.setSuffix(first.getSuffix()); + return dto; + } else { + DEException.throwException(Translator.get("i18n_dataset_plugin_error")); + } } return null; } @@ -445,6 +460,12 @@ public class DatasetSQLManage { } } + private UserFormVO getUserEntity() { + if (getRowPermissionsApi() == null) { + return null; + } + return getRowPermissionsApi().getUserById(AuthUtils.getUser().getUserId()); + } private SQLObj getUnionTable(DatasetTableDTO currentDs, DatasetTableInfoDTO infoDTO, String tableSchema, int index, List parameters, boolean isFromDataSet, boolean isCross, Map dsMap) { SQLObj tableObj; @@ -455,7 +476,7 @@ public class DatasetSQLManage { Provider provider = ProviderFactory.getProvider(dsMap.entrySet().iterator().next().getValue().getType()); // parser sql params and replace default value String s = new String(Base64.getDecoder().decode(infoDTO.getSql())); - String sql = new SqlparserUtils().handleVariableDefaultValue(s, currentDs.getSqlVariableDetails(), false, isFromDataSet, parameters, isCross, dsMap, pluginManage); + String sql = new SqlparserUtils().handleVariableDefaultValue(s, currentDs.getSqlVariableDetails(), false, isFromDataSet, parameters, isCross, dsMap, pluginManage, getUserEntity()); sql = provider.replaceComment(sql); // add table schema if (isCross) { @@ -469,22 +490,24 @@ public class DatasetSQLManage { return tableObj; } - private String putObj2Map(Map dsMap, DatasetTableDTO ds, boolean isCross) throws Exception { + public String putObj2Map(Map dsMap, DatasetTableDTO ds, boolean isCross) { // 通过datasource id校验数据源权限 -// BusiPerCheckDTO dto = new BusiPerCheckDTO(); -// dto.setId(ds.getDatasourceId()); -// dto.setAuthEnum(AuthEnum.READ); -// boolean checked = corePermissionManage.checkAuth(dto); -// if (!checked) { -// DEException.throwException(Translator.get("i18n_no_datasource_permission")); -// } + BusiPerCheckDTO dto = new BusiPerCheckDTO(); + dto.setId(ds.getDatasourceId()); + dto.setAuthEnum(AuthEnum.READ); + boolean checked = corePermissionManage.checkAuth(dto); + if (!checked) { + DEException.throwException(Translator.get("i18n_no_datasource_permission")); + } + + String schemaAlias; if (StringUtils.equalsIgnoreCase(ds.getType(), DatasetTableType.DB) || StringUtils.equalsIgnoreCase(ds.getType(), DatasetTableType.SQL)) { CoreDatasource coreDatasource = dataSourceManage.getCoreDatasource(ds.getDatasourceId()); if (coreDatasource == null) { DEException.throwException(Translator.get("i18n_dataset_ds_error") + ",ID:" + ds.getDatasourceId()); } - if (StringUtils.equalsIgnoreCase("excel", coreDatasource.getType()) || coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.name())) { + if (coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.Excel.name()) || coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.name())) { coreDatasource = engineManage.getDeEngine(); } @@ -522,4 +545,20 @@ public class DatasetSQLManage { } return schemaAlias; } + + public void datasetCrossDefault() { + List allList = datasetGroupManage.getAllList(); + for (DatasetGroupInfoDTO ele : allList) { + mergeDatasetCrossDefault(ele); + CoreDatasetGroup record = new CoreDatasetGroup(); + BeanUtils.copyBean(record, ele); + coreDatasetGroupMapper.updateById(record); + } + } + + public void mergeDatasetCrossDefault(DatasetGroupInfoDTO ele) { + Set allDs = getAllDs(ele.getUnion()); + boolean isCross = allDs.size() > 1; + ele.setIsCross(isCross); + } } diff --git a/core/core-backend/src/main/java/io/gisbi/dataset/manage/DatasetTableFieldManage.java b/core/core-backend/src/main/java/io/gisbi/dataset/manage/DatasetTableFieldManage.java index 6db1d9b..eec4e19 100644 --- a/core/core-backend/src/main/java/io/gisbi/dataset/manage/DatasetTableFieldManage.java +++ b/core/core-backend/src/main/java/io/gisbi/dataset/manage/DatasetTableFieldManage.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import io.gisbi.api.dataset.union.DatasetGroupInfoDTO; import io.gisbi.dataset.dao.auto.entity.CoreDatasetTableField; import io.gisbi.dataset.dao.auto.mapper.CoreDatasetTableFieldMapper; +import io.gisbi.dataset.utils.DatasetUtils; import io.gisbi.dataset.utils.TableUtils; import io.gisbi.engine.constant.ExtFieldConstant; import io.gisbi.engine.func.FunctionConstant; @@ -41,7 +42,8 @@ import java.util.stream.Collectors; public class DatasetTableFieldManage { @Resource private CoreDatasetTableFieldMapper coreDatasetTableFieldMapper; - + @Resource + private PermissionManage permissionManage; @Resource private DatasetSQLManage datasetSQLManage; @Resource @@ -86,10 +88,10 @@ public class DatasetTableFieldManage { checkNameLength(datasetTableFieldDTO.getName()); CoreDatasetTableField coreDatasetTableField = coreDatasetTableFieldMapper.selectById(datasetTableFieldDTO.getId()); CoreDatasetTableField record = transDTO2Record(datasetTableFieldDTO); - if (ObjectUtils.isEmpty(record.getDataeaseName())) { + if (ObjectUtils.isEmpty(record.getGisbiName())) { String n = TableUtils.fieldNameShort(record.getId() + ""); record.setFieldShortName(n); - record.setDataeaseName(n); + record.setgisbiName(n); } if (ObjectUtils.isEmpty(coreDatasetTableField)) { coreDatasetTableFieldMapper.insert(record); @@ -228,7 +230,7 @@ public class DatasetTableFieldManage { List list = transDTO(coreDatasetTableFieldMapper.selectList(wrapper)); Map desensitizationList = new HashMap<>(); -// list = permissionManage.filterColumnPermissions(list, desensitizationList, id, null); + list = permissionManage.filterColumnPermissions(list, desensitizationList, id, null); List dimensionList = list.stream().filter(ele -> StringUtils.equalsIgnoreCase(ele.getGroupType(), "d")).collect(Collectors.toList()); List quotaList = list.stream().filter(ele -> StringUtils.equalsIgnoreCase(ele.getGroupType(), "q")).collect(Collectors.toList()); @@ -240,43 +242,44 @@ public class DatasetTableFieldManage { public List listFieldsWithPermissions(Long id) { List fields = selectByDatasetGroupId(id); -// Map desensitizationList = new HashMap<>(); -// Long userId = AuthUtils.getUser() == null ? null : AuthUtils.getUser().getUserId(); -// List tmp = permissionManage -// .filterColumnPermissions(fields, desensitizationList, id, userId) -// .stream() -// .sorted(Comparator.comparing(DatasetTableFieldDTO::getGroupType)) -// .toList(); -// tmp.forEach(ele -> ele.setDesensitized(desensitizationList.containsKey(ele.getDataeaseName()))); - return fields; + Map desensitizationList = new HashMap<>(); + Long userId = AuthUtils.getUser() == null ? null : AuthUtils.getUser().getUserId(); + List tmp = permissionManage + .filterColumnPermissions(fields, desensitizationList, id, userId) + .stream() + .sorted(Comparator.comparing(DatasetTableFieldDTO::getGroupType)) + .toList(); + tmp.forEach(ele -> ele.setDesensitized(desensitizationList.containsKey(ele.getGisbiName()))); + return tmp; } public List listFieldsWithPermissionsRemoveAgg(Long id) { List fields = selectByDatasetGroupId(id); -// Map desensitizationList = new HashMap<>(); -// Long userId = AuthUtils.getUser() == null ? null : AuthUtils.getUser().getUserId(); -// SQLObj tableObj = new SQLObj(); -// tableObj.setTableAlias(""); -// List tmp = permissionManage -// .filterColumnPermissions(fields, desensitizationList, id, userId) -// .stream() -// .filter(ele -> { -// boolean flag = true; -// if (Objects.equals(ele.getExtField(), ExtFieldConstant.EXT_CALC)) { -// String originField = Utils.calcFieldRegex(ele, tableObj, fields, true, null, Utils.mergeParam(Utils.getParams(fields), null), pluginManage); -// for (String func : FunctionConstant.AGG_FUNC) { -// if (Utils.matchFunction(func, originField)) { -// flag = false; -// break; -// } -// } -// } -// return flag; -// }) -// .sorted(Comparator.comparing(DatasetTableFieldDTO::getGroupType)) -// .toList(); -// tmp.forEach(ele -> ele.setDesensitized(desensitizationList.containsKey(ele.getDataeaseName()))); - return fields; + Map desensitizationList = new HashMap<>(); + Long userId = AuthUtils.getUser() == null ? null : AuthUtils.getUser().getUserId(); + SQLObj tableObj = new SQLObj(); + tableObj.setTableAlias(""); + List tmp = permissionManage + .filterColumnPermissions(fields, desensitizationList, id, userId) + .stream() + .filter(ele -> { + boolean flag = true; + if (Objects.equals(ele.getExtField(), ExtFieldConstant.EXT_CALC)) { + String originField = Utils.calcFieldRegex(ele, tableObj, fields, true, null, Utils.mergeParam(Utils.getParams(fields), null), pluginManage); + for (String func : FunctionConstant.AGG_FUNC) { + if (Utils.matchFunction(func, originField)) { + flag = false; + break; + } + } + } + return flag; + }) + .sorted(Comparator.comparing(DatasetTableFieldDTO::getGroupType)) + .toList(); + tmp.forEach(ele -> ele.setDesensitized(desensitizationList.containsKey(ele.getGisbiName()))); + DatasetUtils.listEncode(tmp); + return tmp; } public DatasetTableFieldDTO transObj(CoreDatasetTableField ele) { diff --git a/core/core-backend/src/main/java/io/gisbi/dataset/manage/PermissionManage.java b/core/core-backend/src/main/java/io/gisbi/dataset/manage/PermissionManage.java index 862991b..43b9751 100644 --- a/core/core-backend/src/main/java/io/gisbi/dataset/manage/PermissionManage.java +++ b/core/core-backend/src/main/java/io/gisbi/dataset/manage/PermissionManage.java @@ -29,8 +29,10 @@ import java.util.stream.Collectors; @Service public class PermissionManage { + @Autowired(required = false) private RowPermissionsApi rowPermissionsApi; + @Autowired(required = false) private ColumnPermissionsApi columnPermissionsApi = null; @Resource @@ -71,14 +73,14 @@ public class PermissionManage { if (CollectionUtils.isNotEmpty(fieldUserColumnPermissionItems)) { if (fieldUserColumnPermissionItems.stream().map(ColumnPermissionItem::getOpt).collect(Collectors.toList()).contains(ColumnPermissionConstants.Desensitization)) { - desensitizationList.put(field.getDataeaseName(), fieldUserColumnPermissionItems.get(0)); + desensitizationList.put(field.getGisbiName(), fieldUserColumnPermissionItems.get(0)); result.add(field); } return; } if (CollectionUtils.isNotEmpty(fieldRoleColumnPermissionItems)) { if (fieldRoleColumnPermissionItems.stream().map(ColumnPermissionItem::getOpt).collect(Collectors.toList()).contains(ColumnPermissionConstants.Desensitization)) { - desensitizationList.put(field.getDataeaseName(), fieldRoleColumnPermissionItems.get(0)); + desensitizationList.put(field.getGisbiName(), fieldRoleColumnPermissionItems.get(0)); result.add(field); } return; diff --git a/core/core-backend/src/main/java/io/gisbi/dataset/server/DatasetDataServer.java b/core/core-backend/src/main/java/io/gisbi/dataset/server/DatasetDataServer.java index 0a4ba5e..be61649 100644 --- a/core/core-backend/src/main/java/io/gisbi/dataset/server/DatasetDataServer.java +++ b/core/core-backend/src/main/java/io/gisbi/dataset/server/DatasetDataServer.java @@ -1,21 +1,15 @@ package io.gisbi.dataset.server; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import io.gisbi.api.dataset.DatasetDataApi; -import io.gisbi.api.dataset.dto.BaseTreeNodeDTO; -import io.gisbi.api.dataset.dto.EnumValueRequest; -import io.gisbi.api.dataset.dto.MultFieldValuesRequest; -import io.gisbi.api.dataset.dto.PreviewSqlDTO; +import io.gisbi.api.dataset.dto.*; import io.gisbi.api.dataset.union.DatasetGroupInfoDTO; -import io.gisbi.api.dataset.dto.EnumObj; -import io.gisbi.dataset.dao.auto.entity.CoreDatasetTable; -import io.gisbi.dataset.dao.auto.entity.CoreDatasetTableField; import io.gisbi.dataset.manage.DatasetDataManage; import io.gisbi.extensions.datasource.dto.DatasetTableDTO; import io.gisbi.extensions.datasource.dto.DatasetTableFieldDTO; import io.gisbi.utils.LogUtil; import jakarta.annotation.Resource; -import org.springframework.web.bind.annotation.*; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; import java.util.List; import java.util.Map; @@ -31,10 +25,9 @@ public class DatasetDataServer implements DatasetDataApi { @Override public Map previewData(DatasetGroupInfoDTO datasetGroupInfoDTO) throws Exception { - return datasetDataManage.previewDataWithLimit(datasetGroupInfoDTO, 0, 100, false); + return datasetDataManage.previewDataWithLimit(datasetGroupInfoDTO, 0, 100, false, true); } - @Override public List tableField(DatasetTableDTO datasetTableDTO) throws Exception { return datasetDataManage.getTableFields(datasetTableDTO); @@ -103,62 +96,4 @@ public class DatasetDataServer implements DatasetDataApi { return null; } } - -// @GetMapping("getTablesByAppId") -// public List> getTablesByAppId(String appid) throws Exception { -// List> result = datasetDataManage.getTablesByAppId(appid); -// return result; -// } - - @GetMapping("getTablesByAppId") - public List> getTablesByAppId(String appid) throws Exception { - List> result = datasetDataManage.getTablesByAppId(appid); - return result; - } - @GetMapping("getFieldsByTableId") - public Map getFieldsByTableId(Long datasourceId,String tablename) throws Exception { - Map result = datasetDataManage.getFieldsByTableId(datasourceId,tablename); - return result; - } - @PostMapping("addTableData") - public boolean addTableData(Long datasourceId, @RequestParam("tableData") String tableData) throws Exception { - boolean result = datasetDataManage.addTableData( - datasourceId, - tableData - ); - return result; - } - - @PostMapping("getTableDataByPk") - public Map getTableDataByPk(Long datasourceId,@RequestParam("whereJson") String whereJson) throws Exception { - Map result=datasetDataManage.getTableDataByPk( - datasourceId, - whereJson - ); - return result; - } - @PostMapping("updateTableData") - public boolean updateTableData(Long datasourceId, @RequestParam("tableData") String tableData) throws Exception { - boolean result = datasetDataManage.updateTableData( - datasourceId, - tableData - ); - return result; - } - @PostMapping("deleteTableData") - public boolean deleteTableData(Long datasourceId, @RequestParam("whereJson") String whereJson) throws Exception { - boolean result = datasetDataManage.deleteTableData( - datasourceId, - whereJson - ); - return result; - } - @PostMapping("queryTableDataPaged") - public Page> queryTableDataPaged(Long datasourceId, @RequestParam("queryJson") String queryJson) throws Exception { - Page> result = datasetDataManage.queryTableDataPaged( - datasourceId, - queryJson - ); - return result; - } } diff --git a/core/core-backend/src/main/java/io/gisbi/dataset/server/DatasetFieldServer.java b/core/core-backend/src/main/java/io/gisbi/dataset/server/DatasetFieldServer.java index ae1d83d..847d56c 100644 --- a/core/core-backend/src/main/java/io/gisbi/dataset/server/DatasetFieldServer.java +++ b/core/core-backend/src/main/java/io/gisbi/dataset/server/DatasetFieldServer.java @@ -6,6 +6,7 @@ import io.gisbi.api.dataset.engine.SQLFunctionDTO; import io.gisbi.api.dataset.engine.SQLFunctionsEnum; import io.gisbi.dataset.manage.DatasetDataManage; import io.gisbi.dataset.manage.DatasetTableFieldManage; +import io.gisbi.dataset.utils.DatasetUtils; import io.gisbi.extensions.datasource.dto.DatasetTableFieldDTO; import jakarta.annotation.Resource; import org.springframework.web.bind.annotation.RequestBody; @@ -40,7 +41,9 @@ public class DatasetFieldServer implements DatasetTableApi { @Override public List listByDatasetGroup(Long id) { - return datasetTableFieldManage.selectByDatasetGroupId(id); + List datasetTableFieldDTOS = datasetTableFieldManage.selectByDatasetGroupId(id); + DatasetUtils.listEncode(datasetTableFieldDTOS); + return datasetTableFieldDTOS; } @Override diff --git a/core/core-backend/src/main/java/io/gisbi/dataset/server/DatasetTreeServer.java b/core/core-backend/src/main/java/io/gisbi/dataset/server/DatasetTreeServer.java index 8315044..a9ae123 100644 --- a/core/core-backend/src/main/java/io/gisbi/dataset/server/DatasetTreeServer.java +++ b/core/core-backend/src/main/java/io/gisbi/dataset/server/DatasetTreeServer.java @@ -1,5 +1,6 @@ package io.gisbi.dataset.server; + import io.gisbi.api.dataset.DatasetTreeApi; import io.gisbi.api.dataset.dto.DataSetExportRequest; import io.gisbi.api.dataset.dto.DatasetNodeDTO; @@ -8,17 +9,21 @@ import io.gisbi.api.dataset.vo.DataSetBarVO; import io.gisbi.constant.LogOT; import io.gisbi.constant.LogST; import io.gisbi.dataset.manage.DatasetGroupManage; +import io.gisbi.exportCenter.manage.ExportCenterDownLoadManage; import io.gisbi.exportCenter.manage.ExportCenterManage; import io.gisbi.extensions.datasource.dto.DatasetTableDTO; -import io.gisbi.extensions.view.dto.SqlVariableDetails; +import io.gisbi.extensions.view.dto.*; import io.gisbi.log.DeLog; import io.gisbi.model.BusiNodeRequest; import io.gisbi.model.BusiNodeVO; import jakarta.annotation.Resource; +import jakarta.servlet.http.HttpServletResponse; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import java.util.List; + +import java.util.*; + @RestController @RequestMapping("datasetTree") @@ -27,24 +32,26 @@ public class DatasetTreeServer implements DatasetTreeApi { private DatasetGroupManage datasetGroupManage; @Resource private ExportCenterManage exportCenterManage; + @Resource + private ExportCenterDownLoadManage exportCenterDownLoadManage; @DeLog(id = "#p0.id", ot = LogOT.MODIFY, st = LogST.DATASET) @Override public DatasetGroupInfoDTO save(DatasetGroupInfoDTO datasetNodeDTO) throws Exception { - return datasetGroupManage.save(datasetNodeDTO, false); + return datasetGroupManage.save(datasetNodeDTO, false, true); } @DeLog(id = "#p0.id", ot = LogOT.MODIFY, st = LogST.DATASET) @Override public DatasetNodeDTO rename(DatasetGroupInfoDTO dto) throws Exception { - return datasetGroupManage.save(dto, true); + return datasetGroupManage.save(dto, true, false); } @DeLog(id = "#p0.id", pid = "#p0.pid", ot = LogOT.CREATE, st = LogST.DATASET) @Override public DatasetNodeDTO create(DatasetGroupInfoDTO dto) throws Exception { - return datasetGroupManage.save(dto, false); + return datasetGroupManage.save(dto, false, true); } @DeLog(id = "#p0.id", ot = LogOT.MODIFY, st = LogST.DATASET) @@ -100,8 +107,12 @@ public class DatasetTreeServer implements DatasetTreeApi { } @Override - public void exportDataset(DataSetExportRequest request) throws Exception { - exportCenterManage.addTask(request.getId(), "dataset", request); + public void exportDataset(DataSetExportRequest request, HttpServletResponse response) throws Exception { + if (request.isGisbiBi()) { + exportCenterDownLoadManage.downloadDataset(request, response); + } else { + exportCenterManage.addTask(request.getId(), "dataset", request); + } } } diff --git a/core/core-backend/src/main/java/io/gisbi/dataset/utils/DatasetUtils.java b/core/core-backend/src/main/java/io/gisbi/dataset/utils/DatasetUtils.java index 763c3bd..1df7b11 100644 --- a/core/core-backend/src/main/java/io/gisbi/dataset/utils/DatasetUtils.java +++ b/core/core-backend/src/main/java/io/gisbi/dataset/utils/DatasetUtils.java @@ -1,15 +1,16 @@ package io.gisbi.dataset.utils; import io.gisbi.api.dataset.dto.BaseTreeNodeDTO; +import io.gisbi.api.dataset.union.DatasetGroupInfoDTO; +import io.gisbi.engine.constant.ExtFieldConstant; +import io.gisbi.extensions.datasource.dto.DatasetTableFieldDTO; +import io.gisbi.extensions.view.dto.ChartViewDTO; import io.gisbi.utils.TreeUtils; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.springframework.util.Assert; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.stream.Collectors; /** @@ -53,4 +54,94 @@ public class DatasetUtils { }); return result; } + + public static String getEncode(String str) { + return Base64.getEncoder().encodeToString(str.getBytes()); + } + + public static String getDecode(String str) { + return new String(Base64.getDecoder().decode(str)); + } + + /** + * 计算字段表达式base64加密 + * + * @param obj + */ + public static void dsEncode(DatasetGroupInfoDTO obj) { + for (DatasetTableFieldDTO dto : obj.getAllFields()) { + if (dto.getExtField().equals(ExtFieldConstant.EXT_CALC)) { + dto.setOriginName(getEncode(dto.getOriginName())); + } + } + } + + /** + * 计算字段表达式base64解密 + * + * @param obj + */ + public static void dsDecode(DatasetGroupInfoDTO obj) { + for (DatasetTableFieldDTO dto : obj.getAllFields()) { + if (dto.getExtField().equals(ExtFieldConstant.EXT_CALC)) { + dto.setOriginName(getDecode(dto.getOriginName())); + } + } + } + + /** + * 计算字段表达式base64加密 + * + * @param fields + */ + public static void listEncode(List fields) { + if (CollectionUtils.isEmpty(fields)) { + return; + } + for (DatasetTableFieldDTO dto : fields) { + if (dto.getExtField().equals(ExtFieldConstant.EXT_CALC)) { + dto.setOriginName(getEncode(dto.getOriginName())); + } + } + } + + /** + * 计算字段表达式base64解密 + * + * @param fields + */ + public static void listDecode(List fields) { + if (CollectionUtils.isEmpty(fields)) { + return; + } + for (DatasetTableFieldDTO dto : fields) { + if (dto.getExtField().equals(ExtFieldConstant.EXT_CALC)) { + dto.setOriginName(getDecode(dto.getOriginName())); + } + } + } + + public static void viewDecode(ChartViewDTO view) { + DatasetUtils.listDecode(view.getXAxis()); + DatasetUtils.listDecode(view.getXAxisExt()); + DatasetUtils.listDecode(view.getYAxis()); + DatasetUtils.listDecode(view.getYAxisExt()); + DatasetUtils.listDecode(view.getExtStack()); + DatasetUtils.listDecode(view.getExtBubble()); + DatasetUtils.listDecode(view.getExtLabel()); + DatasetUtils.listDecode(view.getExtTooltip()); + DatasetUtils.listDecode(view.getExtColor()); + } + + public static void viewEncode(ChartViewDTO view) { + DatasetUtils.listEncode(view.getXAxis()); + DatasetUtils.listEncode(view.getXAxisExt()); + DatasetUtils.listEncode(view.getYAxis()); + DatasetUtils.listEncode(view.getYAxisExt()); + DatasetUtils.listEncode(view.getExtStack()); + DatasetUtils.listEncode(view.getExtBubble()); + DatasetUtils.listEncode(view.getExtLabel()); + DatasetUtils.listEncode(view.getExtTooltip()); + DatasetUtils.listEncode(view.getExtColor()); + } } diff --git a/core/core-backend/src/main/java/io/gisbi/datasource/manage/DataSourceManage.java b/core/core-backend/src/main/java/io/gisbi/datasource/manage/DataSourceManage.java index f02ddfa..4c31c01 100644 --- a/core/core-backend/src/main/java/io/gisbi/datasource/manage/DataSourceManage.java +++ b/core/core-backend/src/main/java/io/gisbi/datasource/manage/DataSourceManage.java @@ -6,11 +6,14 @@ import io.gisbi.commons.constants.OptConstants; import io.gisbi.commons.constants.TaskStatus; import io.gisbi.constant.DataSourceType; import io.gisbi.datasource.dao.auto.entity.CoreDatasource; +import io.gisbi.datasource.dao.auto.mapper.CoreDatasourceMapper; import io.gisbi.datasource.dao.ext.mapper.CoreDatasourceExtMapper; +import io.gisbi.datasource.dao.ext.mapper.DataSourceExtMapper; import io.gisbi.datasource.dao.ext.po.DataSourceNodePO; import io.gisbi.datasource.dao.ext.po.DsItem; import io.gisbi.datasource.dto.DatasourceNodeBO; import io.gisbi.exception.DEException; +import io.gisbi.extensions.datasource.api.PluginManageApi; import io.gisbi.extensions.datasource.dto.DatasourceDTO; import io.gisbi.i18n.Translator; import io.gisbi.model.BusiNodeRequest; @@ -24,23 +27,34 @@ import jakarta.annotation.Resource; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import io.gisbi.datasource.dao.ext.mapper.DataSourceExtMapper; -import io.gisbi.datasource.dao.auto.mapper.CoreDatasourceMapper; + import java.util.ArrayList; import java.util.List; import java.util.Stack; @Component public class DataSourceManage { + @Resource private DataSourceExtMapper dataSourceExtMapper; + @Resource private CoreDatasourceMapper coreDatasourceMapper; - @Resource - private CoreDatasourceExtMapper coreDatasourceExtMapper; + @Resource private CoreOptRecentManage coreOptRecentManage; + + @Resource + private CoreDatasourceExtMapper coreDatasourceExtMapper; + + @Resource + private EngineManage engineManage; + + @Autowired(required = false) + private PluginManageApi pluginManage; + private DatasourceNodeBO rootNode(String appId) { return new DatasourceNodeBO(0L,appId, "root", false, 7, -1L, 0, "mysql"); } @@ -178,9 +192,9 @@ public class DataSourceManage { public CoreDatasource getCoreDatasource(Long id) { -// if (id == -1L) { -// return engineManage.getDeEngine(); -// } + if (id == -1L) { + return engineManage.getDeEngine(); + } return coreDatasourceMapper.selectById(id); } diff --git a/core/core-backend/src/main/java/io/gisbi/datasource/manage/DatasourceSyncManage.java b/core/core-backend/src/main/java/io/gisbi/datasource/manage/DatasourceSyncManage.java index ce9d1dc..5fcc58c 100644 --- a/core/core-backend/src/main/java/io/gisbi/datasource/manage/DatasourceSyncManage.java +++ b/core/core-backend/src/main/java/io/gisbi/datasource/manage/DatasourceSyncManage.java @@ -127,7 +127,11 @@ public class DatasourceSyncManage { record.setQrtzInstance(context.getFireInstanceId()); datasourceMapper.update(record, updateWrapper); } - extractedData(taskId, coreDatasource, updateType, coreDatasourceTask.getSyncRate()); + if (coreDatasource.getType().equalsIgnoreCase("ExcelRemote")) { + extractedExcelData(taskId, coreDatasource, updateType, coreDatasourceTask.getSyncRate()); + } else { + extractedData(taskId, coreDatasource, updateType, coreDatasourceTask.getSyncRate()); + } } catch (Exception e) { LogUtil.error(e); } finally { @@ -173,6 +177,46 @@ public class DatasourceSyncManage { } } + public void extractedExcelData(Long taskId, CoreDatasource coreDatasource, DatasourceServer.UpdateType updateType, String scheduleType) { + DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setDatasource(transDTO(coreDatasource)); + List tables = ExcelUtils.getTables(datasourceRequest); + for (DatasetTableDTO tableDTO : tables) { + CoreDatasourceTaskLog datasetTableTaskLog = datasourceTaskServer.initTaskLog(coreDatasource.getId(), taskId, tableDTO.getTableName(), scheduleType); + datasourceRequest.setTable(tableDTO.getTableName()); + ExcelUtils.getTableFields(datasourceRequest); + List tableFields = ExcelUtils.getTableFields(datasourceRequest); + try { + datasetTableTaskLog.setInfo(datasetTableTaskLog.getInfo() + "/n Begin to sync datatable: " + datasourceRequest.getTable()); + createEngineTable(datasourceRequest.getTable(), tableFields); + if (updateType.equals(DatasourceServer.UpdateType.all_scope)) { + createEngineTable(TableUtils.tmpName(datasourceRequest.getTable()), tableFields); + } + extractExcelData(datasourceRequest, updateType, tableFields); + if (updateType.equals(DatasourceServer.UpdateType.all_scope)) { + replaceTable(datasourceRequest.getTable()); + } + datasetTableTaskLog.setInfo(datasetTableTaskLog.getInfo() + "/n End to sync datatable: " + datasourceRequest.getTable()); + datasetTableTaskLog.setTaskStatus(TaskStatus.Completed.toString()); + datasetTableTaskLog.setEndTime(System.currentTimeMillis()); + } catch (Exception e) { + try { + if (updateType.equals(DatasourceServer.UpdateType.all_scope)) { + dropEngineTable(TableUtils.tmpName(datasourceRequest.getTable())); + } + } catch (Exception ignore) { + } + datasetTableTaskLog.setInfo(datasetTableTaskLog.getInfo() + "/n Failed to sync datatable: " + datasourceRequest.getTable() + ", " + e.getMessage()); + datasetTableTaskLog.setTaskStatus(TaskStatus.Error.toString()); + datasetTableTaskLog.setEndTime(System.currentTimeMillis()); + + e.printStackTrace(); + } finally { + datasourceTaskServer.saveLog(datasetTableTaskLog); + } + } + } + private void updateDsTaskStatus(Long datasourceId) { UpdateWrapper updateWrapper = new UpdateWrapper<>(); updateWrapper.eq("id", datasourceId); diff --git a/core/core-backend/src/main/java/io/gisbi/datasource/manage/EngineManage.java b/core/core-backend/src/main/java/io/gisbi/datasource/manage/EngineManage.java index ec4e2a4..e5f7774 100644 --- a/core/core-backend/src/main/java/io/gisbi/datasource/manage/EngineManage.java +++ b/core/core-backend/src/main/java/io/gisbi/datasource/manage/EngineManage.java @@ -12,6 +12,8 @@ import io.gisbi.extensions.datasource.dto.DatasourceDTO; import io.gisbi.extensions.datasource.dto.DatasourceRequest; import io.gisbi.extensions.datasource.factory.ProviderFactory; import io.gisbi.result.ResultMessage; +import io.gisbi.template.dao.auto.entity.DeTemplateVersion; +import io.gisbi.template.dao.auto.mapper.DeTemplateVersionMapper; import io.gisbi.utils.BeanUtils; import io.gisbi.utils.JsonUtil; import io.gisbi.utils.ModelUtils; @@ -23,6 +25,8 @@ import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; import org.springframework.util.CollectionUtils; +import java.time.LocalDateTime; +import java.time.temporal.ChronoUnit; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -40,9 +44,12 @@ public class EngineManage { @Resource private CoreDatasourceMapper datasourceMapper; - @Value("${gisbi.path.engine:jdbc:h2:/opt/gisbi/desktop_data;AUTO_SERVER=TRUE;AUTO_RECONNECT=TRUE;MODE=MySQL;CASE_INSENSITIVE_IDENTIFIERS=TRUE;DATABASE_TO_UPPER=FALSE}") + @Value("${gisbi.path.engine:jdbc:h2:/opt/gisbi2.0/desktop_data;AUTO_SERVER=TRUE;AUTO_RECONNECT=TRUE;MODE=MySQL;CASE_INSENSITIVE_IDENTIFIERS=TRUE;DATABASE_TO_UPPER=FALSE}") private String engineUrl; + @Resource + private DeTemplateVersionMapper deTemplateVersionMapper; + public CoreDeEngine info() throws DEException { List deEngines = deEngineMapper.selectList(null); @@ -163,7 +170,10 @@ public class EngineManage { QueryWrapper queryWrapper = new QueryWrapper<>(); queryWrapper.eq("id", 985188400292302848L); queryWrapper.ne("create_time", 1715053684176L); - if (!datasourceMapper.exists(queryWrapper) && !ModelUtils.isDesktop()) { + // 版本检查 + QueryWrapper queryVersionWrapper = new QueryWrapper<>(); + queryVersionWrapper.eq("version", "985188400292302848"); + if (!datasourceMapper.exists(queryWrapper) && !deTemplateVersionMapper.exists(queryVersionWrapper) && !ModelUtils.isDesktop()) { Pattern WITH_SQL_FRAGMENT = Pattern.compile("jdbc:mysql://(.*):(\\d+)/(.*)\\?(.*)"); Matcher matcher = WITH_SQL_FRAGMENT.matcher(env.getProperty("spring.datasource.url")); if (!matcher.find()) { @@ -191,6 +201,13 @@ public class EngineManage { initDatasource.setTaskStatus("WaitingForExecution"); datasourceMapper.deleteById(985188400292302848L); datasourceMapper.insert(initDatasource); + + DeTemplateVersion version = new DeTemplateVersion(); + version.setVersion("985188400292302848"); + version.setScript("Demo"); + version.setInstalledOn(LocalDateTime.now().truncatedTo(ChronoUnit.MINUTES)); + version.setSuccess(true); + deTemplateVersionMapper.insert(version); } } diff --git a/core/core-backend/src/main/java/io/gisbi/datasource/provider/ApiUtils.java b/core/core-backend/src/main/java/io/gisbi/datasource/provider/ApiUtils.java index 2d6e508..da8cd3b 100644 --- a/core/core-backend/src/main/java/io/gisbi/datasource/provider/ApiUtils.java +++ b/core/core-backend/src/main/java/io/gisbi/datasource/provider/ApiUtils.java @@ -5,7 +5,9 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.jayway.jsonpath.Configuration; import com.jayway.jsonpath.JsonPath; +import com.jayway.jsonpath.Option; import io.gisbi.extensions.datasource.dto.ApiDefinition; import io.gisbi.extensions.datasource.dto.ApiDefinitionRequest; import io.gisbi.exception.DEException; @@ -26,7 +28,9 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; public class ApiUtils { - + private static Configuration jsonPathConf = Configuration.builder() + .options(Option.DEFAULT_PATH_LEAF_TO_NULL, Option.ALWAYS_RETURN_LIST) + .build(); private static String path = "['%s']"; public static ObjectMapper objectMapper = CommonBeanFactory.getBean(ObjectMapper.class); @@ -35,32 +39,18 @@ public class ApiUtils { private static TypeReference>> listForMapTypeReference = new TypeReference>>() { }; - /** - * 从数据源配置中解析并获取API数据表信息列表 - * - * @param datasourceRequest 数据源请求对象，包含数据源配置信息 - * @return 数据集表DTO列表，包含表名称、显示名称及所属数据源ID - * @throws DEException 当数据处理过程中出现异常时抛出 - */ public static List getApiTables(DatasourceRequest datasourceRequest) throws DEException { List tableDescs = new ArrayList<>(); - // 定义API配置的反序列化类型引用 TypeReference> listTypeReference = new TypeReference>() { }; - - // 从数据源配置中解析API定义列表 List apiDefinitionList = JsonUtil.parseList(datasourceRequest.getDatasource().getConfiguration(), listTypeReference); - for (ApiDefinition apiDefinition : apiDefinitionList) { - // 过滤空对象和参数类型配置 if (apiDefinition == null) { continue; } if (StringUtils.isNotEmpty(apiDefinition.getType()) && apiDefinition.getType().equalsIgnoreCase("params")) { continue; } - - // 构建数据集表信息对象 DatasetTableDTO datasetTableDTO = new DatasetTableDTO(); datasetTableDTO.setTableName(apiDefinition.getDeTableName()); datasetTableDTO.setName(apiDefinition.getName()); @@ -70,25 +60,14 @@ public class ApiUtils { return tableDescs; } - - /** - * 将JSON配置字符串解析为表名映射关系 - * - * @param configration 输入的JSON配置字符串，应包含包含"name"和"deTableName"字段的数组对象 - * @return 返回键值对映射，key为配置中的name字段值，value为对应的deTableName字段值 - * @throws DEException 当JSON解析失败时抛出异常 - */ public static Map getTableNamesMap(String configration) throws DEException { Map result = new HashMap<>(); try { - // 解析JSON根节点并遍历数组元素 JsonNode rootNode = objectMapper.readTree(configration); for (int i = 0; i < rootNode.size(); i++) { - // 提取每个元素的name和deTableName字段构建映射关系 result.put(rootNode.get(i).get("name").asText(), rootNode.get(i).get("deTableName").asText()); } } catch (Exception e) { - // 统一将底层异常转换为领域异常抛出 DEException.throwException(e); } @@ -96,7 +75,6 @@ public class ApiUtils { } - public static Map fetchApiResultField(DatasourceRequest datasourceRequest) throws DEException { Map result = new HashMap<>(); List dataList = new ArrayList<>(); @@ -468,23 +446,34 @@ public class ApiUtils { return response; } - private static void previewNum(List> field) { - for (Map stringObjectMap : field) { + private static void previewNum(List> fields, String response) { + int previewNum = 100; + for (Map field : fields) { JSONArray newArray = new JSONArray(); - if (stringObjectMap.get("value") != null) { - try { - TypeReference listTypeReference = new TypeReference() { - }; - JSONArray array = objectMapper.readValue(stringObjectMap.get("value").toString(), listTypeReference); - if (array.size() > 100) { - for (int i = 0; i < Math.min(100, array.size()); i++) { - newArray.add(array.get(i)); + if (field.get("value") != null) { + Object object = JsonPath.using(jsonPathConf).parse(response).read(field.get("jsonPath").toString()); + int i = 0; + if (object instanceof List) { + for (Object o : (List) object) { + if (Objects.isNull(o)) { + newArray.add(""); + } else { + newArray.add(o.toString()); + } + i++; + if (i >= previewNum) { + break; } - stringObjectMap.put("value", newArray); } - } catch (Exception e) { - + } else { + if (object != null) { + newArray.add(object.toString()); + } } + field.put("value", newArray); + } else { + List> childrenFields = (List>) field.get("children"); + previewNum(childrenFields, response); } } } @@ -527,7 +516,7 @@ public class ApiUtils { rootPath = "$"; handleStr(apiDefinition, response, fields, rootPath); } - previewNum(fields); + previewNum(fields, response); apiDefinition.setJsonFields(fields); return apiDefinition; } else { @@ -574,7 +563,6 @@ public class ApiUtils { }; array = objectMapper.readValue(field.get("value").toString(), listTypeReference); } catch (Exception e) { - e.printStackTrace(); DEException.throwException(e); } array.add(Optional.ofNullable(data.get(field.get("originName"))).orElse("").toString().replaceAll("\n", " ").replaceAll("\r", " ")); @@ -745,6 +733,7 @@ public class ApiUtils { fieldChildren.add(itemChild); } } + field.put("children", fieldChildren); } } @@ -807,9 +796,15 @@ public class ApiUtils { List> columnDataList = new ArrayList<>(); for (int i = 0; i < jsonPaths.size(); i++) { List data = new ArrayList<>(); - Object object = JsonPath.read(result, jsonPaths.get(i)); - if (object instanceof List && jsonPaths.get(i).contains("[*]")) { - data = (List) object; + Object object = JsonPath.using(jsonPathConf).parse(result).read(jsonPaths.get(i)); + if (object instanceof List) { + for (Object o : (List) object) { + if (Objects.isNull(o)) { + data.add(""); + } else { + data.add(o.toString()); + } + } } else { if (object != null) { data.add(object.toString()); diff --git a/core/core-backend/src/main/java/io/gisbi/datasource/provider/CalciteProvider.java b/core/core-backend/src/main/java/io/gisbi/datasource/provider/CalciteProvider.java index a20b026..865eef7 100644 --- a/core/core-backend/src/main/java/io/gisbi/datasource/provider/CalciteProvider.java +++ b/core/core-backend/src/main/java/io/gisbi/datasource/provider/CalciteProvider.java @@ -2,6 +2,7 @@ package io.gisbi.datasource.provider; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.jcraft.jsch.Session; +import io.gisbi.constant.SQLConstants; import io.gisbi.dataset.utils.FieldUtils; import io.gisbi.datasource.dao.auto.entity.CoreDatasource; import io.gisbi.datasource.dao.auto.entity.CoreDriver; @@ -9,7 +10,6 @@ import io.gisbi.datasource.dao.auto.mapper.CoreDatasourceMapper; import io.gisbi.datasource.manage.EngineManage; import io.gisbi.datasource.request.EngineRequest; import io.gisbi.datasource.type.*; -import io.gisbi.constant.SQLConstants; import io.gisbi.exception.DEException; import io.gisbi.extensions.datasource.dto.*; import io.gisbi.extensions.datasource.provider.DriverShim; @@ -31,12 +31,15 @@ import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.parser.SqlParser; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.dbcp2.BasicDataSource; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import java.io.File; import java.io.IOException; +import java.io.Reader; +import java.io.StringReader; import java.math.BigDecimal; import java.net.URL; import java.sql.*; @@ -55,9 +58,9 @@ public class CalciteProvider extends Provider { private EngineManage engineManage; protected ExtendedJdbcClassLoader extendedJdbcClassLoader; private Map customJdbcClassLoaders = new HashMap<>(); - @Value("${gisbi.path.driver:/opt/gisbi/drivers}") + @Value("${gisbi.path.driver:/opt/gisbi2.0/drivers}") private String FILE_PATH; - @Value("${gisbi.path.custom-drivers:/opt/gisbi/custom-drivers/}") + @Value("${gisbi.path.custom-drivers:/opt/gisbi2.0/custom-drivers/}") private String CUSTOM_PATH; private static String split = "DE"; @@ -155,7 +158,7 @@ public class CalciteProvider extends Provider { @Override public Map fetchResultField(DatasourceRequest datasourceRequest) throws DEException { // 不跨数据源 - if (datasourceRequest.getDsList().size() == 1) { + if (datasourceRequest.getIsCross() == null || !datasourceRequest.getIsCross()) { return jdbcFetchResultField(datasourceRequest); } @@ -240,8 +243,65 @@ public class CalciteProvider extends Provider { return fieldList; } + private Map getTableTypeMap(DatasourceRequest datasourceRequest, DatasourceConfiguration datasourceConfiguration, String tableName) throws DEException { + Map map = new HashMap<>(); + String schemaTable = (ObjectUtils.isNotEmpty(datasourceConfiguration.getSchema()) ? (datasourceConfiguration.getSchema() + "`.`") : "") + tableName; + String sql = "SELECT * FROM `$TABLE_NAME$` LIMIT 0 OFFSET 0".replace("$TABLE_NAME$", schemaTable); + sql = transSqlDialect(sql, datasourceRequest.getDsList()); + ResultSet resultSet = null; + try (Connection con = getConnectionFromPool(datasourceRequest.getDatasource().getId()); Statement statement = getStatement(con, 30)) { + resultSet = statement.executeQuery(sql); + + ResultSetMetaData metaData = resultSet.getMetaData(); + int columnCount = metaData.getColumnCount(); + for (int j = 0; j < columnCount; j++) { + String name = StringUtils.lowerCase(metaData.getColumnName(j + 1)); + Integer type = metaData.getColumnType(j + 1); + map.put(name, type); + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (resultSet != null) { + try { + resultSet.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + } + return map; + } + @Override public List fetchTableField(DatasourceRequest datasourceRequest) throws DEException { + if (datasourceRequest.getIsCross() != null && datasourceRequest.getIsCross()) { + List datasetTableFields = new ArrayList<>(); + PreparedStatement statement = null; + ResultSet resultSet = null; + Connection connection = take(); + try { + CalciteConnection calciteConnection = connection.unwrap(CalciteConnection.class); + statement = calciteConnection.prepareStatement(datasourceRequest.getQuery()); + resultSet = statement.executeQuery(); + ResultSetMetaData metaData = resultSet.getMetaData(); + int columnCount = metaData.getColumnCount(); + for (int i = 1; i <= columnCount; i++) { + TableField tableField = new TableField(); + tableField.setOriginName(metaData.getColumnLabel(i)); + tableField.setType(metaData.getColumnTypeName(i)); + tableField.setPrecision(metaData.getPrecision(i)); + int deType = FieldUtils.transType2DeType(tableField.getType()); + tableField.setDeExtractType(deType); + tableField.setDeType(deType); + tableField.setScale(metaData.getScale(i)); + datasetTableFields.add(tableField); + } + } catch (Exception e) { + throw DEException.getException(e.getMessage()); + } + return datasetTableFields; + } List datasetTableFields = new ArrayList<>(); DatasourceSchemaDTO datasourceSchemaDTO = datasourceRequest.getDsList().entrySet().iterator().next().getValue(); datasourceRequest.setDatasource(datasourceSchemaDTO); @@ -281,8 +341,11 @@ public class CalciteProvider extends Provider { } else { resultSet = statement.executeQuery(getTableFiledSql(datasourceRequest)); } + + Map tableTypeMap = getTableTypeMap(datasourceRequest, datasourceConfiguration, table); + while (resultSet.next()) { - TableField tableFieldDesc = getTableFieldDesc(datasourceRequest, resultSet, 3); + TableField tableFieldDesc = getTableFieldDesc(datasourceRequest, resultSet, 3, tableTypeMap); boolean repeat = false; for (TableField ele : datasetTableFields) { if (StringUtils.equalsIgnoreCase(ele.getOriginName(), tableFieldDesc.getOriginName())) { @@ -423,16 +486,36 @@ public class CalciteProvider extends Provider { // schema ResultSet resultSet = null; - try (Connection con = getConnectionFromPool(datasourceRequest.getDatasource().getId()); Statement statement = getPreparedStatement(con, datasourceConfiguration.getQueryTimeout(), datasourceRequest.getQuery(), datasourceRequest.getTableFieldWithValues())) { - if (DatasourceConfiguration.DatasourceType.valueOf(value.getType()) == DatasourceConfiguration.DatasourceType.oracle) { - statement.executeUpdate("ALTER SESSION SET CURRENT_SCHEMA = " + datasourceConfiguration.getSchema()); - } + + try (Connection con = getConnectionFromPool(datasourceRequest.getDatasource().getId())) { + + Statement statement = getStatement(value, con, datasourceRequest, datasourceConfiguration, null); if (CollectionUtils.isNotEmpty(datasourceRequest.getTableFieldWithValues())) { LogUtil.info("execWithPreparedStatement sql: " + datasourceRequest.getQuery()); for (int i = 0; i < datasourceRequest.getTableFieldWithValues().size(); i++) { - ((PreparedStatement) statement).setObject(i + 1, datasourceRequest.getTableFieldWithValues().get(i).getValue(), datasourceRequest.getTableFieldWithValues().get(i).getType()); - LogUtil.info("execWithPreparedStatement param[" + (i + 1) + "]: " + datasourceRequest.getTableFieldWithValues().get(i).getValue()); + try { + Object valueObject = datasourceRequest.getTableFieldWithValues().get(i).getValue(); + + if (valueObject instanceof String + && DatasourceConfiguration.DatasourceType.valueOf(value.getType()) == DatasourceConfiguration.DatasourceType.oracle) { + if (StringUtils.isNotEmpty(datasourceConfiguration.getCharset()) && StringUtils.isNotEmpty(datasourceConfiguration.getTargetCharset())) { + //转换为数据库的字符集 + valueObject = new String(((String) valueObject).getBytes(datasourceConfiguration.getTargetCharset()), datasourceConfiguration.getCharset()); + } + if (datasourceRequest.getTableFieldWithValues().get(i).getType().equals(Types.CLOB)) { + Reader reader = new StringReader((String) valueObject); + ((PreparedStatement) statement).setCharacterStream(i + 1, reader, ((String) valueObject).length()); + } else { + ((PreparedStatement) statement).setObject(i + 1, valueObject, datasourceRequest.getTableFieldWithValues().get(i).getType()); + } + } else { + ((PreparedStatement) statement).setObject(i + 1, valueObject, datasourceRequest.getTableFieldWithValues().get(i).getType()); + } + LogUtil.info("execWithPreparedStatement param[" + (i + 1) + "](" + datasourceRequest.getTableFieldWithValues().get(i).getColumnTypeName() + "): " + datasourceRequest.getTableFieldWithValues().get(i).getValue()); + } catch (SQLException e) { + throw new SQLException(e.getMessage() + ". VALUE: " + datasourceRequest.getTableFieldWithValues().get(i).getValue().toString() + " , TARGET TYPE: " + datasourceRequest.getTableFieldWithValues().get(i).getColumnTypeName()); + } } resultSet = ((PreparedStatement) statement).executeQuery(); } else { @@ -466,15 +549,35 @@ public class CalciteProvider extends Provider { DatasourceConfiguration datasourceConfiguration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), DatasourceConfiguration.class); // schema ResultSet resultSet = null; - try (Connection con = getConnectionFromPool(datasourceRequest.getDatasource().getId()); Statement statement = getPreparedStatement(con, datasourceConfiguration.getQueryTimeout(), datasourceRequest.getQuery(), datasourceRequest.getTableFieldWithValues())) { - if (DatasourceConfiguration.DatasourceType.valueOf(value.getType()) == DatasourceConfiguration.DatasourceType.oracle) { - statement.executeUpdate("ALTER SESSION SET CURRENT_SCHEMA = " + datasourceConfiguration.getSchema()); - } + + try (Connection con = getConnectionFromPool(datasourceRequest.getDatasource().getId())) { + + Statement statement = getStatement(value, con, datasourceRequest, datasourceConfiguration, null); + if (CollectionUtils.isNotEmpty(datasourceRequest.getTableFieldWithValues())) { LogUtil.info("execWithPreparedStatement sql: " + datasourceRequest.getQuery()); for (int i = 0; i < datasourceRequest.getTableFieldWithValues().size(); i++) { - ((PreparedStatement) statement).setObject(i + 1, datasourceRequest.getTableFieldWithValues().get(i).getValue(), datasourceRequest.getTableFieldWithValues().get(i).getType()); - LogUtil.info("execWithPreparedStatement param[" + (i + 1) + "]: " + datasourceRequest.getTableFieldWithValues().get(i).getValue()); + try { + Object valueObject = datasourceRequest.getTableFieldWithValues().get(i).getValue(); + if (valueObject instanceof String + && DatasourceConfiguration.DatasourceType.valueOf(value.getType()) == DatasourceConfiguration.DatasourceType.oracle) { + if (StringUtils.isNotEmpty(datasourceConfiguration.getCharset()) && StringUtils.isNotEmpty(datasourceConfiguration.getTargetCharset())) { + //转换为数据库的字符集 + valueObject = new String(((String) valueObject).getBytes(datasourceConfiguration.getTargetCharset()), datasourceConfiguration.getCharset()); + } + if (datasourceRequest.getTableFieldWithValues().get(i).getType().equals(Types.CLOB)) { + Reader reader = new StringReader((String) valueObject); + ((PreparedStatement) statement).setCharacterStream(i + 1, reader, ((String) valueObject).length()); + } else { + ((PreparedStatement) statement).setObject(i + 1, valueObject, datasourceRequest.getTableFieldWithValues().get(i).getType()); + } + } else { + ((PreparedStatement) statement).setObject(i + 1, valueObject, datasourceRequest.getTableFieldWithValues().get(i).getType()); + } + LogUtil.info("execWithPreparedStatement param[" + (i + 1) + "](" + datasourceRequest.getTableFieldWithValues().get(i).getColumnTypeName() + "): " + datasourceRequest.getTableFieldWithValues().get(i).getValue()); + } catch (SQLException e) { + throw new SQLException(e.getMessage() + ". VALUE: " + datasourceRequest.getTableFieldWithValues().get(i).getValue().toString() + " , TARGET TYPE: " + datasourceRequest.getTableFieldWithValues().get(i).getColumnTypeName()); + } } ((PreparedStatement) statement).execute(); } else { @@ -496,29 +599,80 @@ public class CalciteProvider extends Provider { } } + /** + * 针对Oracle特殊处理 + */ + private Statement getStatement(DatasourceSchemaDTO value, Connection con, DatasourceRequest datasourceRequest, DatasourceConfiguration datasourceConfiguration, String autoIncrementPkName) throws Exception { + Statement statement; + if (DatasourceConfiguration.DatasourceType.valueOf(value.getType()) == DatasourceConfiguration.DatasourceType.oracle) { + statement = getStatement(con, datasourceConfiguration.getQueryTimeout()); + statement.executeUpdate("ALTER SESSION SET CURRENT_SCHEMA = " + datasourceConfiguration.getSchema()); + statement.executeUpdate("ALTER SESSION SET NLS_TIMESTAMP_FORMAT = 'YYYY-MM-DD HH24:MI:SS'"); + //调整字符集 + if (StringUtils.isNotEmpty(datasourceConfiguration.getCharset()) && StringUtils.isNotEmpty(datasourceConfiguration.getTargetCharset())) { + datasourceRequest.setQuery(new String(datasourceRequest.getQuery().getBytes(datasourceConfiguration.getTargetCharset()), datasourceConfiguration.getCharset())); + } + } + statement = getPreparedStatement(con, datasourceConfiguration.getQueryTimeout(), datasourceRequest.getQuery(), datasourceRequest.getTableFieldWithValues(), autoIncrementPkName, datasourceConfiguration); + return statement; + } + @Override - public int executeUpdate(DatasourceRequest datasourceRequest) throws DEException { + public ExecuteResult executeUpdate(DatasourceRequest datasourceRequest, String autoIncrementPkName) throws DEException { DatasourceSchemaDTO value = datasourceRequest.getDsList().entrySet().iterator().next().getValue(); datasourceRequest.setDatasource(value); DatasourceConfiguration datasourceConfiguration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), DatasourceConfiguration.class); // schema ResultSet resultSet = null; - try (Connection con = getConnectionFromPool(datasourceRequest.getDatasource().getId()); Statement statement = getPreparedStatement(con, datasourceConfiguration.getQueryTimeout(), datasourceRequest.getQuery(), datasourceRequest.getTableFieldWithValues())) { - if (DatasourceConfiguration.DatasourceType.valueOf(value.getType()) == DatasourceConfiguration.DatasourceType.oracle) { - statement.executeUpdate("ALTER SESSION SET CURRENT_SCHEMA = " + datasourceConfiguration.getSchema()); - } + try (Connection con = getConnectionFromPool(datasourceRequest.getDatasource().getId())) { + Statement statement = getStatement(value, con, datasourceRequest, datasourceConfiguration, autoIncrementPkName); + + int count = 0; if (CollectionUtils.isNotEmpty(datasourceRequest.getTableFieldWithValues())) { LogUtil.info("execWithPreparedStatement sql: " + datasourceRequest.getQuery()); for (int i = 0; i < datasourceRequest.getTableFieldWithValues().size(); i++) { - ((PreparedStatement) statement).setObject(i + 1, datasourceRequest.getTableFieldWithValues().get(i).getValue(), datasourceRequest.getTableFieldWithValues().get(i).getType()); - LogUtil.info("execWithPreparedStatement param[" + (i + 1) + "]: " + datasourceRequest.getTableFieldWithValues().get(i).getValue()); + try { + Object valueObject = datasourceRequest.getTableFieldWithValues().get(i).getValue(); + + if (valueObject instanceof String + && DatasourceConfiguration.DatasourceType.valueOf(value.getType()) == DatasourceConfiguration.DatasourceType.oracle) { + if (StringUtils.isNotEmpty(datasourceConfiguration.getCharset()) && StringUtils.isNotEmpty(datasourceConfiguration.getTargetCharset())) { + //转换为数据库的字符集 + valueObject = new String(((String) valueObject).getBytes(datasourceConfiguration.getTargetCharset()), datasourceConfiguration.getCharset()); + } + if (datasourceRequest.getTableFieldWithValues().get(i).getType().equals(Types.CLOB)) { + Reader reader = new StringReader((String) valueObject); + ((PreparedStatement) statement).setCharacterStream(i + 1, reader, ((String) valueObject).length()); + } else { + ((PreparedStatement) statement).setObject(i + 1, valueObject, datasourceRequest.getTableFieldWithValues().get(i).getType()); + } + } else { + ((PreparedStatement) statement).setObject(i + 1, valueObject, datasourceRequest.getTableFieldWithValues().get(i).getType()); + } + LogUtil.info("execWithPreparedStatement param[" + (i + 1) + "](" + datasourceRequest.getTableFieldWithValues().get(i).getColumnTypeName() + "): " + datasourceRequest.getTableFieldWithValues().get(i).getValue()); + } catch (SQLException e) { + throw new SQLException(e.getMessage() + ". VALUE: " + datasourceRequest.getTableFieldWithValues().get(i).getValue().toString() + " , TARGET TYPE: " + datasourceRequest.getTableFieldWithValues().get(i).getColumnTypeName()); + } } - return ((PreparedStatement) statement).executeUpdate(); + count = ((PreparedStatement) statement).executeUpdate(); } else { - return statement.executeUpdate(datasourceRequest.getQuery()); + count = statement.executeUpdate(datasourceRequest.getQuery()); } + ExecuteResult result = new ExecuteResult(); + result.setCount(count); + + if (StringUtils.isNotBlank(autoIncrementPkName)) { + List generatedKeys = new ArrayList<>(); + ResultSet keys = statement.getGeneratedKeys(); + while (keys.next()) { + generatedKeys.add(keys.getObject(1).toString()); + } + result.setGeneratedKeys(generatedKeys); + } + + return result; } catch (SQLException e) { DEException.throwException("SQL ERROR: " + e.getMessage()); } catch (Exception e) { @@ -533,7 +687,7 @@ public class CalciteProvider extends Provider { } } - return 0; + return new ExecuteResult(); } private List getField(ResultSet rs, DatasourceRequest datasourceRequest) throws Exception { @@ -542,7 +696,7 @@ public class CalciteProvider extends Provider { int columnCount = metaData.getColumnCount(); for (int j = 0; j < columnCount; j++) { String f = metaData.getColumnName(j + 1); - if (StringUtils.equalsIgnoreCase(f, "DE_ROWNUM")) { + if (StringUtils.containsIgnoreCase(f, "ROWNUM")) { continue; } String l = StringUtils.isNotEmpty(metaData.getColumnLabel(j + 1)) ? metaData.getColumnLabel(j + 1) : f; @@ -558,15 +712,15 @@ public class CalciteProvider extends Provider { } private List getData(ResultSet rs, DatasourceRequest datasourceRequest) throws Exception { - String charset = null; - String targetCharset = "UTF-8"; + String targetCharset = null; + String originCharset = null; if (datasourceRequest != null && datasourceRequest.getDatasource().getType().equalsIgnoreCase("oracle")) { DatasourceConfiguration jdbcConfiguration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), DatasourceConfiguration.class); - if (StringUtils.isNotEmpty(jdbcConfiguration.getCharset()) && !jdbcConfiguration.getCharset().equalsIgnoreCase("Default")) { - charset = jdbcConfiguration.getCharset(); + if (StringUtils.isNotEmpty(jdbcConfiguration.getCharset())) { + originCharset = jdbcConfiguration.getCharset(); } - if (StringUtils.isNotEmpty(jdbcConfiguration.getTargetCharset()) && !jdbcConfiguration.getTargetCharset().equalsIgnoreCase("Default")) { + if (StringUtils.isNotEmpty(jdbcConfiguration.getTargetCharset())) { targetCharset = jdbcConfiguration.getTargetCharset(); } } @@ -583,6 +737,11 @@ public class CalciteProvider extends Provider { row[j] = rs.getDate(j + 1).toString(); } break; + case Types.TIMESTAMP: + if (rs.getTimestamp(j + 1) != null) { + row[j] = rs.getTimestamp(j + 1).toString(); + } + break; case Types.BOOLEAN: row[j] = rs.getBoolean(j + 1) ? "1" : "0"; break; @@ -593,13 +752,17 @@ public class CalciteProvider extends Provider { default: if (metaData.getColumnTypeName(j + 1).toLowerCase().equalsIgnoreCase("blob")) { row[j] = rs.getBlob(j + 1) == null ? "" : rs.getBlob(j + 1).toString(); - } else { - if (charset != null && StringUtils.isNotEmpty(rs.getString(j + 1))) { - String originStr = new String(rs.getString(j + 1).getBytes(charset), targetCharset); - row[j] = new String(originStr.getBytes("UTF-8"), "UTF-8"); + } + if (targetCharset != null && StringUtils.isNotEmpty(rs.getString(j + 1)) && columnType == Types.CLOB) { + if (originCharset == null) { + row[j] = new String(rs.getString(j + 1).getBytes(), targetCharset); } else { - row[j] = rs.getString(j + 1); + row[j] = new String(rs.getString(j + 1).getBytes(originCharset), targetCharset); } + } else if (targetCharset != null && StringUtils.isNotEmpty(rs.getString(j + 1)) && (columnType != Types.NVARCHAR && columnType != Types.NCHAR)) { + row[j] = new String(rs.getBytes(j + 1), targetCharset); + } else { + row[j] = rs.getString(j + 1); } break; @@ -705,7 +868,7 @@ public class CalciteProvider extends Provider { } } - private TableField getTableFieldDesc(DatasourceRequest datasourceRequest, ResultSet resultSet, int commentIndex) throws SQLException { + private TableField getTableFieldDesc(DatasourceRequest datasourceRequest, ResultSet resultSet, int commentIndex, Map tableTypeMap) throws SQLException { TableField tableField = new TableField(); tableField.setOriginName(resultSet.getString(1)); tableField.setType(resultSet.getString(2).toUpperCase()); @@ -718,6 +881,20 @@ public class CalciteProvider extends Provider { tableField.setPrimary(resultSet.getInt(4) > 0); } catch (Exception e) { } + try { + if (StringUtils.endsWithIgnoreCase(datasourceRequest.getDatasource().getType(), "oracle")) { + if (StringUtils.contains(resultSet.getString(5), "nextval") || StringUtils.equalsIgnoreCase(resultSet.getString(5), "GENERATED ALWAYS AS IDENTITY")) { + tableField.setAutoIncrement(true); + } + } else { + tableField.setAutoIncrement(resultSet.getInt(5) > 0); + } + } catch (Exception e) { + } + try { + tableField.setTypeNumber(tableTypeMap.get(StringUtils.lowerCase(tableField.getOriginName()))); + } catch (Exception e) { + } return tableField; } @@ -770,6 +947,13 @@ public class CalciteProvider extends Provider { commonThreadPool.addTask(() -> { try { BasicDataSource dataSource = new BasicDataSource(); + dataSource.setMaxWaitMillis(5 * 1000); + dataSource.setTestWhileIdle(true); + dataSource.setTestOnBorrow(true); + dataSource.setTestOnReturn(true); + dataSource.setTimeBetweenEvictionRunsMillis(60 * 1000); + dataSource.setValidationQuery("select 1"); + dataSource.setValidationQueryTimeout(5); Schema schema = null; DatasourceConfiguration configuration = null; DatasourceConfiguration.DatasourceType datasourceType = DatasourceConfiguration.DatasourceType.valueOf(ds.getType()); @@ -838,6 +1022,7 @@ public class CalciteProvider extends Provider { rootSchema.add(ds.getSchemaAlias(), schema); break; case oracle: + dataSource.setValidationQuery("SELECT 1 FROM DUAL"); configuration = JsonUtil.parseObject(ds.getConfiguration(), Oracle.class); if (StringUtils.isNotBlank(configuration.getUsername())) { dataSource.setUsername(configuration.getUsername()); @@ -856,6 +1041,7 @@ public class CalciteProvider extends Provider { break; case db2: configuration = JsonUtil.parseObject(ds.getConfiguration(), Db2.class); + dataSource.setValidationQuery("select 1 from syscat.tables WHERE TABSCHEMA ='DE_SCHEMA' AND \"TYPE\" = 'T'".replace("DE_SCHEMA", configuration.getSchema())); if (StringUtils.isNotBlank(configuration.getUsername())) { dataSource.setUsername(configuration.getUsername()); } @@ -1022,7 +1208,7 @@ public class CalciteProvider extends Provider { if (database.contains(".")) { sql = "select * from " + datasourceRequest.getTable() + " limit 0 offset 0 "; } else { - sql = String.format("SELECT COLUMN_NAME,DATA_TYPE,COLUMN_COMMENT FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s'", database, datasourceRequest.getTable()); + sql = String.format("SELECT COLUMN_NAME,DATA_TYPE,COLUMN_COMMENT,IF(COLUMN_KEY='PRI',1,0),IF(EXTRA LIKE '%%auto_increment%%',1,0) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s'", database, datasourceRequest.getTable()); } break; case mysql: @@ -1039,21 +1225,49 @@ public class CalciteProvider extends Provider { String[] databasePrams = matcher.group(3).split("\\?"); database = databasePrams[0]; } - sql = String.format("SELECT COLUMN_NAME,DATA_TYPE,COLUMN_COMMENT,IF(COLUMN_KEY='PRI',1,0) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s'", database, datasourceRequest.getTable()); + sql = String.format("SELECT COLUMN_NAME,DATA_TYPE,COLUMN_COMMENT,IF(COLUMN_KEY='PRI',1,0),IF(EXTRA LIKE '%%auto_increment%%',1,0) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s'", database, datasourceRequest.getTable()); break; case oracle: configuration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), Oracle.class); if (StringUtils.isEmpty(configuration.getSchema())) { DEException.throwException(Translator.get("i18n_schema_is_empty")); } - sql = String.format("SELECT a.COLUMN_NAME , a.DATA_TYPE , b.COMMENTS ,0 FROM all_tab_columns a LEFT JOIN all_col_comments b ON a.owner = b.owner AND a.table_name = b.table_name AND a.column_name = b.column_name WHERE a.owner = '%s' AND a.table_name = '%s' ORDER BY a.table_name, a.column_id", configuration.getSchema(), datasourceRequest.getTable()); + sql = String.format(""" + SELECT tc.COLUMN_NAME AS ColumnName, + tc.DATA_TYPE, + cc.COMMENTS, + CASE + WHEN ac.COLUMN_NAME IS NOT NULL THEN 1 + ELSE 0 + END, + tc.DATA_DEFAULT + FROM ALL_TAB_COLUMNS tc + LEFT JOIN (SELECT cols.OWNER, + cols.TABLE_NAME, + cols.COLUMN_NAME + FROM ALL_CONSTRAINTS cons + JOIN + ALL_CONS_COLUMNS cols + ON cons.OWNER = cols.OWNER + AND cons.CONSTRAINT_NAME = cols.CONSTRAINT_NAME + WHERE cons.TABLE_NAME = '%s' + AND cons.CONSTRAINT_TYPE = 'P') ac + ON tc.OWNER = ac.OWNER + AND tc.TABLE_NAME = ac.TABLE_NAME + AND tc.COLUMN_NAME = ac.COLUMN_NAME + LEFT JOIN ALL_COL_COMMENTS cc + ON tc.owner = cc.owner AND tc.table_name = cc.table_name AND tc.column_name = cc.column_name + WHERE tc.TABLE_NAME = '%s' + AND tc.OWNER = '%s' + ORDER BY tc.TABLE_NAME, tc.COLUMN_ID + """, datasourceRequest.getTable(), datasourceRequest.getTable(), configuration.getSchema()); break; case db2: configuration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), Db2.class); if (StringUtils.isEmpty(configuration.getSchema())) { DEException.throwException(Translator.get("i18n_schema_is_empty")); } - sql = String.format("SELECT COLNAME , TYPENAME , REMARKS FROM SYSCAT.COLUMNS WHERE TABSCHEMA = '%s' AND TABNAME = '%s' ", configuration.getSchema(), datasourceRequest.getTable()); + sql = String.format("SELECT COLNAME, TYPENAME, REMARKS, 0, 0 FROM SYSCAT.COLUMNS WHERE TABSCHEMA = '%s' AND TABNAME = '%s' ", configuration.getSchema(), datasourceRequest.getTable()); break; case sqlServer: configuration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), Sqlserver.class); @@ -1061,38 +1275,46 @@ public class CalciteProvider extends Provider { DEException.throwException(Translator.get("i18n_schema_is_empty")); } - sql = String.format("SELECT \n" + " c.name ,t.name ,ep.value, 0 \n" + "FROM \n" + " sys.columns AS c\n" + "LEFT JOIN sys.extended_properties AS ep ON c.object_id = ep.major_id AND c.column_id = ep.minor_id\n" + "LEFT JOIN sys.types AS t ON c.user_type_id = t.user_type_id\n" + "LEFT JOIN sys.objects AS o ON c.object_id = o.object_id\n" + "WHERE o.name = '%s'", datasourceRequest.getTable()); + sql = String.format("SELECT \n" + " c.name ,t.name ,ep.value, 0, 0 \n" + "FROM \n" + " sys.columns AS c\n" + "LEFT JOIN sys.extended_properties AS ep ON c.object_id = ep.major_id AND c.column_id = ep.minor_id\n" + "LEFT JOIN sys.types AS t ON c.user_type_id = t.user_type_id\n" + "LEFT JOIN sys.objects AS o ON c.object_id = o.object_id\n" + "WHERE o.name = '%s'", datasourceRequest.getTable()); break; case pg: configuration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), Pg.class); if (StringUtils.isEmpty(configuration.getSchema())) { DEException.throwException(Translator.get("i18n_schema_is_empty")); } - sql = String.format("SELECT\n" + - " a.attname AS ColumnName,\n" + - " t.typname,\n" + - " b.description AS ColumnDescription,\n" + - " CASE\n" + - " WHEN d.indisprimary THEN 1\n" + - " ELSE 0\n" + - " END\n" + - "FROM\n" + - " pg_class c\n" + - " JOIN pg_attribute a ON a.attrelid = c.oid\n" + - " LEFT JOIN pg_description b ON a.attrelid = b.objoid AND a.attnum = b.objsubid\n" + - " JOIN pg_type t ON a.atttypid = t.oid\n" + - " LEFT JOIN pg_index d ON d.indrelid = a.attrelid AND d.indisprimary AND a.attnum = ANY(d.indkey)\n" + - "where\n" + - " c.relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = '%s')\n" + - " AND c.relname = '%s'\n" + - " AND a.attnum > 0\n" + - " AND NOT a.attisdropped\n" + - "ORDER BY\n" + - " a.attnum;", configuration.getSchema(), datasourceRequest.getTable()); + sql = String.format(""" + SELECT a.attname AS ColumnName, + t.typname, + b.description AS ColumnDescription, + CASE + WHEN d.indisprimary THEN 1 + ELSE 0 + END, + CASE + WHEN pg_get_expr(ad.adbin, ad.adrelid) LIKE 'nextval%%' THEN 1 + """ + ( + datasourceRequest.getDsVersion() > 9 ? """ + WHEN a.attidentity = 'd' THEN 1 + WHEN a.attidentity = 'a' THEN 1 + """ : "") + """ + ELSE 0 + END + FROM pg_class c + JOIN pg_attribute a ON a.attrelid = c.oid + LEFT JOIN pg_attrdef ad ON a.attrelid = ad.adrelid AND a.attnum = ad.adnum + LEFT JOIN pg_description b ON a.attrelid = b.objoid AND a.attnum = b.objsubid + JOIN pg_type t ON a.atttypid = t.oid + LEFT JOIN pg_index d ON d.indrelid = a.attrelid AND d.indisprimary AND a.attnum = ANY (d.indkey) + where c.relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = '%s') + AND c.relname = '%s' + AND a.attnum > 0 + AND NOT a.attisdropped + ORDER BY a.attnum; + """, configuration.getSchema(), datasourceRequest.getTable()); break; case redshift: configuration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), CK.class); - sql = String.format("SELECT\n" + " a.attname AS ColumnName,\n" + " t.typname,\n" + " b.description AS ColumnDescription,\n" + " 0\n" + "FROM\n" + " pg_class c\n" + " JOIN pg_attribute a ON a.attrelid = c.oid\n" + " LEFT JOIN pg_description b ON a.attrelid = b.objoid AND a.attnum = b.objsubid\n" + " JOIN pg_type t ON a.atttypid = t.oid\n" + "WHERE\n" + " c.relname = '%s'\n" + " AND a.attnum > 0\n" + " AND NOT a.attisdropped\n" + "ORDER BY\n" + " a.attnum\n" + " ", datasourceRequest.getTable()); + sql = String.format("SELECT\n" + " a.attname AS ColumnName,\n" + " t.typname,\n" + " b.description AS ColumnDescription,\n" + " 0, 0\n" + "FROM\n" + " pg_class c\n" + " JOIN pg_attribute a ON a.attrelid = c.oid\n" + " LEFT JOIN pg_description b ON a.attrelid = b.objoid AND a.attnum = b.objsubid\n" + " JOIN pg_type t ON a.atttypid = t.oid\n" + "WHERE\n" + " c.relname = '%s'\n" + " AND a.attnum > 0\n" + " AND NOT a.attisdropped\n" + "ORDER BY\n" + " a.attnum\n" + " ", datasourceRequest.getTable()); break; case ck: configuration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), CK.class); @@ -1106,13 +1328,13 @@ public class CalciteProvider extends Provider { String[] databasePrams = matcher.group(3).split("\\?"); database = databasePrams[0]; } - sql = String.format(" SELECT\n" + " name,\n" + " type,\n" + " comment,\n" + " 0\n" + "FROM\n" + " system.columns\n" + "WHERE\n" + " database = '%s' \n" + " AND table = '%s' ", database, datasourceRequest.getTable()); + sql = String.format(" SELECT\n" + " name,\n" + " type,\n" + " comment,\n" + " 0, 0\n" + "FROM\n" + " system.columns\n" + "WHERE\n" + " database = '%s' \n" + " AND table = '%s' ", database, datasourceRequest.getTable()); break; case impala: sql = String.format("DESCRIBE `%s`", datasourceRequest.getTable()); break; case h2: - sql = String.format("SELECT COLUMN_NAME, DATA_TYPE, REMARKS FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = '%s'", datasourceRequest.getTable()); + sql = String.format("SELECT COLUMN_NAME, DATA_TYPE, REMARKS, 0, 0 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = '%s'", datasourceRequest.getTable()); break; default: break; @@ -1170,6 +1392,18 @@ public class CalciteProvider extends Provider { } tableSqls.add("select table_name, comments, owner from all_tab_comments where owner='" + configuration.getSchema() + "' AND table_type = 'TABLE'"); tableSqls.add("select table_name, comments, owner from all_tab_comments where owner='" + configuration.getSchema() + "' AND table_type = 'VIEW'"); + tableSqls.add("SELECT \n" + + " m.mview_name,\n" + + " c.comments\n" + + "FROM \n" + + " ALL_MVIEWS m\n" + + "LEFT JOIN \n" + + " ALL_TAB_COMMENTS c \n" + + "ON \n" + + " m.owner = c.owner \n" + + " AND m.mview_name = c.table_name\n" + + " AND c.table_type = 'MATERIALIZED VIEW'\n" + + "WHERE m.OWNER ='DE_SCHEMA'".replace("DE_SCHEMA", configuration.getSchema())); break; case db2: configuration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), Db2.class); @@ -1192,6 +1426,29 @@ public class CalciteProvider extends Provider { DEException.throwException(Translator.get("i18n_schema_is_empty")); } tableSqls.add("SELECT \n" + " relname AS TableName, \n" + " obj_description(relfilenode::regclass, 'pg_class') AS TableDescription \n" + "FROM \n" + " pg_class \n" + "WHERE \n" + " relkind in ('r','p', 'f') \n" + " AND relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'SCHEMA') ".replace("SCHEMA", configuration.getSchema())); + tableSqls.add("SELECT \n" + + " c.relname AS view_name,\n" + + " COALESCE(d.description, 'No description provided') AS view_description\n" + + "FROM \n" + + " pg_class c\n" + + "JOIN \n" + + " pg_namespace n ON c.relnamespace = n.oid\n" + + "LEFT JOIN \n" + + " pg_description d ON c.oid = d.objoid\n" + + "WHERE \n" + + " c.relkind = 'v' \n" + + " AND n.nspname = 'SCHEMA'".replace("SCHEMA", configuration.getSchema())); + tableSqls.add("SELECT \n" + + " c.relname AS materialized_view_name,\n" + + " COALESCE(d.description, '') AS view_description\n" + + "FROM \n" + + " pg_class c\n" + + "JOIN \n" + + " pg_namespace n ON c.relnamespace = n.oid\n" + + "LEFT JOIN \n" + + " pg_description d ON c.oid = d.objoid\n" + + "WHERE \n" + + " c.relkind = 'm' and n.nspname ='SCHEMA'; ".replace("SCHEMA", configuration.getSchema())); break; case redshift: configuration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), CK.class); @@ -1257,13 +1514,23 @@ public class CalciteProvider extends Provider { } public Statement getPreparedStatement(Connection connection, int queryTimeout, String sql, List values) throws Exception { + return getPreparedStatement(connection, queryTimeout, sql, values, null, null); + } + + public Statement getPreparedStatement(Connection connection, int queryTimeout, String sql, List values, String autoIncrementPkName, DatasourceConfiguration datasourceConfiguration) throws Exception { if (connection == null) { throw new Exception("Failed to get connection!"); } if (CollectionUtils.isNotEmpty(values)) { PreparedStatement stat = null; + String pkName = autoIncrementPkName; try { - stat = connection.prepareStatement(sql); + if (StringUtils.isNotBlank(autoIncrementPkName)) { + String[] generatedColumns = {pkName}; + stat = connection.prepareStatement(sql, generatedColumns); + } else { + stat = connection.prepareStatement(sql); + } stat.setQueryTimeout(queryTimeout); } catch (Exception e) { DEException.throwException(e.getMessage()); @@ -1332,7 +1599,7 @@ public class CalciteProvider extends Provider { public void initConnectionPool() { LogUtil.info("Begin to init datasource pool..."); QueryWrapper datasourceQueryWrapper = new QueryWrapper(); - List coreDatasources = coreDatasourceMapper.selectList(datasourceQueryWrapper).stream().filter(coreDatasource -> !Arrays.asList("folder", "API", "Excel").contains(coreDatasource.getType())).collect(Collectors.toList()); + List coreDatasources = coreDatasourceMapper.selectList(datasourceQueryWrapper).stream().filter(coreDatasource -> !Arrays.asList("folder", "API", "Excel", "ExcelRemote").contains(coreDatasource.getType())).collect(Collectors.toList()); CoreDatasource engine = engineManage.deEngine(); if (engine != null) { coreDatasources.add(engine); @@ -1382,8 +1649,8 @@ public class CalciteProvider extends Provider { buildSchema(datasourceRequest, calciteConnection); } DatasourceConfiguration configuration = JsonUtil.parseObject(datasourceDTO.getConfiguration(), DatasourceConfiguration.class); - if(configuration.isUseSSH()){ - Session session =Provider.getSessions().get(datasourceDTO.getId()); + if (configuration.isUseSSH()) { + Session session = Provider.getSessions().get(datasourceDTO.getId()); session.disconnect(); Provider.getSessions().remove(datasourceDTO.getId()); startSshSession(configuration, null, datasourceDTO.getId()); @@ -1437,6 +1704,7 @@ public class CalciteProvider extends Provider { } JdbcSchema jdbcSchema = rootSchema.getSubSchema(String.format(SQLConstants.SCHEMA, dsId)).unwrap(JdbcSchema.class); BasicDataSource basicDataSource = (BasicDataSource) jdbcSchema.getDataSource(); + basicDataSource.setMaxWaitMillis(5 * 1000); return basicDataSource.getConnection(); } catch (Exception e) { DEException.throwException(Translator.get("i18n_invalid_connection") + e.getMessage()); diff --git a/core/core-backend/src/main/java/io/gisbi/datasource/provider/ExcelUtils.java b/core/core-backend/src/main/java/io/gisbi/datasource/provider/ExcelUtils.java index 9d80147..33f83fd 100644 --- a/core/core-backend/src/main/java/io/gisbi/datasource/provider/ExcelUtils.java +++ b/core/core-backend/src/main/java/io/gisbi/datasource/provider/ExcelUtils.java @@ -4,6 +4,7 @@ package io.gisbi.datasource.provider; import com.alibaba.excel.EasyExcel; import com.alibaba.excel.ExcelReader; import com.alibaba.excel.context.AnalysisContext; +import com.alibaba.excel.enums.CellDataTypeEnum; import com.alibaba.excel.event.AnalysisEventListener; import com.alibaba.excel.metadata.data.ReadCellData; import com.alibaba.excel.read.metadata.ReadSheet; @@ -12,6 +13,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import io.gisbi.api.ds.vo.ExcelFileData; import io.gisbi.api.ds.vo.ExcelSheetData; +import io.gisbi.api.ds.vo.RemoteExcelRequest; import io.gisbi.commons.utils.EncryptUtils; import io.gisbi.datasource.dao.auto.entity.CoreDatasource; import io.gisbi.exception.DEException; @@ -19,6 +21,8 @@ import io.gisbi.extensions.datasource.dto.DatasetTableDTO; import io.gisbi.extensions.datasource.dto.DatasourceDTO; import io.gisbi.extensions.datasource.dto.DatasourceRequest; import io.gisbi.extensions.datasource.dto.TableField; +import io.gisbi.api.ds.vo.ExcelConfiguration; +import io.gisbi.i18n.Translator; import io.gisbi.utils.*; import lombok.Data; import org.apache.commons.lang3.StringUtils; @@ -26,6 +30,9 @@ import org.springframework.util.CollectionUtils; import org.springframework.web.multipart.MultipartFile; import java.io.*; +import java.net.URL; +import java.net.URLConnection; +import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.util.*; import java.util.regex.Matcher; @@ -40,9 +47,9 @@ public class ExcelUtils { public static String getExcelPath() { if (ModelUtils.isDesktop()) { - return ConfigUtils.getConfig("gisbi.path.excel", "/opt/gisbi/data/excel/"); + return ConfigUtils.getConfig("gisbi.path.excel", "/opt/gisbi2.0/data/excel/"); } else { - return "/opt/gisbi/data/excel/"; + return "/opt/gisbi2.0/data/excel/"; } } @@ -53,21 +60,42 @@ public class ExcelUtils { }; public static void mergeSheets(CoreDatasource requestDatasource, DatasourceDTO sourceData) { - List newSheets = JsonUtil.parseList(requestDatasource.getConfiguration(), sheets); - List tableNames = newSheets.stream().map(ExcelSheetData::getDeTableName).collect(Collectors.toList()); - List oldSheets = JsonUtil.parseList(sourceData.getConfiguration(), sheets); - for (ExcelSheetData oldSheet : oldSheets) { - if (!tableNames.contains(oldSheet.getDeTableName())) { - newSheets.add(oldSheet); + if (requestDatasource.getType().equalsIgnoreCase("Excel")) { + List newSheets = JsonUtil.parseList(requestDatasource.getConfiguration(), sheets); + List tableNames = newSheets.stream().map(ExcelSheetData::getDeTableName).collect(Collectors.toList()); + List oldSheets = JsonUtil.parseList(sourceData.getConfiguration(), sheets); + for (ExcelSheetData oldSheet : oldSheets) { + if (!tableNames.contains(oldSheet.getDeTableName())) { + newSheets.add(oldSheet); + } } + requestDatasource.setConfiguration(JsonUtil.toJSONString(newSheets).toString()); + } else { + ExcelConfiguration excelConfiguration = JsonUtil.parseObject(requestDatasource.getConfiguration(), ExcelConfiguration.class); + List newSheets = excelConfiguration.getSheets(); + List tableNames = newSheets.stream().map(ExcelSheetData::getDeTableName).collect(Collectors.toList()); + List oldSheets = JsonUtil.parseObject(sourceData.getConfiguration(), ExcelConfiguration.class).getSheets(); + for (ExcelSheetData oldSheet : oldSheets) { + if (!tableNames.contains(oldSheet.getDeTableName())) { + newSheets.add(oldSheet); + } + } + excelConfiguration.setSheets(newSheets); + requestDatasource.setConfiguration(JsonUtil.toJSONString(excelConfiguration).toString()); } - requestDatasource.setConfiguration(JsonUtil.toJSONString(newSheets).toString()); + } public static List getTables(DatasourceRequest datasourceRequest) throws DEException { List tableDescs = new ArrayList<>(); try { - JsonNode rootNode = objectMapper.readTree(datasourceRequest.getDatasource().getConfiguration()); + String sheets = ""; + if (datasourceRequest.getDatasource().getType().equalsIgnoreCase("Excel")) { + sheets = datasourceRequest.getDatasource().getConfiguration(); + } else { + sheets = objectMapper.readTree(datasourceRequest.getDatasource().getConfiguration()).get("sheets").toString(); + } + JsonNode rootNode = objectMapper.readTree(sheets); for (int i = 0; i < rootNode.size(); i++) { DatasetTableDTO datasetTableDTO = new DatasetTableDTO(); datasetTableDTO.setTableName(rootNode.get(i).get("deTableName").asText()); @@ -83,20 +111,24 @@ public class ExcelUtils { return tableDescs; } - public static Map getTableNamesMap(String configration) throws DEException { + public static Map getTableNamesMap(String type, String configuration) throws DEException { Map result = new HashMap<>(); JsonNode rootNode = null; // 兼容历史未加密信息 + String sheets = configuration; try { - rootNode = objectMapper.readTree((String) EncryptUtils.aesDecrypt(configration)); + if (type.equalsIgnoreCase("ExcelRemote")) { + sheets = objectMapper.readTree(configuration).get("sheets").toString(); + } + rootNode = objectMapper.readTree((String) EncryptUtils.aesDecrypt(sheets)); } catch (Exception e) { try { - rootNode = objectMapper.readTree(configration); + rootNode = objectMapper.readTree(sheets); } catch (Exception ex) { DEException.throwException(ex); } } - if(rootNode != null) { + if (rootNode != null) { for (int i = 0; i < rootNode.size(); i++) { result.put(rootNode.get(i).get("tableName").asText(), rootNode.get(i).get("deTableName").asText()); } @@ -105,6 +137,12 @@ public class ExcelUtils { } public static String getFileName(CoreDatasource datasource) throws DEException { + if (datasource.getType().equalsIgnoreCase("ExcelRemote")) { + ExcelConfiguration excelConfiguration = JsonUtil.parseObject(datasource.getConfiguration(), ExcelConfiguration.class); + for (ExcelSheetData sheet : excelConfiguration.getSheets()) { + return sheet.getFileName(); + } + } JsonNode rootNode = null; try { rootNode = objectMapper.readTree((String) EncryptUtils.aesDecrypt(datasource.getConfiguration())); @@ -120,12 +158,16 @@ public class ExcelUtils { return rootNode.get(i).get("fileName").asText(); } } - - return ""; } public static String getSize(CoreDatasource datasource) throws DEException { + if (datasource.getType().equalsIgnoreCase("ExcelRemote")) { + ExcelConfiguration excelConfiguration = JsonUtil.parseObject(datasource.getConfiguration(), ExcelConfiguration.class); + for (ExcelSheetData sheet : excelConfiguration.getSheets()) { + return sheet.getSize(); + } + } try { JsonNode rootNode = objectMapper.readTree(datasource.getConfiguration()); for (int i = 0; i < rootNode.size(); i++) { @@ -134,35 +176,56 @@ public class ExcelUtils { } catch (Exception e) { DEException.throwException(e); } - return "0 B"; } - public List fetchDataList(DatasourceRequest datasourceRequest) throws DEException { + public List fetchDataList(DatasourceRequest datasourceRequest) throws DEException, IOException { List dataList = new ArrayList<>(); - try { - JsonNode rootNode = objectMapper.readTree(datasourceRequest.getDatasource().getConfiguration()); - for (int i = 0; i < rootNode.size(); i++) { - if (rootNode.get(i).get("deTableName").asText().equalsIgnoreCase(datasourceRequest.getTable())) { - List tableFields = JsonUtil.parseList(rootNode.get(i).get("fields").toString(), TableFieldListTypeReference); - String suffix = rootNode.get(i).get("path").asText().substring(rootNode.get(i).get("path").asText().lastIndexOf(".") + 1); - InputStream inputStream = new FileInputStream(rootNode.get(i).get("path").asText()); + if (datasourceRequest.getDatasource().getType().equalsIgnoreCase("ExcelRemote")) { + ExcelConfiguration excelConfiguration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), ExcelConfiguration.class); + Map fileNames = downLoadRemoteExcel(excelConfiguration); + for (ExcelSheetData sheet : excelConfiguration.getSheets()) { + if (sheet.getDeTableName().equalsIgnoreCase(datasourceRequest.getTable())) { + List tableFields = sheet.getFields(); + String suffix = fileNames.get("fileName").substring(fileNames.get("fileName").lastIndexOf(".") + 1); + InputStream inputStream = new FileInputStream(path + fileNames.get("tranName")); if (StringUtils.equalsIgnoreCase(suffix, "csv")) { BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); reader.readLine();//去掉表头 dataList = csvData(reader, false, tableFields.size()); } else { - dataList = fetchExcelDataList(rootNode.get(i).get("tableName").asText(), inputStream); + dataList = fetchExcelDataList(sheet.getTableName(), inputStream); } } } - } catch (Exception e) { - DEException.throwException(e); + if (StringUtils.isNotEmpty(fileNames.get("tranName"))) { + FileUtils.deleteFile(path + fileNames.get("tranName")); + } + } else { + try { + JsonNode rootNode = objectMapper.readTree(datasourceRequest.getDatasource().getConfiguration()); + for (int i = 0; i < rootNode.size(); i++) { + if (rootNode.get(i).get("deTableName").asText().equalsIgnoreCase(datasourceRequest.getTable())) { + List tableFields = JsonUtil.parseList(rootNode.get(i).get("fields").toString(), TableFieldListTypeReference); + String suffix = rootNode.get(i).get("path").asText().substring(rootNode.get(i).get("path").asText().lastIndexOf(".") + 1); + InputStream inputStream = new FileInputStream(rootNode.get(i).get("path").asText()); + if (StringUtils.equalsIgnoreCase(suffix, "csv")) { + BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); + reader.readLine();//去掉表头 + dataList = csvData(reader, false, tableFields.size()); + } else { + dataList = fetchExcelDataList(rootNode.get(i).get("tableName").asText(), inputStream); + } + } + } + } catch (Exception e) { + DEException.throwException(e); + } } return dataList; } - public List fetchExcelDataList(String sheetName, InputStream inputStream) { + private List fetchExcelDataList(String sheetName, InputStream inputStream) { NoModelDataListener noModelDataListener = new NoModelDataListener(); ExcelReader excelReader = EasyExcel.read(inputStream, noModelDataListener).build(); List sheets = excelReader.excelExecutor().sheetList(); @@ -189,7 +252,13 @@ public class ExcelUtils { TypeReference> listTypeReference = new TypeReference>() { }; try { - JsonNode rootNode = objectMapper.readTree(datasourceRequest.getDatasource().getConfiguration()); + String sheets = ""; + if (datasourceRequest.getDatasource().getType().equalsIgnoreCase("Excel")) { + sheets = datasourceRequest.getDatasource().getConfiguration(); + } else { + sheets = objectMapper.readTree(datasourceRequest.getDatasource().getConfiguration()).get("sheets").toString(); + } + JsonNode rootNode = objectMapper.readTree(sheets); for (int i = 0; i < rootNode.size(); i++) { if (rootNode.get(i).get("deTableName").asText().equalsIgnoreCase(datasourceRequest.getTable())) { tableFields = JsonUtil.parseList(rootNode.get(i).get("fields").toString(), listTypeReference); @@ -201,19 +270,17 @@ public class ExcelUtils { return tableFields; } - public ExcelFileData excelSaveAndParse(MultipartFile file) throws DEException { + public ExcelFileData excelSaveAndParse(MultipartFile file, String createBy) throws DEException { String filename = file.getOriginalFilename(); List excelSheetDataList = null; try { - excelSheetDataList = parseExcel(filename, file.getInputStream(), true); + excelSheetDataList = parseExcel(filename, file.getInputStream(), true, filename); } catch (Exception e) { DEException.throwException(e); } List returnSheetDataList = new ArrayList<>(); returnSheetDataList = excelSheetDataList; - returnSheetDataList = returnSheetDataList.stream() - .filter(excelSheetData -> !CollectionUtils.isEmpty(excelSheetData.getFields())) - .collect(Collectors.toList()); + returnSheetDataList = returnSheetDataList.stream().filter(excelSheetData -> !CollectionUtils.isEmpty(excelSheetData.getFields())).collect(Collectors.toList()); // save file String excelId = UUID.randomUUID().toString(); String filePath = saveFile(file, excelId); @@ -227,7 +294,7 @@ public class ExcelUtils { excelSheetData.setSheetExcelId(excelId); excelSheetData.setFileName(filename); /** - * dataease字段类型：0-文本，1-时间，2-整型数值，3-浮点数值，4-布尔，5-地理位置，6-二进制 + * gisbi字段类型：0-文本，1-时间，2-整型数值，3-浮点数值，4-布尔，5-地理位置，6-二进制 */ for (TableField field : excelSheetData.getFields()) { //TEXT LONG DATETIME DOUBLE @@ -272,17 +339,119 @@ public class ExcelUtils { return excelFileData; } + public static String checkStatus(DatasourceRequest datasourceRequest) throws FileNotFoundException { + ExcelConfiguration excelConfiguration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), ExcelConfiguration.class); + Map fileNames = new HashMap<>(); + try { + fileNames = downLoadRemoteExcel(excelConfiguration); + return "Success"; + } catch (Exception e) { + throw e; + } finally { + if (StringUtils.isNotEmpty(fileNames.get("tranName"))) { + FileUtils.deleteFile(path + fileNames.get("tranName")); + } + } + } + + public ExcelFileData parseRemoteExcel(RemoteExcelRequest remoteExcelRequest) throws DEException, FileNotFoundException { + Map fileNames = downLoadRemoteExcel(remoteExcelRequest); + FileInputStream fileInputStream = new FileInputStream(path + fileNames.get("tranName")); + List returnSheetDataList = new ArrayList<>(); + try { + returnSheetDataList = parseExcel(fileNames.get("tranName"), fileInputStream, true, fileNames.get("fileName")).stream().filter(excelSheetData -> !CollectionUtils.isEmpty(excelSheetData.getFields())).collect(Collectors.toList()); + } catch (Exception e) { + DEException.throwException(e); + } + for (ExcelSheetData excelSheetData : returnSheetDataList) { + excelSheetData.setLastUpdateTime(System.currentTimeMillis()); + excelSheetData.setTableName(excelSheetData.getExcelLabel()); + excelSheetData.setDeTableName("excel_" + excelSheetData.getExcelLabel() + "_" + UUID.randomUUID().toString().replace("-", "").substring(0, 10)); + excelSheetData.setPath(path + fileNames.get("tranName")); + excelSheetData.setSheetId(UUID.randomUUID().toString()); + excelSheetData.setSheetExcelId(fileNames.get("tranName").split("\\.")[0]); + excelSheetData.setFileName(fileNames.get("fileName")); + /** + * gisbi字段类型：0-文本，1-时间，2-整型数值，3-浮点数值，4-布尔，5-地理位置，6-二进制 + */ + for (TableField field : excelSheetData.getFields()) { + //TEXT LONG DATETIME DOUBLE + if (field.getFieldType().equalsIgnoreCase("TEXT")) { + field.setDeType(0); + field.setDeExtractType(0); + } + if (field.getFieldType().equalsIgnoreCase("DATETIME")) { + field.setDeType(1); + field.setDeExtractType(1); + } + if (field.getFieldType().equalsIgnoreCase("LONG")) { + field.setDeType(2); + field.setDeExtractType(2); + } + if (field.getFieldType().equalsIgnoreCase("DOUBLE")) { + field.setDeType(3); + field.setDeExtractType(3); + } + } + long size = 0; + File file = new File(path + fileNames.get("tranName")); + String unit = "B"; + if (file.length() / 1024 == 0) { + size = file.length(); + } + if (0 < file.length() / 1024 && file.length() / 1024 < 1024) { + size = file.length() / 1024; + unit = "KB"; + } + if (1024 <= file.length() / 1024) { + size = file.length() / 1024 / 1024; + unit = "MB"; + } + excelSheetData.setSize(size + " " + unit); + } + + ExcelFileData excelFileData = new ExcelFileData(); + excelFileData.setExcelLabel(fileNames.get("fileName").split("\\.")[0]); + excelFileData.setId(fileNames.get("tranName").split("\\.")[0]); + excelFileData.setPath(path + fileNames.get("tranName")); + excelFileData.setSheets(returnSheetDataList); + if (StringUtils.isNotEmpty(fileNames.get("tranName"))) { + FileUtils.deleteFile(path + fileNames.get("tranName")); + } + return excelFileData; + } + + private static Map downLoadRemoteExcel(ExcelConfiguration remoteExcelRequest) throws DEException, FileNotFoundException { + Map fileNames = new HashMap<>(); + if (remoteExcelRequest.getUrl().trim().startsWith("http")) { + HttpClientConfig httpClientConfig = new HttpClientConfig(); + if (StringUtils.isNotEmpty(remoteExcelRequest.getUserName()) && StringUtils.isNotEmpty(remoteExcelRequest.getPasswd())) { + String authValue = "Basic " + Base64.getUrlEncoder().encodeToString((remoteExcelRequest.getUserName() + ":" + remoteExcelRequest.getPasswd()).getBytes()); + httpClientConfig.addHeader("Authorization", authValue); + } + File p = new File(path); + if (!p.exists()) { + p.mkdirs(); + } + fileNames = HttpClientUtil.downloadFile(remoteExcelRequest.getUrl(), httpClientConfig, path); + } else if (remoteExcelRequest.getUrl().trim().startsWith("ftp")) { + fileNames = downLoadFromFtp(remoteExcelRequest); + } else { + DEException.throwException(Translator.get("i18n_unsupported_protocol")); + } + return fileNames; + } + private static String saveFile(MultipartFile file, String fileNameUUID) throws DEException { String filePath = null; try { String filename = file.getOriginalFilename(); String suffix = filename.substring(filename.lastIndexOf(".") + 1); - String dirPath = path + AuthUtils.getUser().getUserId() + "/"; - File p = new File(dirPath); + File p = new File(path); if (!p.exists()) { p.mkdirs(); } - filePath = dirPath + fileNameUUID + "." + suffix; + filePath = path + fileNameUUID + "." + suffix; File f = new File(filePath); FileOutputStream fileOutputStream = new FileOutputStream(f); fileOutputStream.write(file.getBytes()); @@ -398,7 +567,13 @@ public class ExcelUtils { super.invokeHead(headMap, context); for (Integer key : headMap.keySet()) { ReadCellData cellData = headMap.get(key); - String value = cellData.getStringValue(); + String value = null; + if (cellData.getType().equals(CellDataTypeEnum.STRING)) { + value = cellData.getStringValue(); + } + if (cellData.getType().equals(CellDataTypeEnum.NUMBER)) { + value = cellData.getNumberValue().toString(); + } if (StringUtils.isEmpty(value)) { continue; } @@ -439,7 +614,7 @@ public class ExcelUtils { } - public List parseExcel(String filename, InputStream inputStream, boolean isPreview) throws IOException { + private List parseExcel(String filename, InputStream inputStream, boolean isPreview, String originFilename) throws IOException { List excelSheetDataList = new ArrayList<>(); String suffix = filename.substring(filename.lastIndexOf(".") + 1); if (StringUtils.equalsIgnoreCase(suffix, "xlsx") || StringUtils.equalsIgnoreCase(suffix, "xls")) { @@ -494,21 +669,23 @@ public class ExcelUtils { List fields = new ArrayList<>(); BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8)); String s = reader.readLine();// first line - String[] split = s.split(","); - for (int i = 0; i < split.length; i++) { - String filedName = split[i]; - if (StringUtils.isEmpty(filedName)) { - DEException.throwException("首行行中不允许有空单元格！"); + if (StringUtils.isNotEmpty(s)) { + String[] split = s.split(","); + for (int i = 0; i < split.length; i++) { + String filedName = split[i]; + if (StringUtils.isEmpty(filedName)) { + DEException.throwException(Translator.get("i18n_excel_error_first_row")); + } + if (filedName.startsWith(UFEFF)) { + filedName = filedName.replace(UFEFF, ""); + } + TableField tableFiled = new TableField(); + tableFiled.setName(filedName); + tableFiled.setOriginName(filedName); + tableFiled.setFieldType(null); + tableFiled.setChecked(true); + fields.add(tableFiled); } - if (filedName.startsWith(UFEFF)) { - filedName = filedName.replace(UFEFF, ""); - } - TableField tableFiled = new TableField(); - tableFiled.setName(filedName); - tableFiled.setOriginName(filedName); - tableFiled.setFieldType(null); - tableFiled.setChecked(true); - fields.add(tableFiled); } List data = csvData(reader, isPreview, fields.size()); @@ -535,7 +712,7 @@ public class ExcelUtils { excelSheetData.setFields(fields); excelSheetData.setData(data); excelSheetData.setFileName(filename); - excelSheetData.setExcelLabel(filename.substring(0, filename.lastIndexOf('.'))); + excelSheetData.setExcelLabel(originFilename.substring(0, originFilename.lastIndexOf('.'))); excelSheetDataList.add(excelSheetData); } inputStream.close(); @@ -560,5 +737,74 @@ public class ExcelUtils { return excelSheetDataList; } + public static Map downLoadFromFtp(ExcelConfiguration remoteExcelRequest) { + Map fileNames = new HashMap<>(); + String username = ""; + String password = ""; + String serverAddress = ""; + String filePath = ""; + if (remoteExcelRequest.getUrl().contains("@")) { + String regex = "ftp://([^:]+):([^@]+)@([^/]+)(.*)"; + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(remoteExcelRequest.getUrl()); + if (matcher.find()) { + username = matcher.group(1); + password = matcher.group(2); + serverAddress = matcher.group(3); + filePath = matcher.group(4); + } else { + DEException.throwException(Translator.get("i18n_invalid_address")); + } + } else { + String regex = "ftp://([^/]+)(.*)"; + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(remoteExcelRequest.getUrl()); + if (matcher.find()) { + serverAddress = matcher.group(1); + filePath = matcher.group(2); + } else { + DEException.throwException(Translator.get("i18n_invalid_address")); + } + } + if (StringUtils.isNotEmpty(remoteExcelRequest.getUserName())) { + username = remoteExcelRequest.getUserName(); + } + if (StringUtils.isNotEmpty(remoteExcelRequest.getPasswd())) { + password = remoteExcelRequest.getPasswd(); + } + filePath = filePath.startsWith("/") ? filePath.substring(1) : filePath; + String suffix = filePath.substring(filePath.lastIndexOf(".") + 1); + if (!Arrays.asList("csv", "xlsx", "xls").contains(suffix)) { + DEException.throwException(Translator.get("i18n_unsupported_file_format")); + } + String tranName = UUID.randomUUID().toString() + "." + suffix; + String localFilePath = path + tranName; + fileNames.put("fileName", filePath); + fileNames.put("tranName", tranName); + + try { + URL url; + if (StringUtils.isNotEmpty(username) && StringUtils.isNotEmpty(password)) { + url = new URL("ftp://" + username + ":" + URLEncoder.encode(password, StandardCharsets.UTF_8) + "@" + serverAddress + "/" + filePath); + } else { + url = new URL("ftp://" + serverAddress + "/" + filePath); + } + + URLConnection conn = url.openConnection(); + InputStream inputStream = conn.getInputStream(); + FileOutputStream outputStream = new FileOutputStream(localFilePath); + byte[] buffer = new byte[4096]; + int bytesRead; + while ((bytesRead = inputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, bytesRead); + } + inputStream.close(); + outputStream.close(); + + } catch (IOException e) { + DEException.throwException(Translator.get("i18n_file_download_failed") + ", " + e.getMessage()); + } + return fileNames; + } } diff --git a/core/core-backend/src/main/java/io/gisbi/datasource/provider/H2EngineProvider.java b/core/core-backend/src/main/java/io/gisbi/datasource/provider/H2EngineProvider.java index 55d6b3c..310d0db 100644 --- a/core/core-backend/src/main/java/io/gisbi/datasource/provider/H2EngineProvider.java +++ b/core/core-backend/src/main/java/io/gisbi/datasource/provider/H2EngineProvider.java @@ -46,12 +46,7 @@ public class H2EngineProvider extends EngineProvider { if (tableFields.get(i).isChecked()) { if (StringUtils.isEmpty(strings[i])) { String type = tableFields.get(i).getType() == null ? tableFields.get(i).getFieldType() : tableFields.get(i).getType(); - if (type.equals("LONG") || type.equals("DOUBLE")) { - strings1[length] = "0"; - } else { - strings1[length] = null; - } - + strings1[length] = null; } else { strings1[length] = strings[i].replace("\\", "\\\\").replace("'", "''"); } @@ -61,7 +56,7 @@ public class H2EngineProvider extends EngineProvider { values.append("('").append(String.join("','", Arrays.asList(strings1))) .append("'),"); } - return (insertSql + values.substring(0, values.length() - 1)); + return (insertSql + values.substring(0, values.length() - 1)).replaceAll("'null'", "null"); } diff --git a/core/core-backend/src/main/java/io/gisbi/datasource/provider/MysqlEngineProvider.java b/core/core-backend/src/main/java/io/gisbi/datasource/provider/MysqlEngineProvider.java index 1cafd6f..28d5569 100644 --- a/core/core-backend/src/main/java/io/gisbi/datasource/provider/MysqlEngineProvider.java +++ b/core/core-backend/src/main/java/io/gisbi/datasource/provider/MysqlEngineProvider.java @@ -54,11 +54,7 @@ public class MysqlEngineProvider extends EngineProvider { if (tableFields.get(i).isChecked()) { if (StringUtils.isEmpty(strings[i])) { String type = tableFields.get(i).getType() == null ? tableFields.get(i).getFieldType() : tableFields.get(i).getType(); - if (type.equals("LONG") || type.equals("DOUBLE")) { - strings1[length] = "0"; - } else { - strings1[length] = null; - } + strings1[length] = null; } else { strings1[length] = strings[i].replace("\\", "\\\\").replace("'", "\\'"); } diff --git a/core/core-backend/src/main/java/io/gisbi/datasource/server/DatasourceDriverServer.java b/core/core-backend/src/main/java/io/gisbi/datasource/server/DatasourceDriverServer.java index 05f6c54..7c897c3 100644 --- a/core/core-backend/src/main/java/io/gisbi/datasource/server/DatasourceDriverServer.java +++ b/core/core-backend/src/main/java/io/gisbi/datasource/server/DatasourceDriverServer.java @@ -33,7 +33,7 @@ import java.util.Map; @RequestMapping("/datasourceDriver") public class DatasourceDriverServer implements DatasourceDriverApi { - @Value("${gisbi.path.custom-drivers:/opt/gisbi/custom-drivers/}") + @Value("${gisbi.path.custom-drivers:/opt/gisbi2.0/custom-drivers/}") private String DRIVER_PATH; @Resource diff --git a/core/core-backend/src/main/java/io/gisbi/datasource/server/DatasourceServer.java b/core/core-backend/src/main/java/io/gisbi/datasource/server/DatasourceServer.java index cd1c91b..c0b371f 100644 --- a/core/core-backend/src/main/java/io/gisbi/datasource/server/DatasourceServer.java +++ b/core/core-backend/src/main/java/io/gisbi/datasource/server/DatasourceServer.java @@ -10,8 +10,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.gisbi.api.dataset.dto.PreviewSqlDTO; import io.gisbi.api.ds.DatasourceApi; import io.gisbi.api.ds.vo.*; +import io.gisbi.api.permissions.relation.api.RelationApi; import io.gisbi.commons.constants.TaskStatus; -import io.gisbi.constant.DataSourceType; import io.gisbi.constant.LogOT; import io.gisbi.constant.LogST; import io.gisbi.constant.SQLConstants; @@ -59,10 +59,8 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; +import java.io.*; +import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.*; import java.util.stream.Collectors; @@ -103,11 +101,15 @@ public class DatasourceServer implements DatasourceApi { private CoreUserManage coreUserManage; @Autowired(required = false) private PluginManageApi pluginManage; + @Autowired(required = false) + private RelationApi relationManage; public enum UpdateType { all_scope, add_scope } + public static final List notFullDs = List.of("folder", "Excel", "API"); + private TypeReference> listTypeReference = new TypeReference>() { }; @Resource @@ -122,7 +124,7 @@ public class DatasourceServer implements DatasourceApi { } public boolean checkRepeat(@RequestBody BusiDsRequest dataSourceDTO) { - if (Arrays.asList("Excel", "folder", "es").contains(dataSourceDTO.getType()) || dataSourceDTO.getType().contains("API")) { + if (Arrays.asList("folder", "es").contains(dataSourceDTO.getType()) || dataSourceDTO.getType().contains("API") || dataSourceDTO.getType().contains("Excel")) { return false; } BusiNodeRequest request = new BusiNodeRequest(); @@ -145,7 +147,7 @@ public class DatasourceServer implements DatasourceApi { DatasourceConfiguration configuration = JsonUtil.parseObject(dataSourceDTO.getConfiguration(), DatasourceConfiguration.class); boolean hasRepeat = false; for (CoreDatasource datasource : datasources) { - if (Arrays.asList("API", "Excel", "folder").contains(datasource.getType())) { + if (notFullDs.stream().anyMatch(e -> datasource.getType().contains(e))) { continue; } if (StringUtils.isEmpty(datasource.getConfiguration())) { @@ -207,7 +209,7 @@ public class DatasourceServer implements DatasourceApi { if (StringUtils.isEmpty(dataSourceDTO.getName())) { DEException.throwException("名称不能为空！"); } - CoreDatasource datasource = dataSourceManage.getCoreDatasource(dataSourceDTO.getId()); + CoreDatasource datasource = dataSourceManage.getDatasource(dataSourceDTO.getId()); datasource.setName(dataSourceDTO.getName()); dataSourceDTO.setPid(datasource.getPid()); dataSourceManage.checkName(dataSourceDTO); @@ -243,7 +245,6 @@ public class DatasourceServer implements DatasourceApi { } preCheckDs(dataSourceDTO); dataSourceDTO.setId(IDUtils.snowID()); - dataSourceDTO.setAppId(busiDsRequest.getAppId()); //补充了项目ID dataSourceDTO.setCreateTime(System.currentTimeMillis()); dataSourceDTO.setUpdateTime(System.currentTimeMillis()); try { @@ -278,6 +279,20 @@ public class DatasourceServer implements DatasourceApi { } datasourceSyncManage.extractExcelData(coreDatasource, "all_scope"); } else if (dataSourceDTO.getType().contains(DatasourceConfiguration.DatasourceType.API.name())) { + DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setDatasource(dataSourceDTO); + List tables = (List) invokeMethod(coreDatasource.getType(), "getApiTables", DatasourceRequest.class, datasourceRequest); + checkName(tables.stream().map(DatasetTableDTO::getName).collect(Collectors.toList())); + for (DatasetTableDTO api : tables) { + datasourceRequest.setTable(api.getTableName()); + List tableFields = (List) invokeMethod(coreDatasource.getType(), "getTableFields", DatasourceRequest.class, datasourceRequest); + try { + datasourceSyncManage.createEngineTable(datasourceRequest.getTable(), tableFields); + } catch (Exception e) { + DEException.throwException("Failed to create table " + datasourceRequest.getTable() + ": " + e.getMessage()); + } + } + CoreDatasourceTask coreDatasourceTask = new CoreDatasourceTask(); BeanUtils.copyBean(coreDatasourceTask, dataSourceDTO.getSyncSetting()); coreDatasourceTask.setName(coreDatasource.getName() + "-task"); @@ -295,19 +310,40 @@ public class DatasourceServer implements DatasourceApi { coreDatasourceTask.setTaskStatus(TaskStatus.WaitingForExecution.name()); datasourceTaskServer.insert(coreDatasourceTask); datasourceSyncManage.addSchedule(coreDatasourceTask); + } else if (dataSourceDTO.getType().contains(DatasourceConfiguration.DatasourceType.ExcelRemote.name())) { DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(dataSourceDTO); - List tables = (List) invokeMethod(coreDatasource.getType(), "getApiTables", DatasourceRequest.class, datasourceRequest); - checkName(tables.stream().map(DatasetTableDTO::getName).collect(Collectors.toList())); - for (DatasetTableDTO api : tables) { - datasourceRequest.setTable(api.getTableName()); - List tableFields = (List) invokeMethod(coreDatasource.getType(), "getTableFields", DatasourceRequest.class, datasourceRequest); + List tables = ExcelUtils.getTables(datasourceRequest); + for (DatasetTableDTO table : tables) { + datasourceRequest.setTable(table.getTableName()); + List tableFields = ExcelUtils.getTableFields(datasourceRequest); try { datasourceSyncManage.createEngineTable(datasourceRequest.getTable(), tableFields); } catch (Exception e) { - DEException.throwException("Failed to create table " + datasourceRequest.getTable() + ": " + e.getMessage()); + if (e.getMessage().toLowerCase().contains("Row size too large".toLowerCase())) { + DEException.throwException("文本内容超出最大支持范围： " + datasourceRequest.getTable() + ", " + e.getMessage()); + } else { + DEException.throwException("Failed to create table " + datasourceRequest.getTable() + ", " + e.getMessage()); + } } } + CoreDatasourceTask coreDatasourceTask = new CoreDatasourceTask(); + BeanUtils.copyBean(coreDatasourceTask, dataSourceDTO.getSyncSetting()); + coreDatasourceTask.setName(coreDatasource.getName() + "-task"); + coreDatasourceTask.setDsId(coreDatasource.getId()); + if (coreDatasourceTask.getStartTime() == null) { + coreDatasourceTask.setStartTime(System.currentTimeMillis() - 20 * 1000); + } + if (StringUtils.equalsIgnoreCase(coreDatasourceTask.getSyncRate(), RIGHTNOW.toString())) { + coreDatasourceTask.setCron(null); + } else { + if (coreDatasourceTask.getEndTime() != null && coreDatasourceTask.getEndTime() > 0 && coreDatasourceTask.getStartTime() > coreDatasourceTask.getEndTime()) { + DEException.throwException("结束时间不能小于开始时间！"); + } + } + coreDatasourceTask.setTaskStatus(TaskStatus.WaitingForExecution.name()); + datasourceTaskServer.insert(coreDatasourceTask); + datasourceSyncManage.addSchedule(coreDatasourceTask); } else { checkParams(dataSourceDTO.getConfiguration()); calciteProvider.update(dataSourceDTO); @@ -347,13 +383,14 @@ public class DatasourceServer implements DatasourceApi { datasourceRequest.setDatasource(dataSourceDTO); List toCreateTables = new ArrayList<>(); List toDeleteTables = new ArrayList<>(); - if (dataSourceDTO.getType().contains(DatasourceConfiguration.DatasourceType.API.name())) { + if (dataSourceDTO.getType().contains(DatasourceConfiguration.DatasourceType.API.name()) || dataSourceDTO.getType().equals(DatasourceConfiguration.DatasourceType.ExcelRemote.name())) { requestDatasource.setEnableDataFill(null); - List sourceTableDTOs = (List) invokeMethod(sourceData.getType(), "getApiTables", DatasourceRequest.class, sourceTableRequest); + List sourceTableDTOs = dataSourceDTO.getType().contains(DatasourceConfiguration.DatasourceType.API.name()) ? (List) invokeMethod(sourceData.getType(), "getApiTables", DatasourceRequest.class, sourceTableRequest) : ExcelUtils.getTables(sourceTableRequest); List sourceTables = sourceTableDTOs.stream().map(DatasetTableDTO::getTableName).toList(); - List datasetTableDTOS = (List) invokeMethod(sourceData.getType(), "getApiTables", DatasourceRequest.class, datasourceRequest); - + List datasetTableDTOS = dataSourceDTO.getType().contains(DatasourceConfiguration.DatasourceType.API.name()) ? (List) invokeMethod(sourceData.getType(), "getApiTables", DatasourceRequest.class, datasourceRequest) : ExcelUtils.getTables(datasourceRequest); List tables = datasetTableDTOS.stream().map(DatasetTableDTO::getTableName).collect(Collectors.toList()); + + checkName(datasetTableDTOS.stream().map(DatasetTableDTO::getName).collect(Collectors.toList())); toCreateTables = tables.stream().filter(table -> !sourceTables.contains(table)).collect(Collectors.toList()); toDeleteTables = sourceTables.stream().filter(table -> !tables.contains(table)).collect(Collectors.toList()); @@ -361,10 +398,10 @@ public class DatasourceServer implements DatasourceApi { for (String sourceTable : sourceTables) { if (table.equals(sourceTable)) { datasourceRequest.setTable(table); - List tableFieldList = (List) invokeMethod(datasourceRequest.getDatasource().getType(), "getTableFields", DatasourceRequest.class, datasourceRequest); + List tableFieldList = dataSourceDTO.getType().contains(DatasourceConfiguration.DatasourceType.API.name()) ? (List) invokeMethod(datasourceRequest.getDatasource().getType(), "getTableFields", DatasourceRequest.class, datasourceRequest) : ExcelUtils.getTableFields(datasourceRequest); List tableFields = tableFieldList.stream().map(TableField::getName).sorted().collect(Collectors.toList()); sourceTableRequest.setTable(sourceTable); - List sourceTableFieldList = (List) invokeMethod(sourceTableRequest.getDatasource().getType(), "getTableFields", DatasourceRequest.class, sourceTableRequest); + List sourceTableFieldList = dataSourceDTO.getType().contains(DatasourceConfiguration.DatasourceType.API.name()) ? (List) invokeMethod(sourceTableRequest.getDatasource().getType(), "getTableFields", DatasourceRequest.class, sourceTableRequest) : ExcelUtils.getTableFields(sourceTableRequest); List sourceTableFields = sourceTableFieldList.stream().map(TableField::getName).sorted().collect(Collectors.toList()); if (!String.join(",", tableFields).equals(String.join(",", sourceTableFields))) { toDeleteTables.add(table); @@ -397,7 +434,7 @@ public class DatasourceServer implements DatasourceApi { for (String toCreateTable : toCreateTables) { datasourceRequest.setTable(toCreateTable); try { - datasourceSyncManage.createEngineTable(toCreateTable, (List) invokeMethod(sourceTableRequest.getDatasource().getType(), "getTableFields", DatasourceRequest.class, datasourceRequest)); + datasourceSyncManage.createEngineTable(toCreateTable, dataSourceDTO.getType().contains(DatasourceConfiguration.DatasourceType.API.name()) ? (List) invokeMethod(sourceTableRequest.getDatasource().getType(), "getTableFields", DatasourceRequest.class, datasourceRequest) : ExcelUtils.getTableFields(datasourceRequest)); } catch (Exception e) { DEException.throwException("Failed to create table " + toCreateTable + ", " + e.getMessage()); } @@ -539,12 +576,45 @@ public class DatasourceServer implements DatasourceApi { } } List dsList = datasourceMapper.selectList(queryWrapper); + for (CoreDatasource datasource : dsList) { - try { - list.add(convertCoreDatasource(datasource.getId(), false, datasource)); - } catch (Exception e) { - e.printStackTrace(); + DatasourceDTO datasourceDTO = new DatasourceDTO(); + BeanUtils.copyBean(datasourceDTO, datasource); + + if (datasourceDTO.getType().contains(DatasourceConfiguration.DatasourceType.API.toString())) { + List apiDefinitionList = JsonUtil.parseList(datasourceDTO.getConfiguration(), listTypeReference); + int success = 0; + for (ApiDefinition apiDefinition : apiDefinitionList) { + String status = null; + if (StringUtils.isNotEmpty(datasourceDTO.getStatus())) { + JsonNode jsonNode = null; + try { + jsonNode = objectMapper.readTree(datasourceDTO.getStatus()); + for (JsonNode node : jsonNode) { + if (node.get("name").asText().equals(apiDefinition.getName())) { + status = node.get("status").asText(); + } + } + apiDefinition.setStatus(status); + } catch (Exception ignore) { + } + } + if (StringUtils.isNotEmpty(status) && status.equalsIgnoreCase("Success")) { + success++; + } + } + if (success == apiDefinitionList.size()) { + datasourceDTO.setStatus("Success"); + } else { + if (success > 0 && success < apiDefinitionList.size()) { + datasourceDTO.setStatus("Warning"); + } else { + datasourceDTO.setStatus("Error"); + } + } } + + list.add(datasourceDTO); } return list; } @@ -558,7 +628,7 @@ public class DatasourceServer implements DatasourceApi { @DeLog(id = "#p0", ot = LogOT.DELETE, st = LogST.DATASOURCE) @Override public void delete(Long datasourceId) throws DEException { - Objects.requireNonNull(CommonBeanFactory.getBean(DatasourceServer.class)).recursionDel(datasourceId); + Objects.requireNonNull(io.gisbi.utils.CommonBeanFactory.getBean(DatasourceServer.class)).recursionDel(datasourceId); } public void recursionDel(Long datasourceId) throws DEException { @@ -597,9 +667,8 @@ public class DatasourceServer implements DatasourceApi { datasourceTaskServer.deleteByDSId(datasourceId); } - datasourceMapper.deleteById(datasourceId); - if (!Arrays.asList("API", "Excel", "folder").contains(coreDatasource.getType())) { + if (notFullDs.stream().allMatch(e -> !coreDatasource.getType().contains(e))) { calciteProvider.delete(coreDatasource); } @@ -665,22 +734,42 @@ public class DatasourceServer implements DatasourceApi { datasourceRequest.setDatasource(datasourceDTO); if (coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.name())) { List datasetTableDTOS = (List) invokeMethod(coreDatasource.getType(), "getApiTables", DatasourceRequest.class, datasourceRequest); - datasetTableDTOS.forEach(datasetTableDTO1 -> { - CoreDatasourceTaskLog log = datasourceTaskServer.lastSyncLogForTable(datasetTableDTO.getDatasourceId(), datasetTableDTO1.getTableName()); - if (log != null) { - datasetTableDTO1.setLastUpdateTime(log.getStartTime()); - datasetTableDTO1.setStatus(log.getTaskStatus()); - } - }); return datasetTableDTOS; } - if (coreDatasource.getType().equals("Excel")) { + if (coreDatasource.getType().contains("Excel")) { return ExcelUtils.getTables(datasourceRequest); } Provider provider = ProviderFactory.getProvider(datasourceDTO.getType()); return provider.getTables(datasourceRequest); } + @Override + public List getTableStatus(DatasetTableDTO datasetTableDTO) throws DEException { + CoreDatasource coreDatasource = dataSourceManage.getCoreDatasource(datasetTableDTO.getDatasourceId()); + if (coreDatasource == null) { + DEException.throwException("无效数据源！"); + } + List datasetTableDTOS = new ArrayList<>(); + DatasourceDTO datasourceDTO = new DatasourceDTO(); + BeanUtils.copyBean(datasourceDTO, coreDatasource); + DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setDatasource(datasourceDTO); + if (coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.name())) { + datasetTableDTOS = (List) invokeMethod(coreDatasource.getType(), "getApiTables", DatasourceRequest.class, datasourceRequest); + } + if (coreDatasource.getType().equalsIgnoreCase(DatasourceConfiguration.DatasourceType.ExcelRemote.name())) { + datasetTableDTOS = ExcelUtils.getTables(datasourceRequest); + } + datasetTableDTOS.forEach(datasetTableDTO1 -> { + CoreDatasourceTaskLog log = datasourceTaskServer.lastSyncLogForTable(datasetTableDTO.getDatasourceId(), datasetTableDTO1.getTableName()); + if (log != null) { + datasetTableDTO1.setLastUpdateTime(log.getStartTime()); + datasetTableDTO1.setStatus(log.getTaskStatus()); + } + }); + return datasetTableDTOS; + } + @Override public List getTableField(Map req) throws DEException { String tableName = req.get("tableName"); @@ -693,7 +782,7 @@ public class DatasourceServer implements DatasourceApi { CoreDatasource coreDatasource = dataSourceManage.getCoreDatasource(Long.parseLong(datasourceId)); DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(transDTO(coreDatasource)); - if (coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.name()) || coreDatasource.getType().equals("Excel")) { + if (coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.name()) || coreDatasource.getType().contains("Excel")) { datasourceRequest.setDatasource(transDTO(engineManage.getDeEngine())); DatasourceSchemaDTO datasourceSchemaDTO = new DatasourceSchemaDTO(); BeanUtils.copyBean(datasourceSchemaDTO, engineManage.getDeEngine()); @@ -704,7 +793,7 @@ public class DatasourceServer implements DatasourceApi { Provider provider = ProviderFactory.getProvider(datasourceSchemaDTO.getType()); List tableFields = (List) provider.fetchTableField(datasourceRequest); return tableFields.stream().filter(tableField -> { - return !tableField.getOriginName().equalsIgnoreCase("dataease_uuid"); + return !tableField.getOriginName().equalsIgnoreCase("gisbi_uuid"); }).collect(Collectors.toList()); } @@ -757,13 +846,13 @@ public class DatasourceServer implements DatasourceApi { private static final Integer replace = 0; private static final Integer append = 1; - public ExcelFileData excelUpload(@RequestParam("file") MultipartFile file, @RequestParam("id") long datasourceId, @RequestParam("editType") Integer editType) throws DEException { + public ExcelFileData uploadFile(@RequestParam("file") MultipartFile file, @RequestParam("id") long datasourceId, @RequestParam("editType") Integer editType) throws DEException { CoreDatasource coreDatasource = null; if (ObjectUtils.isNotEmpty(datasourceId) && 0L != datasourceId) { coreDatasource = dataSourceManage.getCoreDatasource(datasourceId); } ExcelUtils excelUtils = new ExcelUtils(); - ExcelFileData excelFileData = excelUtils.excelSaveAndParse(file); + ExcelFileData excelFileData = excelUtils.excelSaveAndParse(file, String.valueOf(AuthUtils.getUser().getUserId())); if (Objects.equals(editType, append)) { //按照excel sheet 名称匹配，替换：0；追加：1 if (coreDatasource != null) { @@ -814,6 +903,39 @@ public class DatasourceServer implements DatasourceApi { return excelFileData; } + public ExcelFileData loadRemoteFile(RemoteExcelRequest remoteExcelRequest) throws DEException, IOException { + remoteExcelRequest.setUserName(new String(Base64.getDecoder().decode(remoteExcelRequest.getUserName()))); + remoteExcelRequest.setPasswd(new String(Base64.getDecoder().decode(remoteExcelRequest.getPasswd()))); + ExcelFileData excelFileData = new ExcelUtils().parseRemoteExcel(remoteExcelRequest); + CoreDatasource coreDatasource = null; + if (ObjectUtils.isNotEmpty(remoteExcelRequest.getDatasourceId()) && 0L != remoteExcelRequest.getDatasourceId()) { + coreDatasource = dataSourceManage.getCoreDatasource(remoteExcelRequest.getDatasourceId()); + } + if (coreDatasource != null) { + DatasourceRequest datasourceRequest = new DatasourceRequest(); + datasourceRequest.setDatasource(transDTO(coreDatasource)); + List datasetTableDTOS = ExcelUtils.getTables(datasourceRequest); + for (ExcelSheetData sheet : excelFileData.getSheets()) { + for (DatasetTableDTO datasetTableDTO : datasetTableDTOS) { + if (excelDataTableName(datasetTableDTO.getTableName()).equals(sheet.getTableName())) { + sheet.setDeTableName(datasetTableDTO.getTableName()); + } + } + } + } + for (ExcelSheetData sheet : excelFileData.getSheets()) { + for (int i = 0; i < sheet.getFields().size() - 1; i++) { + for (int j = i + 1; j < sheet.getFields().size(); j++) { + if (sheet.getFields().get(i).getName().equalsIgnoreCase(sheet.getFields().get(j).getName())) { + DEException.throwException(sheet.getExcelLabel() + Translator.get("i18n_field_name_repeat") + sheet.getFields().get(i).getName()); + } + } + } + } + return excelFileData; + } + + private boolean isEqual(List newTableFields, List oldTableFields) { if (CollectionUtils.isEmpty(newTableFields) || CollectionUtils.isEmpty(oldTableFields)) { return false; @@ -868,6 +990,7 @@ public class DatasourceServer implements DatasourceApi { public ApiDefinition checkApiDatasource(Map request) throws DEException { ApiDefinition apiDefinition = JsonUtil.parseObject(new String(java.util.Base64.getDecoder().decode(request.get("data"))), ApiDefinition.class); + apiDefinition.setType("table"); if (request.keySet().contains("type") && request.get("type").equals("apiStructure")) { apiDefinition.setShowApiStructure(true); } @@ -923,6 +1046,8 @@ public class DatasourceServer implements DatasourceApi { String status = null; if (coreDatasource.getType().startsWith("API")) { status = (String) invokeMethod(coreDatasource.getType(), "checkAPIStatus", DatasourceRequest.class, datasourceRequest); + } else if (coreDatasource.getType().startsWith("Excel")) { + status = ExcelUtils.checkStatus(datasourceRequest); } else { Provider provider = ProviderFactory.getProvider(coreDatasource.getType()); status = provider.checkStatus(datasourceRequest); @@ -949,6 +1074,7 @@ public class DatasourceServer implements DatasourceApi { PreviewSqlDTO previewSqlDTO = new PreviewSqlDTO(); previewSqlDTO.setSql(sql); previewSqlDTO.setDatasourceId(id); + previewSqlDTO.setIsCross(false); return datasetDataManage.previewSql(previewSqlDTO); } @@ -980,7 +1106,12 @@ public class DatasourceServer implements DatasourceApi { CoreDatasource coreDatasource = dataSourceManage.getCoreDatasource(dsId); DatasourceRequest datasourceRequest = new DatasourceRequest(); datasourceRequest.setDatasource(transDTO(coreDatasource)); - List datasetTableDTOS = (List) invokeMethod(coreDatasource.getType(), "getApiTables", DatasourceRequest.class, datasourceRequest); + List datasetTableDTOS = new ArrayList<>(); + if (coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.toString())) { + datasetTableDTOS = (List) invokeMethod(coreDatasource.getType(), "getApiTables", DatasourceRequest.class, datasourceRequest); + } else { + datasetTableDTOS = ExcelUtils.getTables(datasourceRequest); + } for (int i = 0; i < pager.getRecords().size(); i++) { for (int i1 = 0; i1 < datasetTableDTOS.size(); i1++) { if (pager.getRecords().get(i).getTableName().equalsIgnoreCase(datasetTableDTOS.get(i1).getTableName())) { @@ -1186,6 +1317,17 @@ public class DatasourceServer implements DatasourceApi { datasourceDTO.setStatus("Error"); } } + } else { + if (hidePw) { + Provider provider = ProviderFactory.getProvider(datasourceDTO.getType()); + provider.hidePW(datasourceDTO); + } + } + if (datasourceDTO.getType().contains(DatasourceConfiguration.DatasourceType.Excel.toString())) { + datasourceDTO.setFileName(ExcelUtils.getFileName(datasource)); + datasourceDTO.setSize(ExcelUtils.getSize(datasource)); + } + if (datasourceDTO.getType().equalsIgnoreCase(DatasourceConfiguration.DatasourceType.ExcelRemote.name()) || datasourceDTO.getType().contains(DatasourceConfiguration.DatasourceType.API.toString())) { CoreDatasourceTask coreDatasourceTask = datasourceTaskServer.selectByDSId(datasourceDTO.getId()); TaskDTO taskDTO = new TaskDTO(); BeanUtils.copyBean(taskDTO, coreDatasourceTask); @@ -1194,15 +1336,6 @@ public class DatasourceServer implements DatasourceApi { if (task != null) { datasourceDTO.setLastSyncTime(task.getStartTime()); } - } else { - if (hidePw) { - Provider provider = ProviderFactory.getProvider(datasourceDTO.getType()); - provider.hidePW(datasourceDTO); - } - } - if (datasourceDTO.getType().equalsIgnoreCase(DatasourceConfiguration.DatasourceType.Excel.toString())) { - datasourceDTO.setFileName(ExcelUtils.getFileName(datasource)); - datasourceDTO.setSize(ExcelUtils.getSize(datasource)); } datasourceDTO.setConfiguration(RsaUtils.symmetricEncrypt(datasourceDTO.getConfiguration())); datasourceDTO.setCreator(coreUserManage.getUserName(Long.valueOf(datasourceDTO.getCreateBy()))); @@ -1214,7 +1347,7 @@ public class DatasourceServer implements DatasourceApi { BeanUtils.copyBean(datasourceDTO, coreDatasource); try { checkDatasourceStatus(datasourceDTO); - if (!Arrays.asList("API", "Excel", "folder").contains(coreDatasource.getType()) && !coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.name())) { + if (!Arrays.asList("API", "Excel", "folder").contains(coreDatasource.getType()) && !coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.name()) && !coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.ExcelRemote.name())) { calciteProvider.updateDsPoolAfterCheckStatus(datasourceDTO); } } catch (DEException e) { @@ -1294,11 +1427,6 @@ public class DatasourceServer implements DatasourceApi { Object resObj = null; try { Method method = getMethod(dsType, methodName, classes); - if (object instanceof DatasourceRequest && dsType.equals(DataSourceType.APILark.name())) { - Class clazz = Class.forName(DatasourceRequest.class.getName()); - Method setToken = clazz.getMethod("setToken", String.class); - setToken.invoke(object, dataSourceManage.getTenantAccessToken()); - } resObj = method.invoke(null, object); } catch (Exception e) { DEException.throwException(msg(e)); @@ -1312,7 +1440,7 @@ public class DatasourceServer implements DatasourceApi { if (exception.getCause() == null) { return exception.getMessage(); } - if (exception instanceof DEException && !(exception.getCause() instanceof DEException)) { + if (exception instanceof DEException && (!(exception.getCause() instanceof DEException) && !(exception.getCause() instanceof InvocationTargetException))) { return exception.getMessage(); } exception = exception.getCause(); diff --git a/core/core-backend/src/main/java/io/gisbi/datasource/type/H2.java b/core/core-backend/src/main/java/io/gisbi/datasource/type/H2.java index 5058cbf..8abbaae 100644 --- a/core/core-backend/src/main/java/io/gisbi/datasource/type/H2.java +++ b/core/core-backend/src/main/java/io/gisbi/datasource/type/H2.java @@ -1,11 +1,29 @@ package io.gisbi.datasource.type; +import io.gisbi.exception.DEException; import io.gisbi.extensions.datasource.vo.DatasourceConfiguration; import lombok.Data; +import lombok.EqualsAndHashCode; +import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; +import java.util.Arrays; +import java.util.List; + +@EqualsAndHashCode(callSuper = true) @Data @Component("h2") public class H2 extends DatasourceConfiguration { private String driver = "org.h2.Driver"; + private List illegalParameters = Arrays.asList("INIT", "RUNSCRIPT"); + + public String getJdbc() { + for (String illegalParameter : illegalParameters) { + if (jdbc.toUpperCase().replace("\\", "").contains(illegalParameter)) { + DEException.throwException("Has illegal parameter: " + jdbc); + } + } + + return jdbc; + } } diff --git a/core/core-backend/src/main/java/io/gisbi/datasource/type/Mysql.java b/core/core-backend/src/main/java/io/gisbi/datasource/type/Mysql.java index 306f015..f846139 100644 --- a/core/core-backend/src/main/java/io/gisbi/datasource/type/Mysql.java +++ b/core/core-backend/src/main/java/io/gisbi/datasource/type/Mysql.java @@ -27,22 +27,24 @@ public class Mysql extends DatasourceConfiguration { } return getJdbcUrl(); } + String jdbcUrl = ""; if (StringUtils.isEmpty(extraParams.trim())) { - return "jdbc:mysql://HOSTNAME:PORT/DATABASE" + jdbcUrl = "jdbc:mysql://HOSTNAME:PORT/DATABASE" .replace("HOSTNAME", getLHost().trim()) .replace("PORT", getLPort().toString().trim()) .replace("DATABASE", getDataBase().trim()); } else { - for (String illegalParameter : illegalParameters) { - if (URLDecoder.decode(getExtraParams()).toLowerCase().contains(illegalParameter.toLowerCase()) || URLDecoder.decode(getExtraParams()).contains(illegalParameter.toLowerCase())) { - DEException.throwException("Illegal parameter: " + illegalParameter); - } - } - return "jdbc:mysql://HOSTNAME:PORT/DATABASE?EXTRA_PARAMS" + jdbcUrl = "jdbc:mysql://HOSTNAME:PORT/DATABASE?EXTRA_PARAMS" .replace("HOSTNAME", getLHost().trim()) .replace("PORT", getLPort().toString().trim()) .replace("DATABASE", getDataBase().trim()) .replace("EXTRA_PARAMS", getExtraParams().trim()); } + for (String illegalParameter : illegalParameters) { + if (URLDecoder.decode(jdbcUrl).toLowerCase().contains(illegalParameter.toLowerCase()) || URLDecoder.decode(jdbcUrl).contains(illegalParameter.toLowerCase())) { + DEException.throwException("Illegal parameter: " + illegalParameter); + } + } + return jdbcUrl; } } diff --git a/core/core-backend/src/main/java/io/gisbi/datasource/type/Pg.java b/core/core-backend/src/main/java/io/gisbi/datasource/type/Pg.java index 810a410..a7544ea 100644 --- a/core/core-backend/src/main/java/io/gisbi/datasource/type/Pg.java +++ b/core/core-backend/src/main/java/io/gisbi/datasource/type/Pg.java @@ -1,40 +1,58 @@ package io.gisbi.datasource.type; +import io.gisbi.exception.DEException; import io.gisbi.extensions.datasource.vo.DatasourceConfiguration; import lombok.Data; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; +import java.net.URLDecoder; +import java.util.Arrays; +import java.util.List; + @Data @Component("pg") public class Pg extends DatasourceConfiguration { private String driver = "org.postgresql.Driver"; private String extraParams = ""; + private List illegalParameters = Arrays.asList("socketFactory", "socketFactoryArg", "sslfactory", "sslhostnameverifier", "sslpasswordcallback", "authenticationPluginClassName"); public String getJdbc() { - if(StringUtils.isNoneEmpty(getUrlType()) && !getUrlType().equalsIgnoreCase("hostName")){ + if (StringUtils.isNoneEmpty(getUrlType()) && !getUrlType().equalsIgnoreCase("hostName")) { + for (String illegalParameter : illegalParameters) { + if (URLDecoder.decode(getJdbcUrl()).contains(illegalParameter)) { + DEException.throwException("Illegal parameter: " + illegalParameter); + } + } return getJdbcUrl(); } - if(StringUtils.isEmpty(extraParams.trim())){ + String jdbcUrl = ""; + if (StringUtils.isEmpty(extraParams.trim())) { if (StringUtils.isEmpty(getSchema())) { - return "jdbc:postgresql://HOSTNAME:PORT/DATABASE" + jdbcUrl = "jdbc:postgresql://HOSTNAME:PORT/DATABASE" .replace("HOSTNAME", getLHost().trim()) .replace("PORT", getLPort().toString().trim()) .replace("DATABASE", getDataBase().trim()); } else { - return "jdbc:postgresql://HOSTNAME:PORT/DATABASE?currentSchema=SCHEMA" + jdbcUrl = "jdbc:postgresql://HOSTNAME:PORT/DATABASE?currentSchema=SCHEMA" .replace("HOSTNAME", getLHost().trim()) .replace("PORT", getLPort().toString().trim()) .replace("DATABASE", getDataBase().trim()) .replace("SCHEMA", getSchema().trim()); } - }else { - return "jdbc:postgresql://HOSTNAME:PORT/DATABASE?EXTRA_PARAMS" + } else { + jdbcUrl = "jdbc:postgresql://HOSTNAME:PORT/DATABASE?EXTRA_PARAMS" .replace("HOSTNAME", getLHost().trim()) .replace("PORT", getLPort().toString().trim()) .replace("DATABASE", getDataBase().trim()) .replace("EXTRA_PARAMS", getExtraParams().trim()); } + for (String illegalParameter : illegalParameters) { + if (URLDecoder.decode(jdbcUrl).toLowerCase().contains(illegalParameter.toLowerCase()) || URLDecoder.decode(jdbcUrl).contains(illegalParameter.toLowerCase())) { + DEException.throwException("Illegal parameter: " + illegalParameter); + } + } + return jdbcUrl; } } diff --git a/core/core-backend/src/main/java/io/gisbi/datasource/type/Redshift.java b/core/core-backend/src/main/java/io/gisbi/datasource/type/Redshift.java index 830a994..d062613 100644 --- a/core/core-backend/src/main/java/io/gisbi/datasource/type/Redshift.java +++ b/core/core-backend/src/main/java/io/gisbi/datasource/type/Redshift.java @@ -1,23 +1,40 @@ package io.gisbi.datasource.type; +import io.gisbi.exception.DEException; import io.gisbi.extensions.datasource.vo.DatasourceConfiguration; import lombok.Data; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; +import java.net.URLDecoder; +import java.util.Arrays; +import java.util.List; + @Data @Component("redshift") public class Redshift extends DatasourceConfiguration { private String driver = "com.amazon.redshift.jdbc42.Driver"; private String extraParams = ""; + private List illegalParameters = Arrays.asList("socketFactory", "socketFactoryArg", "sslfactory", "sslhostnameverifier", "sslpasswordcallback", "authenticationPluginClassName"); public String getJdbc() { - if(StringUtils.isNoneEmpty(getUrlType()) && !getUrlType().equalsIgnoreCase("hostName")){ + if (StringUtils.isNoneEmpty(getUrlType()) && !getUrlType().equalsIgnoreCase("hostName")) { + for (String illegalParameter : illegalParameters) { + if (URLDecoder.decode(getJdbcUrl()).contains(illegalParameter)) { + DEException.throwException("Illegal parameter: " + illegalParameter); + } + } return getJdbcUrl(); } - return "jdbc:redshift://HOSTNAME:PORT/DATABASE" + String jdbcUrl = "jdbc:redshift://HOSTNAME:PORT/DATABASE" .replace("HOSTNAME", getLHost().trim()) .replace("PORT", getLPort().toString().trim()) .replace("DATABASE", getDataBase().trim()); + for (String illegalParameter : illegalParameters) { + if (URLDecoder.decode(jdbcUrl).contains(illegalParameter)) { + DEException.throwException("Illegal parameter: " + illegalParameter); + } + } + return jdbcUrl; } } diff --git a/core/core-backend/src/main/java/io/gisbi/engine/sql/SQLProvider.java b/core/core-backend/src/main/java/io/gisbi/engine/sql/SQLProvider.java index 17e2d59..e49e746 100644 --- a/core/core-backend/src/main/java/io/gisbi/engine/sql/SQLProvider.java +++ b/core/core-backend/src/main/java/io/gisbi/engine/sql/SQLProvider.java @@ -197,7 +197,7 @@ public class SQLProvider { } public static String sqlLimit(String sql, ChartViewDTO view) { - if (StringUtils.equalsIgnoreCase(view.getType(), "table-info")) { + if (StringUtils.equalsAnyIgnoreCase(view.getType(), "table-info", "table-normal")) { return sql; } if (StringUtils.equalsIgnoreCase(view.getResultMode(), "custom")) { diff --git a/core/core-backend/src/main/java/io/gisbi/engine/trans/CustomWhere2Str.java b/core/core-backend/src/main/java/io/gisbi/engine/trans/CustomWhere2Str.java index 3311cd6..3ff9df7 100644 --- a/core/core-backend/src/main/java/io/gisbi/engine/trans/CustomWhere2Str.java +++ b/core/core-backend/src/main/java/io/gisbi/engine/trans/CustomWhere2Str.java @@ -9,6 +9,7 @@ import io.gisbi.extensions.datasource.dto.DatasetTableFieldDTO; import io.gisbi.extensions.datasource.dto.DatasourceSchemaDTO; import io.gisbi.extensions.datasource.model.SQLMeta; import io.gisbi.extensions.datasource.model.SQLObj; +import io.gisbi.extensions.datasource.vo.DatasourceConfiguration; import io.gisbi.extensions.view.filter.DynamicTimeSetting; import io.gisbi.extensions.view.filter.FilterTreeItem; import io.gisbi.extensions.view.filter.FilterTreeObj; @@ -101,10 +102,10 @@ public class CustomWhere2Str { if (StringUtils.equalsIgnoreCase(dsType, "es")) { originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getOriginName()); } else { - originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getDataeaseName()); + originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getGisbiName()); } } else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 3) { - String groupFieldExp = Utils.transGroupFieldToSql(field, originFields); + String groupFieldExp = Utils.transGroupFieldToSql(field, originFields, isCross, dsMap, pluginManage); // 给计算字段处加一个占位符，后续SQL方言转换后再替换 originName = String.format(SqlPlaceholderConstants.CALC_FIELD_PLACEHOLDER, field.getId()); fieldsDialect.put(originName, groupFieldExp); @@ -115,7 +116,7 @@ public class CustomWhere2Str { if (StringUtils.equalsIgnoreCase(dsType, "es")) { originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getOriginName()); } else { - originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getDataeaseName()); + originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getGisbiName()); } } if (field.getDeType() == 1) { @@ -167,7 +168,9 @@ public class CustomWhere2Str { if (StringUtils.equalsIgnoreCase(item.getFilterType(), "enum")) { if (ObjectUtils.isNotEmpty(item.getEnumValue())) { if ((StringUtils.containsIgnoreCase(field.getType(), "NVARCHAR") - || StringUtils.containsIgnoreCase(field.getType(), "NCHAR")) && !isCross) { + || StringUtils.containsIgnoreCase(field.getType(), "NCHAR")) + && !isCross + && StringUtils.equalsIgnoreCase(dsType, DatasourceConfiguration.DatasourceType.sqlServer.getType())) { res = "(" + whereName + " IN (" + item.getEnumValue().stream().map(str -> "'" + SQLConstants.MSSQL_N_PREFIX + str + "'").collect(Collectors.joining(",")) + "))"; } else { res = "(" + whereName + " IN ('" + String.join("','", item.getEnumValue()) + "'))"; @@ -193,14 +196,18 @@ public class CustomWhere2Str { whereValue = "''"; } else if (StringUtils.containsIgnoreCase(item.getTerm(), "in") || StringUtils.containsIgnoreCase(item.getTerm(), "not in")) { if ((StringUtils.containsIgnoreCase(field.getType(), "NVARCHAR") - || StringUtils.containsIgnoreCase(field.getType(), "NCHAR")) && !isCross) { + || StringUtils.containsIgnoreCase(field.getType(), "NCHAR")) + && !isCross + && StringUtils.equalsIgnoreCase(dsType, DatasourceConfiguration.DatasourceType.sqlServer.getType())) { whereValue = "(" + Arrays.stream(value.split(",")).map(str -> "'" + SQLConstants.MSSQL_N_PREFIX + str + "'").collect(Collectors.joining(",")) + ")"; } else { whereValue = "('" + String.join("','", value.split(",")) + "')"; } } else if (StringUtils.containsIgnoreCase(item.getTerm(), "like")) { if ((StringUtils.containsIgnoreCase(field.getType(), "NVARCHAR") - || StringUtils.containsIgnoreCase(field.getType(), "NCHAR")) && !isCross) { + || StringUtils.containsIgnoreCase(field.getType(), "NCHAR")) + && !isCross + && StringUtils.equalsIgnoreCase(dsType, DatasourceConfiguration.DatasourceType.sqlServer.getType())) { whereValue = "'" + SQLConstants.MSSQL_N_PREFIX + "%" + value + "%'"; } else { whereValue = "'%" + value + "%'"; @@ -246,7 +253,9 @@ public class CustomWhere2Str { } } else { if ((StringUtils.containsIgnoreCase(field.getType(), "NVARCHAR") - || StringUtils.containsIgnoreCase(field.getType(), "NCHAR")) && !isCross) { + || StringUtils.containsIgnoreCase(field.getType(), "NCHAR")) + && !isCross + && StringUtils.equalsIgnoreCase(dsType, DatasourceConfiguration.DatasourceType.sqlServer.getType())) { whereValue = String.format(SQLConstants.WHERE_VALUE_VALUE_CH, value); } else { whereValue = String.format(SQLConstants.WHERE_VALUE_VALUE, value); diff --git a/core/core-backend/src/main/java/io/gisbi/engine/trans/Dimension2SQLObj.java b/core/core-backend/src/main/java/io/gisbi/engine/trans/Dimension2SQLObj.java index 25b8fb8..e7cdc6f 100644 --- a/core/core-backend/src/main/java/io/gisbi/engine/trans/Dimension2SQLObj.java +++ b/core/core-backend/src/main/java/io/gisbi/engine/trans/Dimension2SQLObj.java @@ -57,10 +57,10 @@ public class Dimension2SQLObj { if (StringUtils.equalsIgnoreCase(dsType, "es")) { originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), x.getOriginName()); } else { - originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), x.getDataeaseName()); + originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), x.getGisbiName()); } } else if (ObjectUtils.isNotEmpty(x.getExtField()) && Objects.equals(x.getExtField(), ExtFieldConstant.EXT_GROUP)) { - String groupFieldExp = Utils.transGroupFieldToSql(x, originFields); + String groupFieldExp = Utils.transGroupFieldToSql(x, originFields, isCross, dsMap, pluginManage); // 给计算字段处加一个占位符，后续SQL方言转换后再替换 originField = String.format(SqlPlaceholderConstants.CALC_FIELD_PLACEHOLDER, x.getId()); fieldsDialect.put(originField, groupFieldExp); @@ -71,7 +71,7 @@ public class Dimension2SQLObj { if (StringUtils.equalsIgnoreCase(dsType, "es")) { originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), x.getOriginName()); } else { - originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), x.getDataeaseName()); + originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), x.getGisbiName()); } } String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i); diff --git a/core/core-backend/src/main/java/io/gisbi/engine/trans/ExtWhere2Str.java b/core/core-backend/src/main/java/io/gisbi/engine/trans/ExtWhere2Str.java index 1fef90e..dbd187e 100644 --- a/core/core-backend/src/main/java/io/gisbi/engine/trans/ExtWhere2Str.java +++ b/core/core-backend/src/main/java/io/gisbi/engine/trans/ExtWhere2Str.java @@ -9,6 +9,7 @@ import io.gisbi.extensions.datasource.dto.DatasetTableFieldDTO; import io.gisbi.extensions.datasource.dto.DatasourceSchemaDTO; import io.gisbi.extensions.datasource.model.SQLMeta; import io.gisbi.extensions.datasource.model.SQLObj; +import io.gisbi.extensions.datasource.vo.DatasourceConfiguration; import io.gisbi.extensions.view.dto.ChartExtFilterDTO; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -71,10 +72,10 @@ public class ExtWhere2Str { if (StringUtils.equalsIgnoreCase(dsType, "es")) { originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getOriginName()); } else { - originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getDataeaseName()); + originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getGisbiName()); } } else if (ObjectUtils.isNotEmpty(field.getExtField()) && field.getExtField() == 3) { - String groupFieldExp = Utils.transGroupFieldToSql(field, originFields); + String groupFieldExp = Utils.transGroupFieldToSql(field, originFields, isCross, dsMap, pluginManage); // 给计算字段处加一个占位符，后续SQL方言转换后再替换 originName = String.format(SqlPlaceholderConstants.CALC_FIELD_PLACEHOLDER, field.getId()); fieldsDialect.put(originName, groupFieldExp); @@ -85,7 +86,7 @@ public class ExtWhere2Str { if (StringUtils.equalsIgnoreCase(dsType, "es")) { originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getOriginName()); } else { - originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getDataeaseName()); + originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getGisbiName()); } } @@ -166,14 +167,16 @@ public class ExtWhere2Str { break; } } - if (hasN && !isCross) { + if (hasN && !isCross && StringUtils.equalsIgnoreCase(dsType, DatasourceConfiguration.DatasourceType.sqlServer.getType())) { whereValue = "(" + value.stream().map(str -> "'" + SQLConstants.MSSQL_N_PREFIX + str + "'").collect(Collectors.joining(",")) + ")"; } else { whereValue = "('" + StringUtils.join(value, "','") + "')"; } } else { if ((StringUtils.containsIgnoreCase(request.getDatasetTableField().getType(), "NVARCHAR") - || StringUtils.containsIgnoreCase(request.getDatasetTableField().getType(), "NCHAR")) && !isCross) { + || StringUtils.containsIgnoreCase(request.getDatasetTableField().getType(), "NCHAR")) + && !isCross + && StringUtils.equalsIgnoreCase(dsType, DatasourceConfiguration.DatasourceType.sqlServer.getType())) { whereValue = "(" + value.stream().map(str -> "'" + SQLConstants.MSSQL_N_PREFIX + str + "'").collect(Collectors.joining(",")) + ")"; } else { if (request.getDatasetTableField().getDeType() == 2 || request.getDatasetTableField().getDeType() == 3) { @@ -196,14 +199,16 @@ public class ExtWhere2Str { break; } } - if (hasN && !isCross) { + if (hasN && !isCross && StringUtils.equalsIgnoreCase(dsType, DatasourceConfiguration.DatasourceType.sqlServer.getType())) { whereValue = "'" + SQLConstants.MSSQL_N_PREFIX + "%" + value.get(0) + "%'"; } else { whereValue = "'%" + value.get(0) + "%'"; } } else { if ((StringUtils.containsIgnoreCase(request.getDatasetTableField().getType(), "NVARCHAR") - || StringUtils.containsIgnoreCase(request.getDatasetTableField().getType(), "NCHAR")) && !isCross) { + || StringUtils.containsIgnoreCase(request.getDatasetTableField().getType(), "NCHAR")) + && !isCross + && StringUtils.equalsIgnoreCase(dsType, DatasourceConfiguration.DatasourceType.sqlServer.getType())) { whereValue = "'" + SQLConstants.MSSQL_N_PREFIX + "%" + value.get(0) + "%'"; } else { whereValue = "'%" + value.get(0) + "%'"; @@ -248,8 +253,9 @@ public class ExtWhere2Str { List datasetTableFieldList = request.getDatasetTableFieldList(); boolean hasN = false; for (DatasetTableFieldDTO dto : datasetTableFieldList) { - if (StringUtils.containsIgnoreCase(dto.getType(), "NVARCHAR") - || StringUtils.containsIgnoreCase(dto.getType(), "NCHAR")) { + if ((StringUtils.containsIgnoreCase(dto.getType(), "NVARCHAR") + || StringUtils.containsIgnoreCase(dto.getType(), "NCHAR")) + && StringUtils.equalsIgnoreCase(dsType, DatasourceConfiguration.DatasourceType.sqlServer.getType())) { hasN = true; break; } @@ -261,7 +267,9 @@ public class ExtWhere2Str { } } else { if ((StringUtils.containsIgnoreCase(request.getDatasetTableField().getType(), "NVARCHAR") - || StringUtils.containsIgnoreCase(request.getDatasetTableField().getType(), "NCHAR")) && !isCross) { + || StringUtils.containsIgnoreCase(request.getDatasetTableField().getType(), "NCHAR")) + && !isCross + && StringUtils.equalsIgnoreCase(dsType, DatasourceConfiguration.DatasourceType.sqlServer.getType())) { whereValue = String.format(SQLConstants.WHERE_VALUE_VALUE_CH, value.get(0)); } else { whereValue = String.format(SQLConstants.WHERE_VALUE_VALUE, value.get(0)); diff --git a/core/core-backend/src/main/java/io/gisbi/engine/trans/Field2SQLObj.java b/core/core-backend/src/main/java/io/gisbi/engine/trans/Field2SQLObj.java index c7630de..cee7895 100644 --- a/core/core-backend/src/main/java/io/gisbi/engine/trans/Field2SQLObj.java +++ b/core/core-backend/src/main/java/io/gisbi/engine/trans/Field2SQLObj.java @@ -61,10 +61,10 @@ public class Field2SQLObj { if (StringUtils.equalsIgnoreCase(dsType, "es")) { originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), x.getOriginName()); } else { - originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), x.getDataeaseName()); + originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), x.getGisbiName()); } } else if (ObjectUtils.isNotEmpty(x.getExtField()) && Objects.equals(x.getExtField(), ExtFieldConstant.EXT_GROUP)) { - String groupFieldExp = Utils.transGroupFieldToSql(x, originFields); + String groupFieldExp = Utils.transGroupFieldToSql(x, originFields, isCross, dsMap, pluginManage); // 给计算字段处加一个占位符，后续SQL方言转换后再替换 originField = String.format(SqlPlaceholderConstants.CALC_FIELD_PLACEHOLDER, x.getId()); fieldsDialect.put(originField, groupFieldExp); @@ -75,7 +75,7 @@ public class Field2SQLObj { if (StringUtils.equalsIgnoreCase(dsType, "es")) { originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), x.getOriginName()); } else { - originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), x.getDataeaseName()); + originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), x.getGisbiName()); } } String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i); diff --git a/core/core-backend/src/main/java/io/gisbi/engine/trans/Order2SQLObj.java b/core/core-backend/src/main/java/io/gisbi/engine/trans/Order2SQLObj.java index ce6c12b..ad1bdc2 100644 --- a/core/core-backend/src/main/java/io/gisbi/engine/trans/Order2SQLObj.java +++ b/core/core-backend/src/main/java/io/gisbi/engine/trans/Order2SQLObj.java @@ -63,10 +63,10 @@ public class Order2SQLObj { if (StringUtils.equalsIgnoreCase(dsType, "es")) { originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), f.getOriginName()); } else { - originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), f.getDataeaseName()); + originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), f.getGisbiName()); } } else if (ObjectUtils.isNotEmpty(f.getExtField()) && Objects.equals(f.getExtField(), ExtFieldConstant.EXT_GROUP)) { - String groupFieldExp = Utils.transGroupFieldToSql(f, originFields); + String groupFieldExp = Utils.transGroupFieldToSql(f, originFields, isCross, dsMap, pluginManage); // 给计算字段处加一个占位符，后续SQL方言转换后再替换 originField = String.format(SqlPlaceholderConstants.CALC_FIELD_PLACEHOLDER, f.getId()); fieldsDialect.put(originField, groupFieldExp); @@ -77,7 +77,7 @@ public class Order2SQLObj { if (StringUtils.equalsIgnoreCase(dsType, "es")) { originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), f.getOriginName()); } else { - originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), f.getDataeaseName()); + originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), f.getGisbiName()); } } String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_X_PREFIX, i); diff --git a/core/core-backend/src/main/java/io/gisbi/engine/trans/Quota2SQLObj.java b/core/core-backend/src/main/java/io/gisbi/engine/trans/Quota2SQLObj.java index 012f1e2..f69ef6c 100644 --- a/core/core-backend/src/main/java/io/gisbi/engine/trans/Quota2SQLObj.java +++ b/core/core-backend/src/main/java/io/gisbi/engine/trans/Quota2SQLObj.java @@ -57,10 +57,10 @@ public class Quota2SQLObj { if (StringUtils.equalsIgnoreCase(dsType, "es")) { originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), y.getOriginName()); } else { - originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), y.getDataeaseName()); + originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), y.getGisbiName()); } } else if (ObjectUtils.isNotEmpty(y.getExtField()) && Objects.equals(y.getExtField(), ExtFieldConstant.EXT_GROUP)) { - String groupFieldExp = Utils.transGroupFieldToSql(y, originFields); + String groupFieldExp = Utils.transGroupFieldToSql(y, originFields, isCross, dsMap, pluginManage); // 给计算字段处加一个占位符，后续SQL方言转换后再替换 originField = String.format(SqlPlaceholderConstants.CALC_FIELD_PLACEHOLDER, y.getId()); fieldsDialect.put(originField, groupFieldExp); @@ -71,7 +71,7 @@ public class Quota2SQLObj { if (StringUtils.equalsIgnoreCase(dsType, "es")) { originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), y.getOriginName()); } else { - originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), y.getDataeaseName()); + originField = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), y.getGisbiName()); } } String fieldAlias = String.format(SQLConstants.FIELD_ALIAS_Y_PREFIX, i); diff --git a/core/core-backend/src/main/java/io/gisbi/engine/trans/WhereTree2Str.java b/core/core-backend/src/main/java/io/gisbi/engine/trans/WhereTree2Str.java index a3a4bc9..dce7a0c 100644 --- a/core/core-backend/src/main/java/io/gisbi/engine/trans/WhereTree2Str.java +++ b/core/core-backend/src/main/java/io/gisbi/engine/trans/WhereTree2Str.java @@ -1,8 +1,8 @@ package io.gisbi.engine.trans; import io.gisbi.api.permissions.dataset.dto.DataSetRowPermissionsTreeDTO; -import io.gisbi.engine.constant.ExtFieldConstant; import io.gisbi.constant.SQLConstants; +import io.gisbi.engine.constant.ExtFieldConstant; import io.gisbi.engine.utils.Utils; import io.gisbi.extensions.datasource.api.PluginManageApi; import io.gisbi.extensions.datasource.constant.SqlPlaceholderConstants; @@ -11,6 +11,7 @@ import io.gisbi.extensions.datasource.dto.DatasetTableFieldDTO; import io.gisbi.extensions.datasource.dto.DatasourceSchemaDTO; import io.gisbi.extensions.datasource.model.SQLMeta; import io.gisbi.extensions.datasource.model.SQLObj; +import io.gisbi.extensions.datasource.vo.DatasourceConfiguration; import io.gisbi.extensions.view.dto.DatasetRowPermissionsTreeItem; import io.gisbi.extensions.view.dto.DatasetRowPermissionsTreeObj; import org.apache.commons.collections4.CollectionUtils; @@ -117,10 +118,10 @@ public class WhereTree2Str { if (StringUtils.equalsIgnoreCase(dsType, "es")) { originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getOriginName()); } else { - originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getDataeaseName()); + originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getGisbiName()); } } else if (ObjectUtils.isNotEmpty(field.getExtField()) && Objects.equals(field.getExtField(), ExtFieldConstant.EXT_GROUP)) { - String groupFieldExp = Utils.transGroupFieldToSql(field, originFields); + String groupFieldExp = Utils.transGroupFieldToSql(field, originFields, isCross, dsMap, pluginManage); // 给计算字段处加一个占位符，后续SQL方言转换后再替换 originName = String.format(SqlPlaceholderConstants.CALC_FIELD_PLACEHOLDER, field.getId()); fieldsDialect.put(originName, groupFieldExp); @@ -131,7 +132,7 @@ public class WhereTree2Str { if (StringUtils.equalsIgnoreCase(dsType, "es")) { originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getOriginName()); } else { - originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getDataeaseName()); + originName = String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), field.getGisbiName()); } } if (field.getDeType() == 1) { @@ -180,7 +181,9 @@ public class WhereTree2Str { if (StringUtils.equalsIgnoreCase(item.getFilterType(), "enum")) { if (CollectionUtils.isNotEmpty(item.getEnumValue())) { if ((StringUtils.containsIgnoreCase(field.getType(), "NVARCHAR") - || StringUtils.containsIgnoreCase(field.getType(), "NCHAR")) && !isCross) { + || StringUtils.containsIgnoreCase(field.getType(), "NCHAR")) + && !isCross + && StringUtils.equalsIgnoreCase(dsType, DatasourceConfiguration.DatasourceType.sqlServer.getType())) { res = "(" + whereName + " IN (" + item.getEnumValue().stream().map(str -> "'" + SQLConstants.MSSQL_N_PREFIX + str + "'").collect(Collectors.joining(",")) + "))"; } else { res = "(" + whereName + " IN ('" + String.join("','", item.getEnumValue()) + "'))"; @@ -205,14 +208,18 @@ public class WhereTree2Str { whereValue = "''"; } else if (StringUtils.containsIgnoreCase(item.getTerm(), "in") || StringUtils.containsIgnoreCase(item.getTerm(), "not in")) { if ((StringUtils.containsIgnoreCase(field.getType(), "NVARCHAR") - || StringUtils.containsIgnoreCase(field.getType(), "NCHAR")) && !isCross) { + || StringUtils.containsIgnoreCase(field.getType(), "NCHAR")) + && !isCross + && StringUtils.equalsIgnoreCase(dsType, DatasourceConfiguration.DatasourceType.sqlServer.getType())) { whereValue = "(" + Arrays.stream(value.split(",")).map(str -> "'" + SQLConstants.MSSQL_N_PREFIX + str + "'").collect(Collectors.joining(",")) + ")"; } else { whereValue = "('" + String.join("','", value.split(",")) + "')"; } } else if (StringUtils.containsIgnoreCase(item.getTerm(), "like")) { if ((StringUtils.containsIgnoreCase(field.getType(), "NVARCHAR") - || StringUtils.containsIgnoreCase(field.getType(), "NCHAR")) && !isCross) { + || StringUtils.containsIgnoreCase(field.getType(), "NCHAR")) + && !isCross + && StringUtils.equalsIgnoreCase(dsType, DatasourceConfiguration.DatasourceType.sqlServer.getType())) { whereValue = "'" + SQLConstants.MSSQL_N_PREFIX + "%" + value + "%'"; } else { whereValue = "'%" + value + "%'"; @@ -256,7 +263,9 @@ public class WhereTree2Str { } } else { if ((StringUtils.containsIgnoreCase(field.getType(), "NVARCHAR") - || StringUtils.containsIgnoreCase(field.getType(), "NCHAR")) && !isCross) { + || StringUtils.containsIgnoreCase(field.getType(), "NCHAR")) + && !isCross + && StringUtils.equalsIgnoreCase(dsType, DatasourceConfiguration.DatasourceType.sqlServer.getType())) { whereValue = String.format(SQLConstants.WHERE_VALUE_VALUE_CH, value); } else { whereValue = String.format(SQLConstants.WHERE_VALUE_VALUE, value); diff --git a/core/core-backend/src/main/java/io/gisbi/engine/utils/Utils.java b/core/core-backend/src/main/java/io/gisbi/engine/utils/Utils.java index 31ef6f9..b53fd79 100644 --- a/core/core-backend/src/main/java/io/gisbi/engine/utils/Utils.java +++ b/core/core-backend/src/main/java/io/gisbi/engine/utils/Utils.java @@ -1,7 +1,7 @@ package io.gisbi.engine.utils; -import io.gisbi.engine.constant.ExtFieldConstant; import io.gisbi.constant.SQLConstants; +import io.gisbi.engine.constant.ExtFieldConstant; import io.gisbi.exception.DEException; import io.gisbi.extensions.datasource.api.PluginManageApi; import io.gisbi.extensions.datasource.constant.SqlPlaceholderConstants; @@ -90,9 +90,9 @@ public class Utils { // 计算字段允许二次引用，这里递归查询完整引用链 if (Objects.equals(ele.getExtField(), ExtFieldConstant.EXT_NORMAL)) { if (isCross) { - originField = originField.replaceAll("\\[" + ele.getId() + "]", String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), ele.getDataeaseName())); + originField = originField.replaceAll("\\[" + ele.getId() + "]", String.format(SQLConstants.FIELD_NAME, tableObj.getTableAlias(), ele.getGisbiName())); } else { - originField = originField.replaceAll("\\[" + ele.getId() + "]", datasourceType.getPrefix() + tableObj.getTableAlias() + datasourceType.getSuffix() + "." + datasourceType.getPrefix() + ele.getDataeaseName() + datasourceType.getSuffix()); + originField = originField.replaceAll("\\[" + ele.getId() + "]", datasourceType.getPrefix() + tableObj.getTableAlias() + datasourceType.getSuffix() + "." + datasourceType.getPrefix() + ele.getGisbiName() + datasourceType.getSuffix()); } } else { originField = originField.replaceAll("\\[" + ele.getId() + "]", "(" + ele.getOriginName() + ")"); @@ -161,6 +161,8 @@ public class Utils { return "MM" + split + "dd"; case "H_m_s": return "HH:mm:ss"; + case "y_M_d_H": + return "yyyy" + split + "MM" + split + "dd" + " HH"; case "y_M_d_H_m": return "yyyy" + split + "MM" + split + "dd" + " HH:mm"; case "y_M_d_H_m_s": @@ -496,7 +498,15 @@ public class Utils { return null; } - public static String transGroupFieldToSql(DatasetTableFieldDTO dto, List fields) { + public static String transGroupFieldToSql(DatasetTableFieldDTO dto, List fields, boolean isCross, Map dsMap, PluginManageApi pluginManage) { + // 从fields里取最新的dto + for (DatasetTableFieldDTO fieldDTO : fields) { + if (Objects.equals(dto.getId(), fieldDTO.getId())) { + dto.setGroupList(fieldDTO.getGroupList()); + dto.setOtherGroup(fieldDTO.getOtherGroup()); + break; + } + } // get origin field DatasetTableFieldDTO originField = null; for (DatasetTableFieldDTO ele : fields) { @@ -509,6 +519,22 @@ public class Utils { DEException.throwException("Field not exists"); } + DsTypeDTO datasourceType = null; + if (dsMap != null && dsMap.entrySet().iterator().hasNext()) { + Map.Entry next = dsMap.entrySet().iterator().next(); + datasourceType = getDs(pluginManage, next.getValue().getType()); + } + if (datasourceType == null) { + DEException.throwException("Datasource not exists"); + } + + String fieldName; + if (isCross) { + fieldName = originField.getGisbiName(); + } else { + fieldName = datasourceType.getPrefix() + originField.getGisbiName() + datasourceType.getSuffix(); + } + StringBuilder exp = new StringBuilder(); exp.append(" (CASE "); if (originField.getDeType() == 0) { @@ -516,7 +542,7 @@ public class Utils { exp.append(" WHEN "); for (int i = 0; i < fieldGroupDTO.getText().size(); i++) { String value = fieldGroupDTO.getText().get(i); - exp.append(originField.getDataeaseName()).append(" = ").append("'").append(transValue(value)).append("'"); + exp.append(fieldName).append(" = ").append("'").append(transValue(value)).append("'"); if (i < fieldGroupDTO.getText().size() - 1) { exp.append(" OR "); } @@ -526,17 +552,17 @@ public class Utils { } else if (originField.getDeType() == 1) { for (FieldGroupDTO fieldGroupDTO : dto.getGroupList()) { exp.append(" WHEN "); - exp.append(originField.getDataeaseName()).append(" >= ").append("'").append(fieldGroupDTO.getStartTime()).append("'"); + exp.append(fieldName).append(" >= ").append("'").append(fieldGroupDTO.getStartTime()).append("'"); exp.append(" AND "); - exp.append(originField.getDataeaseName()).append(" <= ").append("'").append(fieldGroupDTO.getEndTime()).append("'"); + exp.append(fieldName).append(" <= ").append("'").append(fieldGroupDTO.getEndTime()).append("'"); exp.append(" THEN '").append(transValue(fieldGroupDTO.getName())).append("'"); } } else if (originField.getDeType() == 2 || originField.getDeType() == 3 || originField.getDeType() == 4) { for (FieldGroupDTO fieldGroupDTO : dto.getGroupList()) { exp.append(" WHEN "); - exp.append(originField.getDataeaseName()).append(StringUtils.equalsIgnoreCase(fieldGroupDTO.getMinTerm(), "le") ? " >= " : " > ").append(fieldGroupDTO.getMin()); + exp.append(fieldName).append(StringUtils.equalsIgnoreCase(fieldGroupDTO.getMinTerm(), "le") ? " >= " : " > ").append(fieldGroupDTO.getMin()); exp.append(" AND "); - exp.append(originField.getDataeaseName()).append(StringUtils.equalsIgnoreCase(fieldGroupDTO.getMaxTerm(), "le") ? " <= " : " < ").append(fieldGroupDTO.getMax()); + exp.append(fieldName).append(StringUtils.equalsIgnoreCase(fieldGroupDTO.getMaxTerm(), "le") ? " <= " : " < ").append(fieldGroupDTO.getMax()); exp.append(" THEN '").append(transValue(fieldGroupDTO.getName())).append("'"); } } diff --git a/core/core-backend/src/main/java/io/gisbi/exportCenter/dao/auto/entity/CoreExportTask.java b/core/core-backend/src/main/java/io/gisbi/exportCenter/dao/auto/entity/CoreExportTask.java index 9a2ce50..307a02f 100644 --- a/core/core-backend/src/main/java/io/gisbi/exportCenter/dao/auto/entity/CoreExportTask.java +++ b/core/core-backend/src/main/java/io/gisbi/exportCenter/dao/auto/entity/CoreExportTask.java @@ -26,7 +26,7 @@ public class CoreExportTask implements Serializable { private String fileSizeUnit; - private String exportFrom; + private Long exportFrom; private String exportStatus; @@ -88,11 +88,11 @@ public class CoreExportTask implements Serializable { this.fileSizeUnit = fileSizeUnit; } - public String getExportFrom() { + public Long getExportFrom() { return exportFrom; } - public void setExportFrom(String exportFrom) { + public void setExportFrom(Long exportFrom) { this.exportFrom = exportFrom; } diff --git a/core/core-backend/src/main/java/io/gisbi/exportCenter/manage/ExportCenterLimitManage.java b/core/core-backend/src/main/java/io/gisbi/exportCenter/manage/ExportCenterLimitManage.java index 09eca65..abbb292 100644 --- a/core/core-backend/src/main/java/io/gisbi/exportCenter/manage/ExportCenterLimitManage.java +++ b/core/core-backend/src/main/java/io/gisbi/exportCenter/manage/ExportCenterLimitManage.java @@ -7,10 +7,10 @@ import org.springframework.stereotype.Component; @Component("exportCenterLimitManage") public class ExportCenterLimitManage { - @Value("${dataease.export.dataset.limit:100000}") + @Value("${gisbi.export.dataset.limit:100000}") private Long datasetLimit; - @Value("${dataease.export.views.limit:100000}") + @Value("${gisbi.export.views.limit:100000}") private Long viewLimit; public Long getExportLimit(String type) { diff --git a/core/core-backend/src/main/java/io/gisbi/exportCenter/manage/ExportCenterManage.java b/core/core-backend/src/main/java/io/gisbi/exportCenter/manage/ExportCenterManage.java index 7d2b0a4..00af2ef 100644 --- a/core/core-backend/src/main/java/io/gisbi/exportCenter/manage/ExportCenterManage.java +++ b/core/core-backend/src/main/java/io/gisbi/exportCenter/manage/ExportCenterManage.java @@ -1,81 +1,46 @@ package io.gisbi.exportCenter.manage; - import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; -import com.fasterxml.jackson.core.type.TypeReference; -import com.google.gson.Gson; -import io.gisbi.api.chart.dto.ViewDetailField; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import io.gisbi.api.chart.request.ChartExcelRequest; -import io.gisbi.api.chart.request.ChartExcelRequestInner; import io.gisbi.api.dataset.dto.DataSetExportRequest; -import io.gisbi.api.dataset.union.DatasetGroupInfoDTO; -import io.gisbi.api.dataset.union.UnionDTO; import io.gisbi.api.export.BaseExportApi; import io.gisbi.api.xpack.dataFilling.DataFillingApi; -import io.gisbi.api.xpack.dataFilling.dto.DataFillFormTableDataRequest; -import io.gisbi.auth.bo.TokenUserBO; -import io.gisbi.chart.dao.auto.mapper.CoreChartViewMapper; -import io.gisbi.chart.server.ChartDataServer; import io.gisbi.commons.utils.ExcelWatermarkUtils; -import io.gisbi.dataset.dao.auto.entity.CoreDatasetGroup; -import io.gisbi.dataset.dao.auto.mapper.CoreDatasetGroupMapper; +import io.gisbi.constant.LogOT; +import io.gisbi.constant.LogST; import io.gisbi.dataset.manage.*; -import io.gisbi.datasource.utils.DatasourceUtils; -import io.gisbi.constant.DeTypeConstants; -import io.gisbi.engine.sql.SQLProvider; -import io.gisbi.engine.trans.Field2SQLObj; -import io.gisbi.engine.trans.Order2SQLObj; -import io.gisbi.engine.trans.Table2SQLObj; -import io.gisbi.engine.trans.WhereTree2Str; -import io.gisbi.engine.utils.Utils; import io.gisbi.exception.DEException; +import io.gisbi.exportCenter.dao.auto.entity.CoreExportDownloadTask; import io.gisbi.exportCenter.dao.auto.entity.CoreExportTask; +import io.gisbi.exportCenter.dao.auto.mapper.CoreExportDownloadTaskMapper; import io.gisbi.exportCenter.dao.auto.mapper.CoreExportTaskMapper; -import io.gisbi.exportCenter.util.ExportCenterUtils; -import io.gisbi.extensions.datasource.api.PluginManageApi; -import io.gisbi.extensions.datasource.dto.DatasetTableFieldDTO; -import io.gisbi.extensions.datasource.dto.DatasourceRequest; -import io.gisbi.extensions.datasource.dto.DatasourceSchemaDTO; -import io.gisbi.extensions.datasource.factory.ProviderFactory; -import io.gisbi.extensions.datasource.model.SQLMeta; -import io.gisbi.extensions.datasource.provider.Provider; -import io.gisbi.extensions.view.dto.*; -import io.gisbi.i18n.Translator; +import io.gisbi.exportCenter.dao.ext.mapper.ExportTaskExtMapper; +import io.gisbi.log.DeLog; import io.gisbi.model.ExportTaskDTO; -import io.gisbi.system.manage.CoreUserManage; import io.gisbi.system.manage.SysParameterManage; import io.gisbi.utils.*; import io.gisbi.visualization.dao.auto.entity.VisualizationWatermark; import io.gisbi.visualization.dao.auto.mapper.VisualizationWatermarkMapper; import io.gisbi.visualization.dao.ext.mapper.ExtDataVisualizationMapper; import io.gisbi.visualization.server.DataVisualizationServer; -import io.gisbi.websocket.WsMessage; -import io.gisbi.websocket.WsService; -import jakarta.annotation.PostConstruct; import jakarta.annotation.Resource; import jakarta.servlet.http.HttpServletResponse; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.ObjectUtils; +import lombok.Data; import org.apache.commons.lang3.StringUtils; import org.apache.poi.ss.usermodel.*; -import org.apache.poi.xssf.streaming.SXSSFWorkbook; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; import io.gisbi.visualization.dto.WatermarkContentDTO; +import io.gisbi.api.permissions.user.vo.UserFormVO; -import java.io.*; import java.net.InetAddress; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; -import java.text.DecimalFormat; import java.util.*; import java.util.concurrent.Future; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; @Component @Transactional(rollbackFor = Exception.class) @@ -83,55 +48,30 @@ public class ExportCenterManage implements BaseExportApi { @Resource private CoreExportTaskMapper exportTaskMapper; @Resource + private CoreExportDownloadTaskMapper coreExportDownloadTaskMapper; + @Resource + private ExportTaskExtMapper exportTaskExtMapper; + @Resource private DatasetGroupManage datasetGroupManage; @Resource DataVisualizationServer dataVisualizationServer; @Resource - private CoreChartViewMapper coreChartViewMapper; - - @Autowired - private WsService wsService; - @Autowired(required = false) - private PluginManageApi pluginManage; + private ExportCenterDownLoadManage exportCenterDownLoadManage; @Resource private SysParameterManage sysParameterManage; - @Value("${dataease.export.core.size:10}") + @Value("${gisbi.export.core.size:10}") private int core; - @Value("${dataease.export.max.size:10}") + @Value("${gisbi.export.max.size:10}") private int max; - - private final static String DATA_URL_TITLE = "data:image/jpeg;base64,"; - @Value("${gisbi.path.exportData:/opt/gisbi/data/exportData/}") + @Value("${gisbi.path.exportData:/opt/gisbi2.0/data/exportData/}") private String exportData_path; @Resource private VisualizationWatermarkMapper watermarkMapper; @Resource private ExtDataVisualizationMapper visualizationMapper; - - public Integer getExtractPageSize() { - return extractPageSize; - } - - @Value("${dataease.export.page.size:50000}") - private Integer extractPageSize; static private List STATUS = Arrays.asList("SUCCESS", "FAILED", "PENDING", "IN_PROGRESS", "ALL"); - private ScheduledThreadPoolExecutor scheduledThreadPoolExecutor; - private int keepAliveSeconds = 600; private Map Running_Task = new HashMap<>(); - @Resource - private ChartDataServer chartDataServer; - @Resource - private CoreDatasetGroupMapper coreDatasetGroupMapper; - @Resource - private CoreUserManage coreUserManage; - @Resource - private DatasetSQLManage datasetSQLManage; - @Resource - private DatasetTableFieldManage datasetTableFieldManage; - @Resource - private DatasetDataManage datasetDataManage; - private final Long sheetLimit = 1000000L; @Autowired(required = false) private DataFillingApi dataFillingApi = null; @@ -140,56 +80,12 @@ public class ExportCenterManage implements BaseExportApi { } - @PostConstruct - public void init() { - scheduledThreadPoolExecutor = new ScheduledThreadPoolExecutor(core); - scheduledThreadPoolExecutor.setKeepAliveTime(keepAliveSeconds, TimeUnit.SECONDS); - scheduledThreadPoolExecutor.setMaximumPoolSize(max); - } - - @Scheduled(fixedRate = 5000) - public void checkRunningTask() { - Iterator> iterator = Running_Task.entrySet().iterator(); - while (iterator.hasNext()) { - Map.Entry entry = iterator.next(); - if (entry.getValue().isDone()) { - iterator.remove(); - try { - CoreExportTask exportTask = exportTaskMapper.selectById(entry.getKey()); - ExportTaskDTO exportTaskDTO = new ExportTaskDTO(); - BeanUtils.copyBean(exportTaskDTO, exportTask); - setExportFromName(exportTaskDTO); - WsMessage message = new WsMessage(exportTask.getUserId(), "/task-export-topic", exportTaskDTO); - wsService.releaseMessage(message); - } catch (Exception e) { - - } - } - } - } - - public void download(String id, HttpServletResponse response) throws Exception { + if (coreExportDownloadTaskMapper.selectById(id) == null) { + DEException.throwException("任务不存在"); + } CoreExportTask exportTask = exportTaskMapper.selectById(id); - OutputStream outputStream = response.getOutputStream(); - response.setContentType("application/vnd.ms-excel"); - - response.setHeader("Content-disposition", "attachment;filename=" + URLEncoder.encode(exportTask.getFileName(), StandardCharsets.UTF_8)); - InputStream fileInputStream; - if (exportTask.getExportTime() < 1730277243491L) { - fileInputStream = new FileInputStream(exportData_path + id + "/" + exportTask.getFileName()); - } else { - fileInputStream = new FileInputStream(exportData_path + id + "/" + id + ".xlsx"); - } - byte[] buffer = new byte[4096]; - int bytesRead; - while ((bytesRead = fileInputStream.read(buffer)) != -1) { - outputStream.write(buffer, 0, bytesRead); - } - outputStream.flush(); - outputStream.close(); - fileInputStream.close(); - response.flushBuffer(); + exportCenterDownLoadManage.download(exportTask, response); } public void delete(String id) { @@ -247,39 +143,69 @@ public class ExportCenterManage implements BaseExportApi { FileUtils.deleteDirectoryRecursively(exportData_path + id); if (exportTask.getExportFromType().equalsIgnoreCase("chart")) { ChartExcelRequest request = JsonUtil.parseObject(exportTask.getParams(), ChartExcelRequest.class); - startViewTask(exportTask, request); + exportCenterDownLoadManage.startViewTask(exportTask, request); } if (exportTask.getExportFromType().equalsIgnoreCase("dataset")) { DataSetExportRequest request = JsonUtil.parseObject(exportTask.getParams(), DataSetExportRequest.class); - startDatasetTask(exportTask, request); + exportCenterDownLoadManage.startDatasetTask(exportTask, request); } if (exportTask.getExportFromType().equalsIgnoreCase("data_filling")) { HashMap request = JsonUtil.parseObject(exportTask.getParams(), HashMap.class); - startDataFillingTask(exportTask, request); + exportCenterDownLoadManage.startDataFillingTask(exportTask, request); } } - public List exportTasks(String status) { + public IPage pager(Page page, String status) { if (!STATUS.contains(status)) { DEException.throwException("Invalid status: " + status); } + QueryWrapper queryWrapper = new QueryWrapper<>(); queryWrapper.eq("user_id", AuthUtils.getUser().getUserId()); + if (!status.equalsIgnoreCase("ALL")) { + queryWrapper.eq("export_status", status); + } queryWrapper.orderByDesc("export_time"); - List exportTasks = exportTaskMapper.selectList(queryWrapper); - List result = new ArrayList<>(); - exportTasks.forEach(exportTask -> { - ExportTaskDTO exportTaskDTO = new ExportTaskDTO(); - BeanUtils.copyBean(exportTaskDTO, exportTask); - if (status.equalsIgnoreCase("ALL") || status.equalsIgnoreCase(exportTaskDTO.getExportStatus())) { - setExportFromAbsName(exportTaskDTO); + IPage pager = exportTaskExtMapper.pager(page, queryWrapper); + + List records = pager.getRecords(); + records.forEach(exportTask -> { + if (status.equalsIgnoreCase("ALL") || status.equalsIgnoreCase(exportTask.getExportStatus())) { + setExportFromAbsName(exportTask); } - if (status.equalsIgnoreCase("ALL") || status.equalsIgnoreCase(exportTaskDTO.getExportStatus())) { - proxy().setOrg(exportTaskDTO); + if (status.equalsIgnoreCase("ALL") || status.equalsIgnoreCase(exportTask.getExportStatus())) { + proxy().setOrg(exportTask); } - result.add(exportTaskDTO); }); + return pager; + } + + public Map exportTasks() { + Map result = new HashMap<>(); + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("user_id", AuthUtils.getUser().getUserId()); + queryWrapper.eq("export_status", "IN_PROGRESS"); + result.put("IN_PROGRESS", exportTaskMapper.selectCount(queryWrapper)); + + queryWrapper.clear(); + queryWrapper.eq("user_id", AuthUtils.getUser().getUserId()); + queryWrapper.eq("export_status", "SUCCESS"); + result.put("SUCCESS", exportTaskMapper.selectCount(queryWrapper)); + + queryWrapper.clear(); + queryWrapper.eq("user_id", AuthUtils.getUser().getUserId()); + queryWrapper.eq("export_status", "FAILED"); + result.put("FAILED", exportTaskMapper.selectCount(queryWrapper)); + + queryWrapper.clear(); + queryWrapper.eq("user_id", AuthUtils.getUser().getUserId()); + queryWrapper.eq("export_status", "PENDING"); + result.put("PENDING", exportTaskMapper.selectCount(queryWrapper)); + + queryWrapper.clear(); + queryWrapper.eq("user_id", AuthUtils.getUser().getUserId()); + result.put("ALL", exportTaskMapper.selectCount(queryWrapper)); return result; } @@ -311,18 +237,6 @@ public class ExportCenterManage implements BaseExportApi { } } - private void setExportFromName(ExportTaskDTO exportTaskDTO) { - if (exportTaskDTO.getExportFromType().equalsIgnoreCase("chart")) { - exportTaskDTO.setExportFromName(coreChartViewMapper.selectById(exportTaskDTO.getExportFrom()).getTitle()); - } - if (exportTaskDTO.getExportFromType().equalsIgnoreCase("dataset")) { - exportTaskDTO.setExportFromName(coreDatasetGroupMapper.selectById(exportTaskDTO.getExportFrom()).getName()); - } - if (exportTaskDTO.getExportFromType().equalsIgnoreCase("data_filling")) { - exportTaskDTO.setExportFromName(getDataFillingApi().get(Long.parseLong(exportTaskDTO.getExportFrom())).getName()); - } - } - private String hostName() { String hostname = null; try { @@ -334,11 +248,11 @@ public class ExportCenterManage implements BaseExportApi { return hostname; } - public void addTask(String exportFrom, String exportFromType, ChartExcelRequest request) { + public void addTask(String exportFrom, String exportFromType, ChartExcelRequest request, String busiFlag) { CoreExportTask exportTask = new CoreExportTask(); - exportTask.setId(UUID.randomUUID().toString()); + exportTask.setId(IDUtils.snowID().toString()); exportTask.setUserId(AuthUtils.getUser().getUserId()); - exportTask.setExportFrom(exportFrom); + exportTask.setExportFrom(Long.valueOf(exportFrom)); exportTask.setExportFromType(exportFromType); exportTask.setExportStatus("PENDING"); exportTask.setFileName(request.getViewName() + ".xlsx"); @@ -347,15 +261,20 @@ public class ExportCenterManage implements BaseExportApi { exportTask.setParams(JsonUtil.toJSONString(request).toString()); exportTask.setExportMachineName(hostName()); exportTaskMapper.insert(exportTask); - startViewTask(exportTask, request); + if (busiFlag.equalsIgnoreCase("dashboard")) { + exportCenterDownLoadManage.startPanelViewTask(exportTask, request); + } else { + exportCenterDownLoadManage.startDataVViewTask(exportTask, request); + } + } public void addTask(Long exportFrom, String exportFromType, DataSetExportRequest request) throws Exception { datasetGroupManage.getDatasetGroupInfoDTO(exportFrom, null); CoreExportTask exportTask = new CoreExportTask(); - exportTask.setId(UUID.randomUUID().toString()); + exportTask.setId(IDUtils.snowID().toString()); exportTask.setUserId(AuthUtils.getUser().getUserId()); - exportTask.setExportFrom(String.valueOf(exportFrom)); + exportTask.setExportFrom(exportFrom); exportTask.setExportFromType(exportFromType); exportTask.setExportStatus("PENDING"); exportTask.setFileName(request.getFilename() + ".xlsx"); @@ -364,16 +283,16 @@ public class ExportCenterManage implements BaseExportApi { exportTask.setParams(JsonUtil.toJSONString(request).toString()); exportTask.setExportMachineName(hostName()); exportTaskMapper.insert(exportTask); - startDatasetTask(exportTask, request); + exportCenterDownLoadManage.startDatasetTask(exportTask, request); } @Override public void addTask(String exportFromId, String exportFromType, HashMap request, Long userId, Long org) { CoreExportTask exportTask = new CoreExportTask(); request.put("org", org); - exportTask.setId(UUID.randomUUID().toString()); + exportTask.setId(IDUtils.snowID().toString()); exportTask.setUserId(userId); - exportTask.setExportFrom(exportFromId); + exportTask.setExportFrom(Long.valueOf(exportFromId)); exportTask.setExportFromType(exportFromType); exportTask.setExportStatus("PENDING"); exportTask.setFileName(request.get("name") + ".xlsx"); @@ -383,399 +302,10 @@ public class ExportCenterManage implements BaseExportApi { exportTask.setExportMachineName(hostName()); exportTaskMapper.insert(exportTask); if (StringUtils.equals(exportFromType, "data_filling")) { - startDataFillingTask(exportTask, request); + exportCenterDownLoadManage.startDataFillingTask(exportTask, request); } } - private void startDataFillingTask(CoreExportTask exportTask, HashMap request) { - - if (ObjectUtils.isEmpty(getDataFillingApi())) { - return; - } - - String dataPath = exportData_path + exportTask.getId(); - File directory = new File(dataPath); - boolean isCreated = directory.mkdir(); - TokenUserBO tokenUserBO = AuthUtils.getUser(); - Future future = scheduledThreadPoolExecutor.submit(() -> { - AuthUtils.setUser(tokenUserBO); - try { - exportTask.setExportStatus("IN_PROGRESS"); - exportTaskMapper.updateById(exportTask); - - getDataFillingApi().writeExcel(dataPath + "/" + exportTask.getId() + ".xlsx", new DataFillFormTableDataRequest().setId(Long.parseLong(exportTask.getExportFrom())).setWithoutLogs(true), exportTask.getUserId(), Long.parseLong(request.get("org").toString())); - - - exportTask.setExportProgress("100"); - exportTask.setExportStatus("SUCCESS"); - - setFileSize(dataPath + "/" + exportTask.getId() + ".xlsx", exportTask); - } catch (Exception e) { - exportTask.setMsg(e.getMessage()); - LogUtil.error("Failed to export data", e); - exportTask.setExportStatus("FAILED"); - } finally { - exportTaskMapper.updateById(exportTask); - } - }); - Running_Task.put(exportTask.getId(), future); - } - - - private void startDatasetTask(CoreExportTask exportTask, DataSetExportRequest request) { - String dataPath = exportData_path + exportTask.getId(); - File directory = new File(dataPath); - boolean isCreated = directory.mkdir(); - - TokenUserBO tokenUserBO = AuthUtils.getUser(); - Future future = scheduledThreadPoolExecutor.submit(() -> { - AuthUtils.setUser(tokenUserBO); - try { - exportTask.setExportStatus("IN_PROGRESS"); - exportTaskMapper.updateById(exportTask); - CoreDatasetGroup coreDatasetGroup = coreDatasetGroupMapper.selectById(exportTask.getExportFrom()); - if (coreDatasetGroup == null) { - throw new Exception("Not found dataset group: " + exportTask.getExportFrom()); - } - DatasetGroupInfoDTO dto = new DatasetGroupInfoDTO(); - BeanUtils.copyBean(dto, coreDatasetGroup); - dto.setUnionSql(null); - List unionDTOList = JsonUtil.parseList(coreDatasetGroup.getInfo(), new TypeReference<>() { - }); - dto.setUnion(unionDTOList); - List dsFields = datasetTableFieldManage.selectByDatasetGroupId(Long.valueOf(exportTask.getExportFrom())); - List allFields = dsFields.stream().map(ele -> { - DatasetTableFieldDTO datasetTableFieldDTO = new DatasetTableFieldDTO(); - BeanUtils.copyBean(datasetTableFieldDTO, ele); - datasetTableFieldDTO.setFieldShortName(ele.getDataeaseName()); - return datasetTableFieldDTO; - }).collect(Collectors.toList()); - - Map sqlMap = datasetSQLManage.getUnionSQLForEdit(dto, null); - String sql = (String) sqlMap.get("sql"); - if (ObjectUtils.isEmpty(allFields)) { - DEException.throwException(Translator.get("i18n_no_fields")); - } - Map desensitizationList = new HashMap<>(); -// allFields = permissionManage.filterColumnPermissions(allFields, desensitizationList, dto.getId(), null); - if (ObjectUtils.isEmpty(allFields)) { - DEException.throwException(Translator.get("i18n_no_column_permission")); - } - dto.setAllFields(allFields); - datasetDataManage.buildFieldName(sqlMap, allFields); - Map dsMap = (Map) sqlMap.get("dsMap"); - DatasourceUtils.checkDsStatus(dsMap); - List dsList = new ArrayList<>(); - for (Map.Entry next : dsMap.entrySet()) { - dsList.add(next.getValue().getType()); - } - boolean needOrder = Utils.isNeedOrder(dsList); - boolean crossDs = Utils.isCrossDs(dsMap); - if (!crossDs) { - if (datasetDataManage.notFullDs.contains(dsMap.entrySet().iterator().next().getValue().getType()) && (boolean) sqlMap.get("isFullJoin")) { - DEException.throwException(Translator.get("i18n_not_full")); - } - sql = Utils.replaceSchemaAlias(sql, dsMap); - } -// List rowPermissionsTree = new ArrayList<>(); -// TokenUserBO user = AuthUtils.getUser(); -// if (user != null) { -// rowPermissionsTree = permissionManage.getRowPermissionsTree(dto.getId(), user.getUserId()); -// } -// if (StringUtils.isNotEmpty(request.getExpressionTree())) { -// Gson gson = new Gson(); -// DatasetRowPermissionsTreeObj datasetRowPermissionsTreeObj = JsonUtil.parseObject(request.getExpressionTree(), DatasetRowPermissionsTreeObj.class); -// permissionManage.getField(datasetRowPermissionsTreeObj); -// DataSetRowPermissionsTreeDTO dataSetRowPermissionsTreeDTO = new DataSetRowPermissionsTreeDTO(); -// dataSetRowPermissionsTreeDTO.setTree(datasetRowPermissionsTreeObj); -// dataSetRowPermissionsTreeDTO.setExportData(true); -// rowPermissionsTree.add(dataSetRowPermissionsTreeDTO); -// } - - Provider provider; - if (crossDs) { - provider = ProviderFactory.getDefaultProvider(); - } else { - provider = ProviderFactory.getProvider(dsList.getFirst()); - } - SQLMeta sqlMeta = new SQLMeta(); - Table2SQLObj.table2sqlobj(sqlMeta, null, "(" + sql + ")", crossDs); - Field2SQLObj.field2sqlObj(sqlMeta, allFields, allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage); -// WhereTree2Str.transFilterTrees(sqlMeta, rowPermissionsTree, allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage); - Order2SQLObj.getOrders(sqlMeta, dto.getSortFields(), allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage); - String replaceSql = provider.rebuildSQL(SQLProvider.createQuerySQL(sqlMeta, false, false, false), sqlMeta, crossDs, dsMap); - Long totalCount = datasetDataManage.getDatasetTotal(dto, replaceSql, null); - Long curLimit = ExportCenterUtils.getExportLimit("dataset"); - totalCount = totalCount > curLimit ? curLimit : totalCount; - - Long sheetCount = (totalCount / sheetLimit) + (totalCount % sheetLimit > 0 ? 1 : 0); - Workbook wb = new SXSSFWorkbook(); - CellStyle cellStyle = wb.createCellStyle(); - Font font = wb.createFont(); - font.setFontHeightInPoints((short) 12); - font.setBold(true); - cellStyle.setFont(font); - cellStyle.setFillForegroundColor(IndexedColors.GREY_25_PERCENT.getIndex()); - cellStyle.setFillPattern(FillPatternType.SOLID_FOREGROUND); - - for (Long s = 1L; s < sheetCount + 1; s++) { - Long sheetSize; - if (s.equals(sheetCount)) { - sheetSize = totalCount - (s - 1) * sheetLimit; - } else { - sheetSize = sheetLimit; - } - Long pageSize = (sheetSize / extractPageSize) + (sheetSize % extractPageSize > 0 ? 1 : 0); - Sheet detailsSheet = null; - List> details = new ArrayList<>(); - for (Long p = 0L; p < pageSize; p++) { - int beforeCount = (int) ((s - 1) * sheetLimit); - String querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, false, beforeCount + p.intValue() * extractPageSize, extractPageSize); - if (pageSize == 1) { - querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, false, 0, sheetSize.intValue()); - } - querySQL = provider.rebuildSQL(querySQL, sqlMeta, crossDs, dsMap); - DatasourceRequest datasourceRequest = new DatasourceRequest(); - datasourceRequest.setQuery(querySQL); - datasourceRequest.setDsList(dsMap); - Map previewData = datasetDataManage.buildPreviewData(provider.fetchResultField(datasourceRequest), allFields, desensitizationList); - List> data = (List>) previewData.get("data"); - if (p.equals(0L)) { - detailsSheet = wb.createSheet("数据" + s); - List header = new ArrayList<>(); - for (DatasetTableFieldDTO field : allFields) { - header.add(field.getName()); - } - details.add(header); - for (Map obj : data) { - List row = new ArrayList<>(); - for (DatasetTableFieldDTO field : allFields) { - String string = (String) obj.get(field.getDataeaseName()); - row.add(string); - } - details.add(row); - } - if (CollectionUtils.isNotEmpty(details)) { - for (int i = 0; i < details.size(); i++) { - Row row = detailsSheet.createRow(i); - List rowData = details.get(i); - if (rowData != null) { - for (int j = 0; j < rowData.size(); j++) { - Cell cell = row.createCell(j); - if (i == 0) { - cell.setCellValue(rowData.get(j)); - cell.setCellStyle(cellStyle); - detailsSheet.setColumnWidth(j, 255 * 20); - } else { - if ((allFields.get(j).getDeType().equals(DeTypeConstants.DE_INT) || allFields.get(j).getDeType() == DeTypeConstants.DE_FLOAT) && StringUtils.isNotEmpty(rowData.get(j))) { - try { - cell.setCellValue(Double.valueOf(rowData.get(j))); - } catch (Exception e) { - cell.setCellValue(rowData.get(j)); - } - } else { - cell.setCellValue(rowData.get(j)); - } - } - } - } - } - } - } else { - details.clear(); - for (Map obj : data) { - List row = new ArrayList<>(); - for (DatasetTableFieldDTO field : allFields) { - String string = (String) obj.get(field.getDataeaseName()); - row.add(string); - } - details.add(row); - } - int lastNum = detailsSheet.getLastRowNum(); - for (int i = 0; i < details.size(); i++) { - Row row = detailsSheet.createRow(i + lastNum + 1); - List rowData = details.get(i); - if (rowData != null) { - for (int j = 0; j < rowData.size(); j++) { - Cell cell = row.createCell(j); - cell.setCellValue(rowData.get(j)); - } - } - } - } - exportTask.setExportStatus("IN_PROGRESS"); - double exportRogress2 = (double) ((double) s - 1) / ((double) sheetCount); - double exportRogress = (double) ((double) (p + 1) / (double) pageSize) * ((double) 1 / sheetCount); - DecimalFormat df = new DecimalFormat("#.##"); - String formattedResult = df.format((exportRogress + exportRogress2) * 100); - exportTask.setExportProgress(formattedResult); - exportTaskMapper.updateById(exportTask); - } - } - this.addWatermarkTools(wb); - FileOutputStream fileOutputStream = new FileOutputStream(dataPath + "/" + exportTask.getId() + ".xlsx"); - wb.write(fileOutputStream); - fileOutputStream.flush(); - fileOutputStream.close(); - wb.close(); - exportTask.setExportProgress("100"); - exportTask.setExportStatus("SUCCESS"); - setFileSize(dataPath + "/" + exportTask.getId() + ".xlsx", exportTask); - - } catch (Exception e) { - LogUtil.error("Failed to export data", e); - exportTask.setMsg(e.getMessage()); - exportTask.setExportStatus("FAILED"); - } finally { - exportTaskMapper.updateById(exportTask); - } - }); - Running_Task.put(exportTask.getId(), future); - } - - - private void startViewTask(CoreExportTask exportTask, ChartExcelRequest request) { - String dataPath = exportData_path + exportTask.getId(); - File directory = new File(dataPath); - boolean isCreated = directory.mkdir(); - TokenUserBO tokenUserBO = AuthUtils.getUser(); - Future future = scheduledThreadPoolExecutor.submit(() -> { -// LicenseUtil.validate(); - AuthUtils.setUser(tokenUserBO); - try { - exportTask.setExportStatus("IN_PROGRESS"); - exportTaskMapper.updateById(exportTask); - Workbook wb = new SXSSFWorkbook(); - CellStyle cellStyle = wb.createCellStyle(); - Font font = wb.createFont(); - font.setFontHeightInPoints((short) 12); - font.setBold(true); - cellStyle.setFont(font); - cellStyle.setFillForegroundColor(IndexedColors.GREY_25_PERCENT.getIndex()); - cellStyle.setFillPattern(FillPatternType.SOLID_FOREGROUND); - - List details = new ArrayList<>(); - Sheet detailsSheet; - Integer sheetIndex = 1; - if ("dataset".equals(request.getDownloadType()) || request.getViewInfo().getType().equalsIgnoreCase("table-info")) { - request.getViewInfo().getChartExtRequest().setPageSize(Long.valueOf(extractPageSize)); - ChartViewDTO chartViewDTO = chartDataServer.findExcelData(request); - for (long i = 1; i < chartViewDTO.getTotalPage() + 1; i++) { - request.getViewInfo().getChartExtRequest().setGoPage(i); - chartDataServer.findExcelData(request); - details.addAll(request.getDetails()); - if (((details.size() + extractPageSize) > sheetLimit) || i == chartViewDTO.getTotalPage()) { - detailsSheet = wb.createSheet("数据" + sheetIndex); - Integer[] excelTypes = request.getExcelTypes(); - details.add(0, request.getHeader()); - ViewDetailField[] detailFields = request.getDetailFields(); - Object[] header = request.getHeader(); - ChartDataServer.setExcelData(detailsSheet, cellStyle, header, details, detailFields, excelTypes, request.getViewInfo(), wb); - sheetIndex++; - details.clear(); - exportTask.setExportStatus("IN_PROGRESS"); - double exportRogress = (double) (i / (chartViewDTO.getTotalPage() + 1)); - DecimalFormat df = new DecimalFormat("#.##"); - String formattedResult = df.format((exportRogress) * 100); - exportTask.setExportProgress(formattedResult); - exportTaskMapper.updateById(exportTask); - } - } - } else { - downloadNotTableInfoData(request, wb); - } - this.addWatermarkTools(wb); - - try (FileOutputStream outputStream = new FileOutputStream(dataPath + "/" + exportTask.getId() + ".xlsx")) { - wb.write(outputStream); - outputStream.flush(); - } - wb.close(); - exportTask.setExportProgress("100"); - exportTask.setExportStatus("SUCCESS"); - setFileSize(dataPath + "/" + exportTask.getId() + ".xlsx", exportTask); - } catch (Exception e) { - exportTask.setMsg(e.getMessage()); - LogUtil.error("Failed to export data", e); - exportTask.setExportStatus("FAILED"); - } finally { - exportTaskMapper.updateById(exportTask); - } - }); - Running_Task.put(exportTask.getId(), future); - } - - private void downloadNotTableInfoData(ChartExcelRequest request, Workbook wb) { - chartDataServer.findExcelData(request); - //给单元格设置样式 - CellStyle cellStyle = wb.createCellStyle(); - Font font = wb.createFont(); - //设置字体大小 - font.setFontHeightInPoints((short) 12); - //设置字体加粗 - font.setBold(true); - //给字体设置样式 - cellStyle.setFont(font); - //设置单元格背景颜色 - cellStyle.setFillForegroundColor(IndexedColors.GREY_25_PERCENT.getIndex()); - //设置单元格填充样式(使用纯色背景颜色填充) - cellStyle.setFillPattern(FillPatternType.SOLID_FOREGROUND); - if (CollectionUtils.isEmpty(request.getMultiInfo())) { - if (request.getViewInfo().getType().equalsIgnoreCase("chart-mix-dual-line")) { - } else { - List details = request.getDetails(); - Integer[] excelTypes = request.getExcelTypes(); - details.add(0, request.getHeader()); - ViewDetailField[] detailFields = request.getDetailFields(); - Object[] header = request.getHeader(); - Sheet detailsSheet = wb.createSheet("数据"); - if (request.getViewInfo().getType().equalsIgnoreCase("table-normal")) { - ChartDataServer.setExcelData(detailsSheet, cellStyle, header, details, detailFields, excelTypes, request.getViewInfo(), wb); - } else { - ChartDataServer.setExcelData(detailsSheet, cellStyle, header, details, detailFields, excelTypes, request.getViewInfo(), null); - } - } - } else { - //多个sheet - for (int i = 0; i < request.getMultiInfo().size(); i++) { - ChartExcelRequestInner requestInner = request.getMultiInfo().get(i); - - List details = requestInner.getDetails(); - Integer[] excelTypes = requestInner.getExcelTypes(); - details.add(0, requestInner.getHeader()); - ViewDetailField[] detailFields = requestInner.getDetailFields(); - Object[] header = requestInner.getHeader(); - //明细sheet - Sheet detailsSheet = wb.createSheet("数据 " + (i + 1)); - ChartDataServer.setExcelData(detailsSheet, cellStyle, header, details, detailFields, excelTypes, request.getViewInfo(), null); - } - } - } - - private void setFileSize(String filePath, CoreExportTask exportTask) { - File file = new File(filePath); - long length = file.length(); - String unit = "Mb"; - Double size = 0.0; - if ((double) length / 1024 / 1024 > 1) { - if ((double) length / 1024 / 1024 / 1024 > 1) { - unit = "Gb"; - size = Double.valueOf(String.format("%.2f", (double) length / 1024 / 1024 / 1024)); - } else { - size = Double.valueOf(String.format("%.2f", (double) length / 1024 / 1024)); - } - - } else { - unit = "Kb"; - size = Double.valueOf(String.format("%.2f", (double) length / 1024)); - } - exportTask.setFileSize(size); - exportTask.setFileSizeUnit(unit); - } - - - private static final String LOG_RETENTION = "30"; - public void cleanLog() { String key = "basic.exportFileLiveTime"; String val = sysParameterManage.singleVal(key); @@ -795,14 +325,46 @@ public class ExportCenterManage implements BaseExportApi { public void addWatermarkTools(Workbook wb) { VisualizationWatermark watermark = watermarkMapper.selectById("system_default"); WatermarkContentDTO watermarkContent = JsonUtil.parseObject(watermark.getSettingContent(), WatermarkContentDTO.class); -// if (watermarkContent.getEnable() && watermarkContent.getExcelEnable()) { -// UserFormVO userInfo = visualizationMapper.queryInnerUserInfo(AuthUtils.getUser().getUserId()); -// // 在主逻辑中添加水印 -// int watermarkPictureIdx = ExcelWatermarkUtils.addWatermarkImage(wb, watermarkContent, userInfo); // 生成水印图片并获取 ID -// for (Sheet sheet : wb) { -// ExcelWatermarkUtils.addWatermarkToSheet(sheet, watermarkPictureIdx); // 为每个 Sheet 添加水印 -// } -// } + if (watermarkContent.getEnable() && watermarkContent.getExcelEnable()) { + UserFormVO userInfo = visualizationMapper.queryInnerUserInfo(AuthUtils.getUser().getUserId()); + // 在主逻辑中添加水印 + int watermarkPictureIdx = ExcelWatermarkUtils.addWatermarkImage(wb, watermarkContent, userInfo); // 生成水印图片并获取 ID + for (Sheet sheet : wb) { + ExcelWatermarkUtils.addWatermarkToSheet(sheet, watermarkPictureIdx); // 为每个 Sheet 添加水印 + } + } + } + + @DeLog(id = "#p0", ot = LogOT.DOWNLOAD, st = LogST.DATA) + public void generateDownloadUri(String id) { + CoreExportDownloadTask coreExportDownloadTask = coreExportDownloadTaskMapper.selectById(id); + if (coreExportDownloadTask != null) { + coreExportDownloadTask.setCreateTime(System.currentTimeMillis()); + coreExportDownloadTaskMapper.updateById(coreExportDownloadTask); + } else { + coreExportDownloadTask = new CoreExportDownloadTask(); + coreExportDownloadTask.setId(id); + coreExportDownloadTask.setCreateTime(System.currentTimeMillis()); + coreExportDownloadTask.setValidTime(5L); + coreExportDownloadTaskMapper.insert(coreExportDownloadTask); + } + } + + + @Scheduled(fixedRate = 60 * 60 * 1000) + public void checkDownLoadInfos() { + coreExportDownloadTaskMapper.selectList(null).forEach(downLoadInfo -> { + if (System.currentTimeMillis() - downLoadInfo.getCreateTime() > downLoadInfo.getValidTime() * 60 * 1000) { + coreExportDownloadTaskMapper.deleteById(downLoadInfo.getId()); + } + }); + } + + @Data + public class DownLoadInfo { + String id; + Long validTime; // 单位：minutes + Long createTime; } } diff --git a/core/core-backend/src/main/java/io/gisbi/exportCenter/server/ExportCenterServer.java b/core/core-backend/src/main/java/io/gisbi/exportCenter/server/ExportCenterServer.java index 2280439..31e07a1 100644 --- a/core/core-backend/src/main/java/io/gisbi/exportCenter/server/ExportCenterServer.java +++ b/core/core-backend/src/main/java/io/gisbi/exportCenter/server/ExportCenterServer.java @@ -1,5 +1,7 @@ package io.gisbi.exportCenter.server; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import io.gisbi.api.exportCenter.ExportCenterApi; import io.gisbi.exportCenter.manage.ExportCenterManage; import io.gisbi.exportCenter.util.ExportCenterUtils; @@ -11,6 +13,7 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; import java.util.List; +import java.util.Map; @RestController @RequestMapping("/exportCenter") @@ -20,8 +23,14 @@ public class ExportCenterServer implements ExportCenterApi { private ExportCenterManage exportCenterManage; @Override - public List exportTasks(String status) { - return exportCenterManage.exportTasks(status); + public Map exportTasks() { + return exportCenterManage.exportTasks(); + } + + @Override + public IPage pager(int goPage, int pageSize, String status) { + Page page = new Page<>(goPage, pageSize); + return exportCenterManage.pager(page, status); } @Override @@ -44,6 +53,12 @@ public class ExportCenterServer implements ExportCenterApi { exportCenterManage.download(id, response); } + @Override + public String generateDownloadUri(String id) throws Exception { + exportCenterManage.generateDownloadUri(id); + return ""; + } + @Override public void retry(String id) { exportCenterManage.retry(id); diff --git a/core/core-backend/src/main/java/io/gisbi/font/manage/FontManage.java b/core/core-backend/src/main/java/io/gisbi/font/manage/FontManage.java index 157ec49..bf435e5 100644 --- a/core/core-backend/src/main/java/io/gisbi/font/manage/FontManage.java +++ b/core/core-backend/src/main/java/io/gisbi/font/manage/FontManage.java @@ -32,7 +32,7 @@ import java.util.UUID; @Component public class FontManage { - @Value("${gisbi.path.font:/opt/gisbi/data/font/}") + @Value("${gisbi.path.font:/opt/gisbi2.0/data/font/}") private String path; @Resource diff --git a/core/core-backend/src/main/java/io/gisbi/license/manage/CoreLicManage.java b/core/core-backend/src/main/java/io/gisbi/license/manage/CoreLicManage.java index 70ab854..077b7b2 100644 --- a/core/core-backend/src/main/java/io/gisbi/license/manage/CoreLicManage.java +++ b/core/core-backend/src/main/java/io/gisbi/license/manage/CoreLicManage.java @@ -6,7 +6,7 @@ import org.springframework.stereotype.Component; @Component public class CoreLicManage { - @Value("1.0.0") + @Value("2.0.0") private String version; public String getVersion() { diff --git a/core/core-backend/src/main/java/io/gisbi/license/server/LicenseServer.java b/core/core-backend/src/main/java/io/gisbi/license/server/LicenseServer.java index 5af0472..2363c10 100644 --- a/core/core-backend/src/main/java/io/gisbi/license/server/LicenseServer.java +++ b/core/core-backend/src/main/java/io/gisbi/license/server/LicenseServer.java @@ -18,6 +18,6 @@ public class LicenseServer { @GetMapping("/version") public String version() { - return "1.0.0"; + return "2.0.0"; } } diff --git a/core/core-backend/src/main/java/io/gisbi/listener/EhCacheStartListener.java b/core/core-backend/src/main/java/io/gisbi/listener/EhCacheStartListener.java index a4ccb70..44cca10 100644 --- a/core/core-backend/src/main/java/io/gisbi/listener/EhCacheStartListener.java +++ b/core/core-backend/src/main/java/io/gisbi/listener/EhCacheStartListener.java @@ -2,24 +2,16 @@ package io.gisbi.listener; import io.gisbi.utils.ConfigUtils; -import org.springframework.beans.factory.config.YamlPropertiesFactoryBean; import org.springframework.context.ApplicationContextInitializer; import org.springframework.context.ConfigurableApplicationContext; -import org.springframework.core.io.ClassPathResource; -import org.springframework.core.io.Resource; - -import java.util.Objects; public class EhCacheStartListener implements ApplicationContextInitializer { @Override public void initialize(ConfigurableApplicationContext applicationContext) { - Resource resource = new ClassPathResource("application.yml"); - YamlPropertiesFactoryBean factory = new YamlPropertiesFactoryBean(); - factory.setResources(resource); - String property = Objects.requireNonNull(factory.getObject()).getProperty("gisbi.login_timeout", "480"); + String property = applicationContext.getEnvironment().getProperty("gisbi.login_timeout", String.class, "480"); System.setProperty("gisbi.login_timeout", property); - String ehcache = ConfigUtils.getConfig("gisbi.path.ehcache", "/opt/gisbi/cache"); + String ehcache = ConfigUtils.getConfig("gisbi.path.ehcache", "/opt/gisbi2.0/cache"); System.setProperty("gisbi.path.ehcache", ehcache); } } diff --git a/core/core-backend/src/main/java/io/gisbi/menu/server/MenuServer.java b/core/core-backend/src/main/java/io/gisbi/menu/server/MenuServer.java index 068e6d2..90b8e8e 100644 --- a/core/core-backend/src/main/java/io/gisbi/menu/server/MenuServer.java +++ b/core/core-backend/src/main/java/io/gisbi/menu/server/MenuServer.java @@ -19,7 +19,6 @@ public class MenuServer implements MenuApi { @Resource private MenuManage menuManage; - @I18n @Override public List query() { List coreMenus = menuManage.coreMenus(); diff --git a/core/core-backend/src/main/java/io/gisbi/resource/ResourceService.java b/core/core-backend/src/main/java/io/gisbi/resource/ResourceService.java index c35102e..22f4746 100644 --- a/core/core-backend/src/main/java/io/gisbi/resource/ResourceService.java +++ b/core/core-backend/src/main/java/io/gisbi/resource/ResourceService.java @@ -2,6 +2,7 @@ package io.gisbi.resource; import io.gisbi.api.permissions.auth.dto.BusiPerCheckDTO; import io.gisbi.constant.AuthEnum; +import io.gisbi.system.manage.CorePermissionManage; import jakarta.annotation.Resource; import org.springframework.stereotype.Component; @@ -10,11 +11,19 @@ import org.springframework.stereotype.Component; */ @Component public class ResourceService { + @Resource + private CorePermissionManage corePermissionManage; public boolean checkPermission(Long id) { BusiPerCheckDTO dto = new BusiPerCheckDTO(); dto.setId(id); dto.setAuthEnum(AuthEnum.READ); - return true; + boolean b; + try { + b = corePermissionManage.checkAuth(dto); + } catch (Exception e) { + b = false; + } + return b; } } diff --git a/core/core-backend/src/main/java/io/gisbi/share/dao/ext/mapper/XpackShareExtMapper.java b/core/core-backend/src/main/java/io/gisbi/share/dao/ext/mapper/XpackShareExtMapper.java index d7c9680..9deec30 100644 --- a/core/core-backend/src/main/java/io/gisbi/share/dao/ext/mapper/XpackShareExtMapper.java +++ b/core/core-backend/src/main/java/io/gisbi/share/dao/ext/mapper/XpackShareExtMapper.java @@ -18,6 +18,7 @@ public interface XpackShareExtMapper { s.id as share_id, v.id as resource_id, v.mobile_layout as ext_flag, + v.status as ext_flag1, v.type, s.creator, s.time, diff --git a/core/core-backend/src/main/java/io/gisbi/share/dao/ext/po/XpackSharePO.java b/core/core-backend/src/main/java/io/gisbi/share/dao/ext/po/XpackSharePO.java index dcaefe6..d9809b6 100644 --- a/core/core-backend/src/main/java/io/gisbi/share/dao/ext/po/XpackSharePO.java +++ b/core/core-backend/src/main/java/io/gisbi/share/dao/ext/po/XpackSharePO.java @@ -30,4 +30,6 @@ public class XpackSharePO implements Serializable { private Integer extFlag; + private Integer extFlag1; + } diff --git a/core/core-backend/src/main/java/io/gisbi/share/interceptor/LinkInterceptor.java b/core/core-backend/src/main/java/io/gisbi/share/interceptor/LinkInterceptor.java index c4660ec..1e1de67 100644 --- a/core/core-backend/src/main/java/io/gisbi/share/interceptor/LinkInterceptor.java +++ b/core/core-backend/src/main/java/io/gisbi/share/interceptor/LinkInterceptor.java @@ -19,8 +19,14 @@ import java.util.List; @Component public class LinkInterceptor implements HandlerInterceptor { - private final static String whiteListText = "/user/ipInfo, /apisix/check, /datasetData/enumValue, /datasetData/enumValueObj, /datasetData/getFieldTree, /dekey, /symmetricKey, /share/validate, /sysParameter/queryOnlineMap, /xpackComponent/viewPlugins, /v3/api-docs, /v3/api-docs/**, /swagger-ui, /swagger-ui/**, /swagger-ui.html, /swagger-resources"; + private final static String whiteListText = "/user/ipInfo, /apisix/check, /datasetData/enumValue, /datasetData/enumValueObj, /datasetData/getFieldTree, /dekey, /symmetricKey, /share/validate, /sysParameter/queryOnlineMap, /xpackComponent/viewPlugins"; + private final static String whiteStartListText = "/dataVisualization/findDvType/"; + + private boolean isWhiteStart(String url) { + List whiteStartList = Arrays.stream(StringUtils.split(whiteStartListText, ",")).map(String::trim).toList(); + return whiteStartList.stream().anyMatch(item -> StringUtils.startsWith(url, item)); + } @Override public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws Exception { @@ -33,6 +39,7 @@ public class LinkInterceptor implements HandlerInterceptor { if (deLinkPermit == null) { List whiteList = Arrays.stream(StringUtils.split(whiteListText, ",")).map(String::trim).toList(); + String requestURI = ServletUtils.request().getRequestURI(); if (StringUtils.startsWith(requestURI, WhitelistUtils.getContextPath())) { requestURI = requestURI.replaceFirst(WhitelistUtils.getContextPath(), ""); @@ -40,7 +47,7 @@ public class LinkInterceptor implements HandlerInterceptor { if (StringUtils.startsWith(requestURI, AuthConstant.DE_API_PREFIX)) { requestURI = requestURI.replaceFirst(AuthConstant.DE_API_PREFIX, ""); } - boolean valid = whiteList.contains(requestURI) || WhitelistUtils.match(requestURI); + boolean valid = whiteList.contains(requestURI) || isWhiteStart(requestURI) || WhitelistUtils.match(requestURI); if (!valid) { DEException.throwException("分享链接Token不支持访问当前url[" + requestURI + "]"); } diff --git a/core/core-backend/src/main/java/io/gisbi/share/manage/XpackShareManage.java b/core/core-backend/src/main/java/io/gisbi/share/manage/XpackShareManage.java index 260702c..b0d9937 100644 --- a/core/core-backend/src/main/java/io/gisbi/share/manage/XpackShareManage.java +++ b/core/core-backend/src/main/java/io/gisbi/share/manage/XpackShareManage.java @@ -204,7 +204,7 @@ public class XpackShareManage { return pos.stream().map(po -> new XpackShareGridVO( po.getShareId(), po.getResourceId(), po.getName(), po.getCreator().toString(), - po.getTime(), po.getExp(), 9, po.getExtFlag(), po.getType())).toList(); + po.getTime(), po.getExp(), 9, po.getExtFlag(),po.getExtFlag1(), po.getType())).toList(); } private XpackShareManage proxy() { diff --git a/core/core-backend/src/main/java/io/gisbi/share/server/XpackShareServer.java b/core/core-backend/src/main/java/io/gisbi/share/server/XpackShareServer.java index 7e12aa2..56066a7 100644 --- a/core/core-backend/src/main/java/io/gisbi/share/server/XpackShareServer.java +++ b/core/core-backend/src/main/java/io/gisbi/share/server/XpackShareServer.java @@ -11,6 +11,7 @@ import io.gisbi.share.dao.auto.entity.XpackShare; import io.gisbi.share.manage.XpackShareManage; import jakarta.annotation.Resource; import org.apache.commons.lang3.ObjectUtils; +import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; diff --git a/core/core-backend/src/main/java/io/gisbi/system/manage/SysParameterManage.java b/core/core-backend/src/main/java/io/gisbi/system/manage/SysParameterManage.java index 77e5f54..543bbea 100644 --- a/core/core-backend/src/main/java/io/gisbi/system/manage/SysParameterManage.java +++ b/core/core-backend/src/main/java/io/gisbi/system/manage/SysParameterManage.java @@ -13,12 +13,12 @@ import io.gisbi.utils.CommonBeanFactory; import io.gisbi.utils.IDUtils; import io.gisbi.utils.SystemSettingUtils; import jakarta.annotation.Resource; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; -import org.springframework.util.CollectionUtils; import java.util.*; import java.util.stream.Collectors; @@ -26,10 +26,10 @@ import java.util.stream.Collectors; @Component public class SysParameterManage { - @Value("${dataease.show-demo-tips:false}") + @Value("${gisbi.show-demo-tips:false}") private boolean showDemoTips; - @Value("${dataease.demo-tips-content:#{null}}") + @Value("${gisbi.demo-tips-content:#{null}}") private String demoTipsContent; private static final String MAP_KEY_PREFIX = "map."; @@ -52,30 +52,61 @@ public class SysParameterManage { return null; } - public OnlineMapEditor queryOnlineMap() { + public OnlineMapEditor queryOnlineMap(String mapType) { + if (StringUtils.isBlank(mapType)) { + List typeList = groupList(MAP_KEY_PREFIX + "mapType"); + mapType = "gaode"; + if (!CollectionUtils.isEmpty(typeList)) { + mapType = typeList.getFirst().getPval(); + } + } + String prefix; + if (!StringUtils.equals(mapType, "gaode")) { + prefix = mapType + "." + MAP_KEY_PREFIX; + } else { + prefix = MAP_KEY_PREFIX; + } var editor = new OnlineMapEditor(); List fields = BeanUtils.getFieldNames(OnlineMapEditor.class); - Map mapVal = groupVal(MAP_KEY_PREFIX); + Map mapVal = groupVal(prefix); fields.forEach(field -> { - String val = mapVal.get(MAP_KEY_PREFIX + field); + String val = mapVal.get(prefix + field); if (StringUtils.isNotBlank(val)) { BeanUtils.setFieldValueByName(editor, field, val, String.class); } }); + + editor.setMapType(mapType); + return editor; } public void saveOnlineMap(OnlineMapEditor editor) { + String mapType = editor.getMapType(); + if (StringUtils.isBlank(mapType)) { + List typeList = groupList(MAP_KEY_PREFIX + "mapType"); + mapType = "gaode"; + if (!CollectionUtils.isEmpty(typeList)) { + mapType = typeList.getFirst().getPval(); + } + } + List fieldNames = BeanUtils.getFieldNames(OnlineMapEditor.class); + String finalMapType = mapType; fieldNames.forEach(field -> { + String prefix = MAP_KEY_PREFIX; + if (!(StringUtils.equals(field, "mapType") || StringUtils.equals(finalMapType, "gaode"))) { + prefix = finalMapType + "." + MAP_KEY_PREFIX; + } + QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.eq("pkey", MAP_KEY_PREFIX + field); + queryWrapper.eq("pkey", prefix + field); CoreSysSetting sysSetting = coreSysSettingMapper.selectOne(queryWrapper); var val = (String) BeanUtils.getFieldValueByName(field, editor); if (ObjectUtils.isEmpty(sysSetting)) { sysSetting = new CoreSysSetting(); sysSetting.setId(IDUtils.snowID()); - sysSetting.setPkey(MAP_KEY_PREFIX + field); + sysSetting.setPkey(prefix + field); sysSetting.setPval(val == null ? "" : val); sysSetting.setType("text"); sysSetting.setSort(1); @@ -135,15 +166,20 @@ public class SysParameterManage { @Transactional public void saveGroup(List vos, String groupKey) { - QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.likeRight("pkey", groupKey); - coreSysSettingMapper.delete(queryWrapper); List sysSettings = vos.stream().filter(vo -> !SystemSettingUtils.xpackSetting(vo.getPkey())).map(item -> { CoreSysSetting sysSetting = BeanUtils.copyBean(new CoreSysSetting(), item); sysSetting.setId(IDUtils.snowID()); return sysSetting; }).collect(Collectors.toList()); - extCoreSysSettingMapper.saveBatch(sysSettings); + if (CollectionUtils.isNotEmpty(sysSettings)) { + QueryWrapper queryWrapper = new QueryWrapper<>(); + sysSettings.forEach(sysSetting -> { + queryWrapper.clear(); + queryWrapper.eq("pkey", sysSetting.getPkey()); + coreSysSettingMapper.delete(queryWrapper); + }); + extCoreSysSettingMapper.saveBatch(sysSettings); + } datasourceServer.addJob(sysSettings); } diff --git a/core/core-backend/src/main/java/io/gisbi/system/server/SysParameterServer.java b/core/core-backend/src/main/java/io/gisbi/system/server/SysParameterServer.java index f57ad8a..9201734 100644 --- a/core/core-backend/src/main/java/io/gisbi/system/server/SysParameterServer.java +++ b/core/core-backend/src/main/java/io/gisbi/system/server/SysParameterServer.java @@ -38,7 +38,12 @@ public class SysParameterServer implements SysParameterApi { @Override public OnlineMapEditor queryOnlineMap() { - return sysParameterManage.queryOnlineMap(); + return sysParameterManage.queryOnlineMap(null); + } + + @Override + public OnlineMapEditor queryOnlineMapByMapType(String type) { + return sysParameterManage.queryOnlineMap(type); } @Override diff --git a/core/core-backend/src/main/java/io/gisbi/template/manage/TemplateCenterManage.java b/core/core-backend/src/main/java/io/gisbi/template/manage/TemplateCenterManage.java index 1af567f..a37c6ed 100644 --- a/core/core-backend/src/main/java/io/gisbi/template/manage/TemplateCenterManage.java +++ b/core/core-backend/src/main/java/io/gisbi/template/manage/TemplateCenterManage.java @@ -7,12 +7,14 @@ import io.gisbi.api.template.dto.TemplateMarketPreviewInfoDTO; import io.gisbi.api.template.response.*; import io.gisbi.api.template.vo.MarketApplicationMetaDataVO; import io.gisbi.api.template.vo.MarketApplicationSpecVO; +import io.gisbi.api.template.vo.MarketLatestReleaseVO; import io.gisbi.api.template.vo.MarketMetaDataVO; import io.gisbi.constant.CommonConstants; import io.gisbi.exception.DEException; import io.gisbi.i18n.Translator; import io.gisbi.operation.manage.CoreOptRecentManage; import io.gisbi.system.manage.SysParameterManage; +import io.gisbi.template.dao.auto.entity.VisualizationTemplateCategoryMap; import io.gisbi.template.dao.auto.mapper.VisualizationTemplateCategoryMapMapper; import io.gisbi.template.dao.ext.ExtVisualizationTemplateMapper; import io.gisbi.utils.HttpClientConfig; @@ -67,7 +69,6 @@ public class TemplateCenterManage { } /** - * @param templateUrl template url * @Description Get template file from template market */ public TemplateManageFileDTO getTemplateFromMarketV2(String templateName) { @@ -101,7 +102,8 @@ public class TemplateCenterManage { config.setConnectTimeout(5000); config.setSocketTimeout(10000); config.setConnectionRequestTimeout(5000); - return HttpClientUtil.get(url, config); + return HttpClientUtil. + get(url, config); } private MarketTemplateV2BaseResponse templateQuery(Map templateParams) { diff --git a/core/core-backend/src/main/java/io/gisbi/template/manage/TemplateLocalParseManage.java b/core/core-backend/src/main/java/io/gisbi/template/manage/TemplateLocalParseManage.java index 7fb349c..7f567a4 100644 --- a/core/core-backend/src/main/java/io/gisbi/template/manage/TemplateLocalParseManage.java +++ b/core/core-backend/src/main/java/io/gisbi/template/manage/TemplateLocalParseManage.java @@ -22,7 +22,7 @@ import java.time.LocalDateTime; import java.time.temporal.ChronoUnit; /** - * @Author bi-coder + * @author : WangJiaHao * @date : 2024/5/7 */ @Service diff --git a/core/core-backend/src/main/java/io/gisbi/visualization/dao/auto/entity/DataVisualizationInfo.java b/core/core-backend/src/main/java/io/gisbi/visualization/dao/auto/entity/DataVisualizationInfo.java index 1d1281a..1282fcb 100644 --- a/core/core-backend/src/main/java/io/gisbi/visualization/dao/auto/entity/DataVisualizationInfo.java +++ b/core/core-backend/src/main/java/io/gisbi/visualization/dao/auto/entity/DataVisualizationInfo.java @@ -17,10 +17,7 @@ public class DataVisualizationInfo implements Serializable { private static final long serialVersionUID = 1L; private Long id; - /** - * 应用ID - */ - private String appId; + /** * 名称 */ @@ -345,7 +342,6 @@ public class DataVisualizationInfo implements Serializable { public String toString() { return "DataVisualizationInfo{" + "id = " + id + - ", appId = " + appId + ", name = " + name + ", pid = " + pid + ", orgId = " + orgId + @@ -372,12 +368,4 @@ public class DataVisualizationInfo implements Serializable { ", checkVersion = " + checkVersion + "}"; } - - public String getAppId() { - return appId; - } - - public void setAppId(String appId) { - this.appId = appId; - } } diff --git a/core/core-backend/src/main/java/io/gisbi/visualization/dao/auto/mapper/VisualizationWatermarkMapper.java b/core/core-backend/src/main/java/io/gisbi/visualization/dao/auto/mapper/VisualizationWatermarkMapper.java index 2bea903..4a683b0 100644 --- a/core/core-backend/src/main/java/io/gisbi/visualization/dao/auto/mapper/VisualizationWatermarkMapper.java +++ b/core/core-backend/src/main/java/io/gisbi/visualization/dao/auto/mapper/VisualizationWatermarkMapper.java @@ -3,6 +3,7 @@ package io.gisbi.visualization.dao.auto.mapper; import io.gisbi.visualization.dao.auto.entity.VisualizationWatermark; import com.baomidou.mybatisplus.core.mapper.BaseMapper; import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Update; /** *

@@ -15,4 +16,8 @@ import org.apache.ibatis.annotations.Mapper; @Mapper public interface VisualizationWatermarkMapper extends BaseMapper { + @Update(""" + UPDATE `visualization_watermark` set `setting_content` = REPLACE(`setting_content`,'"enable":true','"enable":false') + """) + void disable(); } diff --git a/core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/mapper/CoreStoreExtMapper.java b/core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/mapper/CoreStoreExtMapper.java index 80d016e..d354674 100644 --- a/core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/mapper/CoreStoreExtMapper.java +++ b/core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/mapper/CoreStoreExtMapper.java @@ -19,7 +19,8 @@ public interface CoreStoreExtMapper { v.update_by as editor, v.update_time as edit_time, v.name, - v.mobile_layout as ext_flag + v.mobile_layout as ext_flag, + v.status as ext_flag1 from core_store s inner join data_visualization_info v on s.resource_id = v.id ${ew.customSqlSegment} diff --git a/core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/mapper/CoreVisualiationExtMapper.java b/core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/mapper/CoreVisualiationExtMapper.java index d8cbd0b..131a445 100644 --- a/core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/mapper/CoreVisualiationExtMapper.java +++ b/core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/mapper/CoreVisualiationExtMapper.java @@ -14,7 +14,7 @@ import java.util.Set; public interface CoreVisualiationExtMapper { @Select(""" - select id, name, pid, node_type, mobile_layout as extraFlag from data_visualization_info + select id, name, pid, node_type, mobile_layout as extraFlag, status as extraFlag1 from data_visualization_info ${ew.customSqlSegment} """) List queryNodes(@Param("ew") QueryWrapper queryWrapper); diff --git a/core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/mapper/ExtDataVisualizationMapper.java b/core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/mapper/ExtDataVisualizationMapper.java index 95d870c..3989cdd 100644 --- a/core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/mapper/ExtDataVisualizationMapper.java +++ b/core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/mapper/ExtDataVisualizationMapper.java @@ -7,13 +7,16 @@ import io.gisbi.api.visualization.dto.VisualizationViewTableDTO; import io.gisbi.api.visualization.vo.DataVisualizationBaseVO; import io.gisbi.api.visualization.vo.DataVisualizationVO; import io.gisbi.api.visualization.vo.VisualizationReportFilterVO; +import io.gisbi.api.visualization.vo.VisualizationResourceVO; import io.gisbi.chart.dao.auto.entity.CoreChartView; +import io.gisbi.visualization.dao.ext.po.StorePO; import io.gisbi.visualization.dao.ext.po.VisualizationResourcePO; import org.apache.ibatis.annotations.Mapper; import org.apache.ibatis.annotations.Param; import org.apache.ibatis.annotations.Select; import java.util.List; +import java.util.Map; import java.util.Set; @Mapper @@ -29,12 +32,12 @@ public interface ExtDataVisualizationMapper { String findDvType(@Param("dvId") Long dvId); void dvCopy(@Param("sourceDvId") Long sourceDvId,@Param("newDvId") Long newDvId,@Param("copyId") Long copyId); - void viewCopyWithDv(@Param("sourceDvId") Long sourceDvId,@Param("newDvId") Long newDvId,@Param("copyId") Long copyId); + void viewCopyWithDv(@Param("sourceDvId") Long sourceDvId,@Param("newDvId") Long newDvId,@Param("copyId") Long copyId,@Param("resourceTable") String resourceTable); List findViewInfoByCopyId(@Param("copyId") Long copyId); - DataVisualizationVO findDvInfo(@Param("dvId") Long dvId,@Param("dvType") String dvType); + DataVisualizationVO findDvInfo(@Param("dvId") Long dvId,@Param("dvType") String dvType,@Param("resourceTable") String resourceTable); - IPage findRecent(IPage page, @Param("uid") Long uid, @Param("keyword") String keyword, @Param("ew") QueryWrapper ew); + IPage findRecent(IPage page, @Param("uid") Long uid, @Param("keyword") String keyword, @Param("ew") Map ew); void copyLinkJump(@Param("copyId") Long copyId); @@ -50,9 +53,52 @@ public interface ExtDataVisualizationMapper { List queryReportFilter(@Param("dvId") Long dvId,@Param("taskId") Long taskId); - void deleteDataVBatch(@Param("ids") Set ids); + void deleteDataVBatch(@Param("ids") Set ids,@Param("resourceTable") String resourceTable); - void deleteViewsBatch(@Param("ids") Set ids); + void deleteViewsBatch(@Param("ids") Set ids,@Param("resourceTable") String resourceTable); UserFormVO queryInnerUserInfo(@Param("id") Long id); + + void snapshotDataV(@Param("dvId") Long dvId); + + void snapshotViews(@Param("dvId") Long dvId); + + void snapshotLinkJumpTargetViewInfo(@Param("dvId") Long dvId); + + void snapshotLinkJumpInfo(@Param("dvId") Long dvId); + + void snapshotLinkJump(@Param("dvId") Long dvId); + + void snapshotLinkageField(@Param("dvId") Long dvId); + + void snapshotLinkage(@Param("dvId") Long dvId); + + void snapshotOuterParamsTargetViewInfo(@Param("dvId") Long dvId); + + void snapshotOuterParamsInfo(@Param("dvId") Long dvId); + + void snapshotOuterParams(@Param("dvId") Long dvId); + + void restoreDataV(@Param("dvId") Long dvId); + + void restoreViews(@Param("dvId") Long dvId); + + void restoreLinkJumpTargetViewInfo(@Param("dvId") Long dvId); + + void restoreLinkJumpInfo(@Param("dvId") Long dvId); + + void restoreLinkJump(@Param("dvId") Long dvId); + + void restoreLinkageField(@Param("dvId") Long dvId); + + void restoreLinkage(@Param("dvId") Long dvId); + + void restoreOuterParamsTargetViewInfo(@Param("dvId") Long dvId); + + void restoreOuterParamsInfo(@Param("dvId") Long dvId); + + void restoreOuterParams(@Param("dvId") Long dvId); + + @Select("select status from data_visualization_info where id = #{dvId}") + Integer findDvInfoStats(@Param("dvId") Long dvId); } diff --git a/core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/mapper/ExtVisualizationLinkJumpMapper.java b/core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/mapper/ExtVisualizationLinkJumpMapper.java index 9aaddd3..d26a13b 100644 --- a/core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/mapper/ExtVisualizationLinkJumpMapper.java +++ b/core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/mapper/ExtVisualizationLinkJumpMapper.java @@ -14,13 +14,15 @@ import java.util.List; public interface ExtVisualizationLinkJumpMapper { List queryWithDvId(@Param("dvId") Long dvId,@Param("uid") Long uid,@Param("isDesktop") Boolean isDesktop); + List queryWithDvIdSnapshot(@Param("dvId") Long dvId,@Param("uid") Long uid,@Param("isDesktop") Boolean isDesktop); + VisualizationLinkJumpDTO queryWithViewId(@Param("dvId") Long dvId,@Param("viewId") Long viewId,@Param("uid") Long uid,@Param("isDesktop") Boolean isDesktop); - void deleteJumpTargetViewInfo(@Param("dvId") Long dvId,@Param("viewId") Long viewId); + void deleteJumpTargetViewInfoSnapshot(@Param("dvId") Long dvId,@Param("viewId") Long viewId); - void deleteJumpInfo(@Param("dvId") Long dvId,@Param("viewId") Long viewId); + void deleteJumpInfoSnapshot(@Param("dvId") Long dvId,@Param("viewId") Long viewId); - void deleteJump(@Param("dvId") Long dvId,@Param("viewId") Long viewId); + void deleteJumpSnapshot(@Param("dvId") Long dvId,@Param("viewId") Long viewId); void deleteJumpTargetViewInfoWithVisualization(@Param("dvId") Long dvId); @@ -28,8 +30,16 @@ public interface ExtVisualizationLinkJumpMapper { void deleteJumpWithVisualization(@Param("dvId") Long dvId); + void deleteJumpTargetViewInfoWithVisualizationSnapshot(@Param("dvId") Long dvId); + + void deleteJumpInfoWithVisualizationSnapshot(@Param("dvId") Long dvId); + + void deleteJumpWithVisualizationSnapshot(@Param("dvId") Long dvId); + List getTargetVisualizationJumpInfo(@Param("request") VisualizationLinkJumpBaseRequest request); + List getTargetVisualizationJumpInfoSnapshot(@Param("request") VisualizationLinkJumpBaseRequest request); + void copyLinkJump(@Param("copyId")Long copyId); void copyLinkJumpInfo(@Param("copyId")Long copyId); diff --git a/core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/mapper/ExtVisualizationLinkageMapper.java b/core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/mapper/ExtVisualizationLinkageMapper.java index 9033a1c..8ba65ed 100644 --- a/core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/mapper/ExtVisualizationLinkageMapper.java +++ b/core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/mapper/ExtVisualizationLinkageMapper.java @@ -17,6 +17,10 @@ public interface ExtVisualizationLinkageMapper { List getPanelAllLinkageInfo(@Param("dvId") Long dvId); + List getViewLinkageGatherSnapshot(@Param("dvId") Long dvId, @Param("sourceViewId") Long sourceViewId, @Param("targetViewIds") List targetViewIds); + + List getPanelAllLinkageInfoSnapshot(@Param("dvId") Long dvId); + List queryTableField(@Param("table_id") Long tableId); List queryTableFieldWithViewId(@Param("viewId") Long viewId); @@ -25,6 +29,10 @@ public interface ExtVisualizationLinkageMapper { void deleteViewLinkageField(@Param("dvId") Long dvId,@Param("sourceViewId") Long sourceViewId); + void deleteViewLinkageSnapshot(@Param("dvId") Long dvId,@Param("sourceViewId") Long sourceViewId); + + void deleteViewLinkageFieldSnapshot(@Param("dvId") Long dvId,@Param("sourceViewId") Long sourceViewId); + void copyViewLinkage(@Param("copyId") Long copyId); void copyViewLinkageField(@Param("copyId") Long copyId); diff --git a/core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/mapper/ExtVisualizationOuterParamsMapper.java b/core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/mapper/ExtVisualizationOuterParamsMapper.java index 6d07726..08460b2 100644 --- a/core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/mapper/ExtVisualizationOuterParamsMapper.java +++ b/core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/mapper/ExtVisualizationOuterParamsMapper.java @@ -4,6 +4,7 @@ package io.gisbi.visualization.dao.ext.mapper; import io.gisbi.api.dataset.vo.CoreDatasetGroupVO; import io.gisbi.api.visualization.dto.VisualizationOuterParamsDTO; import io.gisbi.api.visualization.dto.VisualizationOuterParamsInfoDTO; +import io.gisbi.visualization.dao.auto.entity.SnapshotVisualizationOuterParamsInfo; import io.gisbi.visualization.dao.auto.entity.VisualizationOuterParamsInfo; import org.apache.ibatis.annotations.Mapper; import org.apache.ibatis.annotations.Param; @@ -13,7 +14,7 @@ import java.util.List; @Mapper public interface ExtVisualizationOuterParamsMapper { - VisualizationOuterParamsDTO queryWithVisualizationId(@Param("visualizationId") String visualizationId); + VisualizationOuterParamsDTO queryWithVisualizationIdSnapshot(@Param("visualizationId") String visualizationId); void deleteOuterParamsTargetWithVisualizationId(@Param("visualizationId") String visualizationId); @@ -21,9 +22,15 @@ public interface ExtVisualizationOuterParamsMapper { void deleteOuterParamsWithVisualizationId(@Param("visualizationId") String visualizationId); + void deleteOuterParamsTargetWithVisualizationIdSnapshot(@Param("visualizationId") String visualizationId); + + void deleteOuterParamsInfoWithVisualizationIdSnapshot(@Param("visualizationId") String visualizationId); + + void deleteOuterParamsWithVisualizationIdSnapshot(@Param("visualizationId") String visualizationId); + List getVisualizationOuterParamsInfo(@Param("visualizationId") String visualizationId); - List getVisualizationOuterParamsInfoBase(@Param("visualizationId") String visualizationId); + List getVisualizationOuterParamsInfoBase(@Param("visualizationId") String visualizationId); List queryDsWithVisualizationId(@Param("visualizationId") String visualizationId); } diff --git a/core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/po/StorePO.java b/core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/po/StorePO.java index 33130a9..34384f2 100644 --- a/core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/po/StorePO.java +++ b/core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/po/StorePO.java @@ -26,4 +26,6 @@ public class StorePO implements Serializable { private Integer extFlag; + private Integer extFlag1; + } diff --git a/core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/po/VisualizationNodePO.java b/core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/po/VisualizationNodePO.java index cced510..c24e3b4 100644 --- a/core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/po/VisualizationNodePO.java +++ b/core/core-backend/src/main/java/io/gisbi/visualization/dao/ext/po/VisualizationNodePO.java @@ -15,5 +15,7 @@ public class VisualizationNodePO implements Serializable { private String nodeType; @Schema(description = "额外标识") private int extraFlag; + @Schema(description = "额外标识1") + private int extraFlag1; } diff --git a/core/core-backend/src/main/java/io/gisbi/visualization/dto/VisualizationNodeBO.java b/core/core-backend/src/main/java/io/gisbi/visualization/dto/VisualizationNodeBO.java index 1eea1ed..80d1814 100644 --- a/core/core-backend/src/main/java/io/gisbi/visualization/dto/VisualizationNodeBO.java +++ b/core/core-backend/src/main/java/io/gisbi/visualization/dto/VisualizationNodeBO.java @@ -20,6 +20,7 @@ public class VisualizationNodeBO implements TreeBaseModel { private Integer weight = 3; private Long pid; private Integer extraFlag; + private Integer extraFlag1; } diff --git a/core/core-backend/src/main/java/io/gisbi/visualization/manage/CoreVisualizationExportManage.java b/core/core-backend/src/main/java/io/gisbi/visualization/manage/CoreVisualizationExportManage.java index 6dc8b49..bb45bd2 100644 --- a/core/core-backend/src/main/java/io/gisbi/visualization/manage/CoreVisualizationExportManage.java +++ b/core/core-backend/src/main/java/io/gisbi/visualization/manage/CoreVisualizationExportManage.java @@ -6,8 +6,8 @@ import io.gisbi.chart.constant.ChartConstants; import io.gisbi.chart.manage.ChartDataManage; import io.gisbi.chart.manage.ChartViewManege; import io.gisbi.constant.CommonConstants; -import io.gisbi.dataset.server.DatasetFieldServer; import io.gisbi.constant.DeTypeConstants; +import io.gisbi.dataset.server.DatasetFieldServer; import io.gisbi.exception.DEException; import io.gisbi.exportCenter.util.ExportCenterUtils; import io.gisbi.extensions.view.dto.ChartExtFilterDTO; @@ -52,15 +52,15 @@ public class CoreVisualizationExportManage { private DatasetFieldServer datasetFieldServer; public String getResourceName(Long dvId, String busiFlag) { - DataVisualizationVO visualization = extDataVisualizationMapper.findDvInfo(dvId, busiFlag); + DataVisualizationVO visualization = extDataVisualizationMapper.findDvInfo(dvId, busiFlag, "core"); if (ObjectUtils.isEmpty(visualization)) DEException.throwException("资源不存在或已经被删除..."); return visualization.getName(); } - public File exportExcel(Long dvId, String busiFlag, List viewIdList, boolean onlyDisplay) throws Exception { - DataVisualizationVO visualization = extDataVisualizationMapper.findDvInfo(dvId, busiFlag); + public File exportExcel(Long dvId, String busiFlag, List viewIdList, boolean onlyDisplay, String filterJson) throws Exception { + DataVisualizationVO visualization = extDataVisualizationMapper.findDvInfo(dvId, busiFlag, "core"); if (ObjectUtils.isEmpty(visualization)) DEException.throwException("资源不存在或已经被删除..."); - List chartViewDTOS = chartViewManege.listBySceneId(dvId); + List chartViewDTOS = chartViewManege.listBySceneId(dvId, CommonConstants.RESOURCE_TABLE.CORE); String componentsJson = visualization.getComponentData(); List> components = JsonUtil.parseList(componentsJson, tokenType); @@ -70,13 +70,15 @@ public class CoreVisualizationExportManage { chartViewDTOS = chartViewDTOS.stream().filter(item -> idList.contains(item.getId()) && viewIdList.contains(item.getId())).collect(Collectors.toList()); } if (CollectionUtils.isEmpty(chartViewDTOS)) return null; - Map chartExtRequestMap = buildViewRequest(visualization, onlyDisplay); + Map chartExtRequestMap = buildViewRequest(filterJson); List sheets = new ArrayList<>(); for (int i = 0; i < chartViewDTOS.size(); i++) { ChartViewDTO view = chartViewDTOS.get(i); - ChartExtRequest extRequest = chartExtRequestMap.get(view.getId().toString()); + ChartExtRequest extRequest = chartExtRequestMap.get(view.getId()); if (ObjectUtils.isNotEmpty(extRequest)) { view.setChartExtRequest(extRequest); + } else { + view.setChartExtRequest(buildDefaultRequest()); } view.getChartExtRequest().setUser(AuthUtils.getUser().getUserId()); view.setTitle((i + 1) + "-" + view.getTitle()); @@ -96,11 +98,11 @@ public class CoreVisualizationExportManage { if (CollectionUtils.isNotEmpty(fields)) { fields.forEach(field -> { Object name = field.getName(); - Object dataeaseName = field.getDataeaseName(); + Object gisbiName = field.getGisbiName(); Object deType = field.getDeType(); - if (ObjectUtils.isNotEmpty(name) && ObjectUtils.isNotEmpty(dataeaseName)) { + if (ObjectUtils.isNotEmpty(name) && ObjectUtils.isNotEmpty(gisbiName)) { heads.add(name.toString()); - headKeys.add(dataeaseName.toString()); + headKeys.add(gisbiName.toString()); if (deType == null) { field.setDeType(DeTypeConstants.DE_STRING); deType = DeTypeConstants.DE_STRING; @@ -187,6 +189,30 @@ public class CoreVisualizationExportManage { private final TypeReference>> tokenType = new TypeReference>>() { }; + private Map buildViewRequest(String filterJson) { + if (StringUtils.isBlank(filterJson)) { + return new HashMap<>(); + } + Map extRequestMap = JsonUtil.parseObject(filterJson, new TypeReference>() { + }); + extRequestMap.forEach((key, chartExtRequest) -> { + chartExtRequest.setQueryFrom("panel"); + chartExtRequest.setResultCount(Math.toIntExact(ExportCenterUtils.getExportLimit("view"))); + chartExtRequest.setResultMode(ChartConstants.VIEW_RESULT_MODE.ALL); + chartExtRequest.setPageSize(ExportCenterUtils.getExportLimit("view")); + }); + return extRequestMap; + } + + private ChartExtRequest buildDefaultRequest() { + ChartExtRequest chartExtRequest = new ChartExtRequest(); + chartExtRequest.setQueryFrom("panel"); + chartExtRequest.setFilter(new ArrayList<>()); + chartExtRequest.setResultCount(Math.toIntExact(ExportCenterUtils.getExportLimit("view"))); + chartExtRequest.setResultMode(ChartConstants.VIEW_RESULT_MODE.ALL); + chartExtRequest.setPageSize(ExportCenterUtils.getExportLimit("view")); + return chartExtRequest; + } private Map buildViewRequest(DataVisualizationVO panelDto, Boolean justView) { String componentsJson = panelDto.getComponentData(); diff --git a/core/core-backend/src/main/java/io/gisbi/visualization/manage/CoreVisualizationManage.java b/core/core-backend/src/main/java/io/gisbi/visualization/manage/CoreVisualizationManage.java index d5fb8fe..4fb21cd 100644 --- a/core/core-backend/src/main/java/io/gisbi/visualization/manage/CoreVisualizationManage.java +++ b/core/core-backend/src/main/java/io/gisbi/visualization/manage/CoreVisualizationManage.java @@ -6,18 +6,23 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import io.gisbi.api.visualization.request.DataVisualizationBaseRequest; import io.gisbi.api.visualization.request.VisualizationWorkbranchQueryRequest; import io.gisbi.api.visualization.vo.VisualizationResourceVO; +import io.gisbi.chart.dao.ext.mapper.ExtChartViewMapper; +import io.gisbi.chart.manage.ChartViewManege; import io.gisbi.commons.constants.DataVisualizationConstants; import io.gisbi.commons.constants.OptConstants; import io.gisbi.constant.BusiResourceEnum; +import io.gisbi.constant.CommonConstants; import io.gisbi.exception.DEException; +import io.gisbi.license.config.XpackInteract; import io.gisbi.model.BusiNodeRequest; import io.gisbi.model.BusiNodeVO; import io.gisbi.operation.manage.CoreOptRecentManage; import io.gisbi.utils.*; import io.gisbi.visualization.dao.auto.entity.DataVisualizationInfo; +import io.gisbi.visualization.dao.auto.entity.SnapshotDataVisualizationInfo; import io.gisbi.visualization.dao.auto.mapper.DataVisualizationInfoMapper; -import io.gisbi.visualization.dao.ext.mapper.CoreVisualiationExtMapper; -import io.gisbi.visualization.dao.ext.mapper.ExtDataVisualizationMapper; +import io.gisbi.visualization.dao.auto.mapper.SnapshotDataVisualizationInfoMapper; +import io.gisbi.visualization.dao.ext.mapper.*; import io.gisbi.visualization.dao.ext.po.VisualizationNodePO; import io.gisbi.visualization.dao.ext.po.VisualizationResourcePO; import io.gisbi.visualization.dto.VisualizationNodeBO; @@ -41,13 +46,31 @@ public class CoreVisualizationManage { @Resource private DataVisualizationInfoMapper mapper; + @Resource + private SnapshotDataVisualizationInfoMapper snapshotMapper; + + @Resource + private ExtVisualizationLinkageMapper linkageMapper; + + @Resource + private ExtVisualizationLinkJumpMapper linkJumpMapper; + + @Resource + private ExtVisualizationOuterParamsMapper outerParamsMapper; + @Resource private ExtDataVisualizationMapper extDataVisualizationMapper; @Resource private CoreOptRecentManage coreOptRecentManage; + @Resource + private ExtChartViewMapper extCoreChartMapper; + @Resource + private ChartViewManege chartViewManege; + + @XpackInteract(value = "visualizationResourceTree", replace = true, invalid = true) public List tree(BusiNodeRequest request) { List nodes = new ArrayList<>(); if (ObjectUtils.isEmpty(request.getLeaf()) || !request.getLeaf()) { @@ -62,6 +85,10 @@ public class CoreVisualizationManage { if (StringUtils.isNotBlank(info)) { queryWrapper.notExists(String.format(info, "data_visualization_info.id")); } + // 如果是编辑界面 只展示已发布的资源 + if(CommonConstants.RESOURCE_TABLE.SNAPSHOT.equals(request.getResourceTable())){ + queryWrapper.in("status", Arrays.asList(1,2)); + } queryWrapper.orderByDesc("create_time"); List pos = extMapper.queryNodes(queryWrapper); if (CollectionUtils.isNotEmpty(pos)) { @@ -70,7 +97,7 @@ public class CoreVisualizationManage { return TreeUtils.mergeTree(nodes, BusiNodeVO.class, false); } - + @XpackInteract(value = "visualizationResourceTree", before = false) public void delete(Long id) { DataVisualizationInfo info = mapper.selectById(id); if (ObjectUtils.isEmpty(info)) { @@ -93,14 +120,16 @@ public class CoreVisualizationManage { } } // 删除可视化资源 - extDataVisualizationMapper.deleteDataVBatch(delIds); + extDataVisualizationMapper.deleteDataVBatch(delIds,CommonConstants.RESOURCE_TABLE.CORE); + extDataVisualizationMapper.deleteDataVBatch(delIds,CommonConstants.RESOURCE_TABLE.SNAPSHOT); // 删除图表信息 - extDataVisualizationMapper.deleteViewsBatch(delIds); + extDataVisualizationMapper.deleteViewsBatch(delIds,CommonConstants.RESOURCE_TABLE.CORE); + extDataVisualizationMapper.deleteViewsBatch(delIds,CommonConstants.RESOURCE_TABLE.SNAPSHOT); coreOptRecentManage.saveOpt(id, OptConstants.OPT_RESOURCE_TYPE.VISUALIZATION, OptConstants.OPT_TYPE.DELETE); } - + @XpackInteract(value = "visualizationResourceTree", before = false) public void move(DataVisualizationBaseRequest request) { if (!request.getMoveFromUpdate()) { DataVisualizationInfo visualizationInfo = new DataVisualizationInfo(); @@ -109,12 +138,15 @@ public class CoreVisualizationManage { DEException.throwException("resource not exist"); } visualizationInfo.setUpdateTime(System.currentTimeMillis()); + SnapshotDataVisualizationInfo snapshotVisualizationInfo = new SnapshotDataVisualizationInfo(); + BeanUtils.copyBean(snapshotVisualizationInfo, visualizationInfo); coreOptRecentManage.saveOpt(visualizationInfo.getId(), OptConstants.OPT_RESOURCE_TYPE.VISUALIZATION, OptConstants.OPT_TYPE.UPDATE); mapper.updateById(visualizationInfo); + snapshotMapper.updateById(snapshotVisualizationInfo); } } - + @XpackInteract(value = "visualizationResourceTree", before = false) public Long innerSave(DataVisualizationInfo visualizationInfo) { visualizationInfo.setVersion(3); return preInnerSave(visualizationInfo); @@ -126,22 +158,42 @@ public class CoreVisualizationManage { visualizationInfo.setId(id); } visualizationInfo.setDeleteFlag(DataVisualizationConstants.DELETE_FLAG.AVAILABLE); + visualizationInfo.setStatus(visualizationInfo.getStatus()); visualizationInfo.setCreateBy(AuthUtils.getUser().getUserId().toString()); visualizationInfo.setUpdateBy(AuthUtils.getUser().getUserId().toString()); visualizationInfo.setCreateTime(System.currentTimeMillis()); visualizationInfo.setUpdateTime(System.currentTimeMillis()); visualizationInfo.setOrgId(AuthUtils.getUser().getDefaultOid()); mapper.insert(visualizationInfo); + // 镜像文件插入 + SnapshotDataVisualizationInfo snapshotVisualizationInfo = new SnapshotDataVisualizationInfo(); + BeanUtils.copyBean(snapshotVisualizationInfo,visualizationInfo); + snapshotMapper.insert(snapshotVisualizationInfo); coreOptRecentManage.saveOpt(visualizationInfo.getId(), OptConstants.OPT_RESOURCE_TYPE.VISUALIZATION, OptConstants.OPT_TYPE.NEW); return visualizationInfo.getId(); } - + @XpackInteract(value = "visualizationResourceTree", before = false) public void innerEdit(DataVisualizationInfo visualizationInfo) { + // 镜像和主表保持名称一致 visualizationInfo.setUpdateTime(System.currentTimeMillis()); visualizationInfo.setUpdateBy(AuthUtils.getUser().getUserId().toString()); visualizationInfo.setVersion(3); - mapper.updateById(visualizationInfo); + // 更新镜像 + SnapshotDataVisualizationInfo snapshotVisualizationInfo = new SnapshotDataVisualizationInfo(); + BeanUtils.copyBean(snapshotVisualizationInfo,visualizationInfo); + snapshotMapper.updateById(snapshotVisualizationInfo); + // 更新主表名称 + DataVisualizationInfo coreVisualizationInfo = new DataVisualizationInfo(); + coreVisualizationInfo.setId(visualizationInfo.getId()); + coreVisualizationInfo.setStatus(visualizationInfo.getStatus()); + coreVisualizationInfo.setPid(visualizationInfo.getPid()); + coreVisualizationInfo.setContentId(visualizationInfo.getContentId()); + coreVisualizationInfo.setName(visualizationInfo.getName()); + coreVisualizationInfo.setUpdateTime(System.currentTimeMillis()); + coreVisualizationInfo.setUpdateBy(AuthUtils.getUser().getUserId().toString()); + coreVisualizationInfo.setVersion(3); + mapper.updateById(coreVisualizationInfo); coreOptRecentManage.saveOpt(visualizationInfo.getId(), OptConstants.OPT_RESOURCE_TYPE.VISUALIZATION, OptConstants.OPT_TYPE.UPDATE); } @@ -150,18 +202,18 @@ public class CoreVisualizationManage { } private VisualizationNodeBO rootNode() { - return new VisualizationNodeBO(0L, "root", false, 7, -1L, 0); + return new VisualizationNodeBO(0L, "root", false, 7, -1L, 0,1); } private VisualizationNodeBO convert(VisualizationNodePO po) { - return new VisualizationNodeBO(po.getId(), po.getName(), StringUtils.equals(po.getNodeType(), "leaf"), 9, po.getPid(), po.getExtraFlag()); + return new VisualizationNodeBO(po.getId(), po.getName(), StringUtils.equals(po.getNodeType(), "leaf"), 9, po.getPid(), po.getExtraFlag(),po.getExtraFlag1()); } public CoreVisualizationManage proxy() { return CommonBeanFactory.getBean(this.getClass()); } - + @XpackInteract(value = "perFilterManage", recursion = true, invalid = true) public IPage query(int pageNum, int pageSize, VisualizationWorkbranchQueryRequest request) { IPage visualizationResourcePOPageIPage = proxy().queryVisualizationPage(pageNum, pageSize, request); if (ObjectUtils.isEmpty(visualizationResourcePOPageIPage)) { @@ -190,20 +242,98 @@ public class CoreVisualizationManage { public IPage queryVisualizationPage(int goPage, int pageSize, VisualizationWorkbranchQueryRequest request) { Long uid = AuthUtils.getUser().getUserId(); - QueryWrapper queryWrapper = new QueryWrapper<>(); + Map params = new HashMap<>(); if (StringUtils.isNotBlank(request.getType())) { BusiResourceEnum busiResourceEnum = BusiResourceEnum.valueOf(request.getType().toUpperCase()); if (ObjectUtils.isEmpty(busiResourceEnum)) { DEException.throwException("type is invalid"); } - queryWrapper.eq("dvResource.type", request.getType()); + params.put("type",request.getType()); } String info = CommunityUtils.getInfo(); if (StringUtils.isNotBlank(info)) { - queryWrapper.notExists(String.format(info, "core_opt_recent.resource_id")); + params.put("info",info); } - queryWrapper.orderBy(true, request.isAsc(), "core_opt_recent.time"); + params.put("isAsc",request.isAsc()); Page page = new Page<>(goPage, pageSize); - return extDataVisualizationMapper.findRecent(page, uid, request.getKeyword(), queryWrapper); + return extDataVisualizationMapper.findRecent(page, uid, request.getKeyword(), params); } + @Transactional + public void removeSnapshot(Long dvId){ + if(dvId != null){ + // 清理历史数据 + Set dvIds = new HashSet<>(); + dvIds.add(dvId); + extDataVisualizationMapper.deleteDataVBatch(dvIds,CommonConstants.RESOURCE_TABLE.SNAPSHOT); + extCoreChartMapper.deleteViewsBySceneId(dvId,CommonConstants.RESOURCE_TABLE.SNAPSHOT); + linkageMapper.deleteViewLinkageFieldSnapshot(dvId,null); + linkageMapper.deleteViewLinkageSnapshot(dvId,null); + linkJumpMapper.deleteJumpTargetViewInfoWithVisualizationSnapshot(dvId); + linkJumpMapper.deleteJumpInfoWithVisualizationSnapshot(dvId); + linkJumpMapper.deleteJumpWithVisualizationSnapshot(dvId); + outerParamsMapper.deleteOuterParamsTargetWithVisualizationIdSnapshot(dvId.toString()); + outerParamsMapper.deleteOuterParamsInfoWithVisualizationIdSnapshot(dvId.toString()); + outerParamsMapper.deleteOuterParamsWithVisualizationIdSnapshot(dvId.toString()); + //xpack 阈值告警 + chartViewManege.removeThreshold(dvId,CommonConstants.RESOURCE_TABLE.SNAPSHOT); + + } + } + @Transactional + public void removeDvCore(Long dvId){ + if(dvId != null){ + // 清理历史数据 + Set dvIds = new HashSet<>(); + dvIds.add(dvId); + extDataVisualizationMapper.deleteDataVBatch(dvIds,CommonConstants.RESOURCE_TABLE.CORE); + extCoreChartMapper.deleteViewsBySceneId(dvId,CommonConstants.RESOURCE_TABLE.CORE); + linkageMapper.deleteViewLinkageField(dvId,null); + linkageMapper.deleteViewLinkage(dvId,null); + linkJumpMapper.deleteJumpTargetViewInfoWithVisualization(dvId); + linkJumpMapper.deleteJumpInfoWithVisualization(dvId); + linkJumpMapper.deleteJumpWithVisualization(dvId); + outerParamsMapper.deleteOuterParamsTargetWithVisualizationId(dvId.toString()); + outerParamsMapper.deleteOuterParamsInfoWithVisualizationId(dvId.toString()); + outerParamsMapper.deleteOuterParamsWithVisualizationId(dvId.toString()); + //xpack 阈值告警 + chartViewManege.removeThreshold(dvId,CommonConstants.RESOURCE_TABLE.CORE); + } + } + + @Transactional + public void dvSnapshotRecover(Long dvId){ + // 清理历史数据 + CoreVisualizationManage proxy = CommonBeanFactory.proxy(this.getClass()); + assert proxy != null; + proxy.removeSnapshot(dvId); + // 导入新数据 + extDataVisualizationMapper.snapshotDataV(dvId); + extDataVisualizationMapper.snapshotViews(dvId); + extDataVisualizationMapper.snapshotLinkJumpTargetViewInfo(dvId); + extDataVisualizationMapper.snapshotLinkJumpInfo(dvId); + extDataVisualizationMapper.snapshotLinkJump(dvId); + extDataVisualizationMapper.snapshotLinkageField(dvId); + extDataVisualizationMapper.snapshotLinkage(dvId); + extDataVisualizationMapper.snapshotOuterParamsTargetViewInfo(dvId); + extDataVisualizationMapper.snapshotOuterParamsInfo(dvId); + extDataVisualizationMapper.snapshotOuterParams(dvId); + //xpack 阈值告警 + chartViewManege.restoreThreshold(dvId,CommonConstants.RESOURCE_TABLE.SNAPSHOT); + } + @Transactional + public void dvRestore(Long dvId){ + extDataVisualizationMapper.restoreDataV(dvId); + extDataVisualizationMapper.restoreViews(dvId); + extDataVisualizationMapper.restoreLinkJumpTargetViewInfo(dvId); + extDataVisualizationMapper.restoreLinkJumpInfo(dvId); + extDataVisualizationMapper.restoreLinkJump(dvId); + extDataVisualizationMapper.restoreLinkageField(dvId); + extDataVisualizationMapper.restoreLinkage(dvId); + extDataVisualizationMapper.restoreOuterParamsTargetViewInfo(dvId); + extDataVisualizationMapper.restoreOuterParamsInfo(dvId); + extDataVisualizationMapper.restoreOuterParams(dvId); + //xpack 阈值告警 + chartViewManege.restoreThreshold(dvId,CommonConstants.RESOURCE_TABLE.CORE); + } + } diff --git a/core/core-backend/src/main/java/io/gisbi/visualization/manage/VisualizationStoreManage.java b/core/core-backend/src/main/java/io/gisbi/visualization/manage/VisualizationStoreManage.java index 4e3dedc..858d69d 100644 --- a/core/core-backend/src/main/java/io/gisbi/visualization/manage/VisualizationStoreManage.java +++ b/core/core-backend/src/main/java/io/gisbi/visualization/manage/VisualizationStoreManage.java @@ -8,6 +8,7 @@ import io.gisbi.api.visualization.request.VisualizationWorkbranchQueryRequest; import io.gisbi.api.visualization.vo.VisualizationStoreVO; import io.gisbi.constant.BusiResourceEnum; import io.gisbi.exception.DEException; +import io.gisbi.license.config.XpackInteract; import io.gisbi.utils.AuthUtils; import io.gisbi.utils.CommonBeanFactory; import io.gisbi.utils.CommunityUtils; @@ -65,6 +66,7 @@ public class VisualizationStoreManage { return coreStoreMapper.exists(queryWrapper); } + @XpackInteract(value = "perFilterManage", recursion = true, invalid = true) public IPage query(int pageNum, int pageSize, VisualizationWorkbranchQueryRequest request) { IPage storePOIPage = proxy().queryStorePage(pageNum, pageSize, request); if (ObjectUtils.isEmpty(storePOIPage)) return null; @@ -88,7 +90,7 @@ public class VisualizationStoreManage { new VisualizationStoreVO( po.getStoreId(), po.getResourceId(), po.getName(), po.getType(), String.valueOf(po.getCreator()), ObjectUtils.isEmpty(po.getEditor()) ? null : String.valueOf(po.getEditor()), - po.getEditTime(), 9, po.getExtFlag())).toList(); + po.getEditTime(), 9, po.getExtFlag(), po.getExtFlag1())).toList(); } public IPage queryStorePage(int goPage, int pageSize, VisualizationWorkbranchQueryRequest request) { diff --git a/core/core-backend/src/main/java/io/gisbi/visualization/manage/VisualizationTemplateExtendDataManage.java b/core/core-backend/src/main/java/io/gisbi/visualization/manage/VisualizationTemplateExtendDataManage.java index e13af39..9397433 100644 --- a/core/core-backend/src/main/java/io/gisbi/visualization/manage/VisualizationTemplateExtendDataManage.java +++ b/core/core-backend/src/main/java/io/gisbi/visualization/manage/VisualizationTemplateExtendDataManage.java @@ -15,7 +15,7 @@ import java.util.List; /** - * @Author bi-coder + * @author : WangJiaHao * @date : 2023/11/13 13:25 */ @Service diff --git a/core/core-backend/src/main/java/io/gisbi/visualization/server/DataVisualizationServer.java b/core/core-backend/src/main/java/io/gisbi/visualization/server/DataVisualizationServer.java index 62124e1..abd5f93 100644 --- a/core/core-backend/src/main/java/io/gisbi/visualization/server/DataVisualizationServer.java +++ b/core/core-backend/src/main/java/io/gisbi/visualization/server/DataVisualizationServer.java @@ -2,7 +2,10 @@ package io.gisbi.visualization.server; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.baomidou.mybatisplus.core.metadata.IPage; +import com.fasterxml.jackson.core.type.TypeReference; import io.gisbi.api.dataset.union.DatasetGroupInfoDTO; +import io.gisbi.api.dataset.union.DatasetTableInfoDTO; +import io.gisbi.api.dataset.union.UnionDTO; import io.gisbi.api.template.dto.TemplateManageFileDTO; import io.gisbi.api.template.dto.VisualizationTemplateExtendDataDTO; import io.gisbi.api.visualization.DataVisualizationApi; @@ -14,6 +17,7 @@ import io.gisbi.api.visualization.vo.*; import io.gisbi.auth.DeLinkPermit; import io.gisbi.chart.dao.auto.entity.CoreChartView; import io.gisbi.chart.dao.auto.mapper.CoreChartViewMapper; +import io.gisbi.chart.dao.ext.mapper.ExtChartViewMapper; import io.gisbi.chart.manage.ChartDataManage; import io.gisbi.chart.manage.ChartViewManege; import io.gisbi.commons.constants.DataVisualizationConstants; @@ -28,11 +32,14 @@ import io.gisbi.dataset.dao.auto.mapper.CoreDatasetTableFieldMapper; import io.gisbi.dataset.dao.auto.mapper.CoreDatasetTableMapper; import io.gisbi.dataset.manage.DatasetDataManage; import io.gisbi.dataset.manage.DatasetGroupManage; +import io.gisbi.dataset.manage.DatasetSQLManage; +import io.gisbi.dataset.utils.DatasetUtils; import io.gisbi.datasource.dao.auto.entity.CoreDatasource; import io.gisbi.datasource.dao.auto.mapper.CoreDatasourceMapper; import io.gisbi.datasource.provider.ExcelUtils; import io.gisbi.datasource.server.DatasourceServer; import io.gisbi.exception.DEException; +import io.gisbi.extensions.datasource.dto.DatasetTableDTO; import io.gisbi.extensions.datasource.vo.DatasourceConfiguration; import io.gisbi.extensions.view.dto.ChartViewDTO; import io.gisbi.i18n.Translator; @@ -50,19 +57,23 @@ import io.gisbi.template.dao.ext.ExtVisualizationTemplateMapper; import io.gisbi.template.manage.TemplateCenterManage; import io.gisbi.utils.*; import io.gisbi.visualization.dao.auto.entity.DataVisualizationInfo; +import io.gisbi.visualization.dao.auto.entity.SnapshotDataVisualizationInfo; import io.gisbi.visualization.dao.auto.entity.VisualizationWatermark; import io.gisbi.visualization.dao.auto.mapper.DataVisualizationInfoMapper; +import io.gisbi.visualization.dao.auto.mapper.SnapshotCoreChartViewMapper; +import io.gisbi.visualization.dao.auto.mapper.SnapshotDataVisualizationInfoMapper; import io.gisbi.visualization.dao.auto.mapper.VisualizationWatermarkMapper; import io.gisbi.visualization.dao.ext.mapper.ExtDataVisualizationMapper; import io.gisbi.visualization.manage.CoreBusiManage; import io.gisbi.visualization.manage.CoreVisualizationManage; import io.gisbi.visualization.utils.VisualizationUtils; import jakarta.annotation.Resource; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.transaction.annotation.Transactional; -import org.springframework.util.CollectionUtils; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -129,7 +140,8 @@ public class DataVisualizationServer implements DataVisualizationApi { @Resource private CoreDatasetTableFieldMapper coreDatasetTableFieldMapper; - @Autowired + + @Resource private CoreDatasourceMapper coreDatasourceMapper; @Resource @@ -143,9 +155,16 @@ public class DataVisualizationServer implements DataVisualizationApi { @Resource private DatasourceServer datasourceServer; + @Resource + private SnapshotDataVisualizationInfoMapper snapshotMapper; + @Resource + private ExtChartViewMapper extChartViewMapper; + @Resource + private DatasetSQLManage datasetSQLManage; + @Override public DataVisualizationVO findCopyResource(Long dvId, String busiFlag) { - DataVisualizationVO result = Objects.requireNonNull(CommonBeanFactory.proxy(this.getClass())).findById(new DataVisualizationBaseRequest(dvId, busiFlag)); + DataVisualizationVO result = Objects.requireNonNull(CommonBeanFactory.proxy(this.getClass())).findById(new DataVisualizationBaseRequest(dvId, busiFlag, CommonConstants.RESOURCE_TABLE.SNAPSHOT,DataVisualizationConstants.QUERY_SOURCE.MAIN_EDIT)); if (result != null && result.getPid() == -1) { return result; } else { @@ -159,7 +178,17 @@ public class DataVisualizationServer implements DataVisualizationApi { public DataVisualizationVO findById(DataVisualizationBaseRequest request) { Long dvId = request.getId(); String busiFlag = request.getBusiFlag(); - DataVisualizationVO result = extDataVisualizationMapper.findDvInfo(dvId, busiFlag); + String resourceTable = request.getResourceTable(); + // 如果是编辑查询 则进行镜像检查 + if (DataVisualizationConstants.QUERY_SOURCE.MAIN_EDIT.equals(request.getSource())) { + QueryWrapper queryWrapper = new QueryWrapper<>(); + queryWrapper.eq("id", dvId); + queryWrapper.in("status", Arrays.asList(CommonConstants.DV_STATUS.UNPUBLISHED, CommonConstants.DV_STATUS.SAVED_UNPUBLISHED)); // 状态为0 未发布 和 2 已保存未发布的 不需要重置镜像 + if (!snapshotMapper.exists(queryWrapper)) { + coreVisualizationManage.dvSnapshotRecover(dvId); + } + } + DataVisualizationVO result = extDataVisualizationMapper.findDvInfo(dvId, busiFlag, resourceTable); if (result != null) { // get creator String userName = coreUserManage.getUserName(Long.valueOf(result.getCreateBy())); @@ -167,9 +196,10 @@ public class DataVisualizationServer implements DataVisualizationApi { result.setCreatorName(userName); } //获取图表信息 - List chartViewDTOS = chartViewManege.listBySceneId(dvId); + List chartViewDTOS = chartViewManege.listBySceneId(dvId, resourceTable); if (!CollectionUtils.isEmpty(chartViewDTOS)) { - Map viewInfo = chartViewDTOS.stream().collect(Collectors.toMap(ChartViewDTO::getId, chartView -> chartView)); + // 增加过滤当前使用的图表信息 + Map viewInfo = chartViewDTOS.stream().filter(item -> result.getComponentData().indexOf("\"id\":\"" + item.getId()) > 0).collect(Collectors.toMap(ChartViewDTO::getId, chartView -> chartView)); result.setCanvasViewInfo(viewInfo); } VisualizationWatermark watermark = watermarkMapper.selectById("system_default"); @@ -208,6 +238,14 @@ public class DataVisualizationServer implements DataVisualizationApi { @Override @Transactional public String saveCanvas(DataVisualizationBaseRequest request) throws Exception { + /* + * 发布兼容逻辑 + * saveCanvas 为初次保存 包括 模板 应用 普通创建 所有变更操作都走snapshot表 + * 1.如果是文件夹直接保存在主表中，如果是仪表板（数据大屏），主表和镜像表各保存一份 主表仅作为权限和预览控制此时主表状态为‘未发布’ + * 2.编辑检查：如果存在未发布的仪表板snapshot，则默认加载snapshot进行编辑所有操作均为snapshot操作 + * 3.发布（重新发布）：将snapshot表中的所有数据复制到主表中，同时变更主表状态为‘已发布’ + * 4.如果对已发布的仪表板编辑并存在已保存的镜像，此时仪表板状态为‘已保存未发布’ + */ boolean isAppSave = false; Long time = System.currentTimeMillis(); // 如果是应用 则新进行应用校验 数据集名称和 数据源名称校验 @@ -220,18 +258,21 @@ public class DataVisualizationServer implements DataVisualizationApi { Map datasourceIdMap = new HashMap<>(); Map> dsTableNamesMap = new HashMap<>(); List newDatasourceId = new ArrayList<>(); + List excelDatasourceId = new ArrayList<>(); + Map excelTableNamesMap = new HashMap<>(); if (appData != null) { isAppSave = true; try { List appCoreDatasourceVO = appData.getDatasourceInfo(); - // app 数据源 excel 表名映射 appCoreDatasourceVO.forEach(datasourceOld -> { newDatasourceId.add(datasourceOld.getSystemDatasourceId()); // Excel 数据表明映射 if (StringUtils.isNotEmpty(datasourceOld.getConfiguration())) { - if (datasourceOld.getType().equals(DatasourceConfiguration.DatasourceType.Excel.name())) { - dsTableNamesMap.put(datasourceOld.getId(), ExcelUtils.getTableNamesMap(datasourceOld.getConfiguration())); + if (datasourceOld.getType().equals(DatasourceConfiguration.DatasourceType.API.name())) { + DEException.throwException(Translator.get("i18n_app_error_no_api")); + } else if (datasourceOld.getType().equals(DatasourceConfiguration.DatasourceType.Excel.name())) { + dsTableNamesMap.put(datasourceOld.getId(), ExcelUtils.getTableNamesMap(datasourceOld.getType(), datasourceOld.getConfiguration())); } else if (datasourceOld.getType().contains(DatasourceConfiguration.DatasourceType.API.name())) { dsTableNamesMap.put(datasourceOld.getId(), (Map) datasourceServer.invokeMethod(datasourceOld.getType(), "getTableNamesMap", String.class, datasourceOld.getConfiguration())); } @@ -243,7 +284,8 @@ public class DataVisualizationServer implements DataVisualizationApi { // Excel 数据表明映射 if (StringUtils.isNotEmpty(datasourceNew.getConfiguration())) { if (datasourceNew.getType().equals(DatasourceConfiguration.DatasourceType.Excel.name())) { - dsTableNamesMap.put(datasourceNew.getId(), ExcelUtils.getTableNamesMap(datasourceNew.getConfiguration())); + dsTableNamesMap.put(datasourceNew.getId(), ExcelUtils.getTableNamesMap(datasourceNew.getType(), datasourceNew.getConfiguration())); + excelDatasourceId.add(datasourceNew.getId()); } else if (datasourceNew.getType().contains(DatasourceConfiguration.DatasourceType.API.name())) { dsTableNamesMap.put(datasourceNew.getId(), (Map) datasourceServer.invokeMethod(datasourceNew.getType(), "getTableNamesMap", String.class, datasourceNew.getConfiguration())); } @@ -257,7 +299,7 @@ public class DataVisualizationServer implements DataVisualizationApi { datasetFolderNewRequest.setName(datasetFolderName); datasetFolderNewRequest.setNodeType("folder"); datasetFolderNewRequest.setPid(datasetFolderPid); - DatasetGroupInfoDTO datasetFolderNew = datasetGroupManage.save(datasetFolderNewRequest, false); + DatasetGroupInfoDTO datasetFolderNew = datasetGroupManage.save(datasetFolderNewRequest, false, false); Long datasetFolderNewId = datasetFolderNew.getId(); //新建数据集 appData.getDatasetGroupsInfo().forEach(appDatasetGroup -> { @@ -332,10 +374,13 @@ public class DataVisualizationServer implements DataVisualizationApi { //表名映射更新 Map appDsTableNamesMap = dsTableNamesMap.get(key); Map systemDsTableNamesMap = dsTableNamesMap.get(value); - if (!CollectionUtils.isEmpty(appDsTableNamesMap) && !CollectionUtils.isEmpty(systemDsTableNamesMap)) { + if (MapUtils.isNotEmpty(appDsTableNamesMap)) { appDsTableNamesMap.forEach((keyName, valueName) -> { - if (StringUtils.isNotEmpty(systemDsTableNamesMap.get(keyName))) { + if (MapUtils.isNotEmpty(systemDsTableNamesMap) && StringUtils.isNotEmpty(systemDsTableNamesMap.get(keyName))) { dsGroup.setInfo(dsGroup.getInfo().replaceAll(valueName, systemDsTableNamesMap.get(keyName))); + excelTableNamesMap.put(valueName, systemDsTableNamesMap.get(keyName)); + } else { + dsGroup.setInfo(dsGroup.getInfo().replaceAll(valueName, "excel_can_not_find")); } }); } @@ -345,6 +390,14 @@ public class DataVisualizationServer implements DataVisualizationApi { dsGroup.setName(dsGroup.getName() + "-" + UUID.randomUUID().toString()); } dsGroupNameSave.add(dsGroup.getName()); + if(dsGroup.getIsCross() == null){ + if(dsGroup.getUnion() == null){ + dsGroup.setUnion(JsonUtil.parseList(dsGroup.getInfo(), new TypeReference<>() { + })); + } + datasetSQLManage.mergeDatasetCrossDefault(dsGroup); + } + excelAdaptor(dsGroup,excelTableNamesMap,excelDatasourceId); datasetGroupManage.innerSave(dsGroup); }); @@ -370,7 +423,7 @@ public class DataVisualizationServer implements DataVisualizationApi { //表名映射更新 Map appDsTableNamesMap = dsTableNamesMap.get(key); Map systemDsTableNamesMap = dsTableNamesMap.get(value); - if (!CollectionUtils.isEmpty(appDsTableNamesMap) && !CollectionUtils.isEmpty(systemDsTableNamesMap)) { + if (MapUtils.isNotEmpty(appDsTableNamesMap) && MapUtils.isNotEmpty(systemDsTableNamesMap)) { appDsTableNamesMap.forEach((keyName, valueName) -> { if (StringUtils.isNotEmpty(systemDsTableNamesMap.get(keyName))) { componentDataStr.set(componentDataStr.get().replaceAll(key.toString(), value.toString())); @@ -392,8 +445,12 @@ public class DataVisualizationServer implements DataVisualizationApi { if (DataVisualizationConstants.RESOURCE_OPT_TYPE.COPY.equals(request.getOptType())) { // 复制更新 新建权限插入 visualizationInfoMapper.deleteById(request.getId()); + snapshotMapper.deleteById(request.getId()); visualizationInfo.setNodeType(DataVisualizationConstants.NODE_TYPE.LEAF); } + // 文件夹走默认发布 非文件夹默认未发布 + visualizationInfo.setStatus(DataVisualizationConstants.NODE_TYPE.FOLDER.equals(visualizationInfo.getNodeType()) + ? CommonConstants.DV_STATUS.PUBLISHED : CommonConstants.DV_STATUS.UNPUBLISHED); Long newDvId = coreVisualizationManage.innerSave(visualizationInfo); request.setId(newDvId); // 还原ID信息 @@ -429,6 +486,26 @@ public class DataVisualizationServer implements DataVisualizationApi { return newDvId.toString(); } + private void excelAdaptor(DatasetGroupInfoDTO dsInfo,Map excelTableNamesMap ,List excelDsId) { + List unionDTOList = JsonUtil.parseList(dsInfo.getInfo(), new TypeReference<>() { + }); + if(CollectionUtils.isNotEmpty(excelDsId) && MapUtils.isNotEmpty(excelTableNamesMap)){ + for(UnionDTO unionDTO : unionDTOList) { + DatasetTableDTO tableDTO =unionDTO.getCurrentDs(); + if(excelDsId.contains(tableDTO.getDatasourceId())){ + DatasetTableInfoDTO infoDTO = JsonUtil.parseObject(tableDTO.getInfo(), DatasetTableInfoDTO.class); + String s = new String(Base64.getDecoder().decode(infoDTO.getSql())); + excelTableNamesMap.forEach((key, value) -> { + infoDTO.setSql(Base64.getEncoder().encodeToString(s.replaceAll(key, value).getBytes())); + }); + tableDTO.setInfo((String)JsonUtil.toJSONString(infoDTO)); + } + + } + } + dsInfo.setInfo((String)JsonUtil.toJSONString(unionDTOList)); + } + @Override public String appCanvasNameCheck(DataVisualizationBaseRequest request) throws Exception { Long datasetFolderPid = request.getDatasetFolderPid(); @@ -436,6 +513,7 @@ public class DataVisualizationServer implements DataVisualizationApi { QueryWrapper queryWrapper = new QueryWrapper<>(); queryWrapper.eq("name", datasetFolderName); queryWrapper.eq("pid", datasetFolderPid); + queryWrapper.eq("node_type", DataVisualizationConstants.NODE_TYPE.FOLDER); if (coreDatasetGroupMapper.exists(queryWrapper)) { return "repeat"; } else { @@ -462,7 +540,10 @@ public class DataVisualizationServer implements DataVisualizationApi { @DeLog(id = "#p0.id", ot = LogOT.MODIFY, stExp = "#p0.type") @Override @Transactional - public void updateCanvas(DataVisualizationBaseRequest request) { + public DataVisualizationVO updateCanvas(DataVisualizationBaseRequest request) { + for (Map.Entry ele : request.getCanvasViewInfo().entrySet()) { + DatasetUtils.viewDecode(ele.getValue()); + } Long dvId = request.getId(); if (dvId == null) { DEException.throwException("ID can not be null"); @@ -485,10 +566,48 @@ public class DataVisualizationServer implements DataVisualizationApi { coreVisualizationManage.move(request); } } + // 状态修改统一为后端操作：历史状态检查 如果 状态为 0（未发布） 或者 2（已发布未保存）则状态不变 + // 如果当前状态为 1 则状态修改为 2（已发布未保存） + Integer curStatus = extDataVisualizationMapper.findDvInfoStats(dvId); + visualizationInfo.setStatus(curStatus == 1 ? CommonConstants.DV_STATUS.SAVED_UNPUBLISHED : curStatus); coreVisualizationManage.innerEdit(visualizationInfo); - //保存图表信息 chartDataManage.saveChartViewFromVisualization(request.getComponentData(), dvId, request.getCanvasViewInfo()); + return new DataVisualizationVO(visualizationInfo.getStatus()); + } + + @Override + @Transactional + public void updatePublishStatus(DataVisualizationBaseRequest request) { + /** + * 如果当前传入状态是1（已发布），则原始状态0（未发布）-》1（已发布）；2（已保存未发布）-》1（已发布） + * 统一处理为1.删除主表数据，2.将镜像表数据统一copy到主表 不删除镜像数据（发布状态后镜像数据和主表数据是保持一致的） + * 其他状态仅更新主表和镜像表状态 + * */ + Long dvId = request.getId(); + DataVisualizationInfo visualizationInfo = new DataVisualizationInfo(); + visualizationInfo.setMobileLayout(request.getMobileLayout()); + visualizationInfo.setId(dvId); + visualizationInfo.setName(request.getName()); + visualizationInfo.setStatus(request.getStatus()); + coreVisualizationManage.innerEdit(visualizationInfo); + if (CommonConstants.DV_STATUS.PUBLISHED == request.getStatus()) { + coreVisualizationManage.removeDvCore(dvId); + coreVisualizationManage.dvRestore(dvId); + // chartViewManege.publishThreshold(dvId, request.getActiveViewIds()); + } else if (CommonConstants.DV_STATUS.UNPUBLISHED == request.getStatus()) { + // chartViewManege.publishThreshold(dvId, request.getActiveViewIds()); + } + } + + @Override + public void recoverToPublished(DataVisualizationBaseRequest request) { + coreVisualizationManage.dvSnapshotRecover(request.getId()); + DataVisualizationInfo visualizationInfo = new DataVisualizationInfo(); + visualizationInfo.setId(request.getId()); + visualizationInfo.setName(request.getName()); + visualizationInfo.setStatus(CommonConstants.DV_STATUS.PUBLISHED); + coreVisualizationManage.innerEdit(visualizationInfo); } /** @@ -527,6 +646,9 @@ public class DataVisualizationServer implements DataVisualizationApi { @Override public List tree(BusiNodeRequest request) { + if (StringUtils.isEmpty(request.getResourceTable())) { + request.setResourceTable(CommonConstants.RESOURCE_TABLE.SNAPSHOT); + } String busiFlag = request.getBusiFlag(); if (busiFlag.equals("dashboard-dataV")) { BusiNodeRequest requestDv = new BusiNodeRequest(); @@ -609,7 +731,8 @@ public class DataVisualizationServer implements DataVisualizationApi { newDv.setPid(request.getPid()); newDv.setCreateTime(System.currentTimeMillis()); // 复制图表 chart_view - extDataVisualizationMapper.viewCopyWithDv(sourceDvId, newDvId, copyId); + extDataVisualizationMapper.viewCopyWithDv(sourceDvId, newDvId, copyId, CommonConstants.RESOURCE_TABLE.CORE); + extDataVisualizationMapper.viewCopyWithDv(sourceDvId, newDvId, copyId, CommonConstants.RESOURCE_TABLE.SNAPSHOT); List viewList = extDataVisualizationMapper.findViewInfoByCopyId(copyId); if (!CollectionUtils.isEmpty(viewList)) { String componentData = newDv.getComponentData(); @@ -660,7 +783,6 @@ public class DataVisualizationServer implements DataVisualizationApi { String name = null; String dvType = null; Integer version = null; - //内部模板新建 if (DataVisualizationConstants.NEW_PANEL_FROM.NEW_INNER_TEMPLATE.equals(newFrom)) { VisualizationTemplate visualizationTemplate = templateMapper.selectById(request.getTemplateId()); @@ -776,10 +898,10 @@ public class DataVisualizationServer implements DataVisualizationApi { @Override public List detailList(Long dvId) { List result = extDataVisualizationMapper.getVisualizationViewDetails(dvId); - DataVisualizationInfo dvInfo = visualizationInfoMapper.selectById(dvId); + SnapshotDataVisualizationInfo dvInfo = snapshotMapper.selectById(dvId); if (dvInfo != null && !CollectionUtils.isEmpty(result)) { String componentData = dvInfo.getComponentData(); - return result.stream().filter(item -> componentData.indexOf(String.valueOf(item.getId())) > 0).toList(); + return result.stream().filter(item -> componentData.indexOf("\"id\":\"" + item.getId()) > 0).toList(); } else { return result; } @@ -810,6 +932,9 @@ public class DataVisualizationServer implements DataVisualizationApi { if (CollectionUtils.isEmpty(datasourceVOInfo)) { DEException.throwException("当前不存在数据源无法导出"); + } else if (datasourceVOInfo.stream() + .anyMatch(datasource -> DatasourceConfiguration.DatasourceType.API.name().equals(datasource.getType()))) { + DEException.throwException(Translator.get("i18n_app_error_no_api")); } List linkageVOInfo = appTemplateMapper.findAppLinkageInfo(dvId); @@ -843,27 +968,28 @@ public class DataVisualizationServer implements DataVisualizationApi { if (AuthUtils.getUser().getDefaultOid() != null) { wrapper.eq("org_id", AuthUtils.getUser().getDefaultOid()); } - if (visualizationInfoMapper.exists(wrapper)) { + List existList = visualizationInfoMapper.selectList(wrapper); + if (CollectionUtils.isNotEmpty(existList) && existList.stream().anyMatch(item -> item.getName().equals(request.getName().trim()))) { DEException.throwException("当前名称已经存在"); } } - public String getAbsPath(String id) { - CoreChartView coreChartView = coreChartViewMapper.selectById(id); - if (coreChartView == null) { + public String getAbsPath(Long id) { + ChartViewDTO viewDTO = chartViewManege.findChartViewAround(String.valueOf(id)); + if (viewDTO == null) { return null; } - if (coreChartView.getSceneId() == null) { - return coreChartView.getTitle(); + if (viewDTO.getPid() == null) { + return viewDTO.getTitle(); } - List parents = getParents(coreChartView.getSceneId()); + List parents = getParents(viewDTO.getPid()); StringBuilder stringBuilder = new StringBuilder(); parents.forEach(ele -> { if (ObjectUtils.isNotEmpty(ele)) { stringBuilder.append(ele.getName()).append("/"); } }); - stringBuilder.append(coreChartView.getTitle()); + stringBuilder.append(viewDTO.getTitle()); return stringBuilder.toString(); } @@ -887,4 +1013,16 @@ public class DataVisualizationServer implements DataVisualizationApi { } } + public List getEnabledViewIds(Long dvId, String resourceTable) { + List result = new ArrayList<>(); + DataVisualizationVO dvInfo = extDataVisualizationMapper.findDvInfo(dvId, null, resourceTable); + List views = extChartViewMapper.selectListCustom(dvId, resourceTable); + if (CollectionUtils.isNotEmpty(views) && dvInfo != null) { + String componentData = dvInfo.getComponentData(); + result = views.stream().filter(item -> componentData.indexOf("\"id\":\"" + item.getId()) > 0).map(CoreChartView::getId) + .collect(Collectors.toList()); + + } + return result; + } } diff --git a/core/core-backend/src/main/java/io/gisbi/visualization/server/StaticResourceServer.java b/core/core-backend/src/main/java/io/gisbi/visualization/server/StaticResourceServer.java index 09de74f..7cf1a21 100644 --- a/core/core-backend/src/main/java/io/gisbi/visualization/server/StaticResourceServer.java +++ b/core/core-backend/src/main/java/io/gisbi/visualization/server/StaticResourceServer.java @@ -34,7 +34,7 @@ import java.util.Map; @RequestMapping("/staticResource") public class StaticResourceServer implements StaticResourceApi { - @Value("${gisbi.path.static-resource:/opt/gisbi/data/static-resource/}") + @Value("${gisbi.path.static-resource:/opt/gisbi2.0/data/static-resource/}") private String staticDir; @Override @@ -113,11 +113,6 @@ public class StaticResourceServer implements StaticResourceApi { return result; } - @Override - public Map urlTest() { - return null; - } - private static boolean isValidSVG(MultipartFile file){ if (file == null || file.isEmpty()) { return false; diff --git a/core/core-backend/src/main/java/io/gisbi/visualization/server/VisualizationLinkJumpService.java b/core/core-backend/src/main/java/io/gisbi/visualization/server/VisualizationLinkJumpService.java index dc12ef8..8d400b9 100644 --- a/core/core-backend/src/main/java/io/gisbi/visualization/server/VisualizationLinkJumpService.java +++ b/core/core-backend/src/main/java/io/gisbi/visualization/server/VisualizationLinkJumpService.java @@ -11,19 +11,14 @@ import io.gisbi.api.visualization.vo.VisualizationViewTableVO; import io.gisbi.auth.DeLinkPermit; import io.gisbi.chart.dao.auto.entity.CoreChartView; import io.gisbi.chart.dao.auto.mapper.CoreChartViewMapper; +import io.gisbi.constant.CommonConstants; import io.gisbi.extensions.datasource.dto.DatasetTableFieldDTO; import io.gisbi.utils.AuthUtils; import io.gisbi.utils.BeanUtils; import io.gisbi.utils.IDUtils; import io.gisbi.utils.ModelUtils; -import io.gisbi.visualization.dao.auto.entity.DataVisualizationInfo; -import io.gisbi.visualization.dao.auto.entity.VisualizationLinkJump; -import io.gisbi.visualization.dao.auto.entity.VisualizationLinkJumpInfo; -import io.gisbi.visualization.dao.auto.entity.VisualizationLinkJumpTargetViewInfo; -import io.gisbi.visualization.dao.auto.mapper.DataVisualizationInfoMapper; -import io.gisbi.visualization.dao.auto.mapper.VisualizationLinkJumpInfoMapper; -import io.gisbi.visualization.dao.auto.mapper.VisualizationLinkJumpMapper; -import io.gisbi.visualization.dao.auto.mapper.VisualizationLinkJumpTargetViewInfoMapper; +import io.gisbi.visualization.dao.auto.entity.*; +import io.gisbi.visualization.dao.auto.mapper.*; import io.gisbi.visualization.dao.ext.mapper.ExtVisualizationLinkJumpMapper; import io.gisbi.visualization.dao.ext.mapper.ExtVisualizationLinkageMapper; import jakarta.annotation.Resource; @@ -59,9 +54,22 @@ public class VisualizationLinkJumpService implements VisualizationLinkJumpApi { @Resource private VisualizationLinkJumpTargetViewInfoMapper visualizationLinkJumpTargetViewInfoMapper; + @Resource + private SnapshotVisualizationLinkJumpMapper snapshotVisualizationLinkJumpMapper; + + @Resource + private SnapshotVisualizationLinkJumpInfoMapper snapshotVisualizationLinkJumpInfoMapper; + + @Resource + private SnapshotVisualizationLinkJumpTargetViewInfoMapper snapshotVisualizationLinkJumpTargetViewInfoMapper; + @Resource private CoreChartViewMapper coreChartViewMapper; + @Resource + private SnapshotCoreChartViewMapper snapshotCoreChartViewMapper; + + @Resource private DataVisualizationInfoMapper dataVisualizationInfoMapper; @@ -73,9 +81,14 @@ public class VisualizationLinkJumpService implements VisualizationLinkJumpApi { @DeLinkPermit //获取仪表板的跳转信息 @Override - public VisualizationLinkJumpBaseResponse queryVisualizationJumpInfo(Long dvId) { + public VisualizationLinkJumpBaseResponse queryVisualizationJumpInfo(Long dvId, String resourceTable) { Map resultBase = new HashMap<>(); - List resultLinkJumpList = extVisualizationLinkJumpMapper.queryWithDvId(dvId, AuthUtils.getUser().getUserId(), ModelUtils.isDesktop()); + List resultLinkJumpList = null; + if (CommonConstants.RESOURCE_TABLE.SNAPSHOT.equals(resourceTable)) { + resultLinkJumpList = extVisualizationLinkJumpMapper.queryWithDvIdSnapshot(dvId, AuthUtils.getUser().getUserId(), ModelUtils.isDesktop()); + } else { + resultLinkJumpList = extVisualizationLinkJumpMapper.queryWithDvId(dvId, AuthUtils.getUser().getUserId(), ModelUtils.isDesktop()); + } Optional.ofNullable(resultLinkJumpList).orElse(new ArrayList<>()).forEach(resultLinkJump -> { if (resultLinkJump.getChecked()) { Long sourceViewId = resultLinkJump.getSourceViewId(); @@ -111,30 +124,30 @@ public class VisualizationLinkJumpService implements VisualizationLinkJumpApi { Assert.notNull(dvId, "dvId cannot be null"); Assert.notNull(viewId, "viewId cannot be null"); //清理原有数据 - extVisualizationLinkJumpMapper.deleteJumpTargetViewInfo(dvId, viewId); - extVisualizationLinkJumpMapper.deleteJumpInfo(dvId, viewId); - extVisualizationLinkJumpMapper.deleteJump(dvId, viewId); + extVisualizationLinkJumpMapper.deleteJumpTargetViewInfoSnapshot(dvId, viewId); + extVisualizationLinkJumpMapper.deleteJumpInfoSnapshot(dvId, viewId); + extVisualizationLinkJumpMapper.deleteJumpSnapshot(dvId, viewId); // 插入新的数据 Long linkJumpId = IDUtils.snowID(); jumpDTO.setId(linkJumpId); - VisualizationLinkJump insertParam = new VisualizationLinkJump(); + SnapshotVisualizationLinkJump insertParam = new SnapshotVisualizationLinkJump(); BeanUtils.copyBean(insertParam, jumpDTO); - visualizationLinkJumpMapper.insert(insertParam); + snapshotVisualizationLinkJumpMapper.insert(insertParam); Optional.ofNullable(jumpDTO.getLinkJumpInfoArray()).orElse(new ArrayList<>()).forEach(linkJumpInfo -> { Long linkJumpInfoId = IDUtils.snowID(); linkJumpInfo.setId(linkJumpInfoId); linkJumpInfo.setLinkJumpId(linkJumpId); - VisualizationLinkJumpInfo insertJumpInfoParam = new VisualizationLinkJumpInfo(); + SnapshotVisualizationLinkJumpInfo insertJumpInfoParam = new SnapshotVisualizationLinkJumpInfo(); BeanUtils.copyBean(insertJumpInfoParam, linkJumpInfo); - visualizationLinkJumpInfoMapper.insert(insertJumpInfoParam); + snapshotVisualizationLinkJumpInfoMapper.insert(insertJumpInfoParam); Optional.ofNullable(linkJumpInfo.getTargetViewInfoList()).orElse(new ArrayList<>()).forEach(targetViewInfo -> { Long targetViewInfoId = IDUtils.snowID(); targetViewInfo.setTargetId(targetViewInfoId); targetViewInfo.setLinkJumpInfoId(linkJumpInfoId); - VisualizationLinkJumpTargetViewInfo insertTargetViewInfoParam = new VisualizationLinkJumpTargetViewInfo(); + SnapshotVisualizationLinkJumpTargetViewInfo insertTargetViewInfoParam = new SnapshotVisualizationLinkJumpTargetViewInfo(); BeanUtils.copyBean(insertTargetViewInfoParam, targetViewInfo); - visualizationLinkJumpTargetViewInfoMapper.insert(insertTargetViewInfoParam); + snapshotVisualizationLinkJumpTargetViewInfoMapper.insert(insertTargetViewInfoParam); }); }); } @@ -142,7 +155,12 @@ public class VisualizationLinkJumpService implements VisualizationLinkJumpApi { @DeLinkPermit("#p0.targetDvId") @Override public VisualizationLinkJumpBaseResponse queryTargetVisualizationJumpInfo(VisualizationLinkJumpBaseRequest request) { - List result = extVisualizationLinkJumpMapper.getTargetVisualizationJumpInfo(request); + List result = null; + if (CommonConstants.RESOURCE_TABLE.SNAPSHOT.equals(request.getResourceTable())) { + result = extVisualizationLinkJumpMapper.getTargetVisualizationJumpInfoSnapshot(request); + } else { + result = extVisualizationLinkJumpMapper.getTargetVisualizationJumpInfo(request); + } return new VisualizationLinkJumpBaseResponse(null, Optional.ofNullable(result).orElse(new ArrayList<>()).stream().filter(item -> StringUtils.isNotEmpty(item.getSourceInfo())).collect(Collectors.toMap(VisualizationLinkJumpDTO::getSourceInfo, VisualizationLinkJumpDTO::getTargetInfoList))); } @@ -161,25 +179,25 @@ public class VisualizationLinkJumpService implements VisualizationLinkJumpApi { outParamsJumpInfo = new ArrayList<>(); componentData = "[]"; } - return new VisualizationComponentDTO(componentData,result,outParamsJumpInfo); + return new VisualizationComponentDTO(componentData, result, outParamsJumpInfo); } @Override public VisualizationLinkJumpBaseResponse updateJumpSetActive(VisualizationLinkJumpBaseRequest request) { - CoreChartView coreChartView = new CoreChartView(); + SnapshotCoreChartView coreChartView = new SnapshotCoreChartView(); coreChartView.setId(Long.valueOf(request.getSourceViewId())); coreChartView.setJumpActive(request.getActiveStatus()); - coreChartViewMapper.updateById(coreChartView); - return queryVisualizationJumpInfo(request.getSourceDvId()); + snapshotCoreChartViewMapper.updateById(coreChartView); + return queryVisualizationJumpInfo(request.getSourceDvId(), CommonConstants.RESOURCE_TABLE.SNAPSHOT); } @Override public void removeJumpSet(VisualizationLinkJumpDTO jumpDTO) { //清理原有数据 - extVisualizationLinkJumpMapper.deleteJumpTargetViewInfo(jumpDTO.getSourceDvId(), jumpDTO.getSourceViewId()); - extVisualizationLinkJumpMapper.deleteJumpInfo(jumpDTO.getSourceDvId(), jumpDTO.getSourceViewId()); - extVisualizationLinkJumpMapper.deleteJump(jumpDTO.getSourceDvId(), jumpDTO.getSourceViewId()); + extVisualizationLinkJumpMapper.deleteJumpTargetViewInfoSnapshot(jumpDTO.getSourceDvId(), jumpDTO.getSourceViewId()); + extVisualizationLinkJumpMapper.deleteJumpInfoSnapshot(jumpDTO.getSourceDvId(), jumpDTO.getSourceViewId()); + extVisualizationLinkJumpMapper.deleteJumpSnapshot(jumpDTO.getSourceDvId(), jumpDTO.getSourceViewId()); } } diff --git a/core/core-backend/src/main/java/io/gisbi/visualization/server/VisualizationLinkageService.java b/core/core-backend/src/main/java/io/gisbi/visualization/server/VisualizationLinkageService.java index 5881d1a..a937e0b 100644 --- a/core/core-backend/src/main/java/io/gisbi/visualization/server/VisualizationLinkageService.java +++ b/core/core-backend/src/main/java/io/gisbi/visualization/server/VisualizationLinkageService.java @@ -9,13 +9,11 @@ import io.gisbi.api.visualization.vo.VisualizationLinkageFieldVO; import io.gisbi.auth.DeLinkPermit; import io.gisbi.chart.dao.auto.entity.CoreChartView; import io.gisbi.chart.dao.auto.mapper.CoreChartViewMapper; +import io.gisbi.constant.CommonConstants; import io.gisbi.utils.BeanUtils; import io.gisbi.utils.IDUtils; -import io.gisbi.visualization.dao.auto.entity.VisualizationLinkage; -import io.gisbi.visualization.dao.auto.entity.VisualizationLinkageField; -import io.gisbi.visualization.dao.auto.mapper.DataVisualizationInfoMapper; -import io.gisbi.visualization.dao.auto.mapper.VisualizationLinkageFieldMapper; -import io.gisbi.visualization.dao.auto.mapper.VisualizationLinkageMapper; +import io.gisbi.visualization.dao.auto.entity.*; +import io.gisbi.visualization.dao.auto.mapper.*; import io.gisbi.visualization.dao.ext.mapper.ExtVisualizationLinkageMapper; import jakarta.annotation.Resource; import org.apache.commons.collections4.CollectionUtils; @@ -41,34 +39,53 @@ public class VisualizationLinkageService implements VisualizationLinkageApi { @Resource private VisualizationLinkageFieldMapper visualizationLinkageFieldMapper; + @Resource + private SnapshotVisualizationLinkageFieldMapper snapshotVisualizationLinkageFieldMapper; + @Resource private VisualizationLinkageMapper visualizationLinkageMapper; + @Resource + private SnapshotVisualizationLinkageMapper snapshotVisualizationLinkageMapper; + @Resource private DataVisualizationInfoMapper dataVisualizationInfoMapper; @Resource private CoreChartViewMapper coreChartViewMapper; + @Resource + private SnapshotCoreChartViewMapper snapshotCoreChartViewMapper; + @Override public Map getViewLinkageGather(VisualizationLinkageRequest request) { if (CollectionUtils.isNotEmpty(request.getTargetViewIds())) { - List linkageDTOList = extVisualizationLinkageMapper.getViewLinkageGather(request.getDvId(), request.getSourceViewId(), request.getTargetViewIds()); - return linkageDTOList.stream().collect(Collectors.toMap(targetViewId ->String.valueOf(targetViewId), PanelViewLinkageDTO -> PanelViewLinkageDTO)); + List linkageDTOList = null; + if (CommonConstants.RESOURCE_TABLE.SNAPSHOT.equals(request.getResourceTable())) { + linkageDTOList = extVisualizationLinkageMapper.getViewLinkageGatherSnapshot(request.getDvId(), request.getSourceViewId(), request.getTargetViewIds()); + } else { + linkageDTOList = extVisualizationLinkageMapper.getViewLinkageGather(request.getDvId(), request.getSourceViewId(), request.getTargetViewIds()); + } + return linkageDTOList.stream().collect(Collectors.toMap(targetViewId -> String.valueOf(targetViewId), PanelViewLinkageDTO -> PanelViewLinkageDTO)); } return new HashMap<>(); } @Override public List getViewLinkageGatherArray(VisualizationLinkageRequest request) { - return extVisualizationLinkageMapper.getViewLinkageGather(request.getDvId(), request.getSourceViewId(), request.getTargetViewIds()); + if (CommonConstants.RESOURCE_TABLE.SNAPSHOT.equals(request.getResourceTable())) { + return extVisualizationLinkageMapper.getViewLinkageGatherSnapshot(request.getDvId(), request.getSourceViewId(), request.getTargetViewIds()); + } else { + return extVisualizationLinkageMapper.getViewLinkageGather(request.getDvId(), request.getSourceViewId(), request.getTargetViewIds()); + } } @Override @Transactional public BaseRspModel saveLinkage(VisualizationLinkageRequest request) { + // 向镜像中保存 Long updateTime = System.currentTimeMillis(); - List linkageInfo = request.getLinkageInfo(); + List linkageInfo = request.getLinkageInfo(); Long sourceViewId = request.getSourceViewId(); Long dvId = request.getDvId(); @@ -76,18 +93,18 @@ public class VisualizationLinkageService implements VisualizationLinkageApi { Assert.notNull(dvId, "dvId can not be null"); // 清理原有关系 - extVisualizationLinkageMapper.deleteViewLinkageField(dvId, sourceViewId); - extVisualizationLinkageMapper.deleteViewLinkage(dvId, sourceViewId); + extVisualizationLinkageMapper.deleteViewLinkageFieldSnapshot(dvId, sourceViewId); + extVisualizationLinkageMapper.deleteViewLinkageSnapshot(dvId, sourceViewId); //重新建立关系 - for (VisualizationLinkageDTO linkageDTO:linkageInfo) { + for (VisualizationLinkageDTO linkageDTO : linkageInfo) { //去掉source view 的信息 - if(sourceViewId.equals(linkageDTO.getTargetViewId())){ + if (sourceViewId.equals(linkageDTO.getTargetViewId())) { continue; } List linkageFields = linkageDTO.getLinkageFields(); Long linkageId = IDUtils.snowID(); - VisualizationLinkage linkage = new VisualizationLinkage(); + SnapshotVisualizationLinkage linkage = new SnapshotVisualizationLinkage(); linkage.setId(linkageId); linkage.setDvId(dvId); linkage.setSourceViewId(sourceViewId); @@ -95,14 +112,14 @@ public class VisualizationLinkageService implements VisualizationLinkageApi { linkage.setUpdatePeople(""); linkage.setUpdateTime(updateTime); linkage.setLinkageActive(linkageDTO.getLinkageActive()); - visualizationLinkageMapper.insert(linkage); + snapshotVisualizationLinkageMapper.insert(linkage); if (CollectionUtils.isNotEmpty(linkageFields) && linkageDTO.getLinkageActive()) { linkageFields.forEach(linkageField -> { linkageField.setId(IDUtils.snowID()); linkageField.setLinkageId(linkageId); linkageField.setUpdateTime(updateTime); - VisualizationLinkageField fieldInsert = new VisualizationLinkageField(); - visualizationLinkageFieldMapper.insert(BeanUtils.copyBean(fieldInsert,linkageField)); + SnapshotVisualizationLinkageField fieldInsert = new SnapshotVisualizationLinkageField(); + snapshotVisualizationLinkageFieldMapper.insert(BeanUtils.copyBean(fieldInsert, linkageField)); }); } } @@ -111,24 +128,29 @@ public class VisualizationLinkageService implements VisualizationLinkageApi { @DeLinkPermit @Override - public Map> getVisualizationAllLinkageInfo(Long dvId) { - List info = extVisualizationLinkageMapper.getPanelAllLinkageInfo(dvId); + public Map> getVisualizationAllLinkageInfo(Long dvId, String resourceTable) { + List info = null; + if (CommonConstants.RESOURCE_TABLE.SNAPSHOT.equals(resourceTable)) { + info = extVisualizationLinkageMapper.getPanelAllLinkageInfoSnapshot(dvId); + }else{ + info = extVisualizationLinkageMapper.getPanelAllLinkageInfo(dvId); + } return Optional.ofNullable(info).orElse(new ArrayList<>()).stream().collect(Collectors.toMap(LinkageInfoDTO::getSourceInfo, LinkageInfoDTO::getTargetInfoList)); } @Override public Map updateLinkageActive(VisualizationLinkageRequest request) { - CoreChartView coreChartView = new CoreChartView(); + SnapshotCoreChartView coreChartView = new SnapshotCoreChartView(); coreChartView.setId(request.getSourceViewId()); coreChartView.setLinkageActive(request.getActiveStatus()); - coreChartViewMapper.updateById(coreChartView); - return getVisualizationAllLinkageInfo(request.getDvId()); + snapshotCoreChartViewMapper.updateById(coreChartView); + return getVisualizationAllLinkageInfo(request.getDvId(),CommonConstants.RESOURCE_TABLE.SNAPSHOT); } @Override public void removeLinkage(VisualizationLinkageRequest request) { // 清理原有关系 - extVisualizationLinkageMapper.deleteViewLinkageField(request.getDvId(), request.getSourceViewId()); - extVisualizationLinkageMapper.deleteViewLinkage(request.getDvId(), request.getSourceViewId()); + extVisualizationLinkageMapper.deleteViewLinkageFieldSnapshot(request.getDvId(), request.getSourceViewId()); + extVisualizationLinkageMapper.deleteViewLinkageSnapshot(request.getDvId(), request.getSourceViewId()); } } diff --git a/core/core-backend/src/main/java/io/gisbi/visualization/server/VisualizationOuterParamsService.java b/core/core-backend/src/main/java/io/gisbi/visualization/server/VisualizationOuterParamsService.java index da35756..d0df75e 100644 --- a/core/core-backend/src/main/java/io/gisbi/visualization/server/VisualizationOuterParamsService.java +++ b/core/core-backend/src/main/java/io/gisbi/visualization/server/VisualizationOuterParamsService.java @@ -9,21 +9,19 @@ import io.gisbi.api.visualization.dto.VisualizationOuterParamsDTO; import io.gisbi.api.visualization.dto.VisualizationOuterParamsInfoDTO; import io.gisbi.api.visualization.response.VisualizationOuterParamsBaseResponse; import io.gisbi.auth.DeLinkPermit; +import io.gisbi.constant.CommonConstants; import io.gisbi.dataset.dao.auto.entity.CoreDatasetTable; import io.gisbi.dataset.dao.auto.mapper.CoreDatasetTableMapper; import io.gisbi.constant.DeTypeConstants; import io.gisbi.extensions.view.dto.SqlVariableDetails; import io.gisbi.utils.BeanUtils; import io.gisbi.utils.JsonUtil; -import io.gisbi.visualization.dao.auto.entity.VisualizationOuterParams; -import io.gisbi.visualization.dao.auto.entity.VisualizationOuterParamsInfo; -import io.gisbi.visualization.dao.auto.entity.VisualizationOuterParamsTargetViewInfo; -import io.gisbi.visualization.dao.auto.mapper.VisualizationOuterParamsInfoMapper; -import io.gisbi.visualization.dao.auto.mapper.VisualizationOuterParamsMapper; -import io.gisbi.visualization.dao.auto.mapper.VisualizationOuterParamsTargetViewInfoMapper; +import io.gisbi.visualization.dao.auto.entity.*; +import io.gisbi.visualization.dao.auto.mapper.*; import io.gisbi.visualization.dao.ext.mapper.ExtVisualizationOuterParamsMapper; import jakarta.annotation.Resource; import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.util.Assert; import org.springframework.util.CollectionUtils; import org.springframework.web.bind.annotation.RequestMapping; @@ -44,50 +42,58 @@ public class VisualizationOuterParamsService implements VisualizationOuterParams private ExtVisualizationOuterParamsMapper extOuterParamsMapper; @Resource private VisualizationOuterParamsMapper outerParamsMapper; + @Resource + private SnapshotVisualizationOuterParamsMapper snapshotOuterParamsMapper; + @Resource private VisualizationOuterParamsInfoMapper outerParamsInfoMapper; + @Resource + private SnapshotVisualizationOuterParamsInfoMapper snapshotOuterParamsInfoMapper; + @Resource private VisualizationOuterParamsTargetViewInfoMapper outerParamsTargetViewInfoMapper; + @Resource + private SnapshotVisualizationOuterParamsTargetViewInfoMapper snapshotOuterParamsTargetViewInfoMapper; + @Resource private CoreDatasetTableMapper coreDatasetTableMapper; + @Autowired + private DataVisualizationServer dataVisualizationServer; + @Autowired + private SnapshotDataVisualizationInfoMapper snapshotDataVisualizationInfoMapper; @Override public VisualizationOuterParamsDTO queryWithVisualizationId(String visualizationId) { - VisualizationOuterParamsDTO visualizationOuterParamsDTO = extOuterParamsMapper.queryWithVisualizationId(visualizationId); + VisualizationOuterParamsDTO visualizationOuterParamsDTO = extOuterParamsMapper.queryWithVisualizationIdSnapshot(visualizationId); return visualizationOuterParamsDTO; } - @Override - public VisualizationOuterParamsDTO queryWithVisualizationIdDS(String dvId) { - return null; - } - @Override public void updateOuterParamsSet(VisualizationOuterParamsDTO outerParamsDTO) { String visualizationId = outerParamsDTO.getVisualizationId(); Assert.notNull(visualizationId, "visualizationId cannot be null"); Map paramsInfoNameIdMap = new HashMap<>(); - List paramsInfoNameIdList = extOuterParamsMapper.getVisualizationOuterParamsInfoBase(visualizationId); + List paramsInfoNameIdList = extOuterParamsMapper.getVisualizationOuterParamsInfoBase(visualizationId); if(!CollectionUtils.isEmpty(paramsInfoNameIdList)){ paramsInfoNameIdMap = paramsInfoNameIdList.stream() - .collect(Collectors.toMap(VisualizationOuterParamsInfo::getParamName, VisualizationOuterParamsInfo::getParamsInfoId)); + .collect(Collectors.toMap(SnapshotVisualizationOuterParamsInfo::getParamName, SnapshotVisualizationOuterParamsInfo::getParamsInfoId)); } //清理原有数据 - extOuterParamsMapper.deleteOuterParamsTargetWithVisualizationId(visualizationId); - extOuterParamsMapper.deleteOuterParamsInfoWithVisualizationId(visualizationId); - extOuterParamsMapper.deleteOuterParamsWithVisualizationId(visualizationId); + extOuterParamsMapper.deleteOuterParamsTargetWithVisualizationIdSnapshot(visualizationId); + extOuterParamsMapper.deleteOuterParamsInfoWithVisualizationIdSnapshot(visualizationId); + extOuterParamsMapper.deleteOuterParamsWithVisualizationIdSnapshot(visualizationId); if(CollectionUtils.isEmpty(outerParamsDTO.getOuterParamsInfoArray())){ return; } // 插入新的数据 String paramsId = UUID.randomUUID().toString(); outerParamsDTO.setParamsId(paramsId); - VisualizationOuterParams newOuterParams = new VisualizationOuterParams(); + SnapshotVisualizationOuterParams newOuterParams = new SnapshotVisualizationOuterParams(); BeanUtils.copyBean(newOuterParams, outerParamsDTO); - outerParamsMapper.insert(newOuterParams); + snapshotOuterParamsMapper.insert(newOuterParams); Map finalParamsInfoNameIdMap = paramsInfoNameIdMap; Optional.ofNullable(outerParamsDTO.getOuterParamsInfoArray()).orElse(new ArrayList<>()).forEach(outerParamsInfo -> { String paramsInfoId = finalParamsInfoNameIdMap.get(outerParamsInfo.getParamName()); @@ -96,17 +102,17 @@ public class VisualizationOuterParamsService implements VisualizationOuterParams } outerParamsInfo.setParamsInfoId(paramsInfoId); outerParamsInfo.setParamsId(paramsId); - VisualizationOuterParamsInfo newOuterParamsInfo = new VisualizationOuterParamsInfo(); + SnapshotVisualizationOuterParamsInfo newOuterParamsInfo = new SnapshotVisualizationOuterParamsInfo(); BeanUtils.copyBean(newOuterParamsInfo, outerParamsInfo); - outerParamsInfoMapper.insert(newOuterParamsInfo); + snapshotOuterParamsInfoMapper.insert(newOuterParamsInfo); String finalParamsInfoId = paramsInfoId; Optional.ofNullable(outerParamsInfo.getTargetViewInfoList()).orElse(new ArrayList<>()).forEach(targetViewInfo -> { String targetViewInfoId = UUID.randomUUID().toString(); targetViewInfo.setTargetId(targetViewInfoId); targetViewInfo.setParamsInfoId(finalParamsInfoId); - VisualizationOuterParamsTargetViewInfo newOuterParamsTargetViewInfo = new VisualizationOuterParamsTargetViewInfo(); + SnapshotVisualizationOuterParamsTargetViewInfo newOuterParamsTargetViewInfo = new SnapshotVisualizationOuterParamsTargetViewInfo(); BeanUtils.copyBean(newOuterParamsTargetViewInfo, targetViewInfo); - outerParamsTargetViewInfoMapper.insert(newOuterParamsTargetViewInfo); + snapshotOuterParamsTargetViewInfoMapper.insert(newOuterParamsTargetViewInfo); }); }); @@ -125,7 +131,12 @@ public class VisualizationOuterParamsService implements VisualizationOuterParams public List queryDsWithVisualizationId(String visualizationId) { List result = extOuterParamsMapper.queryDsWithVisualizationId(visualizationId); if (!CollectionUtils.isEmpty(result)) { + List activeViewIds = dataVisualizationServer.getEnabledViewIds(Long.valueOf(visualizationId), CommonConstants.RESOURCE_TABLE.SNAPSHOT); result.forEach(coreDatasetGroupVO -> { + // 过滤已删除的图表 + if(!CollectionUtils.isEmpty(coreDatasetGroupVO.getDatasetViews())){ + coreDatasetGroupVO.setDatasetViews(coreDatasetGroupVO.getDatasetViews().stream().filter(item ->activeViewIds.contains(item.getChartId())).toList()); + } List fields = coreDatasetGroupVO.getDatasetFields(); QueryWrapper wrapper = new QueryWrapper<>(); wrapper.eq("dataset_group_id", coreDatasetGroupVO.getId()); diff --git a/core/core-backend/src/main/java/io/gisbi/visualization/server/VisualizationStoreServer.java b/core/core-backend/src/main/java/io/gisbi/visualization/server/VisualizationStoreServer.java index 09d662e..db4cbea 100644 --- a/core/core-backend/src/main/java/io/gisbi/visualization/server/VisualizationStoreServer.java +++ b/core/core-backend/src/main/java/io/gisbi/visualization/server/VisualizationStoreServer.java @@ -4,6 +4,7 @@ import com.baomidou.mybatisplus.core.metadata.IPage; import io.gisbi.api.visualization.VisualizationStoreApi; import io.gisbi.api.visualization.request.VisualizationStoreRequest; import io.gisbi.api.visualization.request.VisualizationWorkbranchQueryRequest; +import io.gisbi.api.visualization.vo.VisualizationResourceVO; import io.gisbi.api.visualization.vo.VisualizationStoreVO; import io.gisbi.i18n.Translator; import io.gisbi.visualization.manage.VisualizationStoreManage; diff --git a/core/core-backend/src/main/java/io/gisbi/visualization/utils/VisualizationExcelUtils.java b/core/core-backend/src/main/java/io/gisbi/visualization/utils/VisualizationExcelUtils.java index a90ab34..f2904d2 100644 --- a/core/core-backend/src/main/java/io/gisbi/visualization/utils/VisualizationExcelUtils.java +++ b/core/core-backend/src/main/java/io/gisbi/visualization/utils/VisualizationExcelUtils.java @@ -26,9 +26,9 @@ public class VisualizationExcelUtils { public static String getBaseRoot() { if (ModelUtils.isDesktop()) { - return ConfigUtils.getConfig("gisbi.path.report", "/opt/gisbi/data/report/"); + return ConfigUtils.getConfig("gisbi.path.report", "/opt/gisbi2.0/data/report/"); } else { - return "/opt/gisbi/data/report/"; + return "/opt/gisbi2.0/data/report/"; } } diff --git a/core/core-backend/src/main/resources/application.yml b/core/core-backend/src/main/resources/application.yml index 122ed16..2290c7a 100644 --- a/core/core-backend/src/main/resources/application.yml +++ b/core/core-backend/src/main/resources/application.yml @@ -34,7 +34,7 @@ mybatis: map-underscore-to-camel-case: true logging: file: - path: /opt/gisbi/logs + path: /opt/gisbi2.0/logs # sql日志生产环境注释掉 mybatis-plus: configuration: diff --git a/core/core-backend/src/main/resources/i18n/core_en_US.properties b/core/core-backend/src/main/resources/i18n/core_en_US.properties index 13c7b02..e08cc92 100644 --- a/core/core-backend/src/main/resources/i18n/core_en_US.properties +++ b/core/core-backend/src/main/resources/i18n/core_en_US.properties @@ -203,3 +203,10 @@ i18n_user_new_pwd_error=Password format: 8-20 characters and must include at lea i18n_user_pwd_same_error=Old and new passwords cannot be the same i18n_copilot_ds=Only supports MySQL datasource + +i18n_file_download_failed=File download failed! +i18n_unsupported_file_format=Unsupported file format! +i18n_invalid_address=Invalid address! +i18n_unsupported_protocol=Unsupported protocol! +i18n_excel_error_first_row=Empty cells are not allowed in the middle of the first row! +i18n_app_error_no_api=Current API data source is not supported. diff --git a/core/core-backend/src/main/resources/i18n/core_zh_CN.properties b/core/core-backend/src/main/resources/i18n/core_zh_CN.properties index e7122b8..111555f 100644 --- a/core/core-backend/src/main/resources/i18n/core_zh_CN.properties +++ b/core/core-backend/src/main/resources/i18n/core_zh_CN.properties @@ -129,6 +129,7 @@ i18n_df_gt_check=\u503C\u9700\u8981\u5927\u4E8E %s: %s i18n_df_le_check=\u503C\u9700\u8981\u5C0F\u4E8E\u7B49\u4E8E %s: %s i18n_df_ge_check=\u503C\u9700\u8981\u5927\u4E8E\u7B49\u4E8E %s: %s i18n_df_column_exists=\u5B57\u6BB5: %s \u5DF2\u5B58\u5728 +i18n_df_table_primary_key_not_exists=\u6B64\u8868\u4E0D\u5B58\u5728\u4E3B\u952E\uFF0C\u4E0D\u652F\u6301\u5173\u8054 i18n_wrong_email=\u90AE\u7BB1\u683C\u5F0F\u9519\u8BEF i18n_wrong_tel=\u624B\u673A\u53F7\u7801\u683C\u5F0F\u9519\u8BEF @@ -202,3 +203,10 @@ i18n_user_new_pwd_error=\u5BC6\u7801\u683C\u5F0F\uFF1A8-20\u4F4D\u4E14\u81F3\u5C i18n_user_pwd_same_error=\u65B0\u65E7\u5BC6\u7801\u4E0D\u80FD\u76F8\u540C i18n_copilot_ds=\u5F53\u524D\u4EC5\u652F\u6301MySQL\u6570\u636E\u6E90 + +i18n_file_download_failed=\u4E0B\u8F7D\u6587\u4EF6\u5931\u8D25\uFF01 +i18n_unsupported_file_format=\u4E0D\u652F\u6301\u7684\u6587\u4EF6\u683C\u5F0F\uFF01 +i18n_invalid_address=\u65E0\u6548\u7684\u5730\u5740\uFF01 +i18n_unsupported_protocol=\u4E0D\u652F\u6301\u7684\u534F\u8BAE\uFF01 +i18n_excel_error_first_row=\u9996\u884C\u884C\u4E2D\u4E0D\u5141\u8BB8\u6709\u7A7A\u5355\u5143\u683C\uFF01 +i18n_app_error_no_api=\u5F53\u524D\u4E0D\u652F\u6301API\u6570\u636E\u6E90\u3002 diff --git a/core/core-backend/src/main/resources/i18n/core_zh_TW.properties b/core/core-backend/src/main/resources/i18n/core_zh_TW.properties deleted file mode 100644 index 030a292..0000000 --- a/core/core-backend/src/main/resources/i18n/core_zh_TW.properties +++ /dev/null @@ -1,204 +0,0 @@ -login.validator.name1=\u8CEC\u865F/\u90F5\u7BB1/\u624B\u6A5F\u865F\u4E0D\u80FD\u70BA\u7A7A -login.validator.pwd1=\u5BC6\u78BC\u4E0D\u80FD\u70BA\u7A7A - -i18n_menu.home=\u9996\u9801 -i18n_menu.workbranch=\u5DE5\u4F5C\u81FA -i18n_menu.visualized=\u53EF\u8996\u5316 -i18n_menu.template=\u6A21\u7248 -i18n_menu.application=\u61C9\u7528 -i18n_menu.system=\u7CFB\u7D71\u7BA1\u7406 -i18n_menu.template-market=\u6A21\u677F\u5E02\u5834 -i18n_menu.template-setting=\u6A21\u677F\u7BA1\u7406 -i18n_menu.view=\u6578\u64DA\u5C55\u793A -i18n_menu.data=\u6578\u64DA\u6E96\u5099 -i18n_menu.panel=\u5100\u8868\u677F -i18n_menu.data-filling-manage=\u6578\u64DA\u586B\u5831 -i18n_menu.screen=\u6578\u64DA\u5927\u5C4F -i18n_menu.dataset=\u6578\u64DA\u96C6 -i18n_menu.datasource=\u6578\u64DA\u6E90 -i18n_menu.user=\u7528\u6236\u7BA1\u7406 -i18n_menu.org=\u7D44\u7E54\u7BA1\u7406 -i18n_menu.auth=\u6B0A\u9650\u914D\u7F6E -i18n_menu.report=\u5B9A\u6642\u5831\u544A -i18n_menu.sync=\u540C\u6B65\u7BA1\u7406 -i18n_menu.association=\u8840\u7DE3\u5206\u6790 -i18n_menu.threshold=\u544A\u8B66\u7BA1\u7406 -i18n_menu.webhook=Webhook \u7BA1\u7406 -i18n_menu.summary=\u6982\u89BD -i18n_menu.ds=\u6578\u64DA\u9023\u63A5\u7BA1\u7406 -i18n_menu.task=\u4EFB\u52D9\u7BA1\u7406 -i18n_menu.embedded=\u5D4C\u5165\u5F0F\u7BA1\u7406 -i18n_menu.plugin=\u63D2\u4EF6\u7BA1\u7406 -i18n_menu.platform=\u5E73\u81FA\u5C0D\u63A5 -i18n_menu.appearance=\u5916\u89C0\u914D\u7F6E -i18n_menu.sysVariable=\u7CFB\u7D71\u8B8A\u91CF -i18n_menu.sysTypeface=\u5B57\u9AD4\u7BA1\u7406 -i18n_menu.font=\u5B57\u9AD4\u7BA1\u7406 -i18n_menu.msg-fill=\u586B\u5831\u4EFB\u52D9 -i18n_field_name_repeat=\u6709\u91CD\u5FA9\u5B57\u6BB5\u540D\uFF1A -i18n_pid_not_eq_id=\u79FB\u52D5\u76EE\u6A19\u4E0D\u80FD\u662F\u81EA\u5DF1\u6216\u5B50\u76EE\u9304 -i18n_ds_name_exists=\u8A72\u5206\u7D44\u4E0B\u540D\u7A31\u91CD\u5FA9 -i18n_table_id_can_not_empty=\u67E5\u8A62\u7BC0\u9EDE\u4E0D\u80FD\u70BA\u7A7A -i18n_no_fields=\u5B57\u6BB5\u4E0D\u80FD\u70BA\u7A7A -i18n_no_field=\u8A72\u5B57\u6BB5\u4E0D\u5B58\u5728 -i18n_union_ds_no_checked=\u6C92\u6709\u5B57\u6BB5\u9078\u4E2D -i18n_field_name_duplicated=\u5B57\u6BB5\u540D\u91CD\u5FA9 -i18n_can_not_cross_ds=\u4E0D\u652F\u6301\u8DE8\u6578\u64DA\u96C6\u64CD\u4F5C -i18n_dataset_ds_error=\u7576\u524D\u6578\u64DA\u96C6\u7528\u5230\u7684\u6578\u64DA\u6E90\u5DF2\u88AB\u522A\u9664 -i18n_union_field_can_not_empty=\u95DC\u806F\u5B57\u6BB5\u4E0D\u80FD\u70BA\u7A7A -i18n_table_duplicate=\u76F8\u540C\u7BC0\u9EDE\u9700\u91CD\u65B0\u62D6\u5165\u624D\u80FD\u7E7C\u7E8C\u65B0\u5EFA\u6578\u64DA\u96C6 -i18n_no_column_permission=\u6C92\u6709\u5217\u6B0A\u9650 -i18n_fetch_error=SQL\u57F7\u884C\u5931\u6557\uFF0C\u8ACB\u6AA2\u67E5\u8868\u3001\u5B57\u6BB5\u3001\u95DC\u806F\u95DC\u7CFB\u7B49\u4FE1\u606F\u662F\u5426\u6B63\u78BA\u4E26\u91CD\u65B0\u7DE8\u8F2F\u3002 -i18n_no_datasource_permission=\u7121\u6578\u64DA\u6E90\u8A2A\u554F\u6B0A\u9650 -i18n_no_dataset_permission=\u7121\u6578\u64DA\u96C6\u8A2A\u554F\u6B0A\u9650 -i18n_not_full=\u7576\u524D\u6578\u64DA\u6E90\u4E0D\u652F\u6301\u5168\u9023\u63A5 - -i18n_field_circular_ref=\u5B57\u6BB5\u5B58\u5728\u5FAA\u74B0\u5F15\u7528 - -i18n_chart_not_handler=\u7121\u6CD5\u8655\u7406\u8A72\u5716\u8868\u985E\u578B -i18n_chart_delete=\u5716\u8868\u4E0D\u5B58\u5728 -i18n_no_ds=\u6578\u64DA\u96C6\u4E0D\u5B58\u5728\u6216\u6C92\u6709\u6B0A\u9650 -i18n_datasource_delete=\u6578\u64DA\u6E90\u4E0D\u5B58\u5728 -i18n_gauge_field_change=\u6240\u7528\u5B57\u6BB5\u767C\u751F\u8B8A\u66F4\uFF0C\u8ACB\u91CD\u65B0\u7DE8\u8F2F -i18n_gauge_field_delete=\u6240\u7528\u5B57\u6BB5\u5DF2\u522A\u9664\uFF0C\u8ACB\u91CD\u65B0\u7DE8\u8F2F -i18n_no_id=id\u4E0D\u80FD\u70BA\u7A7A -i18n_name_limit_100=\u540D\u7A31\u4E0D\u80FD\u8D85\u904E100\u5B57\u7B26 -i18n_field_circular_error=\u5B57\u6BB5\u89E3\u6790\u932F\u8AA4\uFF0C\u53EF\u80FD\u539F\u56E0\uFF1A\u5B57\u6BB5\u5DF2\u522A\u9664\u3001\u8A08\u7B97\u5B57\u6BB5\u5F15\u7528\u5C64\u7D1A\u904E\u6DF1\u3001\u5B58\u5728\u5FAA\u74B0\u5F15\u7528\u7B49\uFF0C\u8ACB\u6AA2\u67E5\u8868\u7BC0\u9EDE\u548C\u5B57\u6BB5\u4E26\u91CD\u65B0\u7DE8\u8F2F\u3002 - -i18n_invalid_ds=\u6578\u64DA\u6E90\u7121\u6548 - -i18n_user_disable=\u7528\u6236\u5DF2\u88AB\u7981\u7528\uFF0C\u7121\u6CD5\u767B\u9304 -i18n_login_name_pwd_err=\u7528\u6236\u540D\u6216\u5BC6\u78BC\u932F\u8AA4 -i18n_error_login_type=\u767B\u9304\u985E\u578B\u932F\u8AA4 -i18n_schema_is_empty=schema \u70BA\u7A7A\uFF01 -i18n_table_name_repeat=\u540D\u7A31\u91CD\u5FA9: -i18n_sql_not_empty=sql \u4E0D\u80FD\u70BA\u7A7A -i18n_menu.parameter=\u7CFB\u7D71\u53C3\u6578 -i18n_user_old_pwd_error=\u539F\u59CB\u5BC6\u78BC\u932F\u8AA4 -i18n_menu.toolbox-log=\u64CD\u4F5C\u65E5\u8A8C - -i18n_year=\u5E74 -i18n_month=\u6708 -i18n_day=\u5929 -i18n_hour=\u5C0F\u6642 -i18n_minute=\u5206\u9418 -i18n_second=\u79D2 - -i18n_no_datasource_permission_to_create_column=\u7121\u6578\u64DA\u6E90\u8A2A\u554F\u6B0A\u9650\uFF0C\u7121\u6CD5\u5275\u5EFA\u8868\u5B57\u6BB5 -i18n_df_folder_cannot_to_search=\u6587\u4EF6\u593E\u4E0D\u80FD\u67E5\u8A62\u6578\u64DA -i18n_df_no_primary_key=\u6C92\u6709\u4E3B\u9375 -i18n_df_cannot_operate_folder=\u4E0D\u80FD\u64CD\u4F5C\u6587\u4EF6\u593E -i18n_df_cannot_be_none=[%s] \u4E0D\u80FD\u70BA\u7A7A -i18n_df_value_cannot_be_none=[%s] \u503C: %s \u4E0D\u80FD\u70BA\u7A7A -i18n_df_value_exists_in_database=[%s] \u503C: %s \u5728\u6578\u64DA\u5EAB\u4E2D\u5DF2\u5B58\u5728, \u4E0D\u80FD\u91CD\u5FA9 -i18n_df_data=\u6578\u64DA -i18n_df_start=\u958B\u59CB -i18n_df_end=\u7D50\u675F -i18n_df_datasource_not_found=\u6C92\u6709\u627E\u5230\u6578\u64DA\u6E90 -i18n_df_datasource_does_not_enable_data_filling=\u8A72\u6578\u64DA\u6E90\u6C92\u6709\u555F\u7528\u6578\u64DA\u586B\u5831\u914D\u7F6E -i18n_df_builtin_datasource=\u5167\u5EFA\u6578\u64DA\u5EAB -i18n_df_folder_required=\u76EE\u9304\u5FC5\u9078 -i18n_df_form_not_exists=\u8868\u55AE\u4E0D\u5B58\u5728 -i18n_df_name_can_not_empty=\u540D\u7A31\u4E0D\u80FD\u70BA\u7A7A -i18n_df_template=\u6A21\u677F -i18n_df_task_status_is_null_or_finished=\u4EFB\u52D9\u72C0\u614B\u70BA\u7A7A\u6216\u5DF2\u5B8C\u6210 -i18n_df_task_need_task_id=\u9700\u6307\u5B9A\u4EFB\u52D9ID -i18n_df_not_current_task_user=\u4E0D\u662F\u7576\u524D\u4EFB\u52D9\u7684\u76EE\u6A19\u7528\u6236 -i18n_df_miss_parameter=\u7F3A\u5931\u53C3\u6578 -i18n_df_no_running_instance=\u7576\u524D\u4EFB\u52D9\u66AB\u6642\u7121\u904B\u884C\u5BE6\u4F8B -i18n_df_value=\u503C -i18n_df_format_error=\u683C\u5F0F\u89E3\u6790\u932F\u8AA4 -i18n_df_cannot_earlier_than=\u4E0D\u80FD\u65E9\u65BC -i18n_df_cannot_be_all_null=\u4E0D\u80FD\u53EA\u6709\u4E00\u500B\u70BA\u7A7A -i18n_df_value_not_in_range=\u503C\u4E0D\u5728\u7BC4\u570D\u5167 -i18n_df_value_value_not_in_range=\u503C: %s \u4E0D\u5728\u7BC4\u570D\u5167 -i18n_df_required=\u5FC5\u586B -i18n_df_must_unique=\u4E0D\u5141\u8A31\u91CD\u5FA9\u503C -i18n_df_excel_parsing_error=Excel\u89E3\u6790\u932F\u8AA4 -i18n_df_excel_is_empty=\u8A72Excel\u6C92\u6709\u6578\u64DA -i18n_df_excel_template_column_not_fit=\u6A21\u677F\u5B57\u6BB5\u500B\u6578\u4E0D\u5339\u914D -i18n_df_selection=\u9078\u9805\u503C\u70BA -i18n_df_date_format=\u65E5\u671F\u683C\u5F0F -i18n_df_integer=\u6574\u5F62\u6578\u5B57 -i18n_df_decimal=\u5C0F\u6578\u6578\u5B57 -i18n_df_multiple_value_split=\u591A\u500B\u503C\u4F7F\u7528\u5206\u865F";"\u5206\u5272 -i18n_df_email_type=\u90F5\u7BB1\u683C\u5F0F -i18n_df_phone_type=\u624B\u6A5F\u865F\u683C\u5F0F -i18n_df_lt_check=\u503C\u9700\u8981\u5C0F\u65BC %s: %s -i18n_df_gt_check=\u503C\u9700\u8981\u5927\u65BC %s: %s -i18n_df_le_check=\u503C\u9700\u8981\u5C0F\u65BC\u7B49\u65BC %s: %s -i18n_df_ge_check=\u503C\u9700\u8981\u5927\u65BC\u7B49\u65BC %s: %s -i18n_df_column_exists=\u5B57\u6BB5: %s \u5DF2\u5B58\u5728 - -i18n_wrong_email=\u90F5\u7BB1\u683C\u5F0F\u932F\u8AA4 -i18n_wrong_tel=\u624B\u6A5F\u865F\u78BC\u683C\u5F0F\u932F\u8AA4 - - -i18n_copilot_cross_ds_error=\u8DE8\u6E90\u6578\u64DA\u96C6\u4E0D\u652F\u6301\u8A72\u529F\u80FD - -i18n_template_recommend=\u63A8\u85A6 -i18n_template_recent=\u6700\u8FD1\u4F7F\u7528 - -i18n_default_org=\u9ED8\u8A8D\u7D44\u7E54 -i18n_org_admin=\u7D44\u7E54\u7BA1\u7406\u54E1 -i18n_ordinary_role=\u666E\u901A\u7528\u6236 -i18n_sys_admin=\u7CFB\u7D71\u7BA1\u7406\u54E1 - -i18n_threshold_logic_eq=\u7B49\u65BC -i18n_threshold_logic_not_eq=\u4E0D\u7B49\u65BC -i18n_threshold_logic_lt=\u5C0F\u65BC -i18n_threshold_logic_le=\u5C0F\u65BC\u7B49\u65BC -i18n_threshold_logic_gt=\u5927\u65BC -i18n_threshold_logic_ge=\u5927\u65BC\u7B49\u65BC -i18n_threshold_logic_in=\u5C6C\u65BC -i18n_threshold_logic_not_in=\u4E0D\u5C6C\u65BC -i18n_threshold_logic_like=\u5305\u542B -i18n_threshold_logic_not_like=\u4E0D\u5305\u542B -i18n_threshold_logic_null=\u7A7A -i18n_threshold_logic_not_null=\u4E0D\u7A7A -i18n_threshold_logic_empty=\u7A7A\u5B57\u7B26\u4E32 -i18n_threshold_logic_not_empty=\u975E\u7A7A\u5B57\u7B26\u4E32 -i18n_threshold_logic_between=\u7BC4\u570D\u662F -i18n_threshold_logic_and=\u4E14 -i18n_threshold_logic_or=\u6216 -i18n_threshold_max=\u6700\u5927\u503C -i18n_threshold_min=\u6700\u5C0F\u503C -i18n_threshold_average=\u5E73\u5747\u503C -i18n_time_year=\u5E74 -i18n_time_month=\u6708 -i18n_time_date=\u65E5 -i18n_time_hour=\u6642 -i18n_time_minute=\u5206 -i18n_time_second=\u79D2 -i18n_time_ago=\u524D -i18n_time_later=\u5F8C -i18n_time_year_current=\u7576\u5E74 -i18n_time_year_last=\u53BB\u5E74 -i18n_time_year_next=\u660E\u5E74 -i18n_time_month_current=\u7576\u6708 -i18n_time_month_last=\u4E0A\u500B\u6708 -i18n_time_month_next=\u4E0B\u500B\u6708 -i18n_time_month_start=\u5E74\u521D -i18n_time_month_end=\u5E74\u672B -i18n_time_date_current=\u4ECA\u5929 -i18n_time_date_last=\u6628\u5929 -i18n_time_date_next=\u660E\u5929 -i18n_time_date_start=\u6708\u521D -i18n_time_date_end=\u6708\u672B - -i18n_dataset_create_error=\u6578\u64DA\u96C6\u56E0\u7570\u5E38\u5C0E\u81F4\u7121\u6CD5\u4F7F\u7528\uFF0C\u8ACB\u91CD\u65B0\u5275\u5EFA -i18n_dataset_ds_delete=\u7531\u4E8E\u6578\u64DA\u96C6\u6240\u7528\u7684\u6578\u64DA\u6E90\u5DF2\u88AB\u522A\u9664,\u7121\u6CD5\u986F\u793A\u6578\u64DA\u96C6 -i18n_dataset_plugin_error=\u7576\u524D\u6578\u64DA\u6E90\u63D2\u4EF6\u4E0D\u5B58\u5728 -i18n_dataset_cross_error=\u8DE8\u6E90\u6578\u64DA\u96C6\u4E0D\u652F\u6301\u8A72\u529F\u80FD -i18n_board=\u908A\u6846 -i18n_invalid_connection=\u9023\u63A5\u7121\u6548, -i18n_check_datasource_connection=\u8ACB\u6AA2\u67E5\u6578\u64DA\u6E90\u7684\u6709\u6548\u6027 - -i18n_datasource_not_exists=\u6578\u64DA\u6E90\u4E0D\u5B58\u5728\uFF01 - -i18n_geo_exists=\u5DF2\u5B58\u5728\u540C\u540D\u5340\u57DF\uFF01 -i18n_geo_sub_exists=\u5DF2\u5B58\u5728\u540C\u540D\u5B50\u5340\u57DF\uFF01 -i18n_user_new_pwd_error=\u5BC6\u78BC\u683C\u5F0F\uFF1A8-20\u4F4D\u4E14\u81F3\u5C11\u4E00\u4F4D\u5927\u5BEB\u5B57\u6BCD\u3001\u5C0F\u5BEB\u5B57\u6BCD\u3001\u6578\u5B57\u3001\u7279\u6B8A\u5B57\u7B26 -i18n_user_pwd_same_error=\u65B0\u820A\u5BC6\u78BC\u4E0D\u80FD\u76F8\u540C - -i18n_copilot_ds=\u7576\u524D\u50C5\u652F\u6301MySQL\u6578\u64DA\u6E90 diff --git a/core/core-backend/src/main/resources/mybatis/ExtDataVisualizationMapper.xml b/core/core-backend/src/main/resources/mybatis/ExtDataVisualizationMapper.xml index df88593..4812349 100644 --- a/core/core-backend/src/main/resources/mybatis/ExtDataVisualizationMapper.xml +++ b/core/core-backend/src/main/resources/mybatis/ExtDataVisualizationMapper.xml @@ -2,7 +2,16 @@ - INSERT INTO `core_chart_view` (`id`, + INSERT INTO + + + `snapshot_core_chart_view` + + + `core_chart_view` + + + (`id`, `title`, `scene_id`, `table_id`, @@ -44,7 +53,7 @@ `flow_map_start_name`, `flow_map_end_name`, `ext_color`) - SELECT core_chart_view.`id` + #{copyId} as id, + SELECT ccv.`id` + #{copyId} as id, `title`, #{newDvId} as scene_id, `table_id`, @@ -81,13 +90,21 @@ `refresh_time`, `linkage_active`, `jump_active`, - core_chart_view.`id` as copy_from, + ccv.`id` as copy_from, #{copyId} as copy_id, `flow_map_start_name`, `flow_map_end_name`, `ext_color` - FROM core_chart_view - WHERE core_chart_view.scene_id = #{sourceDvId} + FROM + + + snapshot_core_chart_view ccv + + + core_chart_view ccv + + + WHERE ccv.scene_id = #{sourceDvId} INSERT INTO `data_visualization_info` (`id`, @@ -137,7 +154,7 @@ from data_visualization_info dvInfo where dvInfo.id = #{sourceDvId} - select id, copy_from from core_chart_view where copy_id = #{copyId} @@ -169,11 +186,19 @@ `version`, `content_id`, `check_version` - FROM data_visualization_info - where data_visualization_info.delete_flag = 0 - and data_visualization_info.id = #{dvId} + FROM + + + snapshot_data_visualization_info dvi + + + data_visualization_info dvi + + + where dvi.delete_flag = 0 + and dvi.id = #{dvId} - and data_visualization_info.type = #{dvType} + and dvi.type = #{dvType} @@ -229,7 +254,7 @@ data_visualization_info WHERE data_visualization_info.delete_flag = 0 - AND node_type = 'leaf') dvResource + AND node_type = 'leaf' and data_visualization_info.status != 0 ) dvResource LEFT JOIN core_store ON dvResource.id = core_store.resource_id AND core_store.uid = #{uid} INNER JOIN core_opt_recent ON dvResource.resource_id = core_opt_recent.resource_id @@ -237,38 +262,46 @@ AND LOWER(dvResource.name) LIKE LOWER(CONCAT('%', #{keyword}, '%')) - ${ew.customSqlSegment} + + AND dvResource.type = #{ew.type} + + + AND NOT EXISTS(select 1 from per_busi_resource community where core_opt_recent.resource_id = community.id) + + + order by core_opt_recent.time asc + - INSERT INTO visualization_link_jump (`id`, + INSERT INTO snapshot_visualization_link_jump (`id`, `source_dv_id`, `source_view_id`, `link_jump_info`, `checked`, `copy_from`, `copy_id`) - SELECT visualization_link_jump.`id` + #{copyId} as id, + SELECT snapshot_visualization_link_jump.`id` + #{copyId} as id, dv_view_copy.t_dv_id as source_dv_id, dv_view_copy.t_chart_view_id as source_view_id, - visualization_link_jump.`link_jump_info`, - visualization_link_jump.`checked`, - visualization_link_jump.`id` as copy_from, + snapshot_visualization_link_jump.`link_jump_info`, + snapshot_visualization_link_jump.`checked`, + snapshot_visualization_link_jump.`id` as copy_from, #{copyId} as copy_id - FROM visualization_link_jump + FROM snapshot_visualization_link_jump INNER JOIN (SELECT pvs.scene_id AS s_dv_id, pvs.id AS s_chart_view_id, pvt.scene_id AS t_dv_id, pvt.id AS t_chart_view_id - FROM core_chart_view pvt - INNER JOIN core_chart_view pvs ON pvt.copy_from = pvs.id + FROM snapshot_core_chart_view pvt + INNER JOIN snapshot_core_chart_view pvs ON pvt.copy_from = pvs.id WHERE pvt.copy_id = #{copyId}) dv_view_copy - ON visualization_link_jump.source_dv_id = dv_view_copy.s_dv_id - AND visualization_link_jump.source_view_id = dv_view_copy.s_chart_view_id + ON snapshot_visualization_link_jump.source_dv_id = dv_view_copy.s_dv_id + AND snapshot_visualization_link_jump.source_view_id = dv_view_copy.s_chart_view_id - INSERT INTO visualization_link_jump_info (id, + INSERT INTO snapshot_visualization_link_jump_info (id, link_jump_id, link_type, jump_type, @@ -279,51 +312,51 @@ `attach_params`, copy_from, copy_id) - SELECT visualization_link_jump_info.`id` + #{copyId} as id, + SELECT snapshot_visualization_link_jump_info.`id` + #{copyId} as id, plj_copy.t_id as link_jump_id, - visualization_link_jump_info.`link_type`, - visualization_link_jump_info.`jump_type`, - visualization_link_jump_info.`target_dv_id`, - visualization_link_jump_info.`source_field_id`, - visualization_link_jump_info.`content`, - visualization_link_jump_info.`checked`, - visualization_link_jump_info.`attach_params`, - visualization_link_jump_info.`id` AS copy_from, + snapshot_visualization_link_jump_info.`link_type`, + snapshot_visualization_link_jump_info.`jump_type`, + snapshot_visualization_link_jump_info.`target_dv_id`, + snapshot_visualization_link_jump_info.`source_field_id`, + snapshot_visualization_link_jump_info.`content`, + snapshot_visualization_link_jump_info.`checked`, + snapshot_visualization_link_jump_info.`attach_params`, + snapshot_visualization_link_jump_info.`id` AS copy_from, #{copyId} AS copy_id - FROM visualization_link_jump_info + FROM snapshot_visualization_link_jump_info INNER JOIN (SELECT id AS t_id, copy_from AS s_id - FROM visualization_link_jump + FROM snapshot_visualization_link_jump WHERE copy_id = #{copyId}) plj_copy - ON visualization_link_jump_info.link_jump_id = plj_copy.s_id + ON snapshot_visualization_link_jump_info.link_jump_id = plj_copy.s_id - INSERT INTO visualization_link_jump_target_view_info (`target_id`, + INSERT INTO snapshot_visualization_link_jump_target_view_info (`target_id`, `link_jump_info_id`, `source_field_active_id`, `target_view_id`, `target_field_id`, `copy_from`, `copy_id`) - SELECT visualization_link_jump_target_view_info.`target_id` + #{copyId} as target_id, + SELECT vljtvi.`target_id` + #{copyId} as target_id, plji_copy.t_id as link_jump_info_id, - visualization_link_jump_target_view_info.`source_field_active_id`, - visualization_link_jump_target_view_info.`target_view_id`, - visualization_link_jump_target_view_info.`target_field_id`, - visualization_link_jump_target_view_info.`target_id` AS copy_from, + vljtvi.`source_field_active_id`, + vljtvi.`target_view_id`, + vljtvi.`target_field_id`, + vljtvi.`target_id` AS copy_from, #{copyId} AS copy_id - FROM visualization_link_jump_target_view_info + FROM snapshot_visualization_link_jump_target_view_info vljtvi INNER JOIN (SELECT id AS t_id, copy_from AS s_id FROM visualization_link_jump_info WHERE copy_id = #{copyId}) plji_copy - ON visualization_link_jump_target_view_info.link_jump_info_id = plji_copy.s_id + ON vljtvi.link_jump_info_id = plji_copy.s_id - INSERT INTO visualization_linkage (`id`, + INSERT INTO snapshot_visualization_linkage (`id`, dv_id, source_view_id, target_view_id, @@ -334,7 +367,7 @@ `ext2`, `copy_from`, `copy_id`) - SELECT visualization_linkage.`id` + #{copyId} as `id`, + SELECT snapshot_visualization_linkage.`id` + #{copyId} as `id`, pv_source_copy.t_dv_id AS dv_id, pv_source_copy.t_chart_view_id AS source_view_id, pv_target_copy.t_chart_view_id AS target_view_id, @@ -343,61 +376,61 @@ `linkage_active`, `ext1`, `ext2`, - visualization_linkage.`id` as copy_from, + snapshot_visualization_linkage.`id` as copy_from, #{copyId} as copy_id - FROM visualization_linkage + FROM snapshot_visualization_linkage INNER JOIN (SELECT pvs.scene_id AS s_dv_id, pvs.id AS s_chart_view_id, pvt.scene_id AS t_dv_id, pvt.id AS t_chart_view_id - FROM core_chart_view pvt - INNER JOIN core_chart_view pvs ON pvt.copy_from = pvs.id + FROM snapshot_core_chart_view pvt + INNER JOIN snapshot_core_chart_view pvs ON pvt.copy_from = pvs.id WHERE pvt.copy_id = #{copyId}) pv_source_copy - ON visualization_linkage.dv_id = pv_source_copy.s_dv_id - AND visualization_linkage.source_view_id = pv_source_copy.s_chart_view_id + ON snapshot_visualization_linkage.dv_id = pv_source_copy.s_dv_id + AND snapshot_visualization_linkage.source_view_id = pv_source_copy.s_chart_view_id INNER JOIN (SELECT pvs.scene_id AS s_dv_id, pvs.id AS s_chart_view_id, pvt.scene_id AS t_dv_id, pvt.id AS t_chart_view_id - FROM core_chart_view pvt - INNER JOIN core_chart_view pvs ON pvt.copy_from = pvs.id + FROM snapshot_core_chart_view pvt + INNER JOIN snapshot_core_chart_view pvs ON pvt.copy_from = pvs.id WHERE pvt.copy_id = #{copyId}) pv_target_copy - ON visualization_linkage.dv_id = pv_target_copy.s_dv_id - AND visualization_linkage.target_view_id = pv_target_copy.s_chart_view_id + ON snapshot_visualization_linkage.dv_id = pv_target_copy.s_dv_id + AND snapshot_visualization_linkage.target_view_id = pv_target_copy.s_chart_view_id - INSERT INTO visualization_linkage_field (id, + INSERT INTO snapshot_visualization_linkage_field (id, `linkage_id`, `source_field`, `target_field`, `update_time`, `copy_from`, `copy_id`) - SELECT visualization_linkage_field.`id` + #{copyId} as `id`, + SELECT snapshot_visualization_linkage_field.`id` + #{copyId} as `id`, pvlf_copy.t_id as `linkage_id`, `source_field`, `target_field`, `update_time`, id AS copy_from, #{copyId} as copy_id - FROM visualization_linkage_field + FROM snapshot_visualization_linkage_field INNER JOIN (SELECT id AS t_id, copy_from AS s_id - FROM visualization_linkage + FROM snapshot_visualization_linkage WHERE copy_id = #{copyId}) pvlf_copy - ON visualization_linkage_field.linkage_id = pvlf_copy.s_id + ON snapshot_visualization_linkage_field.linkage_id = pvlf_copy.s_id - + - + @@ -407,22 +440,22 @@ - SELECT `id`, `report_id`, `resource_id`, `dv_type`, `component_id`, `filter_id`, `filter_info`, `filter_version`, `create_time`, `create_user` FROM @@ -433,7 +466,14 @@ DELETE FROM - data_visualization_info dvi + + + snapshot_data_visualization_info dvi + + + data_visualization_info dvi + + WHERE dvi.id IN @@ -443,7 +483,14 @@ DELETE FROM - core_chart_view ccv + + + snapshot_core_chart_view ccv + + + core_chart_view ccv + + WHERE ccv.scene_id IN @@ -451,7 +498,736 @@ - select id,account,name from per_user where id = #{id} - + + + + INSERT into snapshot_data_visualization_info ( + `id`, + `name`, + `pid`, + `org_id`, + `level`, + `node_type`, + `type`, + `canvas_style_data`, + `component_data`, + `mobile_layout`, + `status`, + `self_watermark_status`, + `sort`, + `create_time`, + `create_by`, + `update_time`, + `update_by`, + `remark`, + `source`, + `delete_flag`, + `delete_time`, + `delete_by`, + `version`, + `content_id`, + `check_version` + ) SELECT + `id`, + `name`, + `pid`, + `org_id`, + `level`, + `node_type`, + `type`, + `canvas_style_data`, + `component_data`, + `mobile_layout`, + `status`, + `self_watermark_status`, + `sort`, + `create_time`, + `create_by`, + `update_time`, + `update_by`, + `remark`, + `source`, + `delete_flag`, + `delete_time`, + `delete_by`, + `version`, + `content_id`, + `check_version` + FROM + data_visualization_info + WHERE + id = #{dvId} + + + + INSERT into snapshot_core_chart_view ( + `id`, + `title`, + `scene_id`, + `table_id`, + `type`, + `render`, + `result_count`, + `result_mode`, + `x_axis`, + `x_axis_ext`, + `y_axis`, + `y_axis_ext`, + `ext_stack`, + `ext_bubble`, + `ext_label`, + `ext_tooltip`, + `custom_attr`, + `custom_attr_mobile`, + `custom_style`, + `custom_style_mobile`, + `custom_filter`, + `drill_fields`, + `senior`, + `create_by`, + `create_time`, + `update_time`, + `snapshot`, + `style_priority`, + `chart_type`, + `is_plugin`, + `data_from`, + `view_fields`, + `refresh_view_enable`, + `refresh_unit`, + `refresh_time`, + `linkage_active`, + `jump_active`, + `copy_from`, + `copy_id`, + `aggregate`, + `flow_map_start_name`, + `flow_map_end_name`, + `ext_color`, + `sort_priority` + ) SELECT + `id`, + `title`, + `scene_id`, + `table_id`, + `type`, + `render`, + `result_count`, + `result_mode`, + `x_axis`, + `x_axis_ext`, + `y_axis`, + `y_axis_ext`, + `ext_stack`, + `ext_bubble`, + `ext_label`, + `ext_tooltip`, + `custom_attr`, + `custom_attr_mobile`, + `custom_style`, + `custom_style_mobile`, + `custom_filter`, + `drill_fields`, + `senior`, + `create_by`, + `create_time`, + `update_time`, + `snapshot`, + `style_priority`, + `chart_type`, + `is_plugin`, + `data_from`, + `view_fields`, + `refresh_view_enable`, + `refresh_unit`, + `refresh_time`, + `linkage_active`, + `jump_active`, + `copy_from`, + `copy_id`, + `aggregate`, + `flow_map_start_name`, + `flow_map_end_name`, + `ext_color`, + `sort_priority` + FROM + core_chart_view + WHERE + scene_id = #{dvId} + + + + INSERT into snapshot_visualization_link_jump_target_view_info ( + `target_id`, + `link_jump_info_id`, + `source_field_active_id`, + `target_view_id`, + `target_field_id`, + `copy_from`, + `copy_id`, + `target_type` + ) SELECT + vljtvi.`target_id`, + vljtvi.`link_jump_info_id`, + vljtvi.`source_field_active_id`, + vljtvi.`target_view_id`, + vljtvi.`target_field_id`, + vljtvi.`copy_from`, + vljtvi.`copy_id`, + vljtvi.`target_type` + FROM + visualization_link_jump_target_view_info vljtvi + LEFT JOIN visualization_link_jump_info vlji ON vljtvi.link_jump_info_id = vlji.id + LEFT JOIN visualization_link_jump vlj ON vlji.link_jump_id = vlj.id + WHERE + vlj.source_dv_id = #{dvId} + + + + INSERT into snapshot_visualization_link_jump_info ( + `id`, + `link_jump_id`, + `link_type`, + `jump_type`, + `target_dv_id`, + `source_field_id`, + `content`, + `checked`, + `attach_params`, + `copy_from`, + `copy_id`, + `window_size` + ) SELECT + vlji.`id`, + vlji.`link_jump_id`, + vlji.`link_type`, + vlji.`jump_type`, + vlji.`target_dv_id`, + vlji.`source_field_id`, + vlji.`content`, + vlji.`checked`, + vlji.`attach_params`, + vlji.`copy_from`, + vlji.`copy_id`, + vlji.`window_size` + FROM + visualization_link_jump_info vlji + LEFT JOIN visualization_link_jump vlj ON vlji.link_jump_id = vlj.id + WHERE + vlj.source_dv_id = #{dvId} + + + INSERT into snapshot_visualization_link_jump ( + `id`, + `source_dv_id`, + `source_view_id`, + `link_jump_info`, + `checked`, + `copy_from`, + `copy_id` + ) SELECT + vlj.`id`, + vlj.`source_dv_id`, + vlj.`source_view_id`, + vlj.`link_jump_info`, + vlj.`checked`, + vlj.`copy_from`, + vlj.`copy_id` + FROM + visualization_link_jump vlj + WHERE + vlj.source_dv_id = #{dvId} + + + + INSERT into snapshot_visualization_linkage_field ( + `id`, + `linkage_id`, + `source_field`, + `target_field`, + `update_time`, + `copy_from`, + `copy_id` + ) SELECT + vlf.`id`, + vlf.`linkage_id`, + vlf.`source_field`, + vlf.`target_field`, + vlf.`update_time`, + vlf.`copy_from`, + vlf.`copy_id` + FROM + visualization_linkage_field vlf + LEFT JOIN visualization_linkage vl ON vlf.linkage_id = vl.id + WHERE + vl.dv_id = #{dvId} + + + INSERT into snapshot_visualization_linkage ( + `id`, + `dv_id`, + `source_view_id`, + `target_view_id`, + `update_time`, + `update_people`, + `linkage_active`, + `ext1`, + `ext2`, + `copy_from`, + `copy_id` + ) SELECT + vl.`id`, + vl.`dv_id`, + vl.`source_view_id`, + vl.`target_view_id`, + vl.`update_time`, + vl.`update_people`, + vl.`linkage_active`, + vl.`ext1`, + vl.`ext2`, + vl.`copy_from`, + vl.`copy_id` + FROM + visualization_linkage vl + WHERE + vl.dv_id = #{dvId} + + + INSERT into snapshot_visualization_outer_params_target_view_info ( + `target_id`, + `params_info_id`, + `target_view_id`, + `target_field_id`, + `copy_from`, + `copy_id`, + `target_ds_id` + ) SELECT + voptvi.`target_id`, + voptvi.`params_info_id`, + voptvi.`target_view_id`, + voptvi.`target_field_id`, + voptvi.`copy_from`, + voptvi.`copy_id`, + voptvi.`target_ds_id` + FROM + visualization_outer_params_target_view_info voptvi + LEFT JOIN visualization_outer_params_info vopi ON vopi.params_info_id = voptvi.params_info_id + LEFT JOIN visualization_outer_params vop ON vop.params_id = vopi.params_id + WHERE + vop.visualization_id = #{dvId} + + + INSERT INTO snapshot_visualization_outer_params_info ( + `params_info_id`, + `params_id`, + `param_name`, + `checked`, + `copy_from`, + `copy_id`, + `required`, + `default_value`, + `enabled_default` + ) SELECT + vopi.`params_info_id`, + vopi.`params_id`, + vopi.`param_name`, + vopi.`checked`, + vopi.`copy_from`, + vopi.`copy_id`, + vopi.`required`, + vopi.`default_value`, + vopi.`enabled_default` + FROM + visualization_outer_params_info vopi + LEFT JOIN visualization_outer_params vop ON vop.params_id = vopi.params_id + WHERE + vop.visualization_id = #{dvId} + + + INSERT into snapshot_visualization_outer_params ( + `params_id`, + `visualization_id`, + `checked`, + `remark`, + `copy_from`, + `copy_id` + ) SELECT + vop.`params_id`, + vop.`visualization_id`, + vop.`checked`, + vop.`remark`, + vop.`copy_from`, + vop.`copy_id` + FROM + visualization_outer_params vop + WHERE + vop.visualization_id = #{dvId} + + + + + INSERT INTO data_visualization_info ( + `id`, + `name`, + `pid`, + `org_id`, + `level`, + `node_type`, + `type`, + `canvas_style_data`, + `component_data`, + `mobile_layout`, + `status`, + `self_watermark_status`, + `sort`, + `create_time`, + `create_by`, + `update_time`, + `update_by`, + `remark`, + `source`, + `delete_flag`, + `delete_time`, + `delete_by`, + `version`, + `content_id`, + `check_version` + ) SELECT + `id`, + `name`, + `pid`, + `org_id`, + `level`, + `node_type`, + `type`, + `canvas_style_data`, + `component_data`, + `mobile_layout`, + `status`, + `self_watermark_status`, + `sort`, + `create_time`, + `create_by`, + `update_time`, + `update_by`, + `remark`, + `source`, + `delete_flag`, + `delete_time`, + `delete_by`, + `version`, + `content_id`, + `check_version` + FROM + snapshot_data_visualization_info + WHERE + id = #{dvId} + + + + INSERT INTO core_chart_view ( + `id`, + `title`, + `scene_id`, + `table_id`, + `type`, + `render`, + `result_count`, + `result_mode`, + `x_axis`, + `x_axis_ext`, + `y_axis`, + `y_axis_ext`, + `ext_stack`, + `ext_bubble`, + `ext_label`, + `ext_tooltip`, + `custom_attr`, + `custom_attr_mobile`, + `custom_style`, + `custom_style_mobile`, + `custom_filter`, + `drill_fields`, + `senior`, + `create_by`, + `create_time`, + `update_time`, + `snapshot`, + `style_priority`, + `chart_type`, + `is_plugin`, + `data_from`, + `view_fields`, + `refresh_view_enable`, + `refresh_unit`, + `refresh_time`, + `linkage_active`, + `jump_active`, + `copy_from`, + `copy_id`, + `aggregate`, + `flow_map_start_name`, + `flow_map_end_name`, + `ext_color`, + `sort_priority` + ) SELECT + `id`, + `title`, + `scene_id`, + `table_id`, + `type`, + `render`, + `result_count`, + `result_mode`, + `x_axis`, + `x_axis_ext`, + `y_axis`, + `y_axis_ext`, + `ext_stack`, + `ext_bubble`, + `ext_label`, + `ext_tooltip`, + `custom_attr`, + `custom_attr_mobile`, + `custom_style`, + `custom_style_mobile`, + `custom_filter`, + `drill_fields`, + `senior`, + `create_by`, + `create_time`, + `update_time`, + `snapshot`, + `style_priority`, + `chart_type`, + `is_plugin`, + `data_from`, + `view_fields`, + `refresh_view_enable`, + `refresh_unit`, + `refresh_time`, + `linkage_active`, + `jump_active`, + `copy_from`, + `copy_id`, + `aggregate`, + `flow_map_start_name`, + `flow_map_end_name`, + `ext_color`, + `sort_priority` + FROM + snapshot_core_chart_view + WHERE + scene_id = #{dvId} + + + + INSERT into visualization_link_jump_target_view_info ( + `target_id`, + `link_jump_info_id`, + `source_field_active_id`, + `target_view_id`, + `target_field_id`, + `copy_from`, + `copy_id`, + `target_type` + ) SELECT + vljtvi.`target_id`, + vljtvi.`link_jump_info_id`, + vljtvi.`source_field_active_id`, + vljtvi.`target_view_id`, + vljtvi.`target_field_id`, + vljtvi.`copy_from`, + vljtvi.`copy_id`, + vljtvi.`target_type` + FROM + snapshot_visualization_link_jump_target_view_info vljtvi + LEFT JOIN snapshot_visualization_link_jump_info vlji ON vljtvi.link_jump_info_id = vlji.id + LEFT JOIN snapshot_visualization_link_jump vlj ON vlji.link_jump_id = vlj.id + WHERE + vlj.source_dv_id = #{dvId} + + + + INSERT INTO visualization_link_jump_info ( + `id`, + `link_jump_id`, + `link_type`, + `jump_type`, + `target_dv_id`, + `source_field_id`, + `content`, + `checked`, + `attach_params`, + `copy_from`, + `copy_id`, + `window_size` + ) SELECT + vlji.`id`, + vlji.`link_jump_id`, + vlji.`link_type`, + vlji.`jump_type`, + vlji.`target_dv_id`, + vlji.`source_field_id`, + vlji.`content`, + vlji.`checked`, + vlji.`attach_params`, + vlji.`copy_from`, + vlji.`copy_id`, + vlji.`window_size` + FROM + snapshot_visualization_link_jump_info vlji + LEFT JOIN snapshot_visualization_link_jump vlj ON vlji.link_jump_id = vlj.id + WHERE + vlj.source_dv_id = #{dvId} + + + INSERT INTO visualization_link_jump ( + `id`, + `source_dv_id`, + `source_view_id`, + `link_jump_info`, + `checked`, + `copy_from`, + `copy_id` + ) SELECT + vlj.`id`, + vlj.`source_dv_id`, + vlj.`source_view_id`, + vlj.`link_jump_info`, + vlj.`checked`, + vlj.`copy_from`, + vlj.`copy_id` + FROM + snapshot_visualization_link_jump vlj + WHERE + vlj.source_dv_id = #{dvId} + + + + INSERT INTO visualization_linkage_field ( + `id`, + `linkage_id`, + `source_field`, + `target_field`, + `update_time`, + `copy_from`, + `copy_id` + ) SELECT + vlf.`id`, + vlf.`linkage_id`, + vlf.`source_field`, + vlf.`target_field`, + vlf.`update_time`, + vlf.`copy_from`, + vlf.`copy_id` + FROM + snapshot_visualization_linkage_field vlf + LEFT JOIN snapshot_visualization_linkage vl ON vlf.linkage_id = vl.id + WHERE + vl.dv_id = #{dvId} + + + INSERT INTO visualization_linkage ( + `id`, + `dv_id`, + `source_view_id`, + `target_view_id`, + `update_time`, + `update_people`, + `linkage_active`, + `ext1`, + `ext2`, + `copy_from`, + `copy_id` + ) SELECT + vl.`id`, + vl.`dv_id`, + vl.`source_view_id`, + vl.`target_view_id`, + vl.`update_time`, + vl.`update_people`, + vl.`linkage_active`, + vl.`ext1`, + vl.`ext2`, + vl.`copy_from`, + vl.`copy_id` + FROM + snapshot_visualization_linkage vl + WHERE + vl.dv_id = #{dvId} + + + INSERT into visualization_outer_params_target_view_info ( + `target_id`, + `params_info_id`, + `target_view_id`, + `target_field_id`, + `copy_from`, + `copy_id`, + `target_ds_id` + ) SELECT + voptvi.`target_id`, + voptvi.`params_info_id`, + voptvi.`target_view_id`, + voptvi.`target_field_id`, + voptvi.`copy_from`, + voptvi.`copy_id`, + voptvi.`target_ds_id` + FROM + snapshot_visualization_outer_params_target_view_info voptvi + LEFT JOIN snapshot_visualization_outer_params_info vopi ON vopi.params_info_id = voptvi.params_info_id + LEFT JOIN snapshot_visualization_outer_params vop ON vop.params_id = vopi.params_id + WHERE + vop.visualization_id = #{dvId} + + + INSERT into visualization_outer_params_info ( + `params_info_id`, + `params_id`, + `param_name`, + `checked`, + `copy_from`, + `copy_id`, + `required`, + `default_value`, + `enabled_default` + ) SELECT + vopi.`params_info_id`, + vopi.`params_id`, + vopi.`param_name`, + vopi.`checked`, + vopi.`copy_from`, + vopi.`copy_id`, + vopi.`required`, + vopi.`default_value`, + vopi.`enabled_default` + FROM + snapshot_visualization_outer_params_info vopi + LEFT JOIN snapshot_visualization_outer_params vop ON vop.params_id = vopi.params_id + WHERE + vop.visualization_id = #{dvId} + + + INSERT into visualization_outer_params ( + `params_id`, + `visualization_id`, + `checked`, + `remark`, + `copy_from`, + `copy_id` + ) SELECT + vop.`params_id`, + vop.`visualization_id`, + vop.`checked`, + vop.`remark`, + vop.`copy_from`, + vop.`copy_id` + FROM + snapshot_visualization_outer_params vop + WHERE + vop.visualization_id = #{dvId} + diff --git a/core/core-backend/src/main/resources/mybatis/ExtVisualizationLinkJumpMapper.xml b/core/core-backend/src/main/resources/mybatis/ExtVisualizationLinkJumpMapper.xml index 734afff..c4bff20 100644 --- a/core/core-backend/src/main/resources/mybatis/ExtVisualizationLinkJumpMapper.xml +++ b/core/core-backend/src/main/resources/mybatis/ExtVisualizationLinkJumpMapper.xml @@ -35,8 +35,17 @@ + + + + + + @@ -73,67 +82,132 @@ + + + + - + SELECT ccv.id AS source_view_id, + #{uid} as queryUid, + #{isDesktop} as isDesktop, + vlj.id, + #{dvId} as source_dv_id, vlj.link_jump_info, + ifnull(vlj.checked, 0) AS checked + FROM snapshot_core_chart_view ccv + LEFT JOIN snapshot_visualization_link_jump vlj ON ccv.id = vlj.source_view_id + AND vlj.source_dv_id = #{dvId} + WHERE ccv.id = #{viewId} - - DELETE FROM visualization_link_jump_target_view_info + + DELETE FROM snapshot_visualization_link_jump_target_view_info WHERE link_jump_info_id IN ( SELECT lji.id - FROM visualization_link_jump_info lji - JOIN visualization_link_jump lj ON lji.link_jump_id = lj.id + FROM snapshot_visualization_link_jump_info lji + JOIN snapshot_visualization_link_jump lj ON lji.link_jump_id = lj.id WHERE lj.source_dv_id = #{dvId} AND lj.source_view_id = #{viewId} ); - - DELETE FROM visualization_link_jump_info + + DELETE FROM snapshot_visualization_link_jump_info WHERE link_jump_id IN ( SELECT lj.id - FROM visualization_link_jump lj + FROM snapshot_visualization_link_jump lj WHERE lj.source_dv_id = #{dvId} AND lj.source_view_id = #{viewId} ); - - DELETE FROM visualization_link_jump + + DELETE FROM snapshot_visualization_link_jump WHERE source_dv_id = #{dvId} AND source_view_id = #{viewId} @@ -222,7 +296,16 @@ FROM visualization_link_jump_info lji JOIN visualization_link_jump lj ON lji.link_jump_id = lj.id WHERE lj.source_dv_id = #{dvId} - OR lji.target_dv_id = #{dvId} + ) + + + + DELETE FROM snapshot_visualization_link_jump_target_view_info + WHERE link_jump_info_id IN ( + SELECT lji.id + FROM snapshot_visualization_link_jump_info lji + JOIN snapshot_visualization_link_jump lj ON lji.link_jump_id = lj.id + WHERE lj.source_dv_id = #{dvId} ) @@ -232,7 +315,15 @@ SELECT lj.id FROM visualization_link_jump lj WHERE lj.source_dv_id = #{dvId} - OR lj.target_dv_id = #{dvId} + ) + + + + DELETE FROM snapshot_visualization_link_jump_info + WHERE link_jump_id IN ( + SELECT lj.id + FROM snapshot_visualization_link_jump lj + WHERE lj.source_dv_id = #{dvId} ) @@ -241,6 +332,11 @@ WHERE source_dv_id = #{dvId} + + DELETE FROM snapshot_visualization_link_jump + WHERE source_dv_id = #{dvId} + + + + insert into visualization_link_jump (id, source_dv_id, source_view_id, link_jump_info, `checked`, copy_from, copy_id) diff --git a/core/core-backend/src/main/resources/mybatis/ExtVisualizationLinkageMapper.xml b/core/core-backend/src/main/resources/mybatis/ExtVisualizationLinkageMapper.xml index ba180d2..8ef7418 100644 --- a/core/core-backend/src/main/resources/mybatis/ExtVisualizationLinkageMapper.xml +++ b/core/core-backend/src/main/resources/mybatis/ExtVisualizationLinkageMapper.xml @@ -36,26 +36,48 @@ + + SELECT DISTINCT - CONCAT( visualization_linkage.source_view_id, '#', visualization_linkage_field.source_field ) AS sourceInfo, - CONCAT( visualization_linkage.target_view_id, '#', visualization_linkage_field.target_field ) AS targetInfo + CONCAT( vl.source_view_id, '#', vlf.source_field ) AS sourceInfo, + CONCAT( vl.target_view_id, '#', vlf.target_field ) AS targetInfo FROM - visualization_linkage - LEFT JOIN core_chart_view ON visualization_linkage.source_view_id = core_chart_view.id - LEFT JOIN visualization_linkage_field ON visualization_linkage.id = visualization_linkage_field.linkage_id + visualization_linkage vl + LEFT JOIN core_chart_view ccv ON vl.source_view_id = ccv.id + LEFT JOIN visualization_linkage_field vlf ON vl.id = vlf.linkage_id WHERE - visualization_linkage.dv_id = #{dvId} - AND core_chart_view.linkage_active = 1 - AND visualization_linkage.linkage_active = 1 - AND visualization_linkage_field.id IS NOT NULL + vl.dv_id = #{dvId} + AND ccv.linkage_active = 1 + AND vl.linkage_active = 1 + AND vlf.id IS NOT NULL + + + diff --git a/core/core-backend/src/main/resources/mybatis/ExtVisualizationOuterParamsMapper.xml b/core/core-backend/src/main/resources/mybatis/ExtVisualizationOuterParamsMapper.xml index 33cded2..00dca53 100644 --- a/core/core-backend/src/main/resources/mybatis/ExtVisualizationOuterParamsMapper.xml +++ b/core/core-backend/src/main/resources/mybatis/ExtVisualizationOuterParamsMapper.xml @@ -23,11 +23,11 @@ - + select="getOuterParamsInfoSnapshot"> @@ -60,7 +60,7 @@ - SELECT pop.visualization_id, popi.params_info_id, @@ -73,24 +73,24 @@ poptvi.target_ds_id, poptvi.target_field_id FROM - visualization_outer_params pop - LEFT JOIN visualization_outer_params_info popi ON pop.params_id = popi.params_id - LEFT JOIN visualization_outer_params_target_view_info poptvi ON popi.params_info_id = poptvi.params_info_id + snapshot_visualization_outer_params pop + LEFT JOIN snapshot_visualization_outer_params_info popi ON pop.params_id = popi.params_id + LEFT JOIN snapshot_visualization_outer_params_target_view_info poptvi ON popi.params_info_id = poptvi.params_info_id WHERE pop.visualization_id = #{visualization_id} ORDER BY ifnull( popi.checked, 0 ) DESC - SELECT #{visualizationId} as visualization_id, - ifnull( visualization_outer_params.checked, 0 ) AS checked + ifnull( vop.checked, 0 ) AS checked FROM - data_visualization_info - LEFT JOIN visualization_outer_params ON data_visualization_info.id = visualization_outer_params.visualization_id + snapshot_data_visualization_info dvi + LEFT JOIN snapshot_visualization_outer_params vop ON dvi.id = vop.visualization_id WHERE - data_visualization_info.id = #{visualizationId} + dvi.id = #{visualizationId} @@ -132,6 +132,46 @@ pop.visualization_id = #{visualizationId} + + + DELETE FROM + snapshot_visualization_outer_params_target_view_info poptvi + WHERE + poptvi.params_info_id IN ( + SELECT params_info_id FROM + ( + SELECT poptvi.params_info_id FROM + snapshot_visualization_outer_params_target_view_info poptvi + INNER JOIN snapshot_visualization_outer_params_info popi ON poptvi.params_info_id = popi.params_info_id + INNER JOIN snapshot_visualization_outer_params pop ON popi.params_id = pop.params_id + WHERE pop.visualization_id = #{visualizationId} + ) tmp + ) + + + + DELETE FROM + snapshot_visualization_outer_params_info popi + WHERE + popi.params_id IN ( + SELECT params_id FROM + ( + SELECT popi.params_id FROM + snapshot_visualization_outer_params_info popi + INNER JOIN snapshot_visualization_outer_params pop ON popi.params_id = pop.params_id + WHERE pop.visualization_id = #{visualizationId} + ) tmp + ) + + + + DELETE pop + FROM + snapshot_visualization_outer_params pop + WHERE + pop.visualization_id = #{visualizationId} + + - SELECT vopi.param_name, vopi.params_info_id FROM - visualization_outer_params_info vopi - INNER JOIN visualization_outer_params vop ON vop.params_id = vopi.params_id + snapshot_visualization_outer_params_info vopi + INNER JOIN snapshot_visualization_outer_params vop ON vop.params_id = vopi.params_id WHERE vop.visualization_id = #{visualizationId}