diff --git a/backend/libs/sdk-bundle-2.0.jar b/backend/libs/sdk-bundle-2.0.jar index 3384ade..9b67071 100644 Binary files a/backend/libs/sdk-bundle-2.0.jar and b/backend/libs/sdk-bundle-2.0.jar differ diff --git a/backend/pom.xml b/backend/pom.xml index db9dfa8..4e6a760 100644 --- a/backend/pom.xml +++ b/backend/pom.xml @@ -149,6 +149,12 @@ fastjson 1.2.83 + + + com.h2database + h2 + 2.1.214 + diff --git a/backend/src/main/java/com/stdproject/controller/UserController.java b/backend/src/main/java/com/stdproject/controller/UserController.java index 5d03a58..6ebc503 100644 --- a/backend/src/main/java/com/stdproject/controller/UserController.java +++ b/backend/src/main/java/com/stdproject/controller/UserController.java @@ -223,6 +223,7 @@ private Long jwtExpirationMs; long expireTime = System.currentTimeMillis() + jwtExpirationMs; Map map = new HashMap(); map.put("appid", loginUser.getUser().getAppId()); + map.put("app_name", loginUser.getApp_name()); map.put("userid", userId); map.put("username", loginUser.getUsername()); map.put("nickname", loginUser.getUser().getNickname()); diff --git a/backend/src/main/java/com/stdproject/entity/DeTemplateVersion.java b/backend/src/main/java/com/stdproject/entity/DeTemplateVersion.java new file mode 100644 index 0000000..3610800 --- /dev/null +++ b/backend/src/main/java/com/stdproject/entity/DeTemplateVersion.java @@ -0,0 +1,138 @@ +package com.stdproject.entity; + +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; + +import java.io.Serializable; +import java.time.LocalDateTime; + +/** + *

+ * + *

+ * + * @Author bi-coder + * @since 2024-05-07 + */ +@TableName("de_template_version") +public class DeTemplateVersion implements Serializable { + + private static final long serialVersionUID = 1L; + + @TableId("installed_rank") + private Integer installedRank; + + private String version; + + private String description; + + private String type; + + private String script; + + private Integer checksum; + + private String installedBy; + + private LocalDateTime installedOn; + + private Integer executionTime; + + private Boolean success; + + public Integer getInstalledRank() { + return installedRank; + } + + public void setInstalledRank(Integer installedRank) { + this.installedRank = installedRank; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getScript() { + return script; + } + + public void setScript(String script) { + this.script = script; + } + + public Integer getChecksum() { + return checksum; + } + + public void setChecksum(Integer checksum) { + this.checksum = checksum; + } + + public String getInstalledBy() { + return installedBy; + } + + public void setInstalledBy(String installedBy) { + this.installedBy = installedBy; + } + + public LocalDateTime getInstalledOn() { + return installedOn; + } + + public void setInstalledOn(LocalDateTime installedOn) { + this.installedOn = installedOn; + } + + public Integer getExecutionTime() { + return executionTime; + } + + public void setExecutionTime(Integer executionTime) { + this.executionTime = executionTime; + } + + public Boolean getSuccess() { + return success; + } + + public void setSuccess(Boolean success) { + this.success = success; + } + + @Override + public String toString() { + return "DeTemplateVersion{" + + "installedRank = " + installedRank + + ", version = " + version + + ", description = " + description + + ", type = " + type + + ", script = " + script + + ", checksum = " + checksum + + ", installedBy = " + installedBy + + ", installedOn = " + installedOn + + ", executionTime = " + executionTime + + ", success = " + success + + "}"; + } +} diff --git a/backend/src/main/java/com/stdproject/entity/LoginUser.java b/backend/src/main/java/com/stdproject/entity/LoginUser.java index 2d9f73b..cacc456 100644 --- a/backend/src/main/java/com/stdproject/entity/LoginUser.java +++ b/backend/src/main/java/com/stdproject/entity/LoginUser.java @@ -19,11 +19,12 @@ import java.util.Set; public class LoginUser implements UserDetails { private User user; - + private String app_name; private Set permissions; - public LoginUser(User user, Set permissions) { + public LoginUser(User user,String app_name, Set permissions) { this.user = user; + this.app_name=app_name; this.permissions = permissions; } diff --git a/backend/src/main/java/com/stdproject/entity/User.java b/backend/src/main/java/com/stdproject/entity/User.java index 437904d..e0ef00f 100644 --- a/backend/src/main/java/com/stdproject/entity/User.java +++ b/backend/src/main/java/com/stdproject/entity/User.java @@ -29,6 +29,7 @@ public class User { * 应用ID 关联应用系统 */ private String appId; + /** * 所属组织 */ diff --git a/backend/src/main/java/com/stdproject/mapper/ApplicationMapper.java b/backend/src/main/java/com/stdproject/mapper/ApplicationMapper.java index 68ab2b9..d0704f0 100644 --- a/backend/src/main/java/com/stdproject/mapper/ApplicationMapper.java +++ b/backend/src/main/java/com/stdproject/mapper/ApplicationMapper.java @@ -4,6 +4,7 @@ import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.stdproject.entity.Application; import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; import org.apache.ibatis.annotations.Select; /** @@ -18,4 +19,7 @@ import org.apache.ibatis.annotations.Select; public interface ApplicationMapper extends BaseMapper { @Select("SELECT MAX(CAST(code AS UNSIGNED)) FROM app_application") Integer getMaxCode(); + + @Select("SELECT name app_name FROM app_application where id=#{app_id}") + String getAppName(@Param("app_id") String app_id); } diff --git a/backend/src/main/java/com/stdproject/mapper/CoreDatasourceMapper.java b/backend/src/main/java/com/stdproject/mapper/CoreDatasourceMapper.java index 2e72b2e..d3ad1a1 100644 --- a/backend/src/main/java/com/stdproject/mapper/CoreDatasourceMapper.java +++ b/backend/src/main/java/com/stdproject/mapper/CoreDatasourceMapper.java @@ -1,6 +1,7 @@ package com.stdproject.mapper; import com.baomidou.mybatisplus.core.mapper.BaseMapper; + import com.stdproject.entity.CoreDatasource; import org.apache.ibatis.annotations.Mapper; diff --git a/backend/src/main/java/com/stdproject/mapper/CoreDeEngineMapper.java b/backend/src/main/java/com/stdproject/mapper/CoreDeEngineMapper.java index bd5fe0b..4da44e7 100644 --- a/backend/src/main/java/com/stdproject/mapper/CoreDeEngineMapper.java +++ b/backend/src/main/java/com/stdproject/mapper/CoreDeEngineMapper.java @@ -1,7 +1,6 @@ package com.stdproject.mapper; import com.baomidou.mybatisplus.core.mapper.BaseMapper; - import com.stdproject.entity.CoreDeEngine; import org.apache.ibatis.annotations.Mapper; diff --git a/backend/src/main/java/com/stdproject/mapper/DeTemplateVersionMapper.java b/backend/src/main/java/com/stdproject/mapper/DeTemplateVersionMapper.java new file mode 100644 index 0000000..b679728 --- /dev/null +++ b/backend/src/main/java/com/stdproject/mapper/DeTemplateVersionMapper.java @@ -0,0 +1,18 @@ +package com.stdproject.mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.stdproject.entity.DeTemplateVersion; +import org.apache.ibatis.annotations.Mapper; + +/** + *

+ * Mapper 接口 + *

+ * + * @Author bi-coder + * @since 2024-05-07 + */ +@Mapper +public interface DeTemplateVersionMapper extends BaseMapper { + +} diff --git a/backend/src/main/java/com/stdproject/service/CustomUserDetailsService.java b/backend/src/main/java/com/stdproject/service/CustomUserDetailsService.java index ae3d239..2281292 100644 --- a/backend/src/main/java/com/stdproject/service/CustomUserDetailsService.java +++ b/backend/src/main/java/com/stdproject/service/CustomUserDetailsService.java @@ -4,6 +4,7 @@ import com.stdproject.entity.LoginUser; import com.stdproject.entity.Menu; import com.stdproject.entity.Role; import com.stdproject.entity.User; +import com.stdproject.mapper.ApplicationMapper; import com.stdproject.mapper.MenuMapper; import com.stdproject.mapper.RoleMapper; import lombok.extern.slf4j.Slf4j; @@ -32,6 +33,8 @@ public class CustomUserDetailsService implements UserDetailsService { @Autowired private IUserService appUserService; + @Autowired + private ApplicationMapper applicationMapper; @Autowired private IMenuService appMenuService; @@ -48,8 +51,10 @@ public class CustomUserDetailsService implements UserDetailsService { if (appUser == null) { throw new UsernameNotFoundException("用户不存在: " + username); } + String app_name=applicationMapper.getAppName(appUser.getAppId()); Set permissions = buildUserAuthorities(appUser); - LoginUser loginUser = new LoginUser(appUser,permissions); + LoginUser loginUser = new LoginUser(appUser,app_name,permissions); + return loginUser; } diff --git a/backend/src/main/java/com/stdproject/service/impl/DynamicDataServiceImpl.java b/backend/src/main/java/com/stdproject/service/impl/DynamicDataServiceImpl.java index 3258ba5..f266cf3 100644 --- a/backend/src/main/java/com/stdproject/service/impl/DynamicDataServiceImpl.java +++ b/backend/src/main/java/com/stdproject/service/impl/DynamicDataServiceImpl.java @@ -85,7 +85,7 @@ public class DynamicDataServiceImpl implements IDynamicDataService { log.debug("执行插入数据的SQL: {}", sql); // 执行插入操作 - int result= provider.executeUpdate(datasourceRequest); + int result= provider.executeUpdate(datasourceRequest,null).getCount(); if (result==1) { return true; // process result set @@ -231,7 +231,7 @@ public class DynamicDataServiceImpl implements IDynamicDataService { log.debug("执行更新数据的SQL: {}", sql); // 执行更新操作 - int result= provider.executeUpdate(datasourceRequest); + int result= provider.executeUpdate(datasourceRequest,null).getCount(); if (result==1) { return true; // process result set @@ -271,7 +271,7 @@ public class DynamicDataServiceImpl implements IDynamicDataService { log.debug("执行删除数据的SQL: {}", sql); // 执行删除操作 - int result= provider.executeUpdate(datasourceRequest); + int result= provider.executeUpdate(datasourceRequest,null).getCount(); if (result==1) { return true; // process result set diff --git a/backend/src/main/java/com/stdproject/service/manage/EngineManage.java b/backend/src/main/java/com/stdproject/service/manage/EngineManage.java index 829f6ab..56f6900 100644 --- a/backend/src/main/java/com/stdproject/service/manage/EngineManage.java +++ b/backend/src/main/java/com/stdproject/service/manage/EngineManage.java @@ -1,10 +1,13 @@ package com.stdproject.service.manage; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; + import com.stdproject.entity.CoreDatasource; import com.stdproject.entity.CoreDeEngine; +import com.stdproject.entity.DeTemplateVersion; import com.stdproject.mapper.CoreDatasourceMapper; import com.stdproject.mapper.CoreDeEngineMapper; +import com.stdproject.mapper.DeTemplateVersionMapper; import com.stdproject.service.type.H2; import com.stdproject.service.type.Mysql; import com.stdproject.utils.EncryptUtils; @@ -24,6 +27,8 @@ import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; import org.springframework.util.CollectionUtils; +import java.time.LocalDateTime; +import java.time.temporal.ChronoUnit; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -41,9 +46,12 @@ public class EngineManage { @Resource private CoreDatasourceMapper datasourceMapper; - @Value("${gisbi.path.engine:jdbc:h2:/opt/gisbi/desktop_data;AUTO_SERVER=TRUE;AUTO_RECONNECT=TRUE;MODE=MySQL;CASE_INSENSITIVE_IDENTIFIERS=TRUE;DATABASE_TO_UPPER=FALSE}") + @Value("${gisbi.path.engine:jdbc:h2:/opt/gisbi2.0/desktop_data;AUTO_SERVER=TRUE;AUTO_RECONNECT=TRUE;MODE=MySQL;CASE_INSENSITIVE_IDENTIFIERS=TRUE;DATABASE_TO_UPPER=FALSE}") private String engineUrl; + @Resource + private DeTemplateVersionMapper deTemplateVersionMapper; + public CoreDeEngine info() throws DEException { List deEngines = deEngineMapper.selectList(null); @@ -102,31 +110,44 @@ public class EngineManage { public void initSimpleEngine() throws Exception { initLocalDataSource(); QueryWrapper queryWrapper = new QueryWrapper<>(); - queryWrapper.eq("type", engineType.mysql.name()); + if (ModelUtils.isDesktop()) { + queryWrapper.eq("type", engineType.h2.name()); + } else { + queryWrapper.eq("type", engineType.mysql.name()); + } List deEngines = deEngineMapper.selectList(queryWrapper); if (!CollectionUtils.isEmpty(deEngines)) { return; } CoreDeEngine engine = new CoreDeEngine(); - engine.setType(engineType.mysql.name()); - Mysql mysqlConfiguration = new Mysql(); - Pattern WITH_SQL_FRAGMENT = Pattern.compile("jdbc:mysql://(.*):(\\d+)/(.*)"); - Matcher matcher = WITH_SQL_FRAGMENT.matcher(env.getProperty("spring.datasource.url")); - if (!matcher.find()) { - return; + if (ModelUtils.isDesktop()) { + engine.setType(engineType.h2.name()); + H2 h2 = new H2(); + h2.setJdbc(engineUrl); + h2.setDataBase("PUBLIC"); + h2.setUsername(env.getProperty("spring.datasource.username")); + h2.setPassword(env.getProperty("spring.datasource.password")); + engine.setConfiguration(JsonUtil.toJSONString(h2).toString()); + } else { + engine.setType(engineType.mysql.name()); + Mysql mysqlConfiguration = new Mysql(); + Pattern WITH_SQL_FRAGMENT = Pattern.compile("jdbc:mysql://(.*):(\\d+)/(.*)"); + Matcher matcher = WITH_SQL_FRAGMENT.matcher(env.getProperty("spring.datasource.url")); + if (!matcher.find()) { + return; + } + mysqlConfiguration.setHost(matcher.group(1)); + mysqlConfiguration.setPort(Integer.valueOf(matcher.group(2))); + String[] databasePrams = matcher.group(3).split("\\?"); + mysqlConfiguration.setDataBase(databasePrams[0]); + if (databasePrams.length == 2) { + mysqlConfiguration.setExtraParams(databasePrams[1]); + } + mysqlConfiguration.setUsername(env.getProperty("spring.datasource.username")); + mysqlConfiguration.setPassword(env.getProperty("spring.datasource.password")); + engine.setConfiguration(JsonUtil.toJSONString(mysqlConfiguration).toString()); } - mysqlConfiguration.setHost(matcher.group(1)); - mysqlConfiguration.setPort(Integer.valueOf(matcher.group(2))); - String[] databasePrams = matcher.group(3).split("\\?"); - mysqlConfiguration.setDataBase(databasePrams[0]); - if (databasePrams.length == 2) { - mysqlConfiguration.setExtraParams(databasePrams[1]); - } - mysqlConfiguration.setUsername(env.getProperty("spring.datasource.username")); - mysqlConfiguration.setPassword(env.getProperty("spring.datasource.password")); - String config=JsonUtil.toJSONString(mysqlConfiguration).toString(); - engine.setConfiguration((String) EncryptUtils.aesEncrypt(config)); engine.setName("默认引擎"); engine.setDescription("默认引擎"); deEngineMapper.insert(engine); @@ -180,6 +201,13 @@ public class EngineManage { initDatasource.setTaskStatus("WaitingForExecution"); datasourceMapper.deleteById(985188400292302848L); datasourceMapper.insert(initDatasource); + + DeTemplateVersion version = new DeTemplateVersion(); + version.setVersion("985188400292302848"); + version.setScript("Demo"); + version.setInstalledOn(LocalDateTime.now().truncatedTo(ChronoUnit.MINUTES)); + version.setSuccess(true); + deTemplateVersionMapper.insert(version); } } diff --git a/backend/src/main/java/com/stdproject/service/provider/CalciteProvider.java b/backend/src/main/java/com/stdproject/service/provider/CalciteProvider.java index 79ff034..af0eb1c 100644 --- a/backend/src/main/java/com/stdproject/service/provider/CalciteProvider.java +++ b/backend/src/main/java/com/stdproject/service/provider/CalciteProvider.java @@ -32,12 +32,15 @@ import org.apache.calcite.sql.SqlNode; import org.apache.calcite.sql.parser.SqlParser; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.dbcp2.BasicDataSource; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import java.io.File; import java.io.IOException; +import java.io.Reader; +import java.io.StringReader; import java.math.BigDecimal; import java.net.URL; import java.sql.*; @@ -56,9 +59,9 @@ public class CalciteProvider extends Provider { private EngineManage engineManage; protected ExtendedJdbcClassLoader extendedJdbcClassLoader; private Map customJdbcClassLoaders = new HashMap<>(); - @Value("${gisbi.path.driver:/opt/gisbi/drivers}") + @Value("${gisbi.path.driver:/opt/gisbi2.0/drivers}") private String FILE_PATH; - @Value("${gisbi.path.custom-drivers:/opt/gisbi/custom-drivers/}") + @Value("${gisbi.path.custom-drivers:/opt/gisbi2.0/custom-drivers/}") private String CUSTOM_PATH; private static String split = "DE"; @@ -156,7 +159,7 @@ public class CalciteProvider extends Provider { @Override public Map fetchResultField(DatasourceRequest datasourceRequest) throws BusinessException { // 不跨数据源 - if (datasourceRequest.getDsList().size() == 1) { + if (datasourceRequest.getIsCross() == null || !datasourceRequest.getIsCross()) { return jdbcFetchResultField(datasourceRequest); } @@ -241,8 +244,65 @@ public class CalciteProvider extends Provider { return fieldList; } + private Map getTableTypeMap(DatasourceRequest datasourceRequest, DatasourceConfiguration datasourceConfiguration, String tableName) throws BusinessException { + Map map = new HashMap<>(); + String schemaTable = (ObjectUtils.isNotEmpty(datasourceConfiguration.getSchema()) ? (datasourceConfiguration.getSchema() + "`.`") : "") + tableName; + String sql = "SELECT * FROM `$TABLE_NAME$` LIMIT 0 OFFSET 0".replace("$TABLE_NAME$", schemaTable); + sql = transSqlDialect(sql, datasourceRequest.getDsList()); + ResultSet resultSet = null; + try (Connection con = getConnectionFromPool(datasourceRequest.getDatasource().getId()); Statement statement = getStatement(con, 30)) { + resultSet = statement.executeQuery(sql); + + ResultSetMetaData metaData = resultSet.getMetaData(); + int columnCount = metaData.getColumnCount(); + for (int j = 0; j < columnCount; j++) { + String name = StringUtils.lowerCase(metaData.getColumnName(j + 1)); + Integer type = metaData.getColumnType(j + 1); + map.put(name, type); + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + if (resultSet != null) { + try { + resultSet.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + } + return map; + } + @Override public List fetchTableField(DatasourceRequest datasourceRequest) throws BusinessException { + if (datasourceRequest.getIsCross() != null && datasourceRequest.getIsCross()) { + List datasetTableFields = new ArrayList<>(); + PreparedStatement statement = null; + ResultSet resultSet = null; + Connection connection = take(); + try { + CalciteConnection calciteConnection = connection.unwrap(CalciteConnection.class); + statement = calciteConnection.prepareStatement(datasourceRequest.getQuery()); + resultSet = statement.executeQuery(); + ResultSetMetaData metaData = resultSet.getMetaData(); + int columnCount = metaData.getColumnCount(); + for (int i = 1; i <= columnCount; i++) { + TableField tableField = new TableField(); + tableField.setOriginName(metaData.getColumnLabel(i)); + tableField.setType(metaData.getColumnTypeName(i)); + tableField.setPrecision(metaData.getPrecision(i)); + int deType = FieldUtils.transType2DeType(tableField.getType()); + tableField.setDeExtractType(deType); + tableField.setDeType(deType); + tableField.setScale(metaData.getScale(i)); + datasetTableFields.add(tableField); + } + } catch (Exception e) { + BusinessException.throwException(e.getMessage()); + } + return datasetTableFields; + } List datasetTableFields = new ArrayList<>(); DatasourceSchemaDTO datasourceSchemaDTO = datasourceRequest.getDsList().entrySet().iterator().next().getValue(); datasourceRequest.setDatasource(datasourceSchemaDTO); @@ -282,8 +342,11 @@ public class CalciteProvider extends Provider { } else { resultSet = statement.executeQuery(getTableFiledSql(datasourceRequest)); } + + Map tableTypeMap = getTableTypeMap(datasourceRequest, datasourceConfiguration, table); + while (resultSet.next()) { - TableField tableFieldDesc = getTableFieldDesc(datasourceRequest, resultSet, 3); + TableField tableFieldDesc = getTableFieldDesc(datasourceRequest, resultSet, 3, tableTypeMap); boolean repeat = false; for (TableField ele : datasetTableFields) { if (StringUtils.equalsIgnoreCase(ele.getOriginName(), tableFieldDesc.getOriginName())) { @@ -424,16 +487,36 @@ public class CalciteProvider extends Provider { // schema ResultSet resultSet = null; - try (Connection con = getConnectionFromPool(datasourceRequest.getDatasource().getId()); Statement statement = getPreparedStatement(con, datasourceConfiguration.getQueryTimeout(), datasourceRequest.getQuery(), datasourceRequest.getTableFieldWithValues())) { - if (DatasourceConfiguration.DatasourceType.valueOf(value.getType()) == DatasourceConfiguration.DatasourceType.oracle) { - statement.executeUpdate("ALTER SESSION SET CURRENT_SCHEMA = " + datasourceConfiguration.getSchema()); - } + + try (Connection con = getConnectionFromPool(datasourceRequest.getDatasource().getId())) { + + Statement statement = getStatement(value, con, datasourceRequest, datasourceConfiguration, null); if (CollectionUtils.isNotEmpty(datasourceRequest.getTableFieldWithValues())) { LogUtil.info("execWithPreparedStatement sql: " + datasourceRequest.getQuery()); for (int i = 0; i < datasourceRequest.getTableFieldWithValues().size(); i++) { - ((PreparedStatement) statement).setObject(i + 1, datasourceRequest.getTableFieldWithValues().get(i).getValue(), datasourceRequest.getTableFieldWithValues().get(i).getType()); - LogUtil.info("execWithPreparedStatement param[" + (i + 1) + "]: " + datasourceRequest.getTableFieldWithValues().get(i).getValue()); + try { + Object valueObject = datasourceRequest.getTableFieldWithValues().get(i).getValue(); + + if (valueObject instanceof String + && DatasourceConfiguration.DatasourceType.valueOf(value.getType()) == DatasourceConfiguration.DatasourceType.oracle) { + if (StringUtils.isNotEmpty(datasourceConfiguration.getCharset()) && StringUtils.isNotEmpty(datasourceConfiguration.getTargetCharset())) { + //转换为数据库的字符集 + valueObject = new String(((String) valueObject).getBytes(datasourceConfiguration.getTargetCharset()), datasourceConfiguration.getCharset()); + } + if (datasourceRequest.getTableFieldWithValues().get(i).getType().equals(Types.CLOB)) { + Reader reader = new StringReader((String) valueObject); + ((PreparedStatement) statement).setCharacterStream(i + 1, reader, ((String) valueObject).length()); + } else { + ((PreparedStatement) statement).setObject(i + 1, valueObject, datasourceRequest.getTableFieldWithValues().get(i).getType()); + } + } else { + ((PreparedStatement) statement).setObject(i + 1, valueObject, datasourceRequest.getTableFieldWithValues().get(i).getType()); + } + LogUtil.info("execWithPreparedStatement param[" + (i + 1) + "](" + datasourceRequest.getTableFieldWithValues().get(i).getColumnTypeName() + "): " + datasourceRequest.getTableFieldWithValues().get(i).getValue()); + } catch (SQLException e) { + throw new SQLException(e.getMessage() + ". VALUE: " + datasourceRequest.getTableFieldWithValues().get(i).getValue().toString() + " , TARGET TYPE: " + datasourceRequest.getTableFieldWithValues().get(i).getColumnTypeName()); + } } resultSet = ((PreparedStatement) statement).executeQuery(); } else { @@ -467,15 +550,35 @@ public class CalciteProvider extends Provider { DatasourceConfiguration datasourceConfiguration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), DatasourceConfiguration.class); // schema ResultSet resultSet = null; - try (Connection con = getConnectionFromPool(datasourceRequest.getDatasource().getId()); Statement statement = getPreparedStatement(con, datasourceConfiguration.getQueryTimeout(), datasourceRequest.getQuery(), datasourceRequest.getTableFieldWithValues())) { - if (DatasourceConfiguration.DatasourceType.valueOf(value.getType()) == DatasourceConfiguration.DatasourceType.oracle) { - statement.executeUpdate("ALTER SESSION SET CURRENT_SCHEMA = " + datasourceConfiguration.getSchema()); - } + + try (Connection con = getConnectionFromPool(datasourceRequest.getDatasource().getId())) { + + Statement statement = getStatement(value, con, datasourceRequest, datasourceConfiguration, null); + if (CollectionUtils.isNotEmpty(datasourceRequest.getTableFieldWithValues())) { LogUtil.info("execWithPreparedStatement sql: " + datasourceRequest.getQuery()); for (int i = 0; i < datasourceRequest.getTableFieldWithValues().size(); i++) { - ((PreparedStatement) statement).setObject(i + 1, datasourceRequest.getTableFieldWithValues().get(i).getValue(), datasourceRequest.getTableFieldWithValues().get(i).getType()); - LogUtil.info("execWithPreparedStatement param[" + (i + 1) + "]: " + datasourceRequest.getTableFieldWithValues().get(i).getValue()); + try { + Object valueObject = datasourceRequest.getTableFieldWithValues().get(i).getValue(); + if (valueObject instanceof String + && DatasourceConfiguration.DatasourceType.valueOf(value.getType()) == DatasourceConfiguration.DatasourceType.oracle) { + if (StringUtils.isNotEmpty(datasourceConfiguration.getCharset()) && StringUtils.isNotEmpty(datasourceConfiguration.getTargetCharset())) { + //转换为数据库的字符集 + valueObject = new String(((String) valueObject).getBytes(datasourceConfiguration.getTargetCharset()), datasourceConfiguration.getCharset()); + } + if (datasourceRequest.getTableFieldWithValues().get(i).getType().equals(Types.CLOB)) { + Reader reader = new StringReader((String) valueObject); + ((PreparedStatement) statement).setCharacterStream(i + 1, reader, ((String) valueObject).length()); + } else { + ((PreparedStatement) statement).setObject(i + 1, valueObject, datasourceRequest.getTableFieldWithValues().get(i).getType()); + } + } else { + ((PreparedStatement) statement).setObject(i + 1, valueObject, datasourceRequest.getTableFieldWithValues().get(i).getType()); + } + LogUtil.info("execWithPreparedStatement param[" + (i + 1) + "](" + datasourceRequest.getTableFieldWithValues().get(i).getColumnTypeName() + "): " + datasourceRequest.getTableFieldWithValues().get(i).getValue()); + } catch (SQLException e) { + throw new SQLException(e.getMessage() + ". VALUE: " + datasourceRequest.getTableFieldWithValues().get(i).getValue().toString() + " , TARGET TYPE: " + datasourceRequest.getTableFieldWithValues().get(i).getColumnTypeName()); + } } ((PreparedStatement) statement).execute(); } else { @@ -497,29 +600,80 @@ public class CalciteProvider extends Provider { } } + /** + * 针对Oracle特殊处理 + */ + private Statement getStatement(DatasourceSchemaDTO value, Connection con, DatasourceRequest datasourceRequest, DatasourceConfiguration datasourceConfiguration, String autoIncrementPkName) throws Exception { + Statement statement; + if (DatasourceConfiguration.DatasourceType.valueOf(value.getType()) == DatasourceConfiguration.DatasourceType.oracle) { + statement = getStatement(con, datasourceConfiguration.getQueryTimeout()); + statement.executeUpdate("ALTER SESSION SET CURRENT_SCHEMA = " + datasourceConfiguration.getSchema()); + statement.executeUpdate("ALTER SESSION SET NLS_TIMESTAMP_FORMAT = 'YYYY-MM-DD HH24:MI:SS'"); + //调整字符集 + if (StringUtils.isNotEmpty(datasourceConfiguration.getCharset()) && StringUtils.isNotEmpty(datasourceConfiguration.getTargetCharset())) { + datasourceRequest.setQuery(new String(datasourceRequest.getQuery().getBytes(datasourceConfiguration.getTargetCharset()), datasourceConfiguration.getCharset())); + } + } + statement = getPreparedStatement(con, datasourceConfiguration.getQueryTimeout(), datasourceRequest.getQuery(), datasourceRequest.getTableFieldWithValues(), autoIncrementPkName, datasourceConfiguration); + return statement; + } + @Override - public int executeUpdate(DatasourceRequest datasourceRequest) throws BusinessException { + public ExecuteResult executeUpdate(DatasourceRequest datasourceRequest, String autoIncrementPkName) throws BusinessException { DatasourceSchemaDTO value = datasourceRequest.getDsList().entrySet().iterator().next().getValue(); datasourceRequest.setDatasource(value); DatasourceConfiguration datasourceConfiguration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), DatasourceConfiguration.class); // schema ResultSet resultSet = null; - try (Connection con = getConnectionFromPool(datasourceRequest.getDatasource().getId()); Statement statement = getPreparedStatement(con, datasourceConfiguration.getQueryTimeout(), datasourceRequest.getQuery(), datasourceRequest.getTableFieldWithValues())) { - if (DatasourceConfiguration.DatasourceType.valueOf(value.getType()) == DatasourceConfiguration.DatasourceType.oracle) { - statement.executeUpdate("ALTER SESSION SET CURRENT_SCHEMA = " + datasourceConfiguration.getSchema()); - } + try (Connection con = getConnectionFromPool(datasourceRequest.getDatasource().getId())) { + Statement statement = getStatement(value, con, datasourceRequest, datasourceConfiguration, autoIncrementPkName); + + int count = 0; if (CollectionUtils.isNotEmpty(datasourceRequest.getTableFieldWithValues())) { LogUtil.info("execWithPreparedStatement sql: " + datasourceRequest.getQuery()); for (int i = 0; i < datasourceRequest.getTableFieldWithValues().size(); i++) { - ((PreparedStatement) statement).setObject(i + 1, datasourceRequest.getTableFieldWithValues().get(i).getValue(), datasourceRequest.getTableFieldWithValues().get(i).getType()); - LogUtil.info("execWithPreparedStatement param[" + (i + 1) + "]: " + datasourceRequest.getTableFieldWithValues().get(i).getValue()); + try { + Object valueObject = datasourceRequest.getTableFieldWithValues().get(i).getValue(); + + if (valueObject instanceof String + && DatasourceConfiguration.DatasourceType.valueOf(value.getType()) == DatasourceConfiguration.DatasourceType.oracle) { + if (StringUtils.isNotEmpty(datasourceConfiguration.getCharset()) && StringUtils.isNotEmpty(datasourceConfiguration.getTargetCharset())) { + //转换为数据库的字符集 + valueObject = new String(((String) valueObject).getBytes(datasourceConfiguration.getTargetCharset()), datasourceConfiguration.getCharset()); + } + if (datasourceRequest.getTableFieldWithValues().get(i).getType().equals(Types.CLOB)) { + Reader reader = new StringReader((String) valueObject); + ((PreparedStatement) statement).setCharacterStream(i + 1, reader, ((String) valueObject).length()); + } else { + ((PreparedStatement) statement).setObject(i + 1, valueObject, datasourceRequest.getTableFieldWithValues().get(i).getType()); + } + } else { + ((PreparedStatement) statement).setObject(i + 1, valueObject, datasourceRequest.getTableFieldWithValues().get(i).getType()); + } + LogUtil.info("execWithPreparedStatement param[" + (i + 1) + "](" + datasourceRequest.getTableFieldWithValues().get(i).getColumnTypeName() + "): " + datasourceRequest.getTableFieldWithValues().get(i).getValue()); + } catch (SQLException e) { + throw new SQLException(e.getMessage() + ". VALUE: " + datasourceRequest.getTableFieldWithValues().get(i).getValue().toString() + " , TARGET TYPE: " + datasourceRequest.getTableFieldWithValues().get(i).getColumnTypeName()); + } } - return ((PreparedStatement) statement).executeUpdate(); + count = ((PreparedStatement) statement).executeUpdate(); } else { - return statement.executeUpdate(datasourceRequest.getQuery()); + count = statement.executeUpdate(datasourceRequest.getQuery()); } + ExecuteResult result = new ExecuteResult(); + result.setCount(count); + + if (StringUtils.isNotBlank(autoIncrementPkName)) { + List generatedKeys = new ArrayList<>(); + ResultSet keys = statement.getGeneratedKeys(); + while (keys.next()) { + generatedKeys.add(keys.getObject(1).toString()); + } + result.setGeneratedKeys(generatedKeys); + } + + return result; } catch (SQLException e) { BusinessException.throwException("SQL ERROR: " + e.getMessage()); } catch (Exception e) { @@ -534,7 +688,7 @@ public class CalciteProvider extends Provider { } } - return 0; + return new ExecuteResult(); } private List getField(ResultSet rs, DatasourceRequest datasourceRequest) throws Exception { @@ -543,7 +697,7 @@ public class CalciteProvider extends Provider { int columnCount = metaData.getColumnCount(); for (int j = 0; j < columnCount; j++) { String f = metaData.getColumnName(j + 1); - if (StringUtils.equalsIgnoreCase(f, "DE_ROWNUM")) { + if (StringUtils.containsIgnoreCase(f, "ROWNUM")) { continue; } String l = StringUtils.isNotEmpty(metaData.getColumnLabel(j + 1)) ? metaData.getColumnLabel(j + 1) : f; @@ -559,15 +713,15 @@ public class CalciteProvider extends Provider { } private List getData(ResultSet rs, DatasourceRequest datasourceRequest) throws Exception { - String charset = null; - String targetCharset = "UTF-8"; + String targetCharset = null; + String originCharset = null; if (datasourceRequest != null && datasourceRequest.getDatasource().getType().equalsIgnoreCase("oracle")) { DatasourceConfiguration jdbcConfiguration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), DatasourceConfiguration.class); - if (StringUtils.isNotEmpty(jdbcConfiguration.getCharset()) && !jdbcConfiguration.getCharset().equalsIgnoreCase("Default")) { - charset = jdbcConfiguration.getCharset(); + if (StringUtils.isNotEmpty(jdbcConfiguration.getCharset())) { + originCharset = jdbcConfiguration.getCharset(); } - if (StringUtils.isNotEmpty(jdbcConfiguration.getTargetCharset()) && !jdbcConfiguration.getTargetCharset().equalsIgnoreCase("Default")) { + if (StringUtils.isNotEmpty(jdbcConfiguration.getTargetCharset())) { targetCharset = jdbcConfiguration.getTargetCharset(); } } @@ -584,6 +738,11 @@ public class CalciteProvider extends Provider { row[j] = rs.getDate(j + 1).toString(); } break; + case Types.TIMESTAMP: + if (rs.getTimestamp(j + 1) != null) { + row[j] = rs.getTimestamp(j + 1).toString(); + } + break; case Types.BOOLEAN: row[j] = rs.getBoolean(j + 1) ? "1" : "0"; break; @@ -594,13 +753,17 @@ public class CalciteProvider extends Provider { default: if (metaData.getColumnTypeName(j + 1).toLowerCase().equalsIgnoreCase("blob")) { row[j] = rs.getBlob(j + 1) == null ? "" : rs.getBlob(j + 1).toString(); - } else { - if (charset != null && StringUtils.isNotEmpty(rs.getString(j + 1))) { - String originStr = new String(rs.getString(j + 1).getBytes(charset), targetCharset); - row[j] = new String(originStr.getBytes("UTF-8"), "UTF-8"); + } + if (targetCharset != null && StringUtils.isNotEmpty(rs.getString(j + 1)) && columnType == Types.CLOB) { + if (originCharset == null) { + row[j] = new String(rs.getString(j + 1).getBytes(), targetCharset); } else { - row[j] = rs.getString(j + 1); + row[j] = new String(rs.getString(j + 1).getBytes(originCharset), targetCharset); } + } else if (targetCharset != null && StringUtils.isNotEmpty(rs.getString(j + 1)) && (columnType != Types.NVARCHAR && columnType != Types.NCHAR)) { + row[j] = new String(rs.getBytes(j + 1), targetCharset); + } else { + row[j] = rs.getString(j + 1); } break; @@ -706,7 +869,7 @@ public class CalciteProvider extends Provider { } } - private TableField getTableFieldDesc(DatasourceRequest datasourceRequest, ResultSet resultSet, int commentIndex) throws SQLException { + private TableField getTableFieldDesc(DatasourceRequest datasourceRequest, ResultSet resultSet, int commentIndex, Map tableTypeMap) throws SQLException { TableField tableField = new TableField(); tableField.setOriginName(resultSet.getString(1)); tableField.setType(resultSet.getString(2).toUpperCase()); @@ -719,6 +882,20 @@ public class CalciteProvider extends Provider { tableField.setPrimary(resultSet.getInt(4) > 0); } catch (Exception e) { } + try { + if (StringUtils.endsWithIgnoreCase(datasourceRequest.getDatasource().getType(), "oracle")) { + if (StringUtils.contains(resultSet.getString(5), "nextval") || StringUtils.equalsIgnoreCase(resultSet.getString(5), "GENERATED ALWAYS AS IDENTITY")) { + tableField.setAutoIncrement(true); + } + } else { + tableField.setAutoIncrement(resultSet.getInt(5) > 0); + } + } catch (Exception e) { + } + try { + tableField.setTypeNumber(tableTypeMap.get(StringUtils.lowerCase(tableField.getOriginName()))); + } catch (Exception e) { + } return tableField; } @@ -735,7 +912,6 @@ public class CalciteProvider extends Provider { private void registerDriver() { for (String driverClass : getDriver()) { try { - if(driverClass.equals("org.h2.Driver")) continue;//忽略h2 Driver driver = (Driver) extendedJdbcClassLoader.loadClass(driverClass).newInstance(); DriverManager.registerDriver(new DriverShim(driver)); } catch (Exception e) { @@ -752,6 +928,7 @@ public class CalciteProvider extends Provider { info.setProperty(CalciteConnectionProperty.CASE_SENSITIVE.camelName(), "false"); info.setProperty(CalciteConnectionProperty.PARSER_FACTORY.camelName(), "org.apache.calcite.sql.parser.impl.SqlParserImpl#FACTORY"); info.setProperty(CalciteConnectionProperty.DEFAULT_NULL_COLLATION.camelName(), NullCollation.LAST.name()); + info.setProperty("remarks", "true"); Connection connection = null; try { Class.forName("org.apache.calcite.jdbc.Driver"); @@ -771,6 +948,13 @@ public class CalciteProvider extends Provider { commonThreadPool.addTask(() -> { try { BasicDataSource dataSource = new BasicDataSource(); + dataSource.setMaxWaitMillis(5 * 1000); + dataSource.setTestWhileIdle(true); + dataSource.setTestOnBorrow(true); + dataSource.setTestOnReturn(true); + dataSource.setTimeBetweenEvictionRunsMillis(60 * 1000); + dataSource.setValidationQuery("select 1"); + dataSource.setValidationQueryTimeout(5); Schema schema = null; DatasourceConfiguration configuration = null; DatasourceConfiguration.DatasourceType datasourceType = DatasourceConfiguration.DatasourceType.valueOf(ds.getType()); @@ -839,6 +1023,7 @@ public class CalciteProvider extends Provider { rootSchema.add(ds.getSchemaAlias(), schema); break; case oracle: + dataSource.setValidationQuery("SELECT 1 FROM DUAL"); configuration = JsonUtil.parseObject(ds.getConfiguration(), Oracle.class); if (StringUtils.isNotBlank(configuration.getUsername())) { dataSource.setUsername(configuration.getUsername()); @@ -857,6 +1042,7 @@ public class CalciteProvider extends Provider { break; case db2: configuration = JsonUtil.parseObject(ds.getConfiguration(), Db2.class); + dataSource.setValidationQuery("select 1 from syscat.tables WHERE TABSCHEMA ='DE_SCHEMA' AND \"TYPE\" = 'T'".replace("DE_SCHEMA", configuration.getSchema())); if (StringUtils.isNotBlank(configuration.getUsername())) { dataSource.setUsername(configuration.getUsername()); } @@ -1023,7 +1209,7 @@ public class CalciteProvider extends Provider { if (database.contains(".")) { sql = "select * from " + datasourceRequest.getTable() + " limit 0 offset 0 "; } else { - sql = String.format("SELECT COLUMN_NAME,DATA_TYPE,COLUMN_COMMENT FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s'", database, datasourceRequest.getTable()); + sql = String.format("SELECT COLUMN_NAME,DATA_TYPE,COLUMN_COMMENT,IF(COLUMN_KEY='PRI',1,0),IF(EXTRA LIKE '%%auto_increment%%',1,0) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s'", database, datasourceRequest.getTable()); } break; case mysql: @@ -1040,21 +1226,49 @@ public class CalciteProvider extends Provider { String[] databasePrams = matcher.group(3).split("\\?"); database = databasePrams[0]; } - sql = String.format("SELECT COLUMN_NAME,DATA_TYPE,COLUMN_COMMENT,IF(COLUMN_KEY='PRI',1,0) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s'", database, datasourceRequest.getTable()); + sql = String.format("SELECT COLUMN_NAME,DATA_TYPE,COLUMN_COMMENT,IF(COLUMN_KEY='PRI',1,0),IF(EXTRA LIKE '%%auto_increment%%',1,0) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s'", database, datasourceRequest.getTable()); break; case oracle: configuration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), Oracle.class); if (StringUtils.isEmpty(configuration.getSchema())) { BusinessException.throwException(Translator.get("i18n_schema_is_empty")); } - sql = String.format("SELECT a.COLUMN_NAME , a.DATA_TYPE , b.COMMENTS ,0 FROM all_tab_columns a LEFT JOIN all_col_comments b ON a.owner = b.owner AND a.table_name = b.table_name AND a.column_name = b.column_name WHERE a.owner = '%s' AND a.table_name = '%s' ORDER BY a.table_name, a.column_id", configuration.getSchema(), datasourceRequest.getTable()); + sql = String.format(""" + SELECT tc.COLUMN_NAME AS ColumnName, + tc.DATA_TYPE, + cc.COMMENTS, + CASE + WHEN ac.COLUMN_NAME IS NOT NULL THEN 1 + ELSE 0 + END, + tc.DATA_DEFAULT + FROM ALL_TAB_COLUMNS tc + LEFT JOIN (SELECT cols.OWNER, + cols.TABLE_NAME, + cols.COLUMN_NAME + FROM ALL_CONSTRAINTS cons + JOIN + ALL_CONS_COLUMNS cols + ON cons.OWNER = cols.OWNER + AND cons.CONSTRAINT_NAME = cols.CONSTRAINT_NAME + WHERE cons.TABLE_NAME = '%s' + AND cons.CONSTRAINT_TYPE = 'P') ac + ON tc.OWNER = ac.OWNER + AND tc.TABLE_NAME = ac.TABLE_NAME + AND tc.COLUMN_NAME = ac.COLUMN_NAME + LEFT JOIN ALL_COL_COMMENTS cc + ON tc.owner = cc.owner AND tc.table_name = cc.table_name AND tc.column_name = cc.column_name + WHERE tc.TABLE_NAME = '%s' + AND tc.OWNER = '%s' + ORDER BY tc.TABLE_NAME, tc.COLUMN_ID + """, datasourceRequest.getTable(), datasourceRequest.getTable(), configuration.getSchema()); break; case db2: configuration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), Db2.class); if (StringUtils.isEmpty(configuration.getSchema())) { BusinessException.throwException(Translator.get("i18n_schema_is_empty")); } - sql = String.format("SELECT COLNAME , TYPENAME , REMARKS FROM SYSCAT.COLUMNS WHERE TABSCHEMA = '%s' AND TABNAME = '%s' ", configuration.getSchema(), datasourceRequest.getTable()); + sql = String.format("SELECT COLNAME, TYPENAME, REMARKS, 0, 0 FROM SYSCAT.COLUMNS WHERE TABSCHEMA = '%s' AND TABNAME = '%s' ", configuration.getSchema(), datasourceRequest.getTable()); break; case sqlServer: configuration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), Sqlserver.class); @@ -1062,38 +1276,46 @@ public class CalciteProvider extends Provider { BusinessException.throwException(Translator.get("i18n_schema_is_empty")); } - sql = String.format("SELECT \n" + " c.name ,t.name ,ep.value, 0 \n" + "FROM \n" + " sys.columns AS c\n" + "LEFT JOIN sys.extended_properties AS ep ON c.object_id = ep.major_id AND c.column_id = ep.minor_id\n" + "LEFT JOIN sys.types AS t ON c.user_type_id = t.user_type_id\n" + "LEFT JOIN sys.objects AS o ON c.object_id = o.object_id\n" + "WHERE o.name = '%s'", datasourceRequest.getTable()); + sql = String.format("SELECT \n" + " c.name ,t.name ,ep.value, 0, 0 \n" + "FROM \n" + " sys.columns AS c\n" + "LEFT JOIN sys.extended_properties AS ep ON c.object_id = ep.major_id AND c.column_id = ep.minor_id\n" + "LEFT JOIN sys.types AS t ON c.user_type_id = t.user_type_id\n" + "LEFT JOIN sys.objects AS o ON c.object_id = o.object_id\n" + "WHERE o.name = '%s'", datasourceRequest.getTable()); break; case pg: configuration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), Pg.class); if (StringUtils.isEmpty(configuration.getSchema())) { BusinessException.throwException(Translator.get("i18n_schema_is_empty")); } - sql = String.format("SELECT\n" + - " a.attname AS ColumnName,\n" + - " t.typname,\n" + - " b.description AS ColumnDescription,\n" + - " CASE\n" + - " WHEN d.indisprimary THEN 1\n" + - " ELSE 0\n" + - " END\n" + - "FROM\n" + - " pg_class c\n" + - " JOIN pg_attribute a ON a.attrelid = c.oid\n" + - " LEFT JOIN pg_description b ON a.attrelid = b.objoid AND a.attnum = b.objsubid\n" + - " JOIN pg_type t ON a.atttypid = t.oid\n" + - " LEFT JOIN pg_index d ON d.indrelid = a.attrelid AND d.indisprimary AND a.attnum = ANY(d.indkey)\n" + - "where\n" + - " c.relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = '%s')\n" + - " AND c.relname = '%s'\n" + - " AND a.attnum > 0\n" + - " AND NOT a.attisdropped\n" + - "ORDER BY\n" + - " a.attnum;", configuration.getSchema(), datasourceRequest.getTable()); + sql = String.format(""" + SELECT a.attname AS ColumnName, + t.typname, + b.description AS ColumnDescription, + CASE + WHEN d.indisprimary THEN 1 + ELSE 0 + END, + CASE + WHEN pg_get_expr(ad.adbin, ad.adrelid) LIKE 'nextval%%' THEN 1 + """ + ( + datasourceRequest.getDsVersion() > 9 ? """ + WHEN a.attidentity = 'd' THEN 1 + WHEN a.attidentity = 'a' THEN 1 + """ : "") + """ + ELSE 0 + END + FROM pg_class c + JOIN pg_attribute a ON a.attrelid = c.oid + LEFT JOIN pg_attrdef ad ON a.attrelid = ad.adrelid AND a.attnum = ad.adnum + LEFT JOIN pg_description b ON a.attrelid = b.objoid AND a.attnum = b.objsubid + JOIN pg_type t ON a.atttypid = t.oid + LEFT JOIN pg_index d ON d.indrelid = a.attrelid AND d.indisprimary AND a.attnum = ANY (d.indkey) + where c.relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = '%s') + AND c.relname = '%s' + AND a.attnum > 0 + AND NOT a.attisdropped + ORDER BY a.attnum; + """, configuration.getSchema(), datasourceRequest.getTable()); break; case redshift: configuration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), CK.class); - sql = String.format("SELECT\n" + " a.attname AS ColumnName,\n" + " t.typname,\n" + " b.description AS ColumnDescription,\n" + " 0\n" + "FROM\n" + " pg_class c\n" + " JOIN pg_attribute a ON a.attrelid = c.oid\n" + " LEFT JOIN pg_description b ON a.attrelid = b.objoid AND a.attnum = b.objsubid\n" + " JOIN pg_type t ON a.atttypid = t.oid\n" + "WHERE\n" + " c.relname = '%s'\n" + " AND a.attnum > 0\n" + " AND NOT a.attisdropped\n" + "ORDER BY\n" + " a.attnum\n" + " ", datasourceRequest.getTable()); + sql = String.format("SELECT\n" + " a.attname AS ColumnName,\n" + " t.typname,\n" + " b.description AS ColumnDescription,\n" + " 0, 0\n" + "FROM\n" + " pg_class c\n" + " JOIN pg_attribute a ON a.attrelid = c.oid\n" + " LEFT JOIN pg_description b ON a.attrelid = b.objoid AND a.attnum = b.objsubid\n" + " JOIN pg_type t ON a.atttypid = t.oid\n" + "WHERE\n" + " c.relname = '%s'\n" + " AND a.attnum > 0\n" + " AND NOT a.attisdropped\n" + "ORDER BY\n" + " a.attnum\n" + " ", datasourceRequest.getTable()); break; case ck: configuration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), CK.class); @@ -1107,13 +1329,13 @@ public class CalciteProvider extends Provider { String[] databasePrams = matcher.group(3).split("\\?"); database = databasePrams[0]; } - sql = String.format(" SELECT\n" + " name,\n" + " type,\n" + " comment,\n" + " 0\n" + "FROM\n" + " system.columns\n" + "WHERE\n" + " database = '%s' \n" + " AND table = '%s' ", database, datasourceRequest.getTable()); + sql = String.format(" SELECT\n" + " name,\n" + " type,\n" + " comment,\n" + " 0, 0\n" + "FROM\n" + " system.columns\n" + "WHERE\n" + " database = '%s' \n" + " AND table = '%s' ", database, datasourceRequest.getTable()); break; case impala: sql = String.format("DESCRIBE `%s`", datasourceRequest.getTable()); break; case h2: - sql = String.format("SELECT COLUMN_NAME, DATA_TYPE, REMARKS FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = '%s'", datasourceRequest.getTable()); + sql = String.format("SELECT COLUMN_NAME, DATA_TYPE, REMARKS, 0, 0 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = '%s'", datasourceRequest.getTable()); break; default: break; @@ -1171,6 +1393,18 @@ public class CalciteProvider extends Provider { } tableSqls.add("select table_name, comments, owner from all_tab_comments where owner='" + configuration.getSchema() + "' AND table_type = 'TABLE'"); tableSqls.add("select table_name, comments, owner from all_tab_comments where owner='" + configuration.getSchema() + "' AND table_type = 'VIEW'"); + tableSqls.add("SELECT \n" + + " m.mview_name,\n" + + " c.comments\n" + + "FROM \n" + + " ALL_MVIEWS m\n" + + "LEFT JOIN \n" + + " ALL_TAB_COMMENTS c \n" + + "ON \n" + + " m.owner = c.owner \n" + + " AND m.mview_name = c.table_name\n" + + " AND c.table_type = 'MATERIALIZED VIEW'\n" + + "WHERE m.OWNER ='DE_SCHEMA'".replace("DE_SCHEMA", configuration.getSchema())); break; case db2: configuration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), Db2.class); @@ -1193,6 +1427,29 @@ public class CalciteProvider extends Provider { BusinessException.throwException(Translator.get("i18n_schema_is_empty")); } tableSqls.add("SELECT \n" + " relname AS TableName, \n" + " obj_description(relfilenode::regclass, 'pg_class') AS TableDescription \n" + "FROM \n" + " pg_class \n" + "WHERE \n" + " relkind in ('r','p', 'f') \n" + " AND relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = 'SCHEMA') ".replace("SCHEMA", configuration.getSchema())); + tableSqls.add("SELECT \n" + + " c.relname AS view_name,\n" + + " COALESCE(d.description, 'No description provided') AS view_description\n" + + "FROM \n" + + " pg_class c\n" + + "JOIN \n" + + " pg_namespace n ON c.relnamespace = n.oid\n" + + "LEFT JOIN \n" + + " pg_description d ON c.oid = d.objoid\n" + + "WHERE \n" + + " c.relkind = 'v' \n" + + " AND n.nspname = 'SCHEMA'".replace("SCHEMA", configuration.getSchema())); + tableSqls.add("SELECT \n" + + " c.relname AS materialized_view_name,\n" + + " COALESCE(d.description, '') AS view_description\n" + + "FROM \n" + + " pg_class c\n" + + "JOIN \n" + + " pg_namespace n ON c.relnamespace = n.oid\n" + + "LEFT JOIN \n" + + " pg_description d ON c.oid = d.objoid\n" + + "WHERE \n" + + " c.relkind = 'm' and n.nspname ='SCHEMA'; ".replace("SCHEMA", configuration.getSchema())); break; case redshift: configuration = JsonUtil.parseObject(datasourceRequest.getDatasource().getConfiguration(), CK.class); @@ -1258,13 +1515,23 @@ public class CalciteProvider extends Provider { } public Statement getPreparedStatement(Connection connection, int queryTimeout, String sql, List values) throws Exception { + return getPreparedStatement(connection, queryTimeout, sql, values, null, null); + } + + public Statement getPreparedStatement(Connection connection, int queryTimeout, String sql, List values, String autoIncrementPkName, DatasourceConfiguration datasourceConfiguration) throws Exception { if (connection == null) { throw new Exception("Failed to get connection!"); } if (CollectionUtils.isNotEmpty(values)) { PreparedStatement stat = null; + String pkName = autoIncrementPkName; try { - stat = connection.prepareStatement(sql); + if (StringUtils.isNotBlank(autoIncrementPkName)) { + String[] generatedColumns = {pkName}; + stat = connection.prepareStatement(sql, generatedColumns); + } else { + stat = connection.prepareStatement(sql); + } stat.setQueryTimeout(queryTimeout); } catch (Exception e) { BusinessException.throwException(e.getMessage()); @@ -1333,7 +1600,7 @@ public class CalciteProvider extends Provider { public void initConnectionPool() { LogUtil.info("Begin to init datasource pool..."); QueryWrapper datasourceQueryWrapper = new QueryWrapper(); - List coreDatasources = coreDatasourceMapper.selectList(datasourceQueryWrapper).stream().filter(coreDatasource -> !Arrays.asList("folder", "API", "Excel").contains(coreDatasource.getType())).collect(Collectors.toList()); + List coreDatasources = coreDatasourceMapper.selectList(datasourceQueryWrapper).stream().filter(coreDatasource -> !Arrays.asList("folder", "API", "Excel", "ExcelRemote").contains(coreDatasource.getType())).collect(Collectors.toList()); CoreDatasource engine = engineManage.deEngine(); if (engine != null) { coreDatasources.add(engine); @@ -1384,8 +1651,8 @@ public class CalciteProvider extends Provider { buildSchema(datasourceRequest, calciteConnection); } DatasourceConfiguration configuration = JsonUtil.parseObject(datasourceDTO.getConfiguration(), DatasourceConfiguration.class); - if(configuration.isUseSSH()){ - Session session =Provider.getSessions().get(datasourceDTO.getId()); + if (configuration.isUseSSH()) { + Session session = Provider.getSessions().get(datasourceDTO.getId()); session.disconnect(); Provider.getSessions().remove(datasourceDTO.getId()); startSshSession(configuration, null, datasourceDTO.getId()); @@ -1439,6 +1706,7 @@ public class CalciteProvider extends Provider { } JdbcSchema jdbcSchema = rootSchema.getSubSchema(String.format(SQLConstants.SCHEMA, dsId)).unwrap(JdbcSchema.class); BasicDataSource basicDataSource = (BasicDataSource) jdbcSchema.getDataSource(); + basicDataSource.setMaxWaitMillis(5 * 1000); return basicDataSource.getConnection(); } catch (Exception e) { BusinessException.throwException(Translator.get("i18n_invalid_connection") + e.getMessage()); diff --git a/backend/src/main/java/com/stdproject/service/type/H2.java b/backend/src/main/java/com/stdproject/service/type/H2.java index 18901e7..fb74142 100644 --- a/backend/src/main/java/com/stdproject/service/type/H2.java +++ b/backend/src/main/java/com/stdproject/service/type/H2.java @@ -1,11 +1,29 @@ package com.stdproject.service.type; +import io.gisbi.exception.DEException; import io.gisbi.extensions.datasource.vo.DatasourceConfiguration; import lombok.Data; +import lombok.EqualsAndHashCode; +import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; +import java.util.Arrays; +import java.util.List; + +@EqualsAndHashCode(callSuper = true) @Data @Component("h2") public class H2 extends DatasourceConfiguration { private String driver = "org.h2.Driver"; + private List illegalParameters = Arrays.asList("INIT", "RUNSCRIPT"); + + public String getJdbc() { + for (String illegalParameter : illegalParameters) { + if (jdbc.toUpperCase().replace("\\", "").contains(illegalParameter)) { + DEException.throwException("Has illegal parameter: " + jdbc); + } + } + + return jdbc; + } } diff --git a/backend/src/main/java/com/stdproject/service/type/Mysql.java b/backend/src/main/java/com/stdproject/service/type/Mysql.java index d645038..872edd6 100644 --- a/backend/src/main/java/com/stdproject/service/type/Mysql.java +++ b/backend/src/main/java/com/stdproject/service/type/Mysql.java @@ -27,22 +27,24 @@ public class Mysql extends DatasourceConfiguration { } return getJdbcUrl(); } + String jdbcUrl = ""; if (StringUtils.isEmpty(extraParams.trim())) { - return "jdbc:mysql://HOSTNAME:PORT/DATABASE" + jdbcUrl = "jdbc:mysql://HOSTNAME:PORT/DATABASE" .replace("HOSTNAME", getLHost().trim()) .replace("PORT", getLPort().toString().trim()) .replace("DATABASE", getDataBase().trim()); } else { - for (String illegalParameter : illegalParameters) { - if (URLDecoder.decode(getExtraParams()).toLowerCase().contains(illegalParameter.toLowerCase()) || URLDecoder.decode(getExtraParams()).contains(illegalParameter.toLowerCase())) { - DEException.throwException("Illegal parameter: " + illegalParameter); - } - } - return "jdbc:mysql://HOSTNAME:PORT/DATABASE?EXTRA_PARAMS" + jdbcUrl = "jdbc:mysql://HOSTNAME:PORT/DATABASE?EXTRA_PARAMS" .replace("HOSTNAME", getLHost().trim()) .replace("PORT", getLPort().toString().trim()) .replace("DATABASE", getDataBase().trim()) .replace("EXTRA_PARAMS", getExtraParams().trim()); } + for (String illegalParameter : illegalParameters) { + if (URLDecoder.decode(jdbcUrl).toLowerCase().contains(illegalParameter.toLowerCase()) || URLDecoder.decode(jdbcUrl).contains(illegalParameter.toLowerCase())) { + DEException.throwException("Illegal parameter: " + illegalParameter); + } + } + return jdbcUrl; } } diff --git a/backend/src/main/java/com/stdproject/service/type/Pg.java b/backend/src/main/java/com/stdproject/service/type/Pg.java index 9405869..f46e935 100644 --- a/backend/src/main/java/com/stdproject/service/type/Pg.java +++ b/backend/src/main/java/com/stdproject/service/type/Pg.java @@ -1,40 +1,58 @@ package com.stdproject.service.type; +import io.gisbi.exception.DEException; import io.gisbi.extensions.datasource.vo.DatasourceConfiguration; import lombok.Data; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; +import java.net.URLDecoder; +import java.util.Arrays; +import java.util.List; + @Data @Component("pg") public class Pg extends DatasourceConfiguration { private String driver = "org.postgresql.Driver"; private String extraParams = ""; + private List illegalParameters = Arrays.asList("socketFactory", "socketFactoryArg", "sslfactory", "sslhostnameverifier", "sslpasswordcallback", "authenticationPluginClassName"); public String getJdbc() { - if(StringUtils.isNoneEmpty(getUrlType()) && !getUrlType().equalsIgnoreCase("hostName")){ + if (StringUtils.isNoneEmpty(getUrlType()) && !getUrlType().equalsIgnoreCase("hostName")) { + for (String illegalParameter : illegalParameters) { + if (URLDecoder.decode(getJdbcUrl()).contains(illegalParameter)) { + DEException.throwException("Illegal parameter: " + illegalParameter); + } + } return getJdbcUrl(); } - if(StringUtils.isEmpty(extraParams.trim())){ + String jdbcUrl = ""; + if (StringUtils.isEmpty(extraParams.trim())) { if (StringUtils.isEmpty(getSchema())) { - return "jdbc:postgresql://HOSTNAME:PORT/DATABASE" + jdbcUrl = "jdbc:postgresql://HOSTNAME:PORT/DATABASE" .replace("HOSTNAME", getLHost().trim()) .replace("PORT", getLPort().toString().trim()) .replace("DATABASE", getDataBase().trim()); } else { - return "jdbc:postgresql://HOSTNAME:PORT/DATABASE?currentSchema=SCHEMA" + jdbcUrl = "jdbc:postgresql://HOSTNAME:PORT/DATABASE?currentSchema=SCHEMA" .replace("HOSTNAME", getLHost().trim()) .replace("PORT", getLPort().toString().trim()) .replace("DATABASE", getDataBase().trim()) .replace("SCHEMA", getSchema().trim()); } - }else { - return "jdbc:postgresql://HOSTNAME:PORT/DATABASE?EXTRA_PARAMS" + } else { + jdbcUrl = "jdbc:postgresql://HOSTNAME:PORT/DATABASE?EXTRA_PARAMS" .replace("HOSTNAME", getLHost().trim()) .replace("PORT", getLPort().toString().trim()) .replace("DATABASE", getDataBase().trim()) .replace("EXTRA_PARAMS", getExtraParams().trim()); } + for (String illegalParameter : illegalParameters) { + if (URLDecoder.decode(jdbcUrl).toLowerCase().contains(illegalParameter.toLowerCase()) || URLDecoder.decode(jdbcUrl).contains(illegalParameter.toLowerCase())) { + DEException.throwException("Illegal parameter: " + illegalParameter); + } + } + return jdbcUrl; } } diff --git a/backend/src/main/java/com/stdproject/service/type/Redshift.java b/backend/src/main/java/com/stdproject/service/type/Redshift.java index 81f22d2..d3ff64f 100644 --- a/backend/src/main/java/com/stdproject/service/type/Redshift.java +++ b/backend/src/main/java/com/stdproject/service/type/Redshift.java @@ -1,23 +1,40 @@ package com.stdproject.service.type; +import io.gisbi.exception.DEException; import io.gisbi.extensions.datasource.vo.DatasourceConfiguration; import lombok.Data; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Component; +import java.net.URLDecoder; +import java.util.Arrays; +import java.util.List; + @Data @Component("redshift") public class Redshift extends DatasourceConfiguration { private String driver = "com.amazon.redshift.jdbc42.Driver"; private String extraParams = ""; + private List illegalParameters = Arrays.asList("socketFactory", "socketFactoryArg", "sslfactory", "sslhostnameverifier", "sslpasswordcallback", "authenticationPluginClassName"); public String getJdbc() { - if(StringUtils.isNoneEmpty(getUrlType()) && !getUrlType().equalsIgnoreCase("hostName")){ + if (StringUtils.isNoneEmpty(getUrlType()) && !getUrlType().equalsIgnoreCase("hostName")) { + for (String illegalParameter : illegalParameters) { + if (URLDecoder.decode(getJdbcUrl()).contains(illegalParameter)) { + DEException.throwException("Illegal parameter: " + illegalParameter); + } + } return getJdbcUrl(); } - return "jdbc:redshift://HOSTNAME:PORT/DATABASE" + String jdbcUrl = "jdbc:redshift://HOSTNAME:PORT/DATABASE" .replace("HOSTNAME", getLHost().trim()) .replace("PORT", getLPort().toString().trim()) .replace("DATABASE", getDataBase().trim()); + for (String illegalParameter : illegalParameters) { + if (URLDecoder.decode(jdbcUrl).contains(illegalParameter)) { + DEException.throwException("Illegal parameter: " + illegalParameter); + } + } + return jdbcUrl; } }