package io.gisbi.dataset.manage;

import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import io.gisbi.api.chart.dto.DeSortField;
import io.gisbi.api.dataset.dto.*;
import io.gisbi.api.dataset.union.DatasetGroupInfoDTO;
import io.gisbi.api.dataset.union.DatasetTableInfoDTO;
import io.gisbi.api.permissions.auth.dto.BusiPerCheckDTO;
import io.gisbi.api.permissions.dataset.api.RowPermissionsApi;
import io.gisbi.api.permissions.user.vo.UserFormVO;
import io.gisbi.chart.utils.ChartDataBuild;
import io.gisbi.commons.utils.SqlparserUtils;
import io.gisbi.constant.AuthEnum;
import io.gisbi.constant.SQLConstants;
import io.gisbi.dataset.constant.DatasetTableType;
import io.gisbi.dataset.dao.auto.mapper.CoreDatasetTableFieldMapper;
import io.gisbi.dataset.dao.auto.mapper.CoreDatasetTableMapper;
import io.gisbi.dataset.utils.DatasetUtils;
import io.gisbi.dataset.utils.FieldUtils;
import io.gisbi.dataset.utils.SqlUtils;
import io.gisbi.dataset.utils.TableUtils;
import io.gisbi.datasource.dao.auto.entity.CoreDatasource;
import io.gisbi.datasource.dao.auto.mapper.CoreDatasourceMapper;
import io.gisbi.datasource.manage.DataSourceManage;
import io.gisbi.datasource.manage.EngineManage;
import io.gisbi.datasource.utils.DatasourceUtils;
import io.gisbi.engine.constant.ExtFieldConstant;
import io.gisbi.engine.sql.SQLProvider;
import io.gisbi.engine.trans.*;
import io.gisbi.engine.utils.SQLUtils;
import io.gisbi.engine.utils.Utils;
import io.gisbi.exception.DEException;
import io.gisbi.extensions.datasource.api.PluginManageApi;
import io.gisbi.extensions.datasource.constant.SqlPlaceholderConstants;
import io.gisbi.extensions.datasource.dto.*;
import io.gisbi.extensions.datasource.factory.ProviderFactory;
import io.gisbi.extensions.datasource.model.SQLMeta;
import io.gisbi.extensions.datasource.provider.Provider;
import io.gisbi.extensions.datasource.vo.DatasourceConfiguration;
import io.gisbi.extensions.view.dto.ChartExtFilterDTO;
import io.gisbi.extensions.view.dto.ChartExtRequest;
import io.gisbi.extensions.view.dto.ColumnPermissionItem;
import io.gisbi.extensions.view.dto.SqlVariableDetails;
import io.gisbi.i18n.Translator;
import io.gisbi.system.manage.CorePermissionManage;
import io.gisbi.utils.*;
import jakarta.annotation.Resource;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;

import java.math.BigDecimal;
import java.util.*;
import java.util.stream.Collectors;

import static io.gisbi.chart.manage.ChartDataManage.START_END_SEPARATOR;
import static io.gisbi.dataset.utils.TableUtils.format;

/**
 * @Author bi-coder
 */
@Component
public class DatasetDataManage {
    @Resource
    private DatasetSQLManage datasetSQLManage;
    @Resource
    private CoreDatasourceMapper coreDatasourceMapper;
    @Resource
    private DatasetTableFieldManage datasetTableFieldManage;
    @Resource
    private EngineManage engineManage;
    @Resource
    private DatasetGroupManage datasetGroupManage;
    @Resource
    private DatasetTableSqlLogManage datasetTableSqlLogManage;
    @Autowired(required = false)
    private PluginManageApi pluginManage;
    @Resource
    private CorePermissionManage corePermissionManage;
    @Autowired(required = false)
    private RowPermissionsApi rowPermissionsApi;
    @Resource
    private CoreDatasetTableFieldMapper coreDatasetTableFieldMapper;
    @Resource
    private CoreDatasetTableMapper coreDatasetTableMapper;
    @Resource
    private DataSourceManage dataSourceManage;

    private static Logger logger = LoggerFactory.getLogger(DatasetDataManage.class);

    private RowPermissionsApi getRowPermissionsApi() {
        return rowPermissionsApi;
    }

    public static final List<String> notFullDs = List.of("mysql", "mariadb", "Excel", "API", "H2", "h2");

    public List<DatasetTableFieldDTO> getTableFields(DatasetTableDTO datasetTableDTO) throws Exception {
        List<DatasetTableFieldDTO> list = null;
        List<TableField> tableFields = null;
        String type = datasetTableDTO.getType();
        DatasetTableInfoDTO tableInfoDTO = JsonUtil.parseObject(datasetTableDTO.getInfo(), DatasetTableInfoDTO.class);
        if (StringUtils.equalsIgnoreCase(type, DatasetTableType.DB) || StringUtils.equalsIgnoreCase(type, DatasetTableType.SQL)) {
            CoreDatasource coreDatasource = dataSourceManage.getCoreDatasource(datasetTableDTO.getDatasourceId());
            DatasourceSchemaDTO datasourceSchemaDTO = new DatasourceSchemaDTO();
            if (coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.Excel.name()) || coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.name())) {
                coreDatasource = engineManage.getDeEngine();
            }
            if (StringUtils.isNotEmpty(coreDatasource.getStatus()) && "Error".equalsIgnoreCase(coreDatasource.getStatus())) {
                DEException.throwException(Translator.get("i18n_invalid_ds"));
            }
            BeanUtils.copyBean(datasourceSchemaDTO, coreDatasource);
            datasourceSchemaDTO.setSchemaAlias(String.format(SQLConstants.SCHEMA, datasourceSchemaDTO.getId()));
            Provider provider = ProviderFactory.getProvider(coreDatasource.getType());

            DatasourceRequest datasourceRequest = new DatasourceRequest();
            datasourceRequest.setIsCross(datasetTableDTO.getIsCross());
            datasourceRequest.setDsList(Map.of(datasourceSchemaDTO.getId(), datasourceSchemaDTO));
            String sql;
            if (StringUtils.equalsIgnoreCase(type, DatasetTableType.DB)) {
                // add table schema
                sql = TableUtils.tableName2Sql(datasourceSchemaDTO, tableInfoDTO.getTable()) + " LIMIT 0 OFFSET 0";
                // replace schema alias, trans dialect
                Map map = JsonUtil.parseObject(datasourceSchemaDTO.getConfiguration(), Map.class);
                if (!datasourceRequest.getIsCross()) {
                    if (ObjectUtils.isNotEmpty(map.get("schema"))) {
                        sql = sql.replaceAll(SqlPlaceholderConstants.KEYWORD_PREFIX_REGEX + datasourceSchemaDTO.getSchemaAlias() + SqlPlaceholderConstants.KEYWORD_SUFFIX_REGEX, String.format(format, map.get("schema").toString()) );
                    } else {
                        sql = sql.replaceAll(SqlPlaceholderConstants.KEYWORD_PREFIX_REGEX + datasourceSchemaDTO.getSchemaAlias() + SqlPlaceholderConstants.KEYWORD_SUFFIX_REGEX + "\\.", "");
                    }
                    sql = provider.transSqlDialect(sql, datasourceRequest.getDsList());
                } else {
                    sql = sql.replaceAll(SqlPlaceholderConstants.KEYWORD_PREFIX_REGEX + datasourceSchemaDTO.getSchemaAlias() + SqlPlaceholderConstants.KEYWORD_SUFFIX_REGEX + "\\.", "");
                    String tableSchema = datasetSQLManage.putObj2Map(datasourceRequest.getDsList(), datasetTableDTO, datasourceRequest.getIsCross());
                    sql = SqlUtils.addSchema(sql, tableSchema);
                }
            } else {
                // parser sql params and replace default value
                String s = new String(Base64.getDecoder().decode(tableInfoDTO.getSql()));
                String originSql = new SqlparserUtils().handleVariableDefaultValue(s, datasetTableDTO.getSqlVariableDetails(), true, false, null, datasourceRequest.getIsCross(), datasourceRequest.getDsList(), pluginManage, getUserEntity());
                originSql = provider.replaceComment(originSql);
                // add sql table schema

                if (!datasourceRequest.getIsCross()) {
                    sql = SQLUtils.buildOriginPreviewSql(SqlPlaceholderConstants.TABLE_PLACEHOLDER, 0, 0);
                    sql = provider.transSqlDialect(sql, datasourceRequest.getDsList());
                    // replace placeholder
                    sql = provider.replaceTablePlaceHolder(sql, originSql);
                } else {
                    String tableSchema = datasetSQLManage.putObj2Map(datasourceRequest.getDsList(), datasetTableDTO, datasourceRequest.getIsCross());
                    sql = SqlUtils.addSchema(originSql, tableSchema);
                }
            }
            datasourceRequest.setQuery(sql.replaceAll("\r\n", " ")
                    .replaceAll("\n", " "));
            logger.debug("calcite data table field sql: " + datasourceRequest.getQuery());
            // 获取数据源表的原始字段
            if (StringUtils.equalsIgnoreCase(type, DatasetTableType.DB)) {
                datasourceRequest.setTable(tableInfoDTO.getTable());
            }

            tableFields = provider.fetchTableField(datasourceRequest);
        } else if (StringUtils.equalsIgnoreCase(type, DatasetTableType.Es)) {
            CoreDatasource coreDatasource = dataSourceManage.getCoreDatasource(datasetTableDTO.getDatasourceId());
            Provider provider = ProviderFactory.getProvider(type);
            DatasourceRequest datasourceRequest = new DatasourceRequest();
            DatasourceSchemaDTO datasourceSchemaDTO = new DatasourceSchemaDTO();
            BeanUtils.copyBean(datasourceSchemaDTO, coreDatasource);
            datasourceRequest.setDatasource(datasourceSchemaDTO);
            datasourceRequest.setTable(datasetTableDTO.getTableName());
            tableFields = provider.fetchTableField(datasourceRequest);
        } else {
            // excel,api
            CoreDatasource coreDatasource = engineManage.getDeEngine();
            DatasourceSchemaDTO datasourceSchemaDTO = new DatasourceSchemaDTO();
            BeanUtils.copyBean(datasourceSchemaDTO, coreDatasource);
            datasourceSchemaDTO.setSchemaAlias(String.format(SQLConstants.SCHEMA, datasourceSchemaDTO.getId()));
            Provider provider = ProviderFactory.getDefaultProvider();

            DatasourceRequest datasourceRequest = new DatasourceRequest();
            datasourceRequest.setDsList(Map.of(datasourceSchemaDTO.getId(), datasourceSchemaDTO));
            String sql = TableUtils.tableName2Sql(datasourceSchemaDTO, tableInfoDTO.getTable()) + " LIMIT 0 OFFSET 0";
            // replace schema alias, trans dialect
            sql = Utils.replaceSchemaAlias(sql, datasourceRequest.getDsList());
            sql = provider.transSqlDialect(sql, datasourceRequest.getDsList());
            datasourceRequest.setQuery(sql);
            logger.debug("calcite data table field sql: " + datasourceRequest.getQuery());
            tableFields = provider.fetchTableField(datasourceRequest);
        }
        return transFields(tableFields, true);
    }

    public List<DatasetTableFieldDTO> transFields(List<TableField> tableFields, boolean defaultStatus) {
        return tableFields.stream().map(ele -> {
            DatasetTableFieldDTO dto = new DatasetTableFieldDTO();
            dto.setName(StringUtils.isNotEmpty(ele.getName()) ? ele.getName() : ele.getOriginName());
            dto.setOriginName(ele.getOriginName());
            dto.setChecked(defaultStatus);
            dto.setType(ele.getType());
            int deType = FieldUtils.transType2DeType(ele.getType());
            dto.setDeExtractType(ObjectUtils.isEmpty(ele.getDeExtractType()) ? deType : ele.getDeExtractType());
            dto.setDeType(ObjectUtils.isEmpty(ele.getDeType()) ? deType : ele.getDeType());
            dto.setGroupType(FieldUtils.transDeType2DQ(dto.getDeType()));
            dto.setExtField(0);
            dto.setDescription(StringUtils.isNotEmpty(ele.getName()) ? ele.getName() : null);
            return dto;
        }).collect(Collectors.toList());
    }

    public Map<String, Object> previewDataWithLimit(DatasetGroupInfoDTO datasetGroupInfoDTO, Integer start, Integer count, boolean checkPermission, boolean encode) throws Exception {
        if (encode) {
            DatasetUtils.dsDecode(datasetGroupInfoDTO);
        }

        Map<String, Object> sqlMap = datasetSQLManage.getUnionSQLForEdit(datasetGroupInfoDTO, null);
        String sql = (String) sqlMap.get("sql");

        // 获取allFields
        List<DatasetTableFieldDTO> fields = datasetGroupInfoDTO.getAllFields();
        if (ObjectUtils.isEmpty(fields)) {
            DEException.throwException(Translator.get("i18n_no_fields"));
        }

        Map<String, ColumnPermissionItem> desensitizationList = new HashMap<>();
//        if (checkPermission) {
//            fields = permissionManage.filterColumnPermissions(fields, desensitizationList, datasetGroupInfoDTO.getId(), null);
//            if (ObjectUtils.isEmpty(fields)) {
//                DEException.throwException(Translator.get("i18n_no_column_permission"));
//            }
//        }
        buildFieldName(sqlMap, fields);
        Map<Long, DatasourceSchemaDTO> dsMap = (Map<Long, DatasourceSchemaDTO>) sqlMap.get("dsMap");
        DatasourceUtils.checkDsStatus(dsMap);
        List<String> dsList = new ArrayList<>();
        for (Map.Entry<Long, DatasourceSchemaDTO> next : dsMap.entrySet()) {
            dsList.add(next.getValue().getType());
        }
        boolean needOrder = Utils.isNeedOrder(dsList);
        boolean crossDs = datasetGroupInfoDTO.getIsCross();
        if (!crossDs) {
            if (notFullDs.contains(dsMap.entrySet().iterator().next().getValue().getType()) && (boolean) sqlMap.get("isFullJoin")) {
                DEException.throwException(Translator.get("i18n_not_full"));
            }
            sql = Utils.replaceSchemaAlias(sql, dsMap);
        }
//        List<DataSetRowPermissionsTreeDTO> rowPermissionsTree = new ArrayList<>();
//        TokenUserBO user = AuthUtils.getUser();
//        if (user != null && checkPermission) {
//            rowPermissionsTree = permissionManage.getRowPermissionsTree(datasetGroupInfoDTO.getId(), user.getUserId());
//        }
        Provider provider;
        if (crossDs) {
            provider = ProviderFactory.getDefaultProvider();
        } else {
            provider = ProviderFactory.getProvider(dsList.getFirst());
        }

        // build query sql
        SQLMeta sqlMeta = new SQLMeta();
        Table2SQLObj.table2sqlobj(sqlMeta, null, "(" + sql + ")", crossDs);
        Field2SQLObj.field2sqlObj(sqlMeta, fields, fields, crossDs, dsMap, Utils.getParams(fields), null, pluginManage);
//        WhereTree2Str.transFilterTrees(sqlMeta, null, fields, crossDs, dsMap, Utils.getParams(fields), null, pluginManage);
        Order2SQLObj.getOrders(sqlMeta, datasetGroupInfoDTO.getSortFields(), fields, crossDs, dsMap, Utils.getParams(fields), null, pluginManage);
        String querySQL;
        if (start == null || count == null) {
            querySQL = SQLProvider.createQuerySQL(sqlMeta, false, needOrder, false);
        } else {
            querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, false, start, count);
        }
        querySQL = provider.rebuildSQL(querySQL, sqlMeta, crossDs, dsMap);
        logger.debug("calcite data preview sql: " + querySQL);

        // 通过数据源请求数据
        // 调用数据源的calcite获得data
        DatasourceRequest datasourceRequest = new DatasourceRequest();
        datasourceRequest.setQuery(querySQL);
        datasourceRequest.setDsList(dsMap);
        datasourceRequest.setIsCross(crossDs);
        Map<String, Object> data = provider.fetchResultField(datasourceRequest);

        Map<String, Object> map = new LinkedHashMap<>();
        // 重新构造data
        Map<String, Object> previewData = buildPreviewData(data, fields, desensitizationList, encode);
        map.put("data", previewData);
        if (ObjectUtils.isEmpty(datasetGroupInfoDTO.getId())) {
            map.put("allFields", fields);
        } else {
            List<DatasetTableFieldDTO> fieldList = datasetTableFieldManage.selectByDatasetGroupId(datasetGroupInfoDTO.getId());
            if (encode) {
                DatasetUtils.listEncode(fieldList);
            }
            map.put("allFields", fieldList);
        }
        map.put("sql", Base64.getEncoder().encodeToString(querySQL.getBytes()));
        return map;
    }

    public Long getDatasetTotal(Long datasetGroupId) throws Exception {
        DatasetGroupInfoDTO dto = datasetGroupManage.getForCount(datasetGroupId);
        if (ObjectUtils.isEmpty(dto)) return 0L;
        if (StringUtils.equalsIgnoreCase(dto.getNodeType(), "dataset")) {
            return getDatasetTotal(dto, null, new ChartExtRequest());
        }
        return 0L;
    }

    public Long getDatasetCountWithWhere(Long datasetGroupId) {
        try {
            DatasetGroupInfoDTO datasetGroupInfoDTO = datasetGroupManage.getForCount(datasetGroupId);
            Map<String, Object> sqlMap = datasetSQLManage.getUnionSQLForEdit(datasetGroupInfoDTO, null);
            String sql = (String) sqlMap.get("sql");

            // 获取allFields
            List<DatasetTableFieldDTO> fields = datasetGroupInfoDTO.getAllFields();
            if (ObjectUtils.isEmpty(fields)) {
                DEException.throwException(Translator.get("i18n_no_fields"));
            }

            buildFieldName(sqlMap, fields);

            Map<Long, DatasourceSchemaDTO> dsMap = (Map<Long, DatasourceSchemaDTO>) sqlMap.get("dsMap");
            DatasourceUtils.checkDsStatus(dsMap);
            List<String> dsList = new ArrayList<>();
            for (Map.Entry<Long, DatasourceSchemaDTO> next : dsMap.entrySet()) {
                dsList.add(next.getValue().getType());
            }
            boolean crossDs = datasetGroupInfoDTO.getIsCross();
            if (!crossDs) {
                if (notFullDs.contains(dsMap.entrySet().iterator().next().getValue().getType()) && (boolean) sqlMap.get("isFullJoin")) {
                    DEException.throwException(Translator.get("i18n_not_full"));
                }
                sql = Utils.replaceSchemaAlias(sql, dsMap);
            }

//        List<DataSetRowPermissionsTreeDTO> rowPermissionsTree = new ArrayList<>();
//        TokenUserBO user = AuthUtils.getUser();
//        if (user != null) {
//            rowPermissionsTree = permissionManage.getRowPermissionsTree(datasetGroupInfoDTO.getId(), user.getUserId());
//        }

            Provider provider;
            if (crossDs) {
                provider = ProviderFactory.getDefaultProvider();
            } else {
                provider = ProviderFactory.getProvider(dsList.getFirst());
            }

            // build query sql
            SQLMeta sqlMeta = new SQLMeta();
            Table2SQLObj.table2sqlobj(sqlMeta, null, "(" + sql + ")", crossDs);
            Field2SQLObj.field2sqlObj(sqlMeta, fields, fields, crossDs, dsMap, Utils.getParams(fields), null, pluginManage);
//            WhereTree2Str.transFilterTrees(sqlMeta, rowPermissionsTree, fields, crossDs, dsMap, Utils.getParams(fields), null, pluginManage);
            Order2SQLObj.getOrders(sqlMeta, datasetGroupInfoDTO.getSortFields(), fields, crossDs, dsMap, Utils.getParams(fields), null, pluginManage);
            String replaceSql = provider.rebuildSQL(SQLProvider.createQuerySQL(sqlMeta, false, false, false), sqlMeta, crossDs, dsMap);
            return getDatasetTotal(datasetGroupInfoDTO, replaceSql, null);
        } catch (Exception e) {
            return null;
        }
    }

    public Long getDatasetTotal(DatasetGroupInfoDTO datasetGroupInfoDTO, String s, ChartExtRequest request) throws Exception {
        Map<String, Object> sqlMap = datasetSQLManage.getUnionSQLForEdit(datasetGroupInfoDTO, request);
        Map<Long, DatasourceSchemaDTO> dsMap = (Map<Long, DatasourceSchemaDTO>) sqlMap.get("dsMap");
        boolean crossDs = datasetGroupInfoDTO.getIsCross();
        String sql;
        if (StringUtils.isEmpty(s)) {
            sql = (String) sqlMap.get("sql");
            if (!crossDs) {
                sql = Utils.replaceSchemaAlias(sql, dsMap);
            }
        } else {
            sql = s;
        }
        String querySQL = "SELECT COUNT(*) FROM (" + sql + ") t_a_0";
        logger.debug("calcite data count sql: " + querySQL);

        // 通过数据源请求数据
        // 调用数据源的calcite获得data
        DatasourceRequest datasourceRequest = new DatasourceRequest();
        datasourceRequest.setQuery(querySQL);
        datasourceRequest.setDsList(dsMap);
        datasourceRequest.setIsCross(crossDs);

        Provider provider;
        if (crossDs) {
            provider = ProviderFactory.getDefaultProvider();
        } else {
            provider = ProviderFactory.getProvider(dsMap.entrySet().iterator().next().getValue().getType());
        }
        Map<String, Object> data = provider.fetchResultField(datasourceRequest);
        List<String[]> dataList = (List<String[]>) data.get("data");
        if (ObjectUtils.isNotEmpty(dataList) && ObjectUtils.isNotEmpty(dataList.get(0)) && ObjectUtils.isNotEmpty(dataList.get(0)[0])) {
            return Long.valueOf(dataList.get(0)[0]);
        }
        return 0L;
    }

    public Map<String, Object> previewSqlWithLog(PreviewSqlDTO dto) {
        if (dto == null) {
            return null;
        }
        SqlLogDTO sqlLogDTO = new SqlLogDTO();
        String sql = new String(Base64.getDecoder().decode(dto.getSql()));
        sqlLogDTO.setSql(sql);
        Map<String, Object> map = null;
        try {
            sqlLogDTO.setStartTime(System.currentTimeMillis());
            map = previewSql(dto);
            sqlLogDTO.setEndTime(System.currentTimeMillis());
            sqlLogDTO.setSpend(sqlLogDTO.getEndTime() - sqlLogDTO.getStartTime());
            sqlLogDTO.setStatus("Completed");
        } catch (Exception e) {
            sqlLogDTO.setStatus("Error");
            DEException.throwException(e.getMessage());
        } finally {
            if (ObjectUtils.isNotEmpty(dto.getTableId())) {
                sqlLogDTO.setTableId(dto.getTableId());
                datasetTableSqlLogManage.save(sqlLogDTO);
            }
        }
        return map;
    }

    private UserFormVO getUserEntity() {
        if (getRowPermissionsApi() == null) {
            return null;
        }
        return getRowPermissionsApi().getUserById(AuthUtils.getUser().getUserId());
    }

    public Map<String, Object> previewSql(PreviewSqlDTO dto) throws DEException {
        CoreDatasource coreDatasource = dataSourceManage.getCoreDatasource(dto.getDatasourceId());
        DatasourceSchemaDTO datasourceSchemaDTO = new DatasourceSchemaDTO();
        if (coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.API.name()) || coreDatasource.getType().contains(DatasourceConfiguration.DatasourceType.Excel.name())) {
            BeanUtils.copyBean(datasourceSchemaDTO, engineManage.getDeEngine());
        } else {
            BeanUtils.copyBean(datasourceSchemaDTO, coreDatasource);
        }

        if (StringUtils.isNotEmpty(datasourceSchemaDTO.getStatus()) && "Error".equalsIgnoreCase(datasourceSchemaDTO.getStatus())) {
            DEException.throwException(Translator.get("i18n_invalid_ds"));
        }

        String alias = String.format(SQLConstants.SCHEMA, datasourceSchemaDTO.getId());
        datasourceSchemaDTO.setSchemaAlias(alias);

        Map<Long, DatasourceSchemaDTO> dsMap = new LinkedHashMap<>();
        dsMap.put(datasourceSchemaDTO.getId(), datasourceSchemaDTO);
        DatasourceRequest datasourceRequest = new DatasourceRequest();
        datasourceRequest.setDsList(dsMap);
        datasourceRequest.setIsCross(dto.getIsCross());
        Provider provider = ProviderFactory.getProvider(datasourceSchemaDTO.getType());

        // parser sql params and replace default value

        String s = new String(Base64.getDecoder().decode(dto.getSql()));
        String originSql = new SqlparserUtils().handleVariableDefaultValue(datasetSQLManage.subPrefixSuffixChar(s), dto.getSqlVariableDetails(), true, true, null, dto.getIsCross(), dsMap, pluginManage, getUserEntity());
        originSql = provider.replaceComment(originSql);

        // sql 作为临时表，外层加上limit
        String sql;
        if (dto.getIsCross()) {
            DatasetTableDTO currentDs = new DatasetTableDTO();
            BeanUtils.copyBean(currentDs, dto);
            currentDs.setType("sql");
            String tableSchema = datasetSQLManage.putObj2Map(dsMap, currentDs, dto.getIsCross());
            sql = SqlUtils.addSchema(originSql, tableSchema);
            if (Utils.isNeedOrder(List.of(datasourceSchemaDTO.getType()))) {
                // 先根据sql获取表字段
                String sqlField = SQLUtils.buildOriginPreviewSql(sql, 0, 0);
                datasourceRequest.setQuery(sqlField);

                // 获取数据源表的原始字段
                List<TableField> list = provider.fetchTableField(datasourceRequest);
                if (ObjectUtils.isEmpty(list)) {
                    return null;
                }
                sql = SQLUtils.buildOriginPreviewSqlWithOrderBy(sql, 100, 0, String.format(SQLConstants.FIELD_DOT_FIX, list.get(0).getOriginName()) + " ASC ");
            } else {
                sql = SQLUtils.buildOriginPreviewSql(sql, 100, 0);
            }
        } else {
            if (Utils.isNeedOrder(List.of(datasourceSchemaDTO.getType()))) {
                // 先根据sql获取表字段
                String sqlField = SQLUtils.buildOriginPreviewSql(SqlPlaceholderConstants.TABLE_PLACEHOLDER, 0, 0);

                sqlField = provider.transSqlDialect(sqlField, datasourceRequest.getDsList());
                // replace placeholder
                sqlField = provider.replaceTablePlaceHolder(sqlField, originSql);
                datasourceRequest.setQuery(sqlField);

                // 获取数据源表的原始字段
                List<TableField> list = provider.fetchTableField(datasourceRequest);
                if (ObjectUtils.isEmpty(list)) {
                    return null;
                }
                sql = SQLUtils.buildOriginPreviewSqlWithOrderBy(SqlPlaceholderConstants.TABLE_PLACEHOLDER, 100, 0, String.format(SQLConstants.FIELD_DOT_FIX, list.get(0).getOriginName()) + " ASC ");
            } else {
                sql = SQLUtils.buildOriginPreviewSql(SqlPlaceholderConstants.TABLE_PLACEHOLDER, 100, 0);
            }
            sql = provider.transSqlDialect(sql, datasourceRequest.getDsList());
            // replace placeholder
            sql = provider.replaceTablePlaceHolder(sql, originSql);
        }

        logger.debug("calcite data preview sql: " + sql);
        datasourceRequest.setQuery(sql);
        Map<String, Object> data = provider.fetchResultField(datasourceRequest);
        // 重新构造data
        List<TableField> fList = (List<TableField>) data.get("fields");
        List<DatasetTableFieldDTO> fields = transFields(fList, false);
        Map<String, Object> previewData = buildPreviewData(data, fields, new HashMap<>(), false);
        Map<String, Object> map = new LinkedHashMap<>();
        map.put("data", previewData);
        map.put("sql", Base64.getEncoder().encodeToString(sql.getBytes()));
        return map;
    }

    public Map<String, Object> buildPreviewData(Map<String, Object> data, List<DatasetTableFieldDTO> fields, Map<String, ColumnPermissionItem> desensitizationList, boolean isEncode) {
        Map<String, Object> map = new LinkedHashMap<>();
        List<String[]> dataList = (List<String[]>) data.get("data");
        List<LinkedHashMap<String, Object>> dataObjectList = new ArrayList<>();
        if (ObjectUtils.isNotEmpty(dataList)) {
            for (int i = 0; i < dataList.size(); i++) {
                String[] row = dataList.get(i);
                LinkedHashMap<String, Object> obj = new LinkedHashMap<>();
                if (row.length > 0) {
                    for (int j = 0; j < fields.size(); j++) {
                        String res = row[j];
                        // 如果字段类型是数值类型的小数，则去除科学计数
                        if (fields.get(j).getDeType() == 3 && StringUtils.containsIgnoreCase(res, "E")) {
                            BigDecimal bigDecimal = new BigDecimal(res);
                            res = String.format("%.8f", bigDecimal);
                        }
                        if (desensitizationList.keySet().contains(fields.get(j).getGisbiName())) {
                            obj.put(fields.get(j).getGisbiName(), ChartDataBuild.desensitizationValue(desensitizationList.get(fields.get(j).getGisbiName()), String.valueOf(res)));
                        } else {
                            obj.put(ObjectUtils.isNotEmpty(fields.get(j).getGisbiName()) ?
                                    fields.get(j).getGisbiName() : fields.get(j).getOriginName(), res);
                        }
                    }
                }
                dataObjectList.add(obj);
            }
        }

        if (isEncode) {
            DatasetUtils.listEncode(fields);
        }

        map.put("fields", fields);
        map.put("data", dataObjectList);
        return map;
    }

    public void buildFieldName(Map<String, Object> sqlMap, List<DatasetTableFieldDTO> fields) {
        // 获取内层union sql和字段
        List<DatasetTableFieldDTO> unionFields = (List<DatasetTableFieldDTO>) sqlMap.get("field");
        for (DatasetTableFieldDTO datasetTableFieldDTO : fields) {
            DatasetTableFieldDTO dto = datasetTableFieldManage.selectById(datasetTableFieldDTO.getId());
            if (ObjectUtils.isEmpty(dto)) {
                if (Objects.equals(datasetTableFieldDTO.getExtField(), ExtFieldConstant.EXT_NORMAL)) {
                    for (DatasetTableFieldDTO fieldDTO : unionFields) {
                        if (Objects.equals(datasetTableFieldDTO.getDatasetTableId(), fieldDTO.getDatasetTableId())
                                && Objects.equals(datasetTableFieldDTO.getOriginName(), fieldDTO.getOriginName())) {
                            datasetTableFieldDTO.setGisbiName(fieldDTO.getGisbiName());
                            datasetTableFieldDTO.setFieldShortName(fieldDTO.getFieldShortName());
                        }
                    }
                }
                if (Objects.equals(datasetTableFieldDTO.getExtField(), ExtFieldConstant.EXT_CALC)) {
                    String gisbiName = TableUtils.fieldNameShort(datasetTableFieldDTO.getId() + "_" + datasetTableFieldDTO.getOriginName());
                    datasetTableFieldDTO.setGisbiName(gisbiName);
                    datasetTableFieldDTO.setFieldShortName(gisbiName);
                    datasetTableFieldDTO.setDeExtractType(datasetTableFieldDTO.getDeType());
                }
                if (Objects.equals(datasetTableFieldDTO.getExtField(), ExtFieldConstant.EXT_GROUP)) {
                    String gisbiName = TableUtils.fieldNameShort(datasetTableFieldDTO.getId() + "_" + datasetTableFieldDTO.getOriginName());
                    datasetTableFieldDTO.setGisbiName(gisbiName);
                    datasetTableFieldDTO.setFieldShortName(gisbiName);
                    datasetTableFieldDTO.setDeExtractType(0);
                    datasetTableFieldDTO.setDeType(0);
                    datasetTableFieldDTO.setGroupType("d");
                }
            } else {
                datasetTableFieldDTO.setGisbiName(dto.getGisbiName());
                datasetTableFieldDTO.setFieldShortName(dto.getFieldShortName());
            }
        }
    }

    public List<String> getFieldEnumDs(EnumObj map) throws Exception {
        DatasetTableFieldDTO field = map.getField();
        DatasetGroupInfoDTO datasetGroupInfoDTO = map.getDataset();
        if (field == null) {
            DEException.throwException(Translator.get("i18n_no_field"));
        }
        List<DatasetTableFieldDTO> allFields = new ArrayList<>();

        Map<String, Object> sqlMap = datasetSQLManage.getUnionSQLForEdit(datasetGroupInfoDTO, new ChartExtRequest());
        String sql = (String) sqlMap.get("sql");

        allFields.addAll(datasetGroupInfoDTO.getAllFields());

        Map<Long, DatasourceSchemaDTO> dsMap = (Map<Long, DatasourceSchemaDTO>) sqlMap.get("dsMap");
        boolean crossDs = datasetGroupInfoDTO.getIsCross();
        if (!crossDs) {
            sql = Utils.replaceSchemaAlias(sql, dsMap);
        }

        // build query sql
        SQLMeta sqlMeta = new SQLMeta();
        Table2SQLObj.table2sqlobj(sqlMeta, null, "(" + sql + ")", crossDs);

        // 获取allFields
        List<DatasetTableFieldDTO> fields = Collections.singletonList(field);
        buildFieldName(sqlMap, fields);

        List<String> dsList = new ArrayList<>();
        for (Map.Entry<Long, DatasourceSchemaDTO> next : dsMap.entrySet()) {
            dsList.add(next.getValue().getType());
        }
        boolean needOrder = Utils.isNeedOrder(dsList);

        Provider provider;
        if (crossDs) {
            provider = ProviderFactory.getDefaultProvider();
        } else {
            provider = ProviderFactory.getProvider(dsList.getFirst());
        }

        String dsType = null;
        if (dsMap != null && dsMap.entrySet().iterator().hasNext()) {
            Map.Entry<Long, DatasourceSchemaDTO> next = dsMap.entrySet().iterator().next();
            dsType = next.getValue().getType();
        }

        Field2SQLObj.field2sqlObj(sqlMeta, fields, allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage);
        WhereTree2Str.transFilterTrees(sqlMeta, null, allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage);
        Order2SQLObj.getOrders(sqlMeta, datasetGroupInfoDTO.getSortFields(), allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage);
        String querySQL;
        querySQL = SQLProvider.createQuerySQL(sqlMeta, false, needOrder, !StringUtils.equalsIgnoreCase(dsType, "es"));
        querySQL = provider.rebuildSQL(querySQL, sqlMeta, crossDs, dsMap);
        logger.debug("calcite data enum sql: " + querySQL);

        // 通过数据源请求数据
        // 调用数据源的calcite获得data
        DatasourceRequest datasourceRequest = new DatasourceRequest();
        datasourceRequest.setQuery(querySQL);
        datasourceRequest.setDsList(dsMap);
        datasourceRequest.setIsCross(crossDs);

        Map<String, Object> data = provider.fetchResultField(datasourceRequest);
        List<String[]> dataList = (List<String[]>) data.get("data");
        dataList = dataList.stream().filter(row -> {
            boolean hasEmpty = false;
            for (String s : row) {
                if (StringUtils.isBlank(s)) {
                    hasEmpty = true;
                    break;
                }
            }
            return !hasEmpty;
        }).toList();
        List<String> previewData = new ArrayList<>();
        if (ObjectUtils.isNotEmpty(dataList)) {
            List<String> tmpData = dataList.stream().map(ele -> (ObjectUtils.isNotEmpty(ele) && ele.length > 0) ? ele[0] : null).collect(Collectors.toList());
            if (!CollectionUtils.isEmpty(tmpData)) {
                for (int i = 0; i < tmpData.size(); i++) {
                    String val = tmpData.get(i);
                    if (field.getDeType() == 3 && StringUtils.containsIgnoreCase(val, "E")) {
                        BigDecimal bigDecimal = new BigDecimal(val);
                        val = String.format("%.8f", bigDecimal);
                        tmpData.set(i, val);
                    }
                }
                previewData = tmpData;
            }
        }
        return previewData;
    }

    public List<String> getFieldEnum(MultFieldValuesRequest multFieldValuesRequest) throws Exception {
        // 根据前端传的查询组件field ids，获取所有字段枚举值并去重合并
        List<List<String>> list = new ArrayList<>();
        for (Long id : multFieldValuesRequest.getFieldIds()) {
            DatasetTableFieldDTO field = datasetTableFieldManage.selectById(id);
            if (field == null) {
                DEException.throwException(Translator.get("i18n_no_field"));
            }
            List<DatasetTableFieldDTO> allFields = new ArrayList<>();
            // 根据图表计算字段，获取数据集
            Long datasetGroupId = field.getDatasetGroupId();

            // check permission
            BusiPerCheckDTO dto = new BusiPerCheckDTO();
            dto.setId(datasetGroupId);
            dto.setAuthEnum(AuthEnum.READ);
            boolean checked = corePermissionManage.checkAuth(dto);
            if (!checked) {
                DEException.throwException(Translator.get("i18n_no_dataset_permission"));
            }
            if (field.getChartId() != null) {
                allFields.addAll(datasetTableFieldManage.getChartCalcFields(field.getChartId()));
            }
            DatasetGroupInfoDTO datasetGroupInfoDTO = datasetGroupManage.getDatasetGroupInfoDTO(datasetGroupId, null);

            Map<String, Object> sqlMap = datasetSQLManage.getUnionSQLForEdit(datasetGroupInfoDTO, new ChartExtRequest());
            String sql = (String) sqlMap.get("sql");

            allFields.addAll(datasetGroupInfoDTO.getAllFields());

            Map<Long, DatasourceSchemaDTO> dsMap = (Map<Long, DatasourceSchemaDTO>) sqlMap.get("dsMap");
            boolean crossDs = datasetGroupInfoDTO.getIsCross();
            if (!crossDs) {
                sql = Utils.replaceSchemaAlias(sql, dsMap);
            }

            // build query sql
            SQLMeta sqlMeta = new SQLMeta();
            Table2SQLObj.table2sqlobj(sqlMeta, null, "(" + sql + ")", crossDs);

            // 获取allFields
            List<DatasetTableFieldDTO> fields = Collections.singletonList(field);
            Map<String, ColumnPermissionItem> desensitizationList = new HashMap<>();
//            fields = permissionManage.filterColumnPermissions(fields, desensitizationList, datasetGroupInfoDTO.getId(), null);
            if (ObjectUtils.isEmpty(fields)) {
                DEException.throwException(Translator.get("i18n_no_column_permission"));
            }
            buildFieldName(sqlMap, fields);

            List<String> dsList = new ArrayList<>();
            for (Map.Entry<Long, DatasourceSchemaDTO> next : dsMap.entrySet()) {
                dsList.add(next.getValue().getType());
            }
            boolean needOrder = Utils.isNeedOrder(dsList);

//            List<DataSetRowPermissionsTreeDTO> rowPermissionsTree = new ArrayList<>();
//            TokenUserBO user = AuthUtils.getUser();
//            if (user != null) {
//                rowPermissionsTree = permissionManage.getRowPermissionsTree(datasetGroupInfoDTO.getId(), user.getUserId());
//            }

            Provider provider;
            if (crossDs) {
                provider = ProviderFactory.getDefaultProvider();
            } else {
                provider = ProviderFactory.getProvider(dsList.getFirst());
            }

            String dsType = null;
            if (dsMap != null && dsMap.entrySet().iterator().hasNext()) {
                Map.Entry<Long, DatasourceSchemaDTO> next = dsMap.entrySet().iterator().next();
                dsType = next.getValue().getType();
            }

            Field2SQLObj.field2sqlObj(sqlMeta, fields, allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage);
//            WhereTree2Str.transFilterTrees(sqlMeta, rowPermissionsTree, allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage);
            Order2SQLObj.getOrders(sqlMeta, datasetGroupInfoDTO.getSortFields(), allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage);
            String querySQL;
            if (multFieldValuesRequest.getResultMode() == 0) {
                querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, !StringUtils.equalsIgnoreCase(dsType, "es"), 0, 1000);
            } else {
                querySQL = SQLProvider.createQuerySQL(sqlMeta, false, needOrder, !StringUtils.equalsIgnoreCase(dsType, "es"));
            }
            querySQL = provider.rebuildSQL(querySQL, sqlMeta, crossDs, dsMap);
            logger.debug("calcite data enum sql: " + querySQL);

            // 通过数据源请求数据
            // 调用数据源的calcite获得data
            DatasourceRequest datasourceRequest = new DatasourceRequest();
            datasourceRequest.setQuery(querySQL);
            datasourceRequest.setDsList(dsMap);
            datasourceRequest.setIsCross(crossDs);

            Map<String, Object> data = provider.fetchResultField(datasourceRequest);
            List<String[]> dataList = (List<String[]>) data.get("data");
            dataList = dataList.stream().filter(row -> {
                boolean hasEmpty = false;
                for (String s : row) {
                    if (StringUtils.isBlank(s)) {
                        hasEmpty = true;
                        break;
                    }
                }
                return !hasEmpty;
            }).toList();
            List<String> previewData = new ArrayList<>();
            if (ObjectUtils.isNotEmpty(dataList)) {
                List<String> tmpData = dataList.stream().map(ele -> (ObjectUtils.isNotEmpty(ele) && ele.length > 0) ? ele[0] : null).collect(Collectors.toList());
                if (!CollectionUtils.isEmpty(tmpData)) {
                    for (int i = 0; i < tmpData.size(); i++) {
                        String val = tmpData.get(i);
                        if (field.getDeType() == 3 && StringUtils.containsIgnoreCase(val, "E")) {
                            BigDecimal bigDecimal = new BigDecimal(val);
                            val = String.format("%.8f", bigDecimal);
                            tmpData.set(i, val);
                        }
                    }
                    if (desensitizationList.keySet().contains(field.getGisbiName())) {
                        for (int i = 0; i < tmpData.size(); i++) {
                            previewData.add(ChartDataBuild.desensitizationValue(desensitizationList.get(field.getGisbiName()), tmpData.get(i)));
                        }
                    } else {
                        previewData = tmpData;
                    }
                }
                list.add(previewData);
            }
        }

        // 重新构造data
        Set<String> result = new LinkedHashSet<>();
        for (List<String> l : list) {
            result.addAll(l);
        }
        return result.stream().toList();
    }

    public List<Map<String, Object>> getFieldEnumObj(EnumValueRequest request) throws Exception {
        List<Long> ids = new ArrayList<>();
        if (ObjectUtils.isNotEmpty(request.getQueryId())) {
            ids.add(request.getQueryId());
        }
        if (ObjectUtils.isNotEmpty(request.getDisplayId())) {
            ids.add(request.getDisplayId());
        }

        if (CollectionUtils.isEmpty(ids)) {
            return Collections.emptyList();
        }
        if (ids.size() == 2 && Objects.equals(ids.get(0), ids.get(1))) {
            ids.remove(1);
        }

        SQLMeta sqlMeta = new SQLMeta();
        DatasetGroupInfoDTO datasetGroupInfoDTO = null;
        List<DatasetTableFieldDTO> fields = new ArrayList<>();
        Map<String, Object> sqlMap = null;
        boolean crossDs = false;
        Map<Long, DatasourceSchemaDTO> dsMap = null;

        if (ObjectUtils.isNotEmpty(request.getSortId())) {
            // 如果排序字段和查询字段显示字段不一致，则加入到查询列表中
            if (!request.getSortId().equals(request.getQueryId()) && !request.getSortId().equals(request.getDisplayId())) {
                ids.add(request.getSortId());
            }
        }

        List<DatasetTableFieldDTO> allFields = new ArrayList<>();

        for (Long id : ids) {
            DatasetTableFieldDTO field = datasetTableFieldManage.selectById(id);
            if (field == null) {
                DEException.throwException(Translator.get("i18n_no_field"));
            }

            // 根据图表计算字段，获取数据集
            Long datasetGroupId = field.getDatasetGroupId();

            // check permission
//            BusiPerCheckDTO dto = new BusiPerCheckDTO();
//            dto.setId(datasetGroupId);
//            dto.setAuthEnum(AuthEnum.READ);
//            boolean checked = corePermissionManage.checkAuth(dto);
//            if (!checked) {
//                DEException.throwException(Translator.get("i18n_no_dataset_permission"));
//            }

            if (field.getChartId() != null) {
                allFields.addAll(datasetTableFieldManage.getChartCalcFields(field.getChartId()));
            }
            datasetGroupInfoDTO = datasetGroupManage.getDatasetGroupInfoDTO(datasetGroupId, null);

            sqlMap = datasetSQLManage.getUnionSQLForEdit(datasetGroupInfoDTO, new ChartExtRequest());
            String sql = (String) sqlMap.get("sql");

            allFields.addAll(datasetGroupInfoDTO.getAllFields());

            dsMap = (Map<Long, DatasourceSchemaDTO>) sqlMap.get("dsMap");
            crossDs = datasetGroupInfoDTO.getIsCross();
            if (!crossDs) {
                sql = Utils.replaceSchemaAlias(sql, dsMap);
            }

            // build query sql
            Table2SQLObj.table2sqlobj(sqlMeta, null, "(" + sql + ")", crossDs);
            fields.add(field);
        }

        // 获取allFields
        Map<String, ColumnPermissionItem> desensitizationList = new HashMap<>();
//        fields = permissionManage.filterColumnPermissions(fields, desensitizationList, datasetGroupInfoDTO.getId(), null);
        if (ObjectUtils.isEmpty(fields)) {
            DEException.throwException(Translator.get("i18n_no_column_permission"));
        }
        buildFieldName(sqlMap, fields);

        List<String> dsList = new ArrayList<>();
        for (Map.Entry<Long, DatasourceSchemaDTO> next : dsMap.entrySet()) {
            dsList.add(next.getValue().getType());
        }
        boolean needOrder = Utils.isNeedOrder(dsList);

//        List<DataSetRowPermissionsTreeDTO> rowPermissionsTree = new ArrayList<>();
//        TokenUserBO user = AuthUtils.getUser();
//        if (user != null) {
//            rowPermissionsTree = permissionManage.getRowPermissionsTree(datasetGroupInfoDTO.getId(), user.getUserId());
//        }

        //组件过滤条件
        List<ChartExtFilterDTO> extFilterList = new ArrayList<>();
        if (ObjectUtils.isNotEmpty(request.getFilter())) {
            for (ChartExtFilterDTO filterDTO : request.getFilter()) {
                // 解析多个fieldId,fieldId是一个逗号分隔的字符串
                String fieldId = filterDTO.getFieldId();
                if (filterDTO.getIsTree() == null) {
                    filterDTO.setIsTree(false);
                }

                boolean hasParameters = false;
                List<SqlVariableDetails> sqlVariables = datasetGroupManage.getSqlParams(Arrays.asList(datasetGroupInfoDTO.getId()));
                if (org.apache.commons.collections4.CollectionUtils.isNotEmpty(sqlVariables)) {
                    for (SqlVariableDetails parameter : Optional.ofNullable(filterDTO.getParameters()).orElse(new ArrayList<>())) {
                        String parameterId = StringUtils.endsWith(parameter.getId(), START_END_SEPARATOR) ? parameter.getId().split(START_END_SEPARATOR)[0] : parameter.getId();
                        if (sqlVariables.stream().map(SqlVariableDetails::getId).collect(Collectors.toList()).contains(parameterId)) {
                            hasParameters = true;
                        }
                    }
                }

                if (hasParameters) {
                    continue;
                }

                if (StringUtils.isNotEmpty(fieldId)) {
                    List<Long> fieldIds = Arrays.stream(fieldId.split(",")).map(Long::valueOf).collect(Collectors.toList());

                    if (filterDTO.getIsTree()) {
                        ChartExtFilterDTO filterRequest = new ChartExtFilterDTO();
                        BeanUtils.copyBean(filterRequest, filterDTO);
                        filterRequest.setDatasetTableFieldList(new ArrayList<>());
                        for (Long fId : fieldIds) {
                            DatasetTableFieldDTO datasetTableField = datasetTableFieldManage.selectById(fId);
                            if (datasetTableField == null) {
                                continue;
                            }
                            if (Objects.equals(datasetTableField.getDatasetGroupId(), datasetGroupInfoDTO.getId())) {
                                filterRequest.getDatasetTableFieldList().add(datasetTableField);
                            }
                        }
                        if (ObjectUtils.isNotEmpty(filterRequest.getDatasetTableFieldList())) {
                            extFilterList.add(filterRequest);
                        }
                    } else {
                        for (Long fId : fieldIds) {
                            ChartExtFilterDTO filterRequest = new ChartExtFilterDTO();
                            BeanUtils.copyBean(filterRequest, filterDTO);
                            filterRequest.setFieldId(fId + "");

                            DatasetTableFieldDTO datasetTableField = datasetTableFieldManage.selectById(fId);
                            if (datasetTableField == null) {
                                continue;
                            }
                            filterRequest.setDatasetTableField(datasetTableField);
                            if (Objects.equals(datasetTableField.getDatasetGroupId(), datasetGroupInfoDTO.getId())) {
                                extFilterList.add(filterRequest);
                            }
                        }
                    }
                }
            }
        }

        // 搜索备选项
        if (StringUtils.isNotEmpty(request.getSearchText())) {
            ChartExtFilterDTO dto = new ChartExtFilterDTO();
            DatasetTableFieldDTO field = null;
            if (ids.size() == 1) {
                field = datasetTableFieldManage.selectById(ids.get(0));
            } else {
                field = datasetTableFieldManage.selectById(ids.get(1));
            }
            dto.setDatasetTableField(field);
            dto.setFieldId(field.getId() + "");
            dto.setIsTree(false);
            dto.setOperator("like");
            dto.setValue(List.of(request.getSearchText()));
            extFilterList.add(dto);
        }

        // 排序
        boolean sortDistinct = true;
        if (ObjectUtils.isNotEmpty(request.getSortId())) {
            DatasetTableFieldDTO field = datasetTableFieldManage.selectById(request.getSortId());
            if (field == null) {
                DEException.throwException(Translator.get("i18n_no_field"));
            }
            DeSortField deSortField = new DeSortField();
            BeanUtils.copyBean(deSortField, field);
            deSortField.setOrderDirection(request.getSort());
            datasetGroupInfoDTO.setSortFields(Collections.singletonList(deSortField));
            sortDistinct = false;
        }

        Provider provider;
        if (crossDs) {
            provider = ProviderFactory.getDefaultProvider();
        } else {
            provider = ProviderFactory.getProvider(dsList.getFirst());
        }

        String dsType = null;
        if (dsMap != null && dsMap.entrySet().iterator().hasNext()) {
            Map.Entry<Long, DatasourceSchemaDTO> next = dsMap.entrySet().iterator().next();
            dsType = next.getValue().getType();
        }

        Field2SQLObj.field2sqlObj(sqlMeta, fields, allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage);
        ExtWhere2Str.extWhere2sqlOjb(sqlMeta, extFilterList, allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage);
//        WhereTree2Str.transFilterTrees(sqlMeta, rowPermissionsTree, allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage);
        Order2SQLObj.getOrders(sqlMeta, datasetGroupInfoDTO.getSortFields(), allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage);
        String querySQL;
        if (request.getResultMode() == 0) {
            querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, sortDistinct && ids.size() == 1 && !StringUtils.equalsIgnoreCase(dsType, "es"), 0, 1000);
        } else {
            querySQL = SQLProvider.createQuerySQL(sqlMeta, false, needOrder, sortDistinct && ids.size() == 1 && !StringUtils.equalsIgnoreCase(dsType, "es"));
        }
        querySQL = provider.rebuildSQL(querySQL, sqlMeta, crossDs, dsMap);
        logger.debug("calcite data enum sql: " + querySQL);

        // 通过数据源请求数据
        // 调用数据源的calcite获得data
        DatasourceRequest datasourceRequest = new DatasourceRequest();
        datasourceRequest.setQuery(querySQL);
        datasourceRequest.setDsList(dsMap);
        datasourceRequest.setIsCross(crossDs);

        Map<String, Object> data = provider.fetchResultField(datasourceRequest);
        List<String[]> dataList = (List<String[]>) data.get("data");
        dataList = dataList.stream().filter(row -> {
            boolean hasEmpty = false;
            for (String s : row) {
                if (StringUtils.isBlank(s)) {
                    hasEmpty = true;
                    break;
                }
            }
            return !hasEmpty;
        }).toList();
        Map<String, String[]> distinctData = new LinkedHashMap<>();
        for (String[] arr : dataList) {
            String key = Arrays.toString(arr);
            if (!distinctData.containsKey(key)) {
                distinctData.put(key, arr);
            }
        }

        List<String[]> distinctDataList = new ArrayList<>();
        for (Map.Entry<String, String[]> ele : distinctData.entrySet()) {
            distinctDataList.add(ele.getValue());
        }

        List<Map<String, Object>> previewData = new ArrayList<>();
        if (ObjectUtils.isNotEmpty(distinctDataList)) {
            for (String[] ele : distinctDataList) {
                Map<String, Object> map = new LinkedHashMap<>();
                for (int i = 0; i < fields.size(); i++) {
                    String val = ele[i];
                    DatasetTableFieldDTO field = fields.get(i);
                    if (field.getDeType() == 3 && StringUtils.containsIgnoreCase(val, "E")) {
                        BigDecimal bigDecimal = new BigDecimal(val);
                        val = String.format("%.8f", bigDecimal);
                    }
                    if (desensitizationList.containsKey(field.getGisbiName())) {
                        String str = ChartDataBuild.desensitizationValue(desensitizationList.get(field.getGisbiName()), val);
                        map.put(field.getId() + "", str);
                    } else {
                        map.put(field.getId() + "", val);
                    }
                }
                previewData.add(map);
            }
        }
        return previewData;
    }

    public List<BaseTreeNodeDTO> getFieldValueTree(MultFieldValuesRequest multFieldValuesRequest) throws Exception {
        List<Long> ids = multFieldValuesRequest.getFieldIds();
        if (ids.isEmpty()) {
            DEException.throwException("no field selected.");
        }
        // 根据前端传的查询组件field ids，获取所有字段枚举值并去重合并
        List<List<String>> list = new ArrayList<>();
        List<DatasetTableFieldDTO> fields = new ArrayList<>();

        // 根据图表计算字段，获取数据集
        List<DatasetTableFieldDTO> allFields = new ArrayList<>();
        DatasetTableFieldDTO field = datasetTableFieldManage.selectById(ids.getFirst());
        Long datasetGroupId = field.getDatasetGroupId();
        if (field.getChartId() != null) {
            allFields.addAll(datasetTableFieldManage.getChartCalcFields(field.getChartId()));
        }
        DatasetGroupInfoDTO datasetGroupInfoDTO = datasetGroupManage.getDatasetGroupInfoDTO(datasetGroupId, null);

        Map<String, Object> sqlMap = datasetSQLManage.getUnionSQLForEdit(datasetGroupInfoDTO, new ChartExtRequest());
        String sql = (String) sqlMap.get("sql");

        allFields.addAll(datasetGroupInfoDTO.getAllFields());

        Map<Long, DatasourceSchemaDTO> dsMap = (Map<Long, DatasourceSchemaDTO>) sqlMap.get("dsMap");
        boolean crossDs = datasetGroupInfoDTO.getIsCross();
        if (!crossDs) {
            sql = Utils.replaceSchemaAlias(sql, dsMap);
        }

        // build query sql
        SQLMeta sqlMeta = new SQLMeta();
        Table2SQLObj.table2sqlobj(sqlMeta, null, "(" + sql + ")", crossDs);

        for (Long id : ids) {
            DatasetTableFieldDTO f = datasetTableFieldManage.selectById(id);
            if (f == null) {
                DEException.throwException(Translator.get("i18n_no_field"));
            }
            // 获取allFields
            fields.add(f);
        }

        Map<String, ColumnPermissionItem> desensitizationList = new HashMap<>();
//        fields = permissionManage.filterColumnPermissions(fields, desensitizationList, datasetGroupInfoDTO.getId(), null);
        if (ObjectUtils.isEmpty(fields)) {
            DEException.throwException(Translator.get("i18n_no_column_permission"));
        }
        buildFieldName(sqlMap, fields);

        List<String> dsList = new ArrayList<>();
        for (Map.Entry<Long, DatasourceSchemaDTO> next : dsMap.entrySet()) {
            dsList.add(next.getValue().getType());
        }
        boolean needOrder = Utils.isNeedOrder(dsList);

//        List<DataSetRowPermissionsTreeDTO> rowPermissionsTree = new ArrayList<>();
//        TokenUserBO user = AuthUtils.getUser();
//        if (user != null) {
//            rowPermissionsTree = permissionManage.getRowPermissionsTree(datasetGroupInfoDTO.getId(), user.getUserId());
//        }

        Provider provider;
        if (crossDs) {
            provider = ProviderFactory.getDefaultProvider();
        } else {
            provider = ProviderFactory.getProvider(dsList.getFirst());
        }

        Field2SQLObj.field2sqlObj(sqlMeta, fields, allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage);
//        WhereTree2Str.transFilterTrees(sqlMeta, rowPermissionsTree, allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage);
        Order2SQLObj.getOrders(sqlMeta, datasetGroupInfoDTO.getSortFields(), allFields, crossDs, dsMap, Utils.getParams(allFields), null, pluginManage);
        String querySQL;
        if (multFieldValuesRequest.getResultMode() == 0) {
            querySQL = SQLProvider.createQuerySQLWithLimit(sqlMeta, false, needOrder, false, 0, 1000);
        } else {
            querySQL = SQLProvider.createQuerySQL(sqlMeta, false, needOrder, false);
        }
        querySQL = provider.rebuildSQL(querySQL, sqlMeta, crossDs, dsMap);
        logger.debug("filter tree sql: " + querySQL);

        // 通过数据源请求数据
        // 调用数据源的calcite获得data
        DatasourceRequest datasourceRequest = new DatasourceRequest();
        datasourceRequest.setQuery(querySQL);
        datasourceRequest.setDsList(dsMap);
        datasourceRequest.setIsCross(crossDs);

        Map<String, Object> data = provider.fetchResultField(datasourceRequest);
        List<String[]> rows = (List<String[]>) data.get("data");

        // 重新构造data
        Set<String> pkSet = new HashSet<>();
        rows = rows.stream().filter(row -> {
            boolean hasEmpty = false;
            int emptyCount = 0;
            for (String s : row) {
                if (StringUtils.isBlank(s)) {
                    emptyCount++;
                    hasEmpty = true; // 标记已遇到第一个null
                } else if (hasEmpty) {
                    return false; // 在null后出现非null元素，不符合要求
                }
            }
            return emptyCount != row.length;
        }).toList();
        List<BaseTreeNodeDTO> treeNodes = rows.stream().map(row -> buildTreeNode(row, pkSet)).flatMap(Collection::stream).collect(Collectors.toList());
        List<BaseTreeNodeDTO> tree = DatasetUtils.mergeDuplicateTree(treeNodes, "root");
        return tree;
    }

    private List<BaseTreeNodeDTO> buildTreeNode(String[] row, Set<String> pkSet) {
        List<BaseTreeNodeDTO> nodes = new ArrayList<>();
        List<String> parentPkList = new ArrayList<>();
        for (int i = 0; i < row.length; i++) {
            String text = row[i];
            if (StringUtils.isEmpty(text)) {
                continue;
            }
            parentPkList.add(text);
            String val = String.join(TreeUtils.SEPARATOR, parentPkList);
            String parentVal = i == 0 ? TreeUtils.DEFAULT_ROOT : row[i - 1];
            String pk = String.join(TreeUtils.SEPARATOR, parentPkList);
            if (pkSet.contains(pk)) continue;
            pkSet.add(pk);
            BaseTreeNodeDTO node = new BaseTreeNodeDTO(val, parentVal, StringUtils.isNotBlank(text) ? text.trim() : text, pk + TreeUtils.SEPARATOR + i);
            nodes.add(node);
        }
        return nodes;

    }
//-----------------------下面代码为扩展代码，实现了数据表的增删改查功能--zhengsl at 2025-05-20----------------------------------------------------------------------------------------//
    public List<Map<String,Object>> getTablesByAppId(String appid) throws Exception {
        List<Map<String,Object>> tables = coreDatasetTableMapper.getTablesByAppId(appid);
        return tables;
    }

public Map<String, Object> getFieldsByTableId(Long datasource_id, String tablename) throws Exception {
    CoreDatasource coreDatasource = coreDatasourceMapper.selectById(datasource_id);
    if (coreDatasource == null) {
        DEException.throwException("数据源不存在");
    }

    DatasourceSchemaDTO datasourceSchemaDTO = new DatasourceSchemaDTO();
    BeanUtils.copyBean(datasourceSchemaDTO, coreDatasource);

    Provider provider = ProviderFactory.getProvider(coreDatasource.getType());
    DatasourceRequest datasourceRequest = new DatasourceRequest();
    datasourceRequest.setDsList(Map.of(datasourceSchemaDTO.getId(), datasourceSchemaDTO));
    String dbType = datasourceSchemaDTO.getType().toLowerCase();
    // 构建字段查询SQL（带注释）
    String columnSql = buildColumnQuerySQL(dbType, tablename);
    datasourceRequest.setQuery(columnSql);
    Map<String, Object> columnData = provider.fetchResultField(datasourceRequest);
    List<String[]> columnRows = (List<String[]>) columnData.get("data");

    // 构建主键查询SQL
    String pkSql = buildPrimaryKeyQuerySQL(dbType, tablename);
    datasourceRequest.setQuery(pkSql);
    Map<String, Object> pkData = provider.fetchResultField(datasourceRequest);
    List<String[]> pkRows = (List<String[]>) pkData.get("data");

    Set<String> primaryKeyFields = new HashSet<>();
    for (String[] row : pkRows) {
        if (row.length > 0 && StringUtils.isNotBlank(row[0])) {
            primaryKeyFields.add(row[0]);
        }
    }

    // 构建字段信息
    List<Map<String, Object>> fields = new ArrayList<>();
    for (String[] row : columnRows) {
        String field = row[0];
        String type = row[1];
        int size = 0 ;
        if(!"longtext".equals(type)&&!"longblob".equals(type)){
            size = ObjectUtils.isNotEmpty(row[2])?Integer.parseInt(row[2]):0;
        }

        String comment = row.length > 3 ? row[3] : "";

        Map<String, Object> fieldInfo = new HashMap<>();
        fieldInfo.put("field", field);
        fieldInfo.put("name", StringUtils.isNotBlank(comment) ? comment : field); // 注释优先
        fieldInfo.put("type", type);
        fieldInfo.put("size", size);
        fieldInfo.put("iskey", primaryKeyFields.contains(field));

        fields.add(fieldInfo);
    }

    // 返回结果
    Map<String, Object> result = new HashMap<>();
    result.put("keyfeilds", new ArrayList<>(primaryKeyFields));
    result.put("fields", fields);

    return result;
}

private String buildColumnQuerySQL(String dbType, String tablename) {
    switch (dbType) {
        case "mysql":
            return String.format(
                "SELECT COLUMN_NAME AS field, DATA_TYPE AS type, CHARACTER_MAXIMUM_LENGTH AS size, COLUMN_COMMENT AS comment " +
                "FROM INFORMATION_SCHEMA.COLUMNS " +
                "WHERE TABLE_NAME = '%s'", tablename);
        case "mariadb":
            return String.format(
                "SELECT COLUMN_NAME AS field, DATA_TYPE AS type, COALESCE(CHARACTER_MAXIMUM_LENGTH, 0) AS size, COLUMN_COMMENT AS comment " +
                "FROM INFORMATION_SCHEMA.COLUMNS " +
                "WHERE TABLE_NAME = '%s'", tablename);

        case "postgresql":
            return String.format(
                "SELECT a.attname AS field, " +
                "pg_catalog.format_type(a.atttypid, a.atttypmod) AS type, " +
                "COALESCE(a.attlen, 0) AS size, " +
                "d.description AS comment " +
                "FROM pg_attribute a " +
                "LEFT JOIN pg_description d ON d.objoid = a.attrelid AND d.objsubid = a.attnum " +
                "WHERE a.attrelid = '%s'::regclass AND a.attnum > 0 AND NOT a.attisdropped", tablename);

        case "oracle":
            return String.format(
                "SELECT cols.column_name AS field, cols.data_type AS type, cols.data_length AS size, comm.comments AS comment " +
                "FROM all_cons_columns cons, all_constraints c, all_col_comments comm, all_tab_cols cols " +
                "WHERE cols.table_name = '%s' " +
                "AND cols.column_name = comm.column_name(+) " +
                "AND cols.table_name = comm.table_name(+) " +
                "AND cols.owner = comm.owner(+)", tablename.toUpperCase());

        case "sqlserver":
            return String.format(
                "SELECT c.name AS field, t.name AS type, c.max_length AS size, " +
                "ep.value AS comment " +
                "FROM sys.columns c " +
                "JOIN sys.types t ON c.user_type_id = t.user_type_id " +
                "LEFT JOIN sys.extended_properties ep ON ep.major_id = c.object_id AND ep.minor_id = c.column_id AND ep.name = 'MS_Description' " +
                "WHERE c.object_id = OBJECT_ID('%s')", tablename);

        default:
            throw new UnsupportedOperationException("不支持的数据库类型：" + dbType);
    }
}
private String buildPrimaryKeyQuerySQL(String dbType, String tablename) {
    switch (dbType) {
        case "mysql":
            return String.format(
                "SELECT COLUMN_NAME AS field " +
                "FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE " +
                "WHERE TABLE_NAME = '%s' AND CONSTRAINT_NAME IN (" +
                    "SELECT CONSTRAINT_NAME FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS " +
                    "WHERE TABLE_NAME = '%s' AND CONSTRAINT_TYPE = 'PRIMARY KEY'" +
                ")", tablename, tablename);
        case "mariadb":
            return String.format(
                "SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE " +
                "WHERE TABLE_NAME = '%s' AND CONSTRAINT_NAME IN (" +
                    "SELECT CONSTRAINT_NAME FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS " +
                    "WHERE TABLE_NAME = '%s' AND CONSTRAINT_TYPE = 'PRIMARY KEY'" +
                ")", tablename, tablename);

        case "postgresql":
            return String.format(
                "SELECT a.attname AS field " +
                "FROM pg_index i " +
                "JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey) " +
                "WHERE i.indrelid = '%s'::regclass AND i.indisprimary", tablename);

        case "oracle":
            return String.format(
                "SELECT cols.column_name " +
                "FROM all_constraints c, all_cons_columns cols " +
                "WHERE c.owner = cols.owner " +
                "AND c.constraint_name = cols.constraint_name " +
                "AND c.constraint_type = 'P' " +
                "AND cols.table_name = '%s'", tablename.toUpperCase());

        case "sqlserver":
            return String.format(
                "SELECT COL_NAME(object_id, column_id) AS field " +
                "FROM sys.key_constraints kc " +
                "JOIN sys.index_columns ic ON kc.parent_object_id = ic.object_id AND kc.unique_index_id = ic.index_id " +
                "WHERE kc.type = 'PK' AND kc.parent_object_id = OBJECT_ID('%s')", tablename);

        default:
            throw new UnsupportedOperationException("不支持的数据库类型：" + dbType);
    }
}

  public boolean addTableData(Long datasourceId, String tableData) throws Exception {
    // 根据数据源 id 查询数据源信息，调用通用数据源执行器
    CoreDatasource coreDatasource = coreDatasourceMapper.selectById(datasourceId);
    if (coreDatasource == null) {
        DEException.throwException("数据源不存在");
    }

    // 解析 tableData JSON 字符串  "{ \"tableName\": \"user\", \"data\": [ { \"fieldName\": \"id\", \"fieldType\": \"varchar\", \"IsPrimaryKey\": true, \"fieldValue\": \"0001\" }, { \"fieldName\": \"name\", \"fieldType\": \"varchar\", \"fieldValue\": \"张三\" } ] }";
    Map<String, Object> dataMap = JsonUtil.parseObject(tableData, Map.class);
    String tableName = (String) dataMap.get("tableName");
    List<Map<String, Object>> fieldList = (List<Map<String, Object>>) dataMap.get("data");

    if (fieldList == null || fieldList.isEmpty()) {
        DEException.throwException("没有可插入的数据字段");
    }

    // 构建插入语句
    StringBuilder columns = new StringBuilder();
    StringBuilder values = new StringBuilder();

    for (int i = 0; i < fieldList.size(); i++) {
        Map<String, Object> field = fieldList.get(i);
        String fieldName = (String) field.get("fieldName");
        Object fieldValue = field.get("fieldValue");
        boolean isPrimaryKey = field.get("IsPrimaryKey") != null && (boolean) field.get("IsPrimaryKey");
        if (isPrimaryKey) {
            if (fieldValue == null) {fieldValue= IDUtils.snowID();}
        }
        if (i > 0) {
            columns.append(", ");
            values.append(", ");
        }
        columns.append(fieldName);
        if (fieldValue instanceof String) {
            values.append("'").append(fieldValue).append("'");
        } else {
            values.append(fieldValue);
        }
    }

    String sql = String.format("INSERT INTO %s (%s) VALUES (%s)", tableName, columns.toString(), values.toString());

    // 调用执行器，向数据表中插入 tableData 数据
    DatasourceSchemaDTO datasourceSchemaDTO = new DatasourceSchemaDTO();
    BeanUtils.copyBean(datasourceSchemaDTO, coreDatasource);

    Provider provider = ProviderFactory.getProvider(coreDatasource.getType());
    DatasourceRequest datasourceRequest = new DatasourceRequest();
    datasourceRequest.setQuery(sql);
    datasourceRequest.setDsList(Map.of(datasourceSchemaDTO.getId(), datasourceSchemaDTO));

    logger.debug("执行插入数据的SQL: {}", sql);

    // 执行插入操作
      int result= provider.executeUpdate(datasourceRequest,null).getCount();
      if (result==1) {
          return true;
          // process result set
      } else {
          return false;
      }
}
    public Map<String, Object> getTableDataByPk(Long datasourceId, String condtion) throws Exception {
        // 获取数据源信息
        CoreDatasource coreDatasource = coreDatasourceMapper.selectById(datasourceId);
        if (coreDatasource == null) {
            DEException.throwException("数据源不存在");
        }
        // 解析 JSON 数据
        //condtion={ \"tableName\": \"user\", key:[{ \"fieldName\": \"id\",\"fieldValue\": \"0001\"]
        Map<String, Object> dataMap = JsonUtil.parseObject(condtion, Map.class);
        String tableName = (String) dataMap.get("tableName");
        // 参数校验
        if (StringUtils.isBlank(tableName)) {
            DEException.throwException("表名不能为空");
        }
        List<Map<String, Object>> keyFields = (List<Map<String, Object>>) dataMap.get("key");
        if (CollectionUtils.isEmpty(keyFields)) {
            DEException.throwException("主键字段或值不能为空");
        }

        String whereClause = buildWhereCondition(keyFields);
        String sql = String.format("SELECT * FROM %s WHERE %s", tableName, whereClause);
        // 执行查询操作
        DatasourceSchemaDTO datasourceSchemaDTO = new DatasourceSchemaDTO();
        BeanUtils.copyBean(datasourceSchemaDTO, coreDatasource);

        Provider provider = ProviderFactory.getProvider(coreDatasource.getType());
        DatasourceRequest datasourceRequest = new DatasourceRequest();
        datasourceRequest.setQuery(sql);
        datasourceRequest.setDsList(Map.of(datasourceSchemaDTO.getId(), datasourceSchemaDTO));

        logger.debug("执行查询数据的SQL: {}", sql);

        // 获取查询结果
        Map<String, Object> data = provider.fetchResultField(datasourceRequest);

        // 处理查询结果
        List<String[]> dataList = (List<String[]>) data.get("data");
        List<TableField> fields = (List<TableField>) data.get("fields");

        if (CollectionUtils.isEmpty(dataList) || dataList.size() == 0) {
            return null;
        }

        // 将结果转换为 Map 格式
        String[] row = dataList.get(0);
        Map<String, Object> resultMap = new LinkedHashMap<>();
        for (int i = 0; i < fields.size(); i++) {
             TableField field = fields.get(i);
            String fieldName = field.getOriginName();
            resultMap.put(fieldName, row[i]);
        }
        return resultMap;
    }
    /**
     * 构建 WHERE 条件字符串，用于多字段主键查询/更新/删除操作
     *
     * @param keyFields 主键字段列表，格式如：{ "fieldName": "id", "fieldValue": "0001" }
     * @return SQL WHERE 子句字符串
     */
    private String buildWhereCondition(List<Map<String, Object>> keyFields) {
        StringBuilder whereClause = new StringBuilder();

        for (int i = 0; i < keyFields.size(); i++) {
            Map<String, Object> keyField = keyFields.get(i);
            String fieldName = (String) keyField.get("fieldName");
            Object fieldValue = keyField.get("fieldValue");

            if (i > 0) {
                whereClause.append(" AND ");
            }

            if (fieldValue == null) {
                whereClause.append(fieldName).append(" IS NULL");
            } else if (fieldValue instanceof String) {
                whereClause.append(String.format("%s = '%s'", fieldName, fieldValue));
            } else {
                whereClause.append(String.format("%s = %s", fieldName, fieldValue));
            }
        }

        return whereClause.toString();
    }

  public boolean updateTableData(Long datasourceId, String tableData) throws Exception {
    // 获取数据源信息
    CoreDatasource coreDatasource = coreDatasourceMapper.selectById(datasourceId);
    if (coreDatasource == null) {
        DEException.throwException("数据源不存在");
    }
    //String tableDataJson = "{ \"tableName\": \"user\", key:[{ \"fieldName\": \"id\",\"fieldValue\": \"0001\"], \"data\": [ { \"fieldName\": \"name\", \"fieldType\": \"varchar\", \"fieldValue\": \"李四\" } ] }";
    // 解析 JSON 数据
    Map<String, Object> dataMap = JsonUtil.parseObject(tableData, Map.class);
    String tableName = (String) dataMap.get("tableName");
    if (StringUtils.isBlank(tableName)) {
      DEException.throwException("表名不能为空");
    }
    List<Map<String, Object>> keyFields = (List<Map<String, Object>>) dataMap.get("key");
    if (CollectionUtils.isEmpty(keyFields)) {
      DEException.throwException("主键字段或值不能为空");
    }
    List<Map<String, Object>> fieldList = (List<Map<String, Object>>) dataMap.get("data");
    if (fieldList == null || fieldList.isEmpty()) {
        DEException.throwException("没有可更新的数据字段");
    }
    // 构建 UPDATE 语句
    StringBuilder setClause = new StringBuilder();
    for (int i = 0; i < fieldList.size(); i++) {
        Map<String, Object> field = fieldList.get(i);
        String fieldName = (String) field.get("fieldName");
        Object fieldValue = field.get("fieldValue");

        if (i > 0) {
            setClause.append(", ");
        }
        if (fieldValue instanceof String) {
            setClause.append(String.format("%s = '%s'", fieldName, fieldValue));
        } else {
            setClause.append(String.format("%s = %s", fieldName, fieldValue));
        }
    }

    String whereClause = buildWhereCondition(keyFields);
    String sql = String.format("UPDATE %s SET %s WHERE %s", tableName, setClause, whereClause);
    // 调用执行器执行 SQL
    DatasourceSchemaDTO datasourceSchemaDTO = new DatasourceSchemaDTO();
    BeanUtils.copyBean(datasourceSchemaDTO, coreDatasource);

    Provider provider = ProviderFactory.getProvider(coreDatasource.getType());
    DatasourceRequest datasourceRequest = new DatasourceRequest();
    datasourceRequest.setQuery(sql);
    datasourceRequest.setDsList(Map.of(datasourceSchemaDTO.getId(), datasourceSchemaDTO));

    logger.debug("执行更新数据的SQL: {}", sql);

    // 执行更新操作
      int result= provider.executeUpdate(datasourceRequest,null).getCount();
      if (result==1) {
          return true;
          // process result set
      } else {
          return false;
      }
}
    public boolean deleteTableData(Long datasourceId, String condtion) throws Exception {
        // 获取数据源信息
        CoreDatasource coreDatasource = coreDatasourceMapper.selectById(datasourceId);
        if (coreDatasource == null) {
            DEException.throwException("数据源不存在");        }

        // 解析 JSON 数据
        //String tableDataJson = "{ \"tableName\": \"user\",  key:[{ \"fieldName\": \"id\",\"fieldValue\": \"0001\"] }";
        Map<String, Object> dataMap = JsonUtil.parseObject(condtion, Map.class);
        String tableName = (String) dataMap.get("tableName");
        if (StringUtils.isBlank(tableName)) {
            DEException.throwException("表名不能为空");
        }
        List<Map<String, Object>> keyFields = (List<Map<String, Object>>) dataMap.get("key");
        if (CollectionUtils.isEmpty(keyFields)) {
            DEException.throwException("主键字段或值不能为空");
        }
        String whereClause = buildWhereCondition(keyFields);
        String sql = String.format("DELETE FROM %s WHERE %s", tableName, whereClause);
        // 调用执行器执行 SQL
        DatasourceSchemaDTO datasourceSchemaDTO = new DatasourceSchemaDTO();
        BeanUtils.copyBean(datasourceSchemaDTO, coreDatasource);

        Provider provider = ProviderFactory.getProvider(coreDatasource.getType());
        DatasourceRequest datasourceRequest = new DatasourceRequest();
        datasourceRequest.setQuery(sql);
        datasourceRequest.setDsList(Map.of(datasourceSchemaDTO.getId(), datasourceSchemaDTO));

        logger.debug("执行删除数据的SQL: {}", sql);

        // 执行删除操作
        int result= provider.executeUpdate(datasourceRequest,null).getCount();
        if (result==1) {
            return true;
            // process result set
        } else {
            return false;
        }
    }

   public Page<Map<String, Object>> queryTableDataPaged(Long datasourceId, String condition) throws Exception {
    // 获取数据源信息
    CoreDatasource coreDatasource = coreDatasourceMapper.selectById(datasourceId);
    if (coreDatasource == null) {
        DEException.throwException("数据源不存在");
    }

    // 解析 condition JSON 数据
    Map<String, Object> dataMap = JsonUtil.parseObject(condition, Map.class);
    String tableName = (String) dataMap.get("tableName");
    List<Map<String, Object>> conditionList = (List<Map<String, Object>>) dataMap.get("conditions");

    Integer pageNum = (Integer) dataMap.getOrDefault("pageNum", 1);
    Integer pageSize = (Integer) dataMap.getOrDefault("pageSize", 10);

    if (StringUtils.isBlank(tableName)) {
        DEException.throwException("表名不能为空");
    }

    // 构建 WHERE 条件子句
    StringBuilder whereClause = new StringBuilder();
    if (conditionList != null && !conditionList.isEmpty()) {
        whereClause.append(" WHERE ");
        for (int i = 0; i < conditionList.size(); i++) {
            Map<String, Object> cond = conditionList.get(i);
            String field = (String) cond.get("field");
            String operator = ((String) cond.get("operator")).toLowerCase();
            Object value = cond.get("value");

            if (i > 0) {
                whereClause.append(" AND ");
            }
            switch (operator) {
                case "like":
                    whereClause.append(String.format("%s LIKE '%%%s%%'", field, value));
                    break;
                case "mlike":
                     String fields[]=field.split(",");
                     for(int j=0;j<fields.length;j++){
                         if(j==0){
                             whereClause.append("(");
                         }
                        if(j>0){
                            whereClause.append(" OR ");
                        }
                        whereClause.append(String.format("%s LIKE '%%%s%%'", fields[j], value));
                        if(j==fields.length-1){
                            whereClause.append(")");
                        }
                    }
                    break;
                case "=":
                    appendValue(whereClause, field, value, "=");
                    break;
                case "<":
                    appendValue(whereClause, field, value, "<");
                    break;
                case ">":
                    appendValue(whereClause, field, value, ">");
                    break;
                case "<=":
                    appendValue(whereClause, field, value, "<=");
                    break;
                case ">=":
                    appendValue(whereClause, field, value, ">=");
                    break;
                case "between":
                    String datevales[]=String.valueOf(value).split(",");
                    appendValue(whereClause, field, datevales[0]+" 00:00:00", ">=");
                    whereClause.append(" AND ");
                    appendValue(whereClause, field, datevales[1]+" 23:59:59", "<=");
                    break;
                case "<>":
                    appendValue(whereClause, field, value, "<>");
                    break;
                case "in":
                    if (!(value instanceof List<?>)) {
                        DEException.throwException("IN 操作符要求值为列表类型");
                    }
                    List<?> values = (List<?>) value;
                    String inValues = values.stream()
                            .map(v -> v instanceof String ? "'" + v + "'" : v.toString())
                            .collect(Collectors.joining(", "));
                    whereClause.append(String.format("%s IN (%s)", field, inValues));
                    break;
                default:
                    DEException.throwException("不支持的操作符: " + operator);
            }
        }
    }

    // 构建基础 SQL
    String baseSql = String.format("SELECT * FROM %s%s", tableName, whereClause);
    String dbType = coreDatasource.getType().toLowerCase();
    String pagedSql = buildPagedSQL(baseSql, dbType, pageNum, pageSize);
    String countSql = String.format("SELECT COUNT(*) FROM %s%s", tableName, whereClause);

    DatasourceSchemaDTO schemaDTO = new DatasourceSchemaDTO();
    BeanUtils.copyBean(schemaDTO, coreDatasource);
    Provider provider = ProviderFactory.getProvider(coreDatasource.getType());
    DatasourceRequest request = new DatasourceRequest();
    request.setDsList(Map.of(schemaDTO.getId(), schemaDTO));

    // 查询分页数据
    request.setQuery(pagedSql);
    Map<String, Object> result = provider.fetchResultField(request);
    List<String[]> dataList = (List<String[]>) result.get("data");
    List<TableField> fields = (List<TableField>) result.get("fields");

    // 查询总记录数
    request.setQuery(countSql);
    Map<String, Object> countResult = provider.fetchResultField(request);
    long total = Long.parseLong(((List<String[]>) countResult.get("data")).get(0)[0]);

    // 将数据封装为 Map 形式
    List<Map<String, Object>> records = new ArrayList<>();
    if (!CollectionUtils.isEmpty(dataList)) {
        for (String[] row : dataList) {
            Map<String, Object> rowMap = new LinkedHashMap<>();
            for (int i = 0; i < fields.size() && i < row.length; i++) {
                String fieldName = fields.get(i).getOriginName();
                String fieldValue = row[i];
                rowMap.put(fieldName, fieldValue);
            }
            records.add(rowMap);
        }
    }

    // 返回分页结果
    Page<Map<String, Object>> page = new Page<>();
    page.setCurrent(pageNum);
    page.setSize(pageSize);
    page.setTotal(total);
    page.setRecords(records);

    return page;
}


    private void appendValue(StringBuilder sb, String field, Object value, String op) {
        if (value instanceof String) {
            sb.append(String.format("%s %s '%s'", field, op, value));
        } else {
            sb.append(String.format("%s %s %s", field, op, value));
        }
    }

    private String buildPagedSQL(String baseSql, String dbType, int pageNum, int pageSize) {
        int offset = (pageNum - 1) * pageSize;

        switch (dbType) {
            case "mysql":
            case "mariadb":
                return String.format("%s LIMIT %d OFFSET %d", baseSql, pageSize, offset);
            case "postgresql":
                return String.format("%s LIMIT %d OFFSET %d", baseSql, pageSize, offset);
            case "oracle":
                int start = offset + 1;
                int end = offset + pageSize;
                return String.format("SELECT * FROM (SELECT ROWNUM rn, t.* FROM (%s) t WHERE ROWNUM <= %d) WHERE rn >= %d", baseSql, end, start);
            case "sqlserver":
                return String.format("%s OFFSET %d ROWS FETCH NEXT %d ROWS ONLY", baseSql, offset, pageSize);
            default:
                // 默认使用 MySQL 方式
                return String.format("%s LIMIT %d OFFSET %d", baseSql, pageSize, offset);
        }
    }



}
