Skip to content

(fix | improvement) fix the aliases obtained by intelligent recognition could not be converted in Q&A. modify to allow all LLMs to be called on LLM management page #2312 #2317

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
1985312383 wants to merge 3 commits into
base: master
Choose a base branch
from
Open

(fix | improvement) fix the aliases obtained by intelligent recognition could not be converted in Q&A. modify to allow all LLMs to be called on LLM management page #2312 #2317

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
26adf3e
(fix) fix logical bug #2306
1985312383 Jun 27, 2025
6d1fdd0
(improvement) smart filling support use other LLM and customized prompt
1985312383 Jun 27, 2025
38856ad
(fix) fix buildOntologyQuery to check name、bizName and alias
1985312383 Jun 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 38 additions & 0 deletions common/src/main/java/com/tencent/supersonic/common/config/AliasGenerateParameterConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
package com.tencent.supersonic.common.config;

import com.google.common.collect.Lists;
import com.tencent.supersonic.common.pojo.Parameter;
import org.springframework.stereotype.Service;

import java.util.List;

@Service("AliasGenerateParameterConfig")
public class AliasGenerateParameterConfig extends ParameterConfig {
private static final String MODULE_NAME = "智能填充配置";

public static final Parameter LLM_ALIAS_GENERATION_MODEL =
new Parameter("llm.alias.generation.model", "1",
"别名生成选用模型", "用于指标、维度等别名生成", "string", MODULE_NAME);

public static final String ALIAS_GENERATE_TUTORIAL =
"#Role: You are a professional data analyst specializing in metrics and dimensions.\n"
+ "#Task: You will be provided with metadata about a metric or dimension, please help "
+ "generate a few aliases in the same language as its name.\n"
+ "#Rules:\n"
+ "1. Please do not generate aliases like xxx1, xxx2, xxx3.\n"
+ "2. Please do not generate aliases that are the same as the original names of metrics/dimensions.\n"
+ "3. Please pay attention to the quality of the generated aliases and "
+ "avoid creating aliases that look like test data.\n"
+ "4. Please output as a json string array.\n"
+ "#Output:";

public static final Parameter LLM_ALIAS_GENERATION_PROMPT =
new Parameter("llm.alias.generation.prompt", ALIAS_GENERATE_TUTORIAL,
"别名生成Prompt", "用于指标、维度等别名生成的Prompt", "longText", MODULE_NAME);

@Override
public List<Parameter> getSysParameters() {
return Lists.newArrayList(LLM_ALIAS_GENERATION_MODEL, LLM_ALIAS_GENERATION_PROMPT);
}
}

4 changes: 4 additions & 0 deletions common/src/main/java/com/tencent/supersonic/common/config/SystemConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,4 +74,8 @@ public List<Parameter> getParameters() {
}
return defaultParameters;
}

public String getAliasGeneratePrompt() {
return getParameterByName(AliasGenerateParameterConfig.LLM_ALIAS_GENERATION_PROMPT.getName());
}
}
22 changes: 22 additions & 0 deletions common/src/main/java/dev/langchain4j/provider/ModelProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,14 @@
package dev.langchain4j.provider;

import com.tencent.supersonic.common.config.ChatModel;
import com.tencent.supersonic.common.config.EmbeddingModelParameterConfig;
import com.tencent.supersonic.common.pojo.ChatModelConfig;
import com.tencent.supersonic.common.pojo.EmbeddingModelConfig;
import com.tencent.supersonic.common.util.ContextUtils;
import com.tencent.supersonic.common.service.ChatModelService;
import com.tencent.supersonic.common.config.SystemConfig;
import com.tencent.supersonic.common.service.SystemConfigService;
import com.tencent.supersonic.common.config.AliasGenerateParameterConfig;
import dev.langchain4j.model.chat.ChatLanguageModel;
import dev.langchain4j.model.embedding.EmbeddingModel;
import org.apache.commons.lang3.StringUtils;
Expand All @@ -30,6 +35,23 @@ public static ChatLanguageModel getChatModel() {
public static ChatLanguageModel getChatModel(ChatModelConfig modelConfig) {
if (modelConfig == null || StringUtils.isBlank(modelConfig.getProvider())
|| StringUtils.isBlank(modelConfig.getBaseUrl())) {
SystemConfigService systemConfigService = ContextUtils.getBean(SystemConfigService.class);
SystemConfig systemConfig = systemConfigService.getSystemConfig();
String modelId = systemConfig.getParameterByName(
AliasGenerateParameterConfig.LLM_ALIAS_GENERATION_MODEL.getName());
if (StringUtils.isNotBlank(modelId)) {
ChatModelService chatModelService = ContextUtils.getBean(ChatModelService.class);
ChatModel chatModel = chatModelService.getChatModel(Integer.parseInt(modelId));
if (chatModel != null) {
modelConfig = chatModel.getConfig();
} else {
modelConfig = DEMO_CHAT_MODEL;
}
} else {
modelConfig = DEMO_CHAT_MODEL;
}
}
if (modelConfig == null) {
modelConfig = DEMO_CHAT_MODEL;
}
ModelFactory modelFactory = factories.get(modelConfig.getProvider().toUpperCase());
Expand Down
6 changes: 6 additions & 0 deletions ...src/main/java/com/tencent/supersonic/headless/chat/corrector/LLMPhysicalSqlCorrector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,12 @@ public void doCorrect(ChatQueryContext chatQueryContext, SemanticParseInfo seman
return;
}

// Check if querySQL is null to avoid template variable error
String querySQL = semanticParseInfo.getSqlInfo().getQuerySQL();
if (StringUtils.isBlank(querySQL)) {
return;
}

ChatLanguageModel chatLanguageModel =
ModelProvider.getChatModel(chatApp.getChatModelConfig());
PhysicalSqlExtractor extractor =
Expand Down
101 changes: 80 additions & 21 deletions .../src/main/java/com/tencent/supersonic/headless/core/translator/parser/SqlQueryParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,18 +56,44 @@ public void parse(QueryStatement queryStatement) throws Exception {
ontologyQuery.getMetrics().forEach(m -> {
ontologyMetricsDimensions.add(m.getName());
ontologyBizNameMetricsDimensions.add(m.getBizName());
// Add aliases to the semantic fields
if (StringUtils.isNotBlank(m.getAlias())) {
List<String> aliasList = SchemaItem.getAliasList(m.getAlias());
ontologyMetricsDimensions.addAll(aliasList);
ontologyBizNameMetricsDimensions.addAll(aliasList);
}
});
ontologyQuery.getDimensions().forEach(d -> {
ontologyMetricsDimensions.add(d.getName());
ontologyBizNameMetricsDimensions.add(d.getBizName());
// Add aliases to the semantic fields
if (StringUtils.isNotBlank(d.getAlias())) {
List<String> aliasList = SchemaItem.getAliasList(d.getAlias());
ontologyMetricsDimensions.addAll(aliasList);
ontologyBizNameMetricsDimensions.addAll(aliasList);
}
});
// check if there are fields not matched with any metric or dimension

if (!(queryFieldsSet.containsAll(ontologyMetricsDimensions)
|| queryFieldsSet.containsAll(ontologyBizNameMetricsDimensions))) {
if (!(ontologyMetricsDimensions.containsAll(queryFieldsSet)
|| ontologyBizNameMetricsDimensions.containsAll(queryFieldsSet))) {
List<String> semanticFields = Lists.newArrayList();
ontologyQuery.getMetrics().forEach(m -> semanticFields.add(m.getName()));
ontologyQuery.getDimensions().forEach(d -> semanticFields.add(d.getName()));
ontologyQuery.getMetrics().forEach(m -> {
semanticFields.add(m.getName());
// Add aliases to semantic fields for error message
if (StringUtils.isNotBlank(m.getAlias())) {
List<String> aliasList = SchemaItem.getAliasList(m.getAlias());
semanticFields.addAll(aliasList);
}
});
ontologyQuery.getDimensions().forEach(d -> {
semanticFields.add(d.getName());
// Add aliases to semantic fields for error message
if (StringUtils.isNotBlank(d.getAlias())) {
List<String> aliasList = SchemaItem.getAliasList(d.getAlias());
semanticFields.addAll(aliasList);
}
});
String errMsg =
String.format("Querying columns[%s] not matched with semantic fields[%s].",
queryFields, semanticFields);
Expand Down Expand Up @@ -200,13 +226,20 @@ private OntologyQuery buildOntologyQuery(Ontology ontology, List<String> queryFi
ontology.getMetricMap().entrySet().forEach(entry -> {
String modelName = entry.getKey();
entry.getValue().forEach(m -> {
if (fields.contains(m.getName()) || fields.contains(m.getBizName())) {
Set<String> allNamesForMetric = new HashSet<>();
allNamesForMetric.add(m.getName());
allNamesForMetric.add(m.getBizName());
if (StringUtils.isNotBlank(m.getAlias())) {
allNamesForMetric.addAll(SchemaItem.getAliasList(m.getAlias()));
}
Set<String> matchedFields = new HashSet<>(fields);
matchedFields.retainAll(allNamesForMetric);
if (!matchedFields.isEmpty()) {
ontologyQuery.getModelMap().put(modelName,
ontology.getModelMap().get(modelName));
ontologyQuery.getMetricMap().computeIfAbsent(modelName, k -> Sets.newHashSet())
.add(m);
fields.remove(m.getName());
fields.remove(m.getBizName());
fields.removeAll(matchedFields);
}
});
});
Expand All @@ -217,13 +250,20 @@ private OntologyQuery buildOntologyQuery(Ontology ontology, List<String> queryFi
.forEach(entry -> {
String modelName = entry.getKey();
entry.getValue().forEach(d -> {
if (fields.contains(d.getName()) || fields.contains(d.getBizName())) {
Set<String> allNamesForDimension = new HashSet<>();
allNamesForDimension.add(d.getName());
allNamesForDimension.add(d.getBizName());
if (StringUtils.isNotBlank(d.getAlias())) {
allNamesForDimension.addAll(SchemaItem.getAliasList(d.getAlias()));
}
Set<String> matchedFields = new HashSet<>(fields);
matchedFields.retainAll(allNamesForDimension);
if (!matchedFields.isEmpty()) {
ontologyQuery.getModelMap().put(modelName,
ontology.getModelMap().get(modelName));
ontologyQuery.getDimensionMap()
.computeIfAbsent(modelName, k -> Sets.newHashSet()).add(d);
fields.remove(d.getName());
fields.remove(d.getBizName());
fields.removeAll(matchedFields);
}
});
});
Expand All @@ -232,21 +272,33 @@ private OntologyQuery buildOntologyQuery(Ontology ontology, List<String> queryFi
// is needed.
if (!fields.isEmpty()) {
Map<String, Set<DimSchemaResp>> model2dims = new HashMap<>();
Map<String, Set<String>> model2matchedFields = new HashMap<>();
ontology.getDimensionMap().entrySet().forEach(entry -> {
String modelName = entry.getKey();
entry.getValue().forEach(d -> {
if (fields.contains(d.getName()) || fields.contains(d.getBizName())) {
Set<String> allNamesForDimension = new HashSet<>();
allNamesForDimension.add(d.getName());
allNamesForDimension.add(d.getBizName());
if (StringUtils.isNotBlank(d.getAlias())) {
allNamesForDimension.addAll(SchemaItem.getAliasList(d.getAlias()));
}
Set<String> matchedFieldsInQuery = new HashSet<>(fields);
matchedFieldsInQuery.retainAll(allNamesForDimension);
if (!matchedFieldsInQuery.isEmpty()) {
model2dims.computeIfAbsent(modelName, k -> Sets.newHashSet()).add(d);
model2matchedFields.computeIfAbsent(modelName, k -> Sets.newHashSet())
.addAll(matchedFieldsInQuery);
}
});
});
Optional<Map.Entry<String, Set<DimSchemaResp>>> modelEntry = model2dims.entrySet()
.stream().filter(entry -> entry.getValue().size() == fields.size()).findFirst();
if (modelEntry.isPresent()) {
ontologyQuery.getDimensionMap().put(modelEntry.get().getKey(),
modelEntry.get().getValue());
ontologyQuery.getModelMap().put(modelEntry.get().getKey(),
ontology.getModelMap().get(modelEntry.get().getKey()));
Optional<String> bestModel = model2matchedFields.entrySet().stream()
.filter(entry -> entry.getValue().size() == fields.size())
.map(Map.Entry::getKey).findFirst();
if (bestModel.isPresent()) {
String modelName = bestModel.get();
ontologyQuery.getDimensionMap().put(modelName, model2dims.get(modelName));
ontologyQuery.getModelMap().put(modelName,
ontology.getModelMap().get(modelName));
fields.clear();
}
}
Expand All @@ -258,13 +310,20 @@ private OntologyQuery buildOntologyQuery(Ontology ontology, List<String> queryFi
String modelName = entry.getKey();
if (!ontologyQuery.getDimensionMap().containsKey(modelName)) {
entry.getValue().forEach(d -> {
if (fields.contains(d.getName()) || fields.contains(d.getBizName())) {
Set<String> allNamesForDimension = new HashSet<>();
allNamesForDimension.add(d.getName());
allNamesForDimension.add(d.getBizName());
if (StringUtils.isNotBlank(d.getAlias())) {
allNamesForDimension.addAll(SchemaItem.getAliasList(d.getAlias()));
}
Set<String> matchedFields = new HashSet<>(fields);
matchedFields.retainAll(allNamesForDimension);
if (!matchedFields.isEmpty()) {
ontologyQuery.getModelMap().put(modelName,
ontology.getModelMap().get(modelName));
ontologyQuery.getDimensionMap()
.computeIfAbsent(modelName, k -> Sets.newHashSet()).add(d);
fields.remove(d.getName());
fields.remove(d.getBizName());
fields.removeAll(matchedFields);
}
});
}
Expand Down
69 changes: 24 additions & 45 deletions ...c/main/java/com/tencent/supersonic/headless/server/service/impl/DimensionServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -314,58 +314,37 @@ private List<DimensionResp> convertList(List<DimensionDO> dimensionDOS) {

@Override
public List<String> mockAlias(DimensionReq dimensionReq, String mockType, User user) {
String mockAlias = aliasGenerateHelper.generateAlias(mockType, dimensionReq.getName(),
dimensionReq.getBizName(), "", dimensionReq.getDescription());
String ret = aliasGenerateHelper.extractJsonStringFromAiMessage(mockAlias);
return JSONObject.parseObject(ret, new TypeReference<List<String>>() {});
String name = dimensionReq.getName();
ModelResp modelResp = modelService.getModel(dimensionReq.getModelId());
if (Objects.isNull(modelResp)) {
return Lists.newArrayList();
}
String modelName = modelResp.getName();
if (StringUtils.isBlank(name) || StringUtils.isBlank(modelName)) {
return Lists.newArrayList();
}
return aliasGenerateHelper.generateAlias(modelName, name, "dimension");
}

@Override
public List<DimValueMap> mockDimensionValueAlias(DimensionReq dimensionReq, User user) {
ModelResp modelResp = modelService.getModel(dimensionReq.getModelId());
ModelDetail modelDetail = modelResp.getModelDetail();
String sqlQuery = modelDetail.getSqlQuery();
if (ModelDefineType.TABLE_QUERY.getName().equals(modelDetail.getQueryType())) {
String tableQuery = modelDetail.getTableQuery();
sqlQuery = "SELECT * FROM " + tableQuery;
String json = JsonUtil.toString(dimensionReq.getDimValueMaps());
if (StringUtils.isEmpty(json)) {
return Collections.emptyList();
}
DatabaseResp database = databaseService.getDatabase(modelResp.getDatabaseId());

String sql = "select ai_talk." + dimensionReq.getBizName() + " from (" + sqlQuery
+ ") as ai_talk group by ai_talk." + dimensionReq.getBizName();
SemanticQueryResp semanticQueryResp = databaseService.executeSql(sql, database);
List<Map<String, Object>> resultList = semanticQueryResp.getResultList();
List<String> valueList = new ArrayList<>();
for (Map<String, Object> stringObjectMap : resultList) {
String value = String.valueOf(stringObjectMap.get(dimensionReq.getBizName()));
valueList.add(value);
}
String json = aliasGenerateHelper.generateDimensionValueAlias(JSON.toJSONString(valueList));
log.info("return llm res is :{}", json);
String ret = aliasGenerateHelper.extractJsonStringFromAiMessage(json);
JSONObject jsonObject = JSON.parseObject(ret);
List<DimValueMap> dimValueMapsResp = new ArrayList<>();
int i = 0;
for (Map<String, Object> stringObjectMap : resultList) {
DimValueMap dimValueMap = new DimValueMap();
dimValueMap.setTechName(String.valueOf(stringObjectMap.get(dimensionReq.getBizName())));
try {
String tran = jsonObject.getJSONArray("tran").getString(i);
dimValueMap.setBizName(tran);
} catch (Exception exception) {
dimValueMap.setBizName("");
}
try {
dimValueMap.setAlias(jsonObject.getJSONObject("alias")
.getJSONArray(stringObjectMap.get(dimensionReq.getBizName()) + "")
.toJavaList(String.class));
} catch (Exception exception) {
dimValueMap.setAlias(null);
String mockAlias = aliasGenerateHelper.generateDimensionValueAlias(json);
String ret = aliasGenerateHelper.extractJsonStringFromAiMessage(mockAlias);
Map<String, List<String>> aliasMap =
(Map<String, List<String>>) JSONObject.parseObject(ret, Map.class).get("alias");
List<DimValueMap> dimValueMaps = dimensionReq.getDimValueMaps();
if (aliasMap != null) {
for (DimValueMap dimValueMap : dimValueMaps) {
if (aliasMap.containsKey(dimValueMap.getBizName())) {
dimValueMap.setAlias(aliasMap.get(dimValueMap.getBizName()));
}
}
dimValueMapsResp.add(dimValueMap);
i++;
}
return dimValueMapsResp;
return dimValueMaps;
}

private void checkExist(List<DimensionReq> dimensionReqs) {
Expand Down
16 changes: 10 additions & 6 deletions .../src/main/java/com/tencent/supersonic/headless/server/service/impl/MetricServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -473,12 +473,16 @@ public MetricResp getMetric(Long id) {

@Override
public List<String> mockAlias(MetricBaseReq metricReq, String mockType, User user) {

String mockAlias = aliasGenerateHelper.generateAlias(mockType, metricReq.getName(),
metricReq.getBizName(), "", metricReq.getDescription());
String ret = mockAlias.replaceAll("`", "").replace("json", "").replace("\n", "")
.replace(" ", "");
return JSONObject.parseObject(ret, new TypeReference<List<String>>() {});
String name = metricReq.getName();
ModelResp modelResp = modelService.getModel(metricReq.getModelId());
if (Objects.isNull(modelResp)) {
return Lists.newArrayList();
}
String modelName = modelResp.getName();
if (StringUtils.isBlank(name) || StringUtils.isBlank(modelName)) {
return Lists.newArrayList();
}
return aliasGenerateHelper.generateAlias(modelName, name, "metric");
}

@Override
Expand Down
Loading
Loading