Skip to content

feature: added flag which will add DEFECT_COUNT column to dq execute api #4537

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gs-jp1 merged 1 commit into from
Mar 11, 2026
+326 −34
Merged

feature: added flag which will add DEFECT_COUNT column to dq execute api #4537

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions ...pi/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecute.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -384,7 +384,7 @@ private org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.Lambd
{
return DataQualityLambdaGenerator.generateRelationValidationMainQueryRowCount(pureModel, dataQualityExecuteInput.packagePath);
}
return DataQualityLambdaGenerator.generateLambda(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.getValidationNames(), dataQualityExecuteInput.runQuery, dataQualityExecuteInput.defectsLimit, dataQualityExecuteInput.enrichDQColumns, queryType);
return DataQualityLambdaGenerator.generateLambda(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.getValidationNames(), dataQualityExecuteInput.runQuery, dataQualityExecuteInput.defectsLimit, dataQualityExecuteInput.enrichDQColumns, dataQualityExecuteInput.includeTotalDefectCount, queryType);
}

@POST
Expand All @@ -398,7 +398,7 @@ public Response lambda(@Context HttpServletRequest request, DataQualityExecuteTr
// 1. load pure model from PureModelContext
PureModel pureModel = this.modelManager.loadModel(dataQualityExecuteInput.model, dataQualityExecuteInput.clientVersion, identity, null);
// 2. call DQ PURE func to generate lambda
org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambda(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.getValidationNames(), dataQualityExecuteInput.runQuery, dataQualityExecuteInput.defectsLimit, dataQualityExecuteInput.enrichDQColumns, null);
org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambda(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.getValidationNames(), dataQualityExecuteInput.runQuery, dataQualityExecuteInput.defectsLimit, dataQualityExecuteInput.enrichDQColumns, dataQualityExecuteInput.includeTotalDefectCount, null);
LambdaFunction lambda = DataQualityLambdaGenerator.transformLambda(dqLambdaFunction, pureModel, this.extensions);
return ManageConstantResult.manageResult(identity.getName(), lambda, objectMapper);
}
Expand Down
1 change: 1 addition & 0 deletions ...n/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecuteTrialInput.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ public class DataQualityExecuteTrialInput
@JsonProperty
public String packagePath;
public Integer defectsLimit;
public boolean includeTotalDefectCount = false;
public List<ParameterValue> lambdaParameterValues;
@Deprecated
public String validationName; //this should be replaced by validationNames - remove once all usages migrated
Expand Down
12 changes: 6 additions & 6 deletions .../main/java/org/finos/legend/engine/generation/dataquality/DataQualityLambdaGenerator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,21 +43,21 @@
public class DataQualityLambdaGenerator
{

public static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction<Object> generateLambda(PureModel pureModel, String qualifiedPath, Set<String> validationNames, Boolean runQuery, Integer resultLimit, boolean enrichDQColumns, String queryType)
public static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction<Object> generateLambda(PureModel pureModel, String qualifiedPath, Set<String> validationNames, Boolean runQuery, Integer resultLimit, boolean enrichDQColumns, boolean includeTotalDefectCount, String queryType)
{
PackageableElement packageableElement = pureModel.getPackageableElement(qualifiedPath);
return generateLambda(pureModel, packageableElement, validationNames, runQuery, resultLimit, enrichDQColumns, queryType);
return generateLambda(pureModel, packageableElement, validationNames, runQuery, resultLimit, enrichDQColumns, includeTotalDefectCount, queryType);
}

public static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction<Object> generateLambda(PureModel pureModel, PackageableElement packageableElement, Set<String> validationNames, Boolean runQuery, Integer resultLimit, boolean enrichDQColumns, String queryType)
public static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction<Object> generateLambda(PureModel pureModel, PackageableElement packageableElement, Set<String> validationNames, Boolean runQuery, Integer resultLimit, boolean enrichDQColumns, boolean includeTotalDefectCount, String queryType)
{
if (packageableElement instanceof Root_meta_external_dataquality_DataQuality)
{
return generateModelConstraintLambda(pureModel, null, (Root_meta_external_dataquality_DataQuality) packageableElement);
}
else if (packageableElement instanceof Root_meta_external_dataquality_DataQualityRelationValidation)
{
return generateRelationValidationLambda(pureModel, (Root_meta_external_dataquality_DataQualityRelationValidation) packageableElement, validationNames, runQuery, resultLimit, enrichDQColumns, queryType);
return generateRelationValidationLambda(pureModel, (Root_meta_external_dataquality_DataQualityRelationValidation) packageableElement, validationNames, runQuery, resultLimit, enrichDQColumns, includeTotalDefectCount, queryType);
}
throw new EngineException("Unsupported Dataquality element! " + packageableElement.getClass().getSimpleName(), ExceptionCategory.USER_EXECUTION_ERROR);
}
Expand All @@ -67,7 +67,7 @@ private static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.functio
return core_dataquality_generation_dataquality.Root_meta_external_dataquality_generateDataQualityQuery_DataQuality_1__Integer_$0_1$__LambdaFunction_1_(packageableElement, Objects.isNull(queryLimit) ? null : queryLimit.longValue(), pureModel.getExecutionSupport());
}

private static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction<Object> generateRelationValidationLambda(PureModel pureModel, Root_meta_external_dataquality_DataQualityRelationValidation packageableElement, Set<String> validationNames, Boolean runQuery, Integer resultLimit, boolean enrichDQColumns, String queryType)
private static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction<Object> generateRelationValidationLambda(PureModel pureModel, Root_meta_external_dataquality_DataQualityRelationValidation packageableElement, Set<String> validationNames, Boolean runQuery, Integer resultLimit, boolean enrichDQColumns, boolean includeTotalDefectCount, String queryType)
{
if (isNotBlank(queryType) && !core_dataquality_generation_dataquality.Root_meta_external_dataquality_isEndingWithFromFunction_FunctionDefinition_1__Boolean_1_(packageableElement._query(), pureModel.getExecutionSupport()))
{
Expand All @@ -77,7 +77,7 @@ private static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.functio
{
return (org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction<Object>) packageableElement._query();
}
return (org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction<Object>) core_dataquality_generation_dataquality.Root_meta_external_dataquality_generateDataqualityRelationValidationLambda_DataQualityRelationValidation_1__String_MANY__Integer_$0_1$__Boolean_1__LambdaFunction_1_(packageableElement, Sets.immutable.ofAll(validationNames), Objects.isNull(resultLimit) ? null : resultLimit.longValue(), enrichDQColumns, pureModel.getExecutionSupport());
return (org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction<Object>) core_dataquality_generation_dataquality.Root_meta_external_dataquality_generateDataqualityRelationValidationLambda_DataQualityRelationValidation_1__String_MANY__Integer_$0_1$__Boolean_1__Boolean_1__Boolean_1__Boolean_1__LambdaFunction_1_(packageableElement, Sets.immutable.ofAll(validationNames), Objects.isNull(resultLimit) ? null : resultLimit.longValue(), enrichDQColumns, false, false, includeTotalDefectCount, pureModel.getExecutionSupport());
}

public static LambdaFunction transformLambda(org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction<?> lambda, PureModel pureModel, Function<PureModel, RichIterable<? extends Root_meta_pure_extension_Extension>> extensions)
Expand Down
2 changes: 1 addition & 1 deletion ...egend/engine/generation/dataquality/DataQualityValidationArtifactGenerationExtension.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ public List<Artifact> generate(PackageableElement packageableElement, PureModel

Function<PureModel, RichIterable<? extends Root_meta_pure_extension_Extension>> routerExtensions = (PureModel p) -> PureCoreExtensionLoader.extensions().flatCollect(e -> e.extraPureCoreExtensions(p.getExecutionSupport()));
// 1. call DQ PURE func to generate lambda
LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambda(pureModel, packageableElement, null, false, null, true, "generate artifacts for dq");
LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambda(pureModel, packageableElement, null, false, null, true, false, "generate artifacts for dq");
// 2. Generate Plan from the lambda generated in the previous step
SingleExecutionPlan singleExecutionPlan = PlanGenerator.generateExecutionPlan(dqLambdaFunction, null, null, null, pureModel, clientVersion, PlanPlatform.JAVA, null,
routerExtensions.apply(pureModel), LegendPlanTransformers.transformers);// since lambda has from function we dont need mapping, runtime and context
Expand Down
6 changes: 3 additions & 3 deletions ...t/java/org/finos/legend/engine/generation/dataquality/TestDataQualityLambdaGenerator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,15 +181,15 @@ private String generateLambda(String modelString)
PureModelContextData modelData = loadWithModel(modelString);
PureModel model = Compiler.compile(modelData, DeploymentMode.TEST_IGNORE_FUNCTION_MATCH, Identity.getAnonymousIdentity().getName());
Function<PureModel, RichIterable<? extends Root_meta_pure_extension_Extension>> routerExtensions = extensions();
LambdaFunction<?> lambdaFunction = DataQualityLambdaGenerator.generateLambda(model, "meta::dataquality::Validation", null, false, null, true, "test");
LambdaFunction<?> lambdaFunction = DataQualityLambdaGenerator.generateLambda(model, "meta::dataquality::Validation", null, false, null, true, false, "test");
return DataQualityLambdaGenerator.transformLambdaAsJson(lambdaFunction, model, routerExtensions);
}

private LambdaFunction generateLambdaDefinition(String modelString, String validationName, String queryType)
private LambdaFunction<?> generateLambdaDefinition(String modelString, String validationName, String queryType)
{
PureModelContextData modelData = loadWithModel(modelString);
PureModel model = Compiler.compile(modelData, DeploymentMode.TEST_IGNORE_FUNCTION_MATCH, Identity.getAnonymousIdentity().getName());
return DataQualityLambdaGenerator.generateLambda(model, "meta::dataquality::Validation", Sets.mutable.of(validationName), false, null, true, queryType);
return DataQualityLambdaGenerator.generateLambda(model, "meta::dataquality::Validation", Sets.mutable.of(validationName), false, null, true, false, queryType);
}

private static PureModelContextData loadWithModel(String code)
Expand Down
Loading
Loading