Skip to content

Commit 9b2c3ab

Browse files
dramaladramala
authored
feature: added flag which will add DEFECT_COUNT column to dq execute api (#4537)
1 parent 3c8cc8b commit 9b2c3ab

File tree

8 files changed

+326
-34
lines changed

8 files changed

+326
-34
lines changed

‎legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecute.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -384,7 +384,7 @@ private org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.Lambd
384384
{
385385
return DataQualityLambdaGenerator.generateRelationValidationMainQueryRowCount(pureModel, dataQualityExecuteInput.packagePath);
386386
}
387-
return DataQualityLambdaGenerator.generateLambda(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.getValidationNames(), dataQualityExecuteInput.runQuery, dataQualityExecuteInput.defectsLimit, dataQualityExecuteInput.enrichDQColumns, queryType);
387+
return DataQualityLambdaGenerator.generateLambda(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.getValidationNames(), dataQualityExecuteInput.runQuery, dataQualityExecuteInput.defectsLimit, dataQualityExecuteInput.enrichDQColumns, dataQualityExecuteInput.includeTotalDefectCount, queryType);
388388
}
389389

390390
@POST
@@ -398,7 +398,7 @@ public Response lambda(@Context HttpServletRequest request, DataQualityExecuteTr
398398
// 1. load pure model from PureModelContext
399399
PureModel pureModel = this.modelManager.loadModel(dataQualityExecuteInput.model, dataQualityExecuteInput.clientVersion, identity, null);
400400
// 2. call DQ PURE func to generate lambda
401-
org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambda(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.getValidationNames(), dataQualityExecuteInput.runQuery, dataQualityExecuteInput.defectsLimit, dataQualityExecuteInput.enrichDQColumns, null);
401+
org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambda(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.getValidationNames(), dataQualityExecuteInput.runQuery, dataQualityExecuteInput.defectsLimit, dataQualityExecuteInput.enrichDQColumns, dataQualityExecuteInput.includeTotalDefectCount, null);
402402
LambdaFunction lambda = DataQualityLambdaGenerator.transformLambda(dqLambdaFunction, pureModel, this.extensions);
403403
return ManageConstantResult.manageResult(identity.getName(), lambda, objectMapper);
404404
}

‎legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecuteTrialInput.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ public class DataQualityExecuteTrialInput
3434
@JsonProperty
3535
public String packagePath;
3636
public Integer defectsLimit;
37+
public boolean includeTotalDefectCount = false;
3738
public List<ParameterValue> lambdaParameterValues;
3839
@Deprecated
3940
public String validationName; //this should be replaced by validationNames - remove once all usages migrated

‎legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/main/java/org/finos/legend/engine/generation/dataquality/DataQualityLambdaGenerator.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -43,21 +43,21 @@
4343
public class DataQualityLambdaGenerator
4444
{
4545

46-
public static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction<Object> generateLambda(PureModel pureModel, String qualifiedPath, Set<String> validationNames, Boolean runQuery, Integer resultLimit, boolean enrichDQColumns, String queryType)
46+
public static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction<Object> generateLambda(PureModel pureModel, String qualifiedPath, Set<String> validationNames, Boolean runQuery, Integer resultLimit, boolean enrichDQColumns, boolean includeTotalDefectCount, String queryType)
4747
{
4848
PackageableElement packageableElement = pureModel.getPackageableElement(qualifiedPath);
49-
return generateLambda(pureModel, packageableElement, validationNames, runQuery, resultLimit, enrichDQColumns, queryType);
49+
return generateLambda(pureModel, packageableElement, validationNames, runQuery, resultLimit, enrichDQColumns, includeTotalDefectCount, queryType);
5050
}
5151

52-
public static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction<Object> generateLambda(PureModel pureModel, PackageableElement packageableElement, Set<String> validationNames, Boolean runQuery, Integer resultLimit, boolean enrichDQColumns, String queryType)
52+
public static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction<Object> generateLambda(PureModel pureModel, PackageableElement packageableElement, Set<String> validationNames, Boolean runQuery, Integer resultLimit, boolean enrichDQColumns, boolean includeTotalDefectCount, String queryType)
5353
{
5454
if (packageableElement instanceof Root_meta_external_dataquality_DataQuality)
5555
{
5656
return generateModelConstraintLambda(pureModel, null, (Root_meta_external_dataquality_DataQuality) packageableElement);
5757
}
5858
else if (packageableElement instanceof Root_meta_external_dataquality_DataQualityRelationValidation)
5959
{
60-
return generateRelationValidationLambda(pureModel, (Root_meta_external_dataquality_DataQualityRelationValidation) packageableElement, validationNames, runQuery, resultLimit, enrichDQColumns, queryType);
60+
return generateRelationValidationLambda(pureModel, (Root_meta_external_dataquality_DataQualityRelationValidation) packageableElement, validationNames, runQuery, resultLimit, enrichDQColumns, includeTotalDefectCount, queryType);
6161
}
6262
throw new EngineException("Unsupported Dataquality element! " + packageableElement.getClass().getSimpleName(), ExceptionCategory.USER_EXECUTION_ERROR);
6363
}
@@ -67,7 +67,7 @@ private static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.functio
6767
return core_dataquality_generation_dataquality.Root_meta_external_dataquality_generateDataQualityQuery_DataQuality_1__Integer_$0_1$__LambdaFunction_1_(packageableElement, Objects.isNull(queryLimit) ? null : queryLimit.longValue(), pureModel.getExecutionSupport());
6868
}
6969

70-
private static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction<Object> generateRelationValidationLambda(PureModel pureModel, Root_meta_external_dataquality_DataQualityRelationValidation packageableElement, Set<String> validationNames, Boolean runQuery, Integer resultLimit, boolean enrichDQColumns, String queryType)
70+
private static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction<Object> generateRelationValidationLambda(PureModel pureModel, Root_meta_external_dataquality_DataQualityRelationValidation packageableElement, Set<String> validationNames, Boolean runQuery, Integer resultLimit, boolean enrichDQColumns, boolean includeTotalDefectCount, String queryType)
7171
{
7272
if (isNotBlank(queryType) && !core_dataquality_generation_dataquality.Root_meta_external_dataquality_isEndingWithFromFunction_FunctionDefinition_1__Boolean_1_(packageableElement._query(), pureModel.getExecutionSupport()))
7373
{
@@ -77,7 +77,7 @@ private static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.functio
7777
{
7878
return (org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction<Object>) packageableElement._query();
7979
}
80-
return (org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction<Object>) core_dataquality_generation_dataquality.Root_meta_external_dataquality_generateDataqualityRelationValidationLambda_DataQualityRelationValidation_1__String_MANY__Integer_$0_1$__Boolean_1__LambdaFunction_1_(packageableElement, Sets.immutable.ofAll(validationNames), Objects.isNull(resultLimit) ? null : resultLimit.longValue(), enrichDQColumns, pureModel.getExecutionSupport());
80+
return (org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction<Object>) core_dataquality_generation_dataquality.Root_meta_external_dataquality_generateDataqualityRelationValidationLambda_DataQualityRelationValidation_1__String_MANY__Integer_$0_1$__Boolean_1__Boolean_1__Boolean_1__Boolean_1__LambdaFunction_1_(packageableElement, Sets.immutable.ofAll(validationNames), Objects.isNull(resultLimit) ? null : resultLimit.longValue(), enrichDQColumns, false, false, includeTotalDefectCount, pureModel.getExecutionSupport());
8181
}
8282

8383
public static LambdaFunction transformLambda(org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction<?> lambda, PureModel pureModel, Function<PureModel, RichIterable<? extends Root_meta_pure_extension_Extension>> extensions)

‎legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/main/java/org/finos/legend/engine/generation/dataquality/DataQualityValidationArtifactGenerationExtension.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ public List<Artifact> generate(PackageableElement packageableElement, PureModel
8080

8181
Function<PureModel, RichIterable<? extends Root_meta_pure_extension_Extension>> routerExtensions = (PureModel p) -> PureCoreExtensionLoader.extensions().flatCollect(e -> e.extraPureCoreExtensions(p.getExecutionSupport()));
8282
// 1. call DQ PURE func to generate lambda
83-
LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambda(pureModel, packageableElement, null, false, null, true, "generate artifacts for dq");
83+
LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambda(pureModel, packageableElement, null, false, null, true, false, "generate artifacts for dq");
8484
// 2. Generate Plan from the lambda generated in the previous step
8585
SingleExecutionPlan singleExecutionPlan = PlanGenerator.generateExecutionPlan(dqLambdaFunction, null, null, null, pureModel, clientVersion, PlanPlatform.JAVA, null,
8686
routerExtensions.apply(pureModel), LegendPlanTransformers.transformers);// since lambda has from function we dont need mapping, runtime and context

‎legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/test/java/org/finos/legend/engine/generation/dataquality/TestDataQualityLambdaGenerator.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -181,15 +181,15 @@ private String generateLambda(String modelString)
181181
PureModelContextData modelData = loadWithModel(modelString);
182182
PureModel model = Compiler.compile(modelData, DeploymentMode.TEST_IGNORE_FUNCTION_MATCH, Identity.getAnonymousIdentity().getName());
183183
Function<PureModel, RichIterable<? extends Root_meta_pure_extension_Extension>> routerExtensions = extensions();
184-
LambdaFunction<?> lambdaFunction = DataQualityLambdaGenerator.generateLambda(model, "meta::dataquality::Validation", null, false, null, true, "test");
184+
LambdaFunction<?> lambdaFunction = DataQualityLambdaGenerator.generateLambda(model, "meta::dataquality::Validation", null, false, null, true, false, "test");
185185
return DataQualityLambdaGenerator.transformLambdaAsJson(lambdaFunction, model, routerExtensions);
186186
}
187187

188-
private LambdaFunction generateLambdaDefinition(String modelString, String validationName, String queryType)
188+
private LambdaFunction<?> generateLambdaDefinition(String modelString, String validationName, String queryType)
189189
{
190190
PureModelContextData modelData = loadWithModel(modelString);
191191
PureModel model = Compiler.compile(modelData, DeploymentMode.TEST_IGNORE_FUNCTION_MATCH, Identity.getAnonymousIdentity().getName());
192-
return DataQualityLambdaGenerator.generateLambda(model, "meta::dataquality::Validation", Sets.mutable.of(validationName), false, null, true, queryType);
192+
return DataQualityLambdaGenerator.generateLambda(model, "meta::dataquality::Validation", Sets.mutable.of(validationName), false, null, true, false, queryType);
193193
}
194194

195195
private static PureModelContextData loadWithModel(String code)

0 commit comments

Comments
 (0)