diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecute.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecute.java index 3b46d809b08..ec10a0f5a89 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecute.java +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecute.java @@ -384,7 +384,7 @@ private org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.Lambd { return DataQualityLambdaGenerator.generateRelationValidationMainQueryRowCount(pureModel, dataQualityExecuteInput.packagePath); } - return DataQualityLambdaGenerator.generateLambda(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.getValidationNames(), dataQualityExecuteInput.runQuery, dataQualityExecuteInput.defectsLimit, dataQualityExecuteInput.enrichDQColumns, queryType); + return DataQualityLambdaGenerator.generateLambda(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.getValidationNames(), dataQualityExecuteInput.runQuery, dataQualityExecuteInput.defectsLimit, dataQualityExecuteInput.enrichDQColumns, dataQualityExecuteInput.includeTotalDefectCount, queryType); } @POST @@ -398,7 +398,7 @@ public Response lambda(@Context HttpServletRequest request, DataQualityExecuteTr // 1. load pure model from PureModelContext PureModel pureModel = this.modelManager.loadModel(dataQualityExecuteInput.model, dataQualityExecuteInput.clientVersion, identity, null); // 2. call DQ PURE func to generate lambda - org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambda(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.getValidationNames(), dataQualityExecuteInput.runQuery, dataQualityExecuteInput.defectsLimit, dataQualityExecuteInput.enrichDQColumns, null); + org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambda(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.getValidationNames(), dataQualityExecuteInput.runQuery, dataQualityExecuteInput.defectsLimit, dataQualityExecuteInput.enrichDQColumns, dataQualityExecuteInput.includeTotalDefectCount, null); LambdaFunction lambda = DataQualityLambdaGenerator.transformLambda(dqLambdaFunction, pureModel, this.extensions); return ManageConstantResult.manageResult(identity.getName(), lambda, objectMapper); } diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecuteTrialInput.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecuteTrialInput.java index 5542dc0508b..17399e62ee5 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecuteTrialInput.java +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecuteTrialInput.java @@ -34,6 +34,7 @@ public class DataQualityExecuteTrialInput @JsonProperty public String packagePath; public Integer defectsLimit; + public boolean includeTotalDefectCount = false; public List lambdaParameterValues; @Deprecated public String validationName; //this should be replaced by validationNames - remove once all usages migrated diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/main/java/org/finos/legend/engine/generation/dataquality/DataQualityLambdaGenerator.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/main/java/org/finos/legend/engine/generation/dataquality/DataQualityLambdaGenerator.java index d458a2149f6..341bb7ce555 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/main/java/org/finos/legend/engine/generation/dataquality/DataQualityLambdaGenerator.java +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/main/java/org/finos/legend/engine/generation/dataquality/DataQualityLambdaGenerator.java @@ -43,13 +43,13 @@ public class DataQualityLambdaGenerator { - public static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction generateLambda(PureModel pureModel, String qualifiedPath, Set validationNames, Boolean runQuery, Integer resultLimit, boolean enrichDQColumns, String queryType) + public static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction generateLambda(PureModel pureModel, String qualifiedPath, Set validationNames, Boolean runQuery, Integer resultLimit, boolean enrichDQColumns, boolean includeTotalDefectCount, String queryType) { PackageableElement packageableElement = pureModel.getPackageableElement(qualifiedPath); - return generateLambda(pureModel, packageableElement, validationNames, runQuery, resultLimit, enrichDQColumns, queryType); + return generateLambda(pureModel, packageableElement, validationNames, runQuery, resultLimit, enrichDQColumns, includeTotalDefectCount, queryType); } - public static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction generateLambda(PureModel pureModel, PackageableElement packageableElement, Set validationNames, Boolean runQuery, Integer resultLimit, boolean enrichDQColumns, String queryType) + public static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction generateLambda(PureModel pureModel, PackageableElement packageableElement, Set validationNames, Boolean runQuery, Integer resultLimit, boolean enrichDQColumns, boolean includeTotalDefectCount, String queryType) { if (packageableElement instanceof Root_meta_external_dataquality_DataQuality) { @@ -57,7 +57,7 @@ public static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function } else if (packageableElement instanceof Root_meta_external_dataquality_DataQualityRelationValidation) { - return generateRelationValidationLambda(pureModel, (Root_meta_external_dataquality_DataQualityRelationValidation) packageableElement, validationNames, runQuery, resultLimit, enrichDQColumns, queryType); + return generateRelationValidationLambda(pureModel, (Root_meta_external_dataquality_DataQualityRelationValidation) packageableElement, validationNames, runQuery, resultLimit, enrichDQColumns, includeTotalDefectCount, queryType); } throw new EngineException("Unsupported Dataquality element! " + packageableElement.getClass().getSimpleName(), ExceptionCategory.USER_EXECUTION_ERROR); } @@ -67,7 +67,7 @@ private static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.functio return core_dataquality_generation_dataquality.Root_meta_external_dataquality_generateDataQualityQuery_DataQuality_1__Integer_$0_1$__LambdaFunction_1_(packageableElement, Objects.isNull(queryLimit) ? null : queryLimit.longValue(), pureModel.getExecutionSupport()); } - private static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction generateRelationValidationLambda(PureModel pureModel, Root_meta_external_dataquality_DataQualityRelationValidation packageableElement, Set validationNames, Boolean runQuery, Integer resultLimit, boolean enrichDQColumns, String queryType) + private static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction generateRelationValidationLambda(PureModel pureModel, Root_meta_external_dataquality_DataQualityRelationValidation packageableElement, Set validationNames, Boolean runQuery, Integer resultLimit, boolean enrichDQColumns, boolean includeTotalDefectCount, String queryType) { if (isNotBlank(queryType) && !core_dataquality_generation_dataquality.Root_meta_external_dataquality_isEndingWithFromFunction_FunctionDefinition_1__Boolean_1_(packageableElement._query(), pureModel.getExecutionSupport())) { @@ -77,7 +77,7 @@ private static org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.functio { return (org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction) packageableElement._query(); } - return (org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction) core_dataquality_generation_dataquality.Root_meta_external_dataquality_generateDataqualityRelationValidationLambda_DataQualityRelationValidation_1__String_MANY__Integer_$0_1$__Boolean_1__LambdaFunction_1_(packageableElement, Sets.immutable.ofAll(validationNames), Objects.isNull(resultLimit) ? null : resultLimit.longValue(), enrichDQColumns, pureModel.getExecutionSupport()); + return (org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction) core_dataquality_generation_dataquality.Root_meta_external_dataquality_generateDataqualityRelationValidationLambda_DataQualityRelationValidation_1__String_MANY__Integer_$0_1$__Boolean_1__Boolean_1__Boolean_1__Boolean_1__LambdaFunction_1_(packageableElement, Sets.immutable.ofAll(validationNames), Objects.isNull(resultLimit) ? null : resultLimit.longValue(), enrichDQColumns, false, false, includeTotalDefectCount, pureModel.getExecutionSupport()); } public static LambdaFunction transformLambda(org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction lambda, PureModel pureModel, Function> extensions) diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/main/java/org/finos/legend/engine/generation/dataquality/DataQualityValidationArtifactGenerationExtension.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/main/java/org/finos/legend/engine/generation/dataquality/DataQualityValidationArtifactGenerationExtension.java index cc48ab3d0d8..3fad4190f47 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/main/java/org/finos/legend/engine/generation/dataquality/DataQualityValidationArtifactGenerationExtension.java +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/main/java/org/finos/legend/engine/generation/dataquality/DataQualityValidationArtifactGenerationExtension.java @@ -80,7 +80,7 @@ public List generate(PackageableElement packageableElement, PureModel Function> routerExtensions = (PureModel p) -> PureCoreExtensionLoader.extensions().flatCollect(e -> e.extraPureCoreExtensions(p.getExecutionSupport())); // 1. call DQ PURE func to generate lambda - LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambda(pureModel, packageableElement, null, false, null, true, "generate artifacts for dq"); + LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambda(pureModel, packageableElement, null, false, null, true, false, "generate artifacts for dq"); // 2. Generate Plan from the lambda generated in the previous step SingleExecutionPlan singleExecutionPlan = PlanGenerator.generateExecutionPlan(dqLambdaFunction, null, null, null, pureModel, clientVersion, PlanPlatform.JAVA, null, routerExtensions.apply(pureModel), LegendPlanTransformers.transformers);// since lambda has from function we dont need mapping, runtime and context diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/test/java/org/finos/legend/engine/generation/dataquality/TestDataQualityLambdaGenerator.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/test/java/org/finos/legend/engine/generation/dataquality/TestDataQualityLambdaGenerator.java index e6b9c675464..eab858f6d1c 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/test/java/org/finos/legend/engine/generation/dataquality/TestDataQualityLambdaGenerator.java +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/test/java/org/finos/legend/engine/generation/dataquality/TestDataQualityLambdaGenerator.java @@ -181,15 +181,15 @@ private String generateLambda(String modelString) PureModelContextData modelData = loadWithModel(modelString); PureModel model = Compiler.compile(modelData, DeploymentMode.TEST_IGNORE_FUNCTION_MATCH, Identity.getAnonymousIdentity().getName()); Function> routerExtensions = extensions(); - LambdaFunction lambdaFunction = DataQualityLambdaGenerator.generateLambda(model, "meta::dataquality::Validation", null, false, null, true, "test"); + LambdaFunction lambdaFunction = DataQualityLambdaGenerator.generateLambda(model, "meta::dataquality::Validation", null, false, null, true, false, "test"); return DataQualityLambdaGenerator.transformLambdaAsJson(lambdaFunction, model, routerExtensions); } - private LambdaFunction generateLambdaDefinition(String modelString, String validationName, String queryType) + private LambdaFunction generateLambdaDefinition(String modelString, String validationName, String queryType) { PureModelContextData modelData = loadWithModel(modelString); PureModel model = Compiler.compile(modelData, DeploymentMode.TEST_IGNORE_FUNCTION_MATCH, Identity.getAnonymousIdentity().getName()); - return DataQualityLambdaGenerator.generateLambda(model, "meta::dataquality::Validation", Sets.mutable.of(validationName), false, null, true, queryType); + return DataQualityLambdaGenerator.generateLambda(model, "meta::dataquality::Validation", Sets.mutable.of(validationName), false, null, true, false, queryType); } private static PureModelContextData loadWithModel(String code) diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure-test/src/main/resources/core_dataquality_test/dataquality_test.pure b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure-test/src/main/resources/core_dataquality_test/dataquality_test.pure index f843def9650..9c81255f9e1 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure-test/src/main/resources/core_dataquality_test/dataquality_test.pure +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure-test/src/main/resources/core_dataquality_test/dataquality_test.pure @@ -363,6 +363,115 @@ function <> meta::external::dataquality::tests::testLambdaGeneration_ } // -------------------------------------- row level tests for backward compatibility end ------------------------------------------------------------------------------------------------ +// ----------------------- includeTotalDefectCount tests --------------------------------------------------------------------------------------------------- + +function <> meta::external::dataquality::tests::testLambdaGeneration_relationValidation_row_level_withTotalDefectCountAndLimit():Boolean[1] +{ + doRelationTestWithTotalDefectCountAndLimit( + '###DataQualityValidation' + + ' DataQualityRelationValidation meta::external::dataquality::tests::domain::RelationValidation' + + ' {' + + ' query: #>{meta::external::dataquality::tests::domain::db.personTable}#->select(~[FIRSTNAME, LASTNAME])->extend(~fullname:x|[$x.FIRSTNAME->toOne(), $x.LASTNAME->toOne()]->joinStrings())->from(meta::external::dataquality::tests::domain::DataQualityRuntime);' + + ' validations: [' + + ' {' + + ' name: \'validFirstName\';' + + ' description: \'first name should not be empty\';' + + ' assertion: row2|$row2.FIRSTNAME->isNotEmpty();' + + ' type: ROW_LEVEL;' + + ' }' + + ' ];' + + ' }', + 'validFirstName', + {|#>{meta::external::dataquality::tests::domain::db.personTable}#->select(~[FIRSTNAME, LASTNAME])->extend(~fullname:x|[$x.FIRSTNAME->meta::pure::functions::multiplicity::toOne(), $x.LASTNAME->meta::pure::functions::multiplicity::toOne()]->joinStrings()) + ->filter(row2|$row2.FIRSTNAME->meta::pure::functions::collection::isNotEmpty()->meta::pure::functions::boolean::not()) + ->extend(~DQ_RULE_NAME: row | 'validFirstName') + ->extend(~DQ_LOGICAL_DEFECT_ID: row | meta::pure::functions::hash::hash(if($row.FIRSTNAME->isEmpty(), | '', |$row.FIRSTNAME->toOne()->toString()) + if($row.LASTNAME->isEmpty(), | '', |$row.LASTNAME->toOne()->toString()) + if($row.fullname->isEmpty(), | '', |$row.fullname->toOne()->toString()) + if($row.DQ_RULE_NAME->isEmpty(), | '', |$row.DQ_RULE_NAME->toOne()->toString()), meta::pure::functions::hash::HashType.MD5)) + ->extend(~DQ_DEFECT_ID: row | generateGuid()) + ->extend(~DEFECT_COUNT : x | $x.FIRSTNAME : y | $y->size()) + ->take(100)}, + 100, + 'meta::external::dataquality::tests::domain::DataQualityRuntime' + ) +} + +function <> meta::external::dataquality::tests::testLambdaGeneration_relationValidation_aggregate_withTotalDefectCountAndLimit():Boolean[1] +{ + doRelationTestWithTotalDefectCountAndLimit( + '###DataQualityValidation' + + ' DataQualityRelationValidation meta::external::dataquality::tests::domain::RelationValidation' + + ' {' + + ' query: #>{meta::external::dataquality::tests::domain::db.personTable}#->select(~[FIRSTNAME, LASTNAME])->from(meta::external::dataquality::tests::domain::DataQualityRuntime);' + + ' validations: [' + + ' {' + + ' name: \'invalidFirstName\';' + + ' description: \'first name is not valid\';' + + ' assertion: rel|$rel->filter(row|$row.FIRSTNAME == \'invalid\')->meta::external::dataquality::assertRelationEmpty(~[FIRSTNAME]);' + + ' type: AGGREGATE;' + + ' }' + + ' ];' + + ' }', + 'invalidFirstName', + {|{rel: meta::pure::metamodel::relation::Relation<(FIRSTNAME:meta::pure::precisePrimitives::Varchar(200),LASTNAME:meta::pure::precisePrimitives::Varchar(200))>[1] | + $rel->filter(row|$row.FIRSTNAME == 'invalid') + ->select(~[FIRSTNAME]) + ->extend(~DQ_RULE_NAME: row | 'invalidFirstName') + ->extend(~DQ_LOGICAL_DEFECT_ID: row | meta::pure::functions::hash::hash(if($row.FIRSTNAME->isEmpty(), | '', |$row.FIRSTNAME->toOne()->toString()) + if($row.DQ_RULE_NAME->isEmpty(), | '', |$row.DQ_RULE_NAME->toOne()->toString()), meta::pure::functions::hash::HashType.MD5)) + ->extend(~DQ_DEFECT_ID: row | generateGuid()) + ->extend(~DEFECT_COUNT : x | $x.FIRSTNAME : y | $y->size())} + ->meta::pure::functions::lang::eval(#>{meta::external::dataquality::tests::domain::db.personTable}#->meta::pure::functions::relation::select(~[FIRSTNAME,LASTNAME])) + ->limit(100);}, + 100, + 'meta::external::dataquality::tests::domain::DataQualityRuntime' + ) +} + +function <> meta::external::dataquality::tests::testLambdaGeneration_relationValidation_aggregate_withTotalDefectCountAndLimit_twoValidations():Boolean[1] +{ + doRelationTestWithTotalDefectCountAndLimit( + '###DataQualityValidation' + + ' DataQualityRelationValidation meta::external::dataquality::tests::domain::RelationValidation' + + ' {' + + ' query: #>{meta::external::dataquality::tests::domain::db.personTable}#->select(~[FIRSTNAME, LASTNAME])->from(meta::external::dataquality::tests::domain::DataQualityRuntime);' + + ' validations: [' + + ' {' + + ' name: \'invalidFirstName\';' + + ' description: \'first name is not valid\';' + + ' assertion: rel|$rel->filter(row|$row.FIRSTNAME == \'invalid\')->meta::external::dataquality::assertRelationEmpty(~[FIRSTNAME]);' + + ' type: AGGREGATE;' + + ' },' + + ' {' + + ' name: \'invalidLastName\';' + + ' description: \'last name is not valid\';' + + ' assertion: rel|$rel->filter(row|$row.LASTNAME == \'invalid\')->meta::external::dataquality::assertRelationEmpty(~[FIRSTNAME,LASTNAME]);' + + ' type: AGGREGATE;' + + ' }' + + ' ];' + + ' }', + [], + {|let sourceQuery = #>{meta::external::dataquality::tests::domain::db.personTable}#->meta::pure::functions::relation::select(~[FIRSTNAME,LASTNAME]); + {rel: meta::pure::metamodel::relation::Relation<(FIRSTNAME:meta::pure::precisePrimitives::Varchar(200),LASTNAME:meta::pure::precisePrimitives::Varchar(200))>[1] | + $rel->filter(row|$row.LASTNAME == 'invalid') + ->select(~[FIRSTNAME,LASTNAME]) + ->extend(~DQ_RULE_NAME: row | 'invalidLastName') + ->extend(~DQ_LOGICAL_DEFECT_ID: row | meta::pure::functions::hash::hash(if($row.FIRSTNAME->isEmpty(), | '', |$row.FIRSTNAME->toOne()->toString()) + if($row.LASTNAME->isEmpty(), | '', |$row.LASTNAME->toOne()->toString()) + if($row.DQ_RULE_NAME->isEmpty(), | '', |$row.DQ_RULE_NAME->toOne()->toString()), meta::pure::functions::hash::HashType.MD5)) + ->extend(~DQ_DEFECT_ID: row | generateGuid()) + ->extend(~DEFECT_COUNT : x | $x.FIRSTNAME : y | $y->size())} + ->meta::pure::functions::lang::eval($sourceQuery)->limit(100)->concatenate( + {rel: meta::pure::metamodel::relation::Relation<(FIRSTNAME:meta::pure::precisePrimitives::Varchar(200),LASTNAME:meta::pure::precisePrimitives::Varchar(200))>[1] | + $rel->filter(row|$row.FIRSTNAME == 'invalid') + ->select(~[FIRSTNAME]) + ->extend(~DQ_RULE_NAME: row | 'invalidFirstName') + ->extend(~DQ_LOGICAL_DEFECT_ID: row | meta::pure::functions::hash::hash(if($row.FIRSTNAME->isEmpty(), | '', |$row.FIRSTNAME->toOne()->toString()) + if($row.DQ_RULE_NAME->isEmpty(), | '', |$row.DQ_RULE_NAME->toOne()->toString()), meta::pure::functions::hash::HashType.MD5)) + ->extend(~DQ_DEFECT_ID: row | generateGuid()) + ->extend(~DEFECT_COUNT : x | $x.FIRSTNAME : y | $y->size())} + ->meta::pure::functions::lang::eval($sourceQuery)->limit(100));}, + 100, + 'meta::external::dataquality::tests::domain::DataQualityRuntime' + ) +} + +// -------------------------------------- includeTotalDefectCount tests end ------------------------------------------------------------------------------------------------ + function <> meta::external::dataquality::tests::testLambdaGeneration_relationValidationwithAssertRelationNotEmpty():Boolean[1] { @@ -440,8 +549,8 @@ function <> meta::external::dataquality::tests::testLambdaGeneration_ ->extend(~DQ_RULE_NAME: row | 'firstNameMissing') ->extend(~DQ_LOGICAL_DEFECT_ID: row | meta::pure::functions::hash::hash(if($row.FIRSTNAME->isEmpty(), | '', |$row.FIRSTNAME->toOne()->toString()) + if($row.LASTNAME->isEmpty(), | '', |$row.LASTNAME->toOne()->toString()) + if($row.AGE->isEmpty(), | '', |$row.AGE->toOne()->toString()) + if($row.DQ_RULE_NAME->isEmpty(), | '', |$row.DQ_RULE_NAME->toOne()->toString()), meta::pure::functions::hash::HashType.MD5)) ->extend(~DQ_DEFECT_ID: row | generateGuid())} - ->meta::pure::functions::lang::eval(#>{meta::external::dataquality::tests::domain::db.personTable}#->meta::pure::functions::relation::select(~[FIRSTNAME,LASTNAME,AGE]) - ->meta::pure::functions::relation::limit(100));}, + ->meta::pure::functions::lang::eval(#>{meta::external::dataquality::tests::domain::db.personTable}#->meta::pure::functions::relation::select(~[FIRSTNAME,LASTNAME,AGE])) + ->meta::pure::functions::relation::limit(100);}, 100, 'meta::external::dataquality::tests::domain::DataQualityRuntime' ); @@ -470,8 +579,8 @@ function <> meta::external::dataquality::tests::testLambdaGeneration_ ->extend(~DQ_RULE_NAME: row | 'invalidAge') ->extend(~DQ_LOGICAL_DEFECT_ID: row | meta::pure::functions::hash::hash(if($row.FIRSTNAME->isEmpty(), | '', |$row.FIRSTNAME->toOne()->toString()) + if($row.AGE->isEmpty(), | '', |$row.AGE->toOne()->toString()) + if($row.DQ_RULE_NAME->isEmpty(), | '', |$row.DQ_RULE_NAME->toOne()->toString()), meta::pure::functions::hash::HashType.MD5)) ->extend(~DQ_DEFECT_ID: row | generateGuid())} - ->meta::pure::functions::lang::eval(#>{meta::external::dataquality::tests::domain::db.personTable}#->meta::pure::functions::relation::select(~[FIRSTNAME,LASTNAME,AGE]) - ->meta::pure::functions::relation::limit(100));}, + ->meta::pure::functions::lang::eval(#>{meta::external::dataquality::tests::domain::db.personTable}#->meta::pure::functions::relation::select(~[FIRSTNAME,LASTNAME,AGE])) + ->meta::pure::functions::relation::limit(100);}, 100, 'meta::external::dataquality::tests::domain::DataQualityRuntime' ); @@ -710,6 +819,58 @@ function <> meta::external::dataquality::tests::testLambdaGeneration_ ) } +function <> meta::external::dataquality::tests::testLambdaGeneration_relationValidation_all_validations_withLimit():Boolean[1] +{ + doRelationTestWithLimit( + '###DataQualityValidation' + + ' DataQualityRelationValidation meta::external::dataquality::tests::domain::RelationValidation' + + ' {' + + ' query: #>{meta::external::dataquality::tests::domain::db.personTable}#->select(~[FIRSTNAME, LASTNAME])->from(meta::external::dataquality::tests::domain::DataQualityRuntime);' + + ' validations: [' + + ' {' + + ' name: \'notEmpty\';' + + ' description: \'data is not empty\';' + + ' assertion: rel|$rel->meta::external::dataquality::assertRelationNotEmpty();' + + ' },' + + ' {' + + ' name: \'invalidFirstName\';' + + ' description: \'first name is not valid\';' + + ' assertion: rel|$rel->filter(row|$row.FIRSTNAME == \'invalid\')->meta::external::dataquality::assertRelationEmpty(~[FIRSTNAME]);' + + ' },' + + ' {' + + ' name: \'invalidLastName\';' + + ' description: \'last name is not valid\';' + + ' assertion: rel|$rel->filter(row|$row.LASTNAME == \'invalid\')->meta::external::dataquality::assertRelationEmpty(~[FIRSTNAME,LASTNAME]);' + + ' }' + + ' ];' + + ' }', + [], + {|eval(|let sourceQuery = #>{meta::external::dataquality::tests::domain::db.personTable}#->meta::pure::functions::relation::select(~[FIRSTNAME,LASTNAME]); + {rel: meta::pure::metamodel::relation::Relation<(FIRSTNAME:meta::pure::precisePrimitives::Varchar(200),LASTNAME:meta::pure::precisePrimitives::Varchar(200))>[1] | + $rel->filter(row|$row.LASTNAME == 'invalid') + ->select(~[FIRSTNAME,LASTNAME]) + ->extend(~DQ_RULE_NAME: row | 'invalidLastName') + ->extend(~DQ_LOGICAL_DEFECT_ID: row | meta::pure::functions::hash::hash(if($row.FIRSTNAME->isEmpty(), | '', |$row.FIRSTNAME->toOne()->toString()) + if($row.LASTNAME->isEmpty(), | '', |$row.LASTNAME->toOne()->toString()) + if($row.DQ_RULE_NAME->isEmpty(), | '', |$row.DQ_RULE_NAME->toOne()->toString()), meta::pure::functions::hash::HashType.MD5)) + ->extend(~DQ_DEFECT_ID: row | generateGuid())} + ->meta::pure::functions::lang::eval($sourceQuery)->meta::pure::functions::relation::limit(100)->concatenate( + {rel: meta::pure::metamodel::relation::Relation<(FIRSTNAME:meta::pure::precisePrimitives::Varchar(200),LASTNAME:meta::pure::precisePrimitives::Varchar(200))>[1] | + $rel->filter(row|$row.FIRSTNAME == 'invalid') + ->select(~[FIRSTNAME]) + ->extend(~DQ_RULE_NAME: row | 'invalidFirstName') + ->extend(~DQ_LOGICAL_DEFECT_ID: row | meta::pure::functions::hash::hash(if($row.FIRSTNAME->isEmpty(), | '', |$row.FIRSTNAME->toOne()->toString()) + if($row.DQ_RULE_NAME->isEmpty(), | '', |$row.DQ_RULE_NAME->toOne()->toString()), meta::pure::functions::hash::HashType.MD5)) + ->extend(~DQ_DEFECT_ID: row | generateGuid())} + ->meta::pure::functions::lang::eval($sourceQuery)->meta::pure::functions::relation::limit(100)->concatenate( + {rel: meta::pure::metamodel::relation::Relation<(FIRSTNAME:meta::pure::precisePrimitives::Varchar(200),LASTNAME:meta::pure::precisePrimitives::Varchar(200))>[1] | + $rel->aggregate(~COUNT : x | 1 : y | $y->count())->filter(row|$row.COUNT == 0) + ->extend(~DQ_RULE_NAME: row | 'notEmpty') + ->extend(~DQ_LOGICAL_DEFECT_ID: row | meta::pure::functions::hash::hash(if($row.COUNT->isEmpty(), | '', |$row.COUNT->toOne()->toString()) + if($row.DQ_RULE_NAME->isEmpty(), | '', |$row.DQ_RULE_NAME->toOne()->toString()), meta::pure::functions::hash::HashType.MD5)) + ->extend(~DQ_DEFECT_ID: row | generateGuid())} + ->meta::pure::functions::lang::eval($sourceQuery)->meta::pure::functions::relation::limit(100)));)}, + 100, + 'meta::external::dataquality::tests::domain::DataQualityRuntime' + ) +} + function <> meta::external::dataquality::tests::testLambdaGeneration_relationValidation_constraint():Boolean[1] { doRelationTestWithoutDQColumns( diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure-test/src/main/resources/core_dataquality_test/dataquality_test_utils.pure b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure-test/src/main/resources/core_dataquality_test/dataquality_test_utils.pure index ea367b0ae89..3d2ca2b336a 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure-test/src/main/resources/core_dataquality_test/dataquality_test_utils.pure +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure-test/src/main/resources/core_dataquality_test/dataquality_test_utils.pure @@ -91,11 +91,11 @@ function meta::external::dataquality::tests::doRelationRowCountTest(dq:String[1] assertLambdaEquals($expected, $actual->cast(@LambdaFunction), $packageableRuntimeAsString); } -function meta::external::dataquality::tests::doRelationTestWithLimit(dq:String[1], validationName:String[1], expected:FunctionDefinition[1], resultLimit: Integer[1], packageableRuntimeAsString: String[1]):Boolean[1] +function meta::external::dataquality::tests::doRelationTestWithLimit(dq:String[1], validationNames:String[*], expected:FunctionDefinition[1], resultLimit: Integer[1], packageableRuntimeAsString: String[1]):Boolean[1] { let dataqualityRelationValidation = $dq->loadDataQualityRelationValidation(); - let actual = $dataqualityRelationValidation->meta::external::dataquality::generateDataqualityRelationValidationLambda($validationName, $resultLimit); + let actual = $dataqualityRelationValidation->meta::external::dataquality::generateDataqualityRelationValidationLambda($validationNames, $resultLimit); assertLambdaEquals($expected, $actual->cast(@LambdaFunction), $packageableRuntimeAsString); } @@ -107,6 +107,15 @@ function meta::external::dataquality::tests::doRelationTestWithoutDQColumns(dq:S let actual = $dataqualityRelationValidation->meta::external::dataquality::generateDataqualityRelationValidationLambda($validationName, [], false); assertLambdaEquals($expected, $actual->cast(@LambdaFunction), $packageableRuntimeAsString); +} + +function meta::external::dataquality::tests::doRelationTestWithTotalDefectCountAndLimit(dq:String[1], validationNames:String[*], expected:FunctionDefinition[1], resultLimit: Integer[1], packageableRuntimeAsString: String[1]):Boolean[1] +{ + let dataqualityRelationValidation = $dq->loadDataQualityRelationValidation(); + + let actual = $dataqualityRelationValidation->meta::external::dataquality::generateDataqualityRelationValidationLambda($validationNames, $resultLimit, true, false, true, true); + + assertLambdaEquals($expected, $actual->cast(@LambdaFunction), $packageableRuntimeAsString, false); } @@ -144,7 +153,6 @@ function meta::external::dataquality::tests::assertLambdaEquals(expected:Functio function meta::external::dataquality::tests::assertLambdaEquals(expected:FunctionDefinition[1], actual:FunctionDefinition[1], packageableRuntimeAsString: String[1], fromExpected: Boolean[1]): Boolean[1] { let config = grammarConfig(); - assertLambdaEquals($expected->printFunctionDefinition($config, ^meta::pure::metamodel::serialization::grammar::GContext(space='')), $actual->printFunctionDefinition($config, ^meta::pure::metamodel::serialization::grammar::GContext(space='')), $packageableRuntimeAsString, $fromExpected); } diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality/generation/dataquality.pure b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality/generation/dataquality.pure index d52c9875f1d..2c1c8e1336a 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality/generation/dataquality.pure +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality/generation/dataquality.pure @@ -482,9 +482,9 @@ function <> meta::external::dataquality::constraintToDqRule(cons } -function meta::external::dataquality::generateDataqualityRelationValidationLambda(dqRelationValidation: meta::external::dataquality::DataQualityRelationValidation[1], validationName: String[1], defectsLimit:Integer[0..1]): LambdaFunction[1] +function meta::external::dataquality::generateDataqualityRelationValidationLambda(dqRelationValidation: meta::external::dataquality::DataQualityRelationValidation[1], validationNames: String[*], defectsLimit:Integer[0..1]): LambdaFunction[1] { - $dqRelationValidation->meta::external::dataquality::generateDataqualityRelationValidationLambda($validationName, $defectsLimit, true) + $dqRelationValidation->meta::external::dataquality::generateDataqualityRelationValidationLambda($validationNames, $defectsLimit, true) } function meta::external::dataquality::generateDataqualityRelationValidationLambda(dqRelationValidation: meta::external::dataquality::DataQualityRelationValidation[1], validationNames: String[*], defectsLimit:Integer[0..1], enrichDQColumns:Boolean[1]): LambdaFunction[1] @@ -493,6 +493,11 @@ function meta::external::dataquality::generateDataqualityRelationValidationLambd } function meta::external::dataquality::generateDataqualityRelationValidationLambda(dqRelationValidation: meta::external::dataquality::DataQualityRelationValidation[1], validationNames: String[*], defectsLimit:Integer[0..1], enrichDQColumns:Boolean[1], castDQColumnsToPrimitive:Boolean[1], removeFrom:Boolean[1]): LambdaFunction[1] +{ + generateDataqualityRelationValidationLambda($dqRelationValidation, $validationNames, $defectsLimit, $enrichDQColumns, $castDQColumnsToPrimitive, $removeFrom, false); +} + +function meta::external::dataquality::generateDataqualityRelationValidationLambda(dqRelationValidation: meta::external::dataquality::DataQualityRelationValidation[1], validationNames: String[*], defectsLimit:Integer[0..1], enrichDQColumns:Boolean[1], castDQColumnsToPrimitive:Boolean[1], removeFrom:Boolean[1], includeTotalDefectCount:Boolean[1]): LambdaFunction[1] { //if validation name provided only generate query for that validation else generate query for all validations let selectedValidations = if ($validationNames->isEmpty(), @@ -515,7 +520,7 @@ function meta::external::dataquality::generateDataqualityRelationValidationLambd | ^VariableExpression(name = 'sourceQuery', genericType = $query.genericType, multiplicity = PureOne), | $query ); - let validationSubQueries = $selectedValidations->map(val | $queryInput->generateDataqualityRelationValidationLambda($val, $defectsLimit, $enrichDQColumns, $castDQColumnsToPrimitive, $removeFrom)); + let validationSubQueries = $selectedValidations->map(val | $queryInput->generateDataqualityRelationValidationLambda($val, $defectsLimit, $enrichDQColumns, $castDQColumnsToPrimitive, $removeFrom, $includeTotalDefectCount)); let validationsUnion = $validationSubQueries->tail()->fold({val1, val2 | buildUnionExpression($val1->evaluateAndDeactivate(), $val2->evaluateAndDeactivate())}, $validationSubQueries->head()->toOne()->evaluateAndDeactivate()); let expressionSequence = if ($multipleValidations, | [buildLetExpression('sourceQuery', $query), $validationsUnion], @@ -531,14 +536,23 @@ function meta::external::dataquality::generateDataqualityRelationValidationLambd function meta::external::dataquality::generateDataqualityRelationValidationLambda(query: ValueSpecification[1], selectedValidation: RelationValidation[1], defectsLimit:Integer[0..1], enrichDQColumns:Boolean[1], castDQColumnsToPrimitive:Boolean[1], removeFrom:Boolean[1]): FunctionExpression[1] { - if($selectedValidation.type->isNotEmpty() && $selectedValidation.type == 'ROW_LEVEL', - | $query->generateDataqualityRelationValidationLambda_RowLevel($selectedValidation, $defectsLimit, $enrichDQColumns, $castDQColumnsToPrimitive), // for backward compatibility - | $query->generateDataqualityRelationValidationLambda_Aggregate($selectedValidation, $defectsLimit, $enrichDQColumns, $castDQColumnsToPrimitive)); + $query->generateDataqualityRelationValidationLambda($selectedValidation, $defectsLimit, $enrichDQColumns, $castDQColumnsToPrimitive, $removeFrom, false); } +function meta::external::dataquality::generateDataqualityRelationValidationLambda(query: ValueSpecification[1], selectedValidation: RelationValidation[1], defectsLimit:Integer[0..1], enrichDQColumns:Boolean[1], castDQColumnsToPrimitive:Boolean[1], removeFrom:Boolean[1], includeTotalDefectCount:Boolean[1]): FunctionExpression[1] +{ + if($selectedValidation.type->isNotEmpty() && $selectedValidation.type == 'ROW_LEVEL', + | $query->generateDataqualityRelationValidationLambda_RowLevel($selectedValidation, $defectsLimit, $enrichDQColumns, $castDQColumnsToPrimitive, $includeTotalDefectCount), + | $query->generateDataqualityRelationValidationLambda_Aggregate($selectedValidation, $defectsLimit, $enrichDQColumns, $castDQColumnsToPrimitive, $includeTotalDefectCount)); +} // row level validation for backward compatibility function meta::external::dataquality::generateDataqualityRelationValidationLambda_RowLevel(query: ValueSpecification[1], selectedValidation: RelationValidation[1], defectsLimit:Integer[0..1], enrichDQColumns:Boolean[1], castDQColumnsToPrimitive:Boolean[1]): FunctionExpression[1] +{ + $query->generateDataqualityRelationValidationLambda_RowLevel($selectedValidation, $defectsLimit, $enrichDQColumns, $castDQColumnsToPrimitive, false); +} + +function meta::external::dataquality::generateDataqualityRelationValidationLambda_RowLevel(query: ValueSpecification[1], selectedValidation: RelationValidation[1], defectsLimit:Integer[0..1], enrichDQColumns:Boolean[1], castDQColumnsToPrimitive:Boolean[1], includeTotalDefectCount:Boolean[1]): FunctionExpression[1] { let inputRelType = $query->evaluateAndDeactivate().genericType.typeArguments.rawType->cast(@meta::pure::metamodel::relation::RelationType)->toOne(); let withValidationFilterExpression = $query->toOne()->evaluateAndDeactivate()->buildRelationFilterExpressionForRowLevelVal($selectedValidation, $inputRelType); @@ -546,13 +560,24 @@ function meta::external::dataquality::generateDataqualityRelationValidationLambd // assertion - enrich DQ columns let withDQColumnsAndNewRelationType = if($enrichDQColumns, | $withValidationFilterExpression->enrichDQColumns($inputRelType, $selectedValidation, $castDQColumnsToPrimitive), | ^Pair>(first=$withValidationFilterExpression, second=$inputRelType);); + // add total defect count column if requested (computed before limit so it reflects all defects) + let withDefectCount = if($includeTotalDefectCount, + | $withDQColumnsAndNewRelationType.first->enrichTotalDefectCountColumn($withDQColumnsAndNewRelationType.second, $castDQColumnsToPrimitive);, + | $withDQColumnsAndNewRelationType + ); + if($defectsLimit->isEmpty(), - | $withDQColumnsAndNewRelationType.first, - | $withDQColumnsAndNewRelationType.first->buildLimitFilterExpression($defectsLimit->toOne(), ^GenericType(rawType=Relation, typeArguments = ^GenericType(rawType=$withDQColumnsAndNewRelationType.second))) + | $withDefectCount.first, + | $withDefectCount.first->buildLimitFilterExpression($defectsLimit->toOne(), ^GenericType(rawType=Relation, typeArguments = ^GenericType(rawType=$withDefectCount.second))) ); } function meta::external::dataquality::generateDataqualityRelationValidationLambda_Aggregate(query: ValueSpecification[1], selectedValidation: RelationValidation[1], resultLimit:Integer[0..1], enrichDQcolumns:Boolean[0..1], castDQColumnsToPrimitive:Boolean[1]): FunctionExpression[1] +{ + $query->generateDataqualityRelationValidationLambda_Aggregate($selectedValidation, $resultLimit, $enrichDQcolumns, $castDQColumnsToPrimitive, false); +} + +function meta::external::dataquality::generateDataqualityRelationValidationLambda_Aggregate(query: ValueSpecification[1], selectedValidation: RelationValidation[1], resultLimit:Integer[0..1], enrichDQcolumns:Boolean[0..1], castDQColumnsToPrimitive:Boolean[1], includeTotalDefectCount:Boolean[1]): FunctionExpression[1] { // assertion transformation let poppedAssertion = $selectedValidation->popAssertionForProjection()->evaluateAndDeactivate(); @@ -563,14 +588,111 @@ function meta::external::dataquality::generateDataqualityRelationValidationLambd // assertion - enrich DQ columns by default let withDQColumnsAndNewRelationType = if($enrichDQcolumns->isEmpty() || $enrichDQcolumns->toOne(), | $poppedAssertion->enrichDQColumns($inputRelType, $selectedValidation, $castDQColumnsToPrimitive), | ^Pair>(first=$poppedAssertion, second=$inputRelType);); - // main dataset query transformation - let queryWithLimitExpression = if($resultLimit->isEmpty(), - | $query->evaluateAndDeactivate(), - | $query->evaluateAndDeactivate()->buildRelationLimitFilterExpression($resultLimit->toOne(), $query->genericType()) - ); + // add total defect count column if requested (computed before limit so it reflects all defects) + let withDefectCount = if($includeTotalDefectCount, + | $withDQColumnsAndNewRelationType.first->enrichTotalDefectCountColumn($withDQColumnsAndNewRelationType.second, $castDQColumnsToPrimitive);, + | $withDQColumnsAndNewRelationType + ); + + // compose transformed assertion with source query (no limit on source - limit applied after) + let evalExpression = $withDefectCount.first->buildEvalExpression($query->evaluateAndDeactivate()->cast(@ValueSpecification), $selectedValidation, getRelationType($selectedValidation), $withDefectCount.second); + + // apply limit after the eval so it limits defect rows, not source rows + if($resultLimit->isEmpty(), + | $evalExpression, + | $evalExpression->buildRelationLimitFilterExpression($resultLimit->toOne(), ^GenericType(rawType=Relation, typeArguments = ^GenericType(rawType=$withDefectCount.second))) + ); +} + +function meta::external::dataquality::enrichTotalDefectCountColumn(f:ValueSpecification[1], inputRelType:meta::pure::metamodel::relation::RelationType[1], castDQColumnsToPrimitive:Boolean[1]): Pair>[1] +{ + let newRelTypeWithDefectCount = $inputRelType->addColumns(~[DEFECT_COUNT: Integer[1]])->evaluateAndDeactivate(); + let newRelTypeOnlyDefectCount = ^meta::pure::metamodel::relation::RelationType(columns = $newRelTypeWithDefectCount.columns->filter(col | 'DEFECT_COUNT' == ($col.name->toOne()))); + + // Build the first lambda: {x | $x.} - maps each row to the rule name + let aggColSpecFirstParam = buildAggColSpecDefectCountFirstParam($inputRelType); + + // Get the actual column type for the second param + let firstCol = $inputRelType.columns->first()->toOne(); + let firstColGenericType = $firstCol.classifierGenericType.typeArguments->at(1); + + // Build the second lambda: {y | $y->size()} - counts the mapped values using size + let aggColSpecSecondParam = buildAggColSpecDefectCountSecondParam($firstColGenericType); + + // Build the aggColSpec expression + let aggColSpec = ^SimpleFunctionExpression + ( + func = aggColSpec_Function_1__Function_1__String_1__T_1__AggColSpec_1_, + multiplicity = PureOne, + genericType = ^GenericType(rawType = AggColSpec, typeArguments = [$aggColSpecFirstParam.genericType, $aggColSpecSecondParam.genericType, ^GenericType(rawType=$newRelTypeOnlyDefectCount)]), + importGroup = system::imports::coreImport, + parametersValues = [ + $aggColSpecFirstParam, + $aggColSpecSecondParam, + ^InstanceValue(genericType = ^GenericType(rawType = String), multiplicity = PureOne, values = 'DEFECT_COUNT')->evaluateAndDeactivate(), + ^InstanceValue(genericType = ^GenericType(rawType = $newRelTypeOnlyDefectCount), multiplicity = PureOne)->evaluateAndDeactivate() + ] + )->evaluateAndDeactivate(); + + // Build the extend expression: $input->extend(aggColSpec) + let withDefectCount = ^SimpleFunctionExpression + ( + func = extend_Relation_1__AggColSpec_1__Relation_1_, + functionName = 'extend', + multiplicity = PureOne, + genericType = ^GenericType(rawType=Relation, typeArguments = ^GenericType(rawType=$newRelTypeWithDefectCount)), + importGroup = system::imports::coreImport, + parametersValues = [ + $f, + $aggColSpec + ] + )->evaluateAndDeactivate(); + + ^Pair>(first=$withDefectCount, second=$newRelTypeWithDefectCount); +} + + +// Build the first lambda for defect count: {x | $x.} - maps each row to the first column value +function meta::external::dataquality::buildAggColSpecDefectCountFirstParam(relType: meta::pure::metamodel::relation::RelationType[1]):InstanceValue[1] +{ + let firstCol = $relType.columns->first()->toOne(); + let firstColName = $firstCol.name->toOne(); + let firstColGenericType = $firstCol.classifierGenericType.typeArguments->at(1); + let expression = ^VariableExpression(name = 'x', genericType = ^GenericType(rawType = $relType), multiplicity = PureOne); + let colAccess = ^SimpleFunctionExpression + ( + func = $firstCol, + multiplicity = ZeroOne, + genericType = $firstColGenericType, + importGroup = system::imports::coreImport, + parametersValues = [$expression], + propertyName = ^InstanceValue(genericType = ^GenericType(rawType=Any), multiplicity = ZeroOne, values = [$firstColName]) + )->evaluateAndDeactivate(); + ^InstanceValue( + genericType = ^GenericType(rawType = LambdaFunction, typeArguments = ^GenericType(rawType = ^FunctionType(parameters = $expression, returnMultiplicity = ZeroOne, returnType = $firstColGenericType))), + multiplicity = PureOne, + values = lambda(^FunctionType(returnMultiplicity = ZeroOne, returnType = $firstColGenericType, parameters = $expression), $colAccess) + )->evaluateAndDeactivate(); +} - // compose above transformed query and assertion - $withDQColumnsAndNewRelationType.first->buildEvalExpression($queryWithLimitExpression->cast(@ValueSpecification), $selectedValidation, getRelationType($selectedValidation), $withDQColumnsAndNewRelationType.second); +// Build the second lambda for defect count: {y | $y->size()} - counts using size +function meta::external::dataquality::buildAggColSpecDefectCountSecondParam(colGenericType: GenericType[1]):InstanceValue[1] +{ + let expression = ^VariableExpression(name = 'y', genericType = $colGenericType, multiplicity = ZeroMany); + let sizeCall = ^SimpleFunctionExpression + ( + func = size_Any_MANY__Integer_1_, + functionName = 'size', + multiplicity = PureOne, + genericType = ^GenericType(rawType = Integer), + importGroup = system::imports::coreImport, + parametersValues = [$expression] + )->evaluateAndDeactivate(); + ^InstanceValue( + genericType = ^GenericType(rawType = LambdaFunction, typeArguments = ^GenericType(rawType = ^FunctionType(parameters = $expression, returnMultiplicity = PureOne, returnType = ^GenericType(rawType = Integer)))), + multiplicity = PureOne, + values = lambda(^FunctionType(returnMultiplicity = PureOne, returnType = ^GenericType(rawType = Integer), parameters = $expression), $sizeCall) + )->evaluateAndDeactivate(); } function meta::external::dataquality::buildUnionExpression(param1: FunctionExpression[1], param2: FunctionExpression[1]): FunctionExpression[1]