Skip to content

Commit f308b98

Browse files
sahil37sahil37
authored
add support for assertion functions in dq (finos#3755)
1 parent 6d5f505 commit f308b98

File tree

8 files changed

+231
-93
lines changed

8 files changed

+231
-93
lines changed

‎legend-engine-xts-dataquality/legend-engine-xt-dataquality-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/DataQualityCompilerExtension.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -86,12 +86,14 @@
8686
import java.util.Set;
8787
import java.util.stream.Collectors;
8888

89+
import static org.finos.legend.engine.language.pure.compiler.toPureGraph.handlers.Handlers.SelectColInference;
90+
8991
public class DataQualityCompilerExtension implements CompilerExtension
9092
{
9193

9294
private static final String RELATION_ROW_LEVEL_VAL_TYPE = "ROW_LEVEL";
9395
private static final String RELATION_ASSERTION_INPUT_PARAM = "rel";
94-
private static final ImmutableSet<String> ASSERTION_END_PERMITTED_FUNC_NAMES = Sets.immutable.of("relationEmpty", "relationNotEmpty");
96+
private static final ImmutableSet<String> ASSERTION_END_PERMITTED_FUNC_NAMES = Sets.immutable.of("assertRelationEmpty", "assertRelationNotEmpty");
9597

9698
@Override
9799
public MutableList<String> group()
@@ -263,7 +265,7 @@ private void validateAggregateAssertion(RelationValidation relationalValidation,
263265
{
264266
if (!(assertion._expressionSequence().getLast() instanceof Root_meta_pure_metamodel_valuespecification_SimpleFunctionExpression_Impl) || !ASSERTION_END_PERMITTED_FUNC_NAMES.contains(((Root_meta_pure_metamodel_valuespecification_SimpleFunctionExpression_Impl) assertion._expressionSequence().getLast())._functionName))
265267
{
266-
throw new EngineException("Assertion should end in either relationEmpty or relationNotEmpty functions", relationalValidation.assertion.sourceInformation, EngineErrorType.COMPILATION);
268+
throw new EngineException("Assertion should end in either assertRelationEmpty or assertRelationNotEmpty functions", relationalValidation.assertion.sourceInformation, EngineErrorType.COMPILATION);
267269
}
268270
}
269271

@@ -492,6 +494,8 @@ public List<Function<Handlers, List<FunctionExpressionBuilderRegistrationInfo>>>
492494
Lists.mutable.with(
493495
new FunctionExpressionBuilderRegistrationInfo(null, handlers.m(handlers.h("meta::external::dataquality::relationEmpty_Relation_1__Boolean_1_", false, ps -> handlers.res("Boolean", "one"), ps -> ps.size() == 1 && handlers.typeOne(ps.get(0), "Relation")))),
494496
new FunctionExpressionBuilderRegistrationInfo(null, handlers.m(handlers.h("meta::external::dataquality::relationNotEmpty_Relation_1__Boolean_1_", false, ps -> handlers.res("Boolean", "one"), ps -> ps.size() == 1 && handlers.typeOne(ps.get(0), "Relation")))),
497+
new FunctionExpressionBuilderRegistrationInfo(null, handlers.grp(SelectColInference, handlers.h("meta::external::dataquality::assertRelationEmpty_Relation_1__ColSpecArray_1__Boolean_1_", false, ps -> handlers.res("Boolean", "one"), ps -> true))),
498+
new FunctionExpressionBuilderRegistrationInfo(null, handlers.m(handlers.h("meta::external::dataquality::assertRelationNotEmpty_Relation_1__Boolean_1_", false, ps -> handlers.res("Boolean", "one"), ps -> ps.size() == 1 && handlers.typeOne(ps.get(0), "Relation")))),
495499
// row count
496500
new FunctionExpressionBuilderRegistrationInfo(null, handlers.m(handlers.h("meta::external::dataquality::rowCountGreaterThan_Relation_1__Number_1__Boolean_1_", false, ps -> handlers.res("Boolean", "one"), ps -> ps.size() == 2 && handlers.typeOne(ps.get(0), "Relation") && handlers.typeOne(ps.get(1), Sets.mutable.with("Number", "Integer", "Float", "Decimal"))))),
497501
new FunctionExpressionBuilderRegistrationInfo(null, handlers.m(handlers.h("meta::external::dataquality::rowCountGreaterThanEqual_Relation_1__Number_1__Boolean_1_", false, ps -> handlers.res("Boolean", "one"), ps -> ps.size() == 2 && handlers.typeOne(ps.get(0), "Relation") && handlers.typeOne(ps.get(1), Sets.mutable.with("Number", "Integer", "Float", "Decimal"))))),

‎legend-engine-xts-dataquality/legend-engine-xt-dataquality-compiler/src/test/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/TestDataQualityCompilationFromGrammar.java‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -212,7 +212,7 @@ public void testRelationValidation()
212212
" {\n" +
213213
" name: 'nonEmptyDataset';\n" +
214214
" description: 'dataset cannot be empty';\n" +
215-
" assertion: rel|$rel->relationNotEmpty();\n" +
215+
" assertion: rel|$rel->assertRelationNotEmpty();\n" +
216216
" }\n" +
217217
" ];\n" +
218218
"}");
@@ -227,7 +227,7 @@ public void testRelationValidation()
227227
" {\n" +
228228
" name: 'nonEmptyDataset';\n" +
229229
" description: 'dataset cannot be empty';\n" +
230-
" assertion: rel|$rel->relationNotEmpty();\n" +
230+
" assertion: rel|$rel->assertRelationNotEmpty();\n" +
231231
" }\n" +
232232
" ];\n" +
233233
"}");
@@ -241,7 +241,7 @@ public void testRelationValidation()
241241
" {\n" +
242242
" name: 'nonEmptyDataset';\n" +
243243
" description: 'dataset cannot be empty';\n" +
244-
" assertion: rel|$rel->relationNotEmpty();\n" +
244+
" assertion: rel|$rel->assertRelationNotEmpty();\n" +
245245
" type: AGGREGATE;\n" +
246246
" }\n" +
247247
" ];\n" +
@@ -256,7 +256,7 @@ public void testRelationValidation()
256256
" {\n" +
257257
" name: 'nonEmptyDataset';\n" +
258258
" description: 'dataset cannot be empty';\n" +
259-
" assertion: {firstName:String[1], rel| $rel->relationNotEmpty()};\n" +
259+
" assertion: {firstName:String[1], rel| $rel->assertRelationNotEmpty()};\n" +
260260
" type: AGGREGATE;\n" +
261261
" }\n" +
262262
" ];\n" +
@@ -275,7 +275,7 @@ public void testRelationValidation_query_params()
275275
" {\n" +
276276
" name: 'nonEmptyDataset';\n" +
277277
" description: 'dataset cannot be empty';\n" +
278-
" assertion: {firstName:String[1], rel| $rel->relationNotEmpty()};\n" +
278+
" assertion: {firstName:String[1], rel| $rel->assertRelationNotEmpty()};\n" +
279279
" type: AGGREGATE;\n" +
280280
" }\n" +
281281
" ];\n" +
@@ -290,10 +290,10 @@ public void testRelationValidation_query_params()
290290
" {\n" +
291291
" name: 'nonEmptyDataset';\n" +
292292
" description: 'dataset cannot be empty';\n" +
293-
" assertion: {firstName, rel| $rel->relationNotEmpty()};\n" +
293+
" assertion: {firstName, rel| $rel->assertRelationNotEmpty()};\n" +
294294
" }\n" +
295295
" ];\n" +
296-
"}", "COMPILATION error at [111:36-61]: Invalid parameter specified on assertion - firstName. Please validate that it is exactly the same as query parameters(name/type/multiplicity)");
296+
"}", "COMPILATION error at [111:36-67]: Invalid parameter specified on assertion - firstName. Please validate that it is exactly the same as query parameters(name/type/multiplicity)");
297297

298298
TestCompilationFromGrammar.TestCompilationFromGrammarTestSuite.test(COMPILATION_PREREQUISITE_CODE +
299299
"###DataQualityValidation\n" +
@@ -304,7 +304,7 @@ public void testRelationValidation_query_params()
304304
" {\n" +
305305
" name: 'nonEmptyDataset';\n" +
306306
" description: 'dataset cannot be empty';\n" +
307-
" assertion: {lastName:String[1], rel| $rel->relationNotEmpty()};\n" +
307+
" assertion: {lastName:String[1], rel| $rel->assertRelationNotEmpty()};\n" +
308308
" }\n" +
309309
" ];\n" +
310310
"}", "COMPILATION error at [111:22-39]: Invalid parameter specified on assertion - lastName. Please validate that it is exactly the same as query parameters(name/type/multiplicity)");
@@ -318,10 +318,10 @@ public void testRelationValidation_query_params()
318318
" {\n" +
319319
" name: 'nonEmptyDataset';\n" +
320320
" description: 'dataset cannot be empty';\n" +
321-
" assertion: {firstName:String[1] | $rel->relationNotEmpty()};\n" +
321+
" assertion: {firstName:String[1] | $rel->assertRelationNotEmpty()};\n" +
322322
" }\n" +
323323
" ];\n" +
324-
"}", "COMPILATION error at [111:42-67]: Assertion param 'rel' is missing!");
324+
"}", "COMPILATION error at [111:42-73]: Assertion param 'rel' is missing!");
325325
}
326326

327327
@Test
@@ -354,7 +354,7 @@ public void testRelationValidation_assertionValidation()
354354
" assertion: rel|$rel->filter(r|$r.FIRSTNAME->isNotEmpty());\n" +
355355
" }\n" +
356356
" ];\n" +
357-
"}", "COMPILATION error at [111:24-66]: Assertion should end in either relationEmpty or relationNotEmpty functions");
357+
"}", "COMPILATION error at [111:24-66]: Assertion should end in either assertRelationEmpty or assertRelationNotEmpty functions");
358358
}
359359

360360

‎legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/test/java/org/finos/legend/engine/generation/dataquality/TestDataQualityLambdaGenerator.java‎

Lines changed: 2 additions & 2 deletions
Large diffs are not rendered by default.

‎legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/test/java/org/finos/legend/engine/language/dataquality/grammar/to/TestDataQualityRoundtrip.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,7 @@ public void testRelationalValidation()
144144
" {\n" +
145145
" name: 'testValidation';\n" +
146146
" description: 'test validation';\n" +
147-
" assertion: rel|$rel->relationNotEmpty();\n" +
147+
" assertion: rel|$rel->assertRelationNotEmpty();\n" +
148148
" type: AGGREGATE;\n" +
149149
" }\n" +
150150
" ];\n" +
@@ -158,7 +158,7 @@ public void testRelationalValidation()
158158
" {\n" +
159159
" name: 'testValidation';\n" +
160160
" description: 'test validation';\n" +
161-
" assertion: rel|$rel->relationNotEmpty();\n" +
161+
" assertion: rel|$rel->assertRelationNotEmpty();\n" +
162162
" }\n" +
163163
" ];\n" +
164164
"}\n");
@@ -175,7 +175,7 @@ public void testRelationValidation_query_params()
175175
" {\n" +
176176
" name: 'testValidation';\n" +
177177
" description: 'test validation';\n" +
178-
" assertion: {name: String[1], rel|$rel->relationNotEmpty()};\n" +
178+
" assertion: {name: String[1], rel|$rel->assertRelationNotEmpty()};\n" +
179179
" }\n" +
180180
" ];\n" +
181181
"}\n");
@@ -188,7 +188,7 @@ public void testRelationValidation_query_params()
188188
" {\n" +
189189
" name: 'testValidation';\n" +
190190
" description: 'test validation';\n" +
191-
" assertion: {businessDate: Date[1], rel|$rel->relationNotEmpty()};\n" +
191+
" assertion: {businessDate: Date[1], rel|$rel->assertRelationNotEmpty()};\n" +
192192
" }\n" +
193193
" ];\n" +
194194
"}\n");

‎legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure-test/src/main/resources/core_dataquality_test/dataquality_relation_helper_test.pure‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,3 +86,22 @@ function <<test.Test>> meta::external::dataquality::tests::rowCountEqual():Boole
8686
$rel->meta::external::dataquality::rowCountEqual(3)->assert();
8787
}
8888

89+
function <<test.Test>> meta::external::dataquality::tests::assertRelationEmptyWhenEmpty():Boolean[1]
90+
{
91+
let rel = #TDS
92+
val, str
93+
94+
#;
95+
96+
$rel->meta::external::dataquality::assertRelationEmpty(~[val]);
97+
}
98+
99+
function <<test.Test>> meta::external::dataquality::tests::assertRelationNotEmptyWhenNotEmpty():Boolean[1]
100+
{
101+
let rel = #TDS
102+
val, str
103+
123, name
104+
#;
105+
106+
$rel->meta::external::dataquality::assertRelationNotEmpty();
107+
}

0 commit comments

Comments
 (0)